@genex-ai/cli-demo 1.36.7 → 1.36.8-dev.912
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +637 -221
- package/package.json +1 -1
- package/templates/skills/genex-cover/SKILL.md +3 -0
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-monetization/SKILL.md +1 -9
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -981,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
985
|
${CONTRACT_END}
|
|
986
986
|
`;
|
|
987
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -6344,7 +6344,9 @@ async function fetchOwnerProject(apiUrl, token, slug) {
|
|
|
6344
6344
|
return {
|
|
6345
6345
|
status: project.status ?? "draft",
|
|
6346
6346
|
coverUrl: project.thumbnailUrl ?? null,
|
|
6347
|
-
coverSource: "coverSource" in project ? project.coverSource ?? null : void 0
|
|
6347
|
+
coverSource: "coverSource" in project ? project.coverSource ?? null : void 0,
|
|
6348
|
+
clipUrl: project.coverClipUrl ?? null,
|
|
6349
|
+
clipSource: project.coverClipSource ?? null
|
|
6348
6350
|
};
|
|
6349
6351
|
} catch {
|
|
6350
6352
|
return null;
|
|
@@ -6525,6 +6527,134 @@ async function uploadCover(args) {
|
|
|
6525
6527
|
const view = { coverUrl: body?.coverUrl ?? null, coverSource: body?.coverSource ?? null };
|
|
6526
6528
|
return body?.applied === false ? { kind: "outranked", ...view } : { kind: "applied", ...view };
|
|
6527
6529
|
}
|
|
6530
|
+
var CLIP_MAX_BYTES = 8 * 1024 * 1024;
|
|
6531
|
+
var BUILD_COMMIT = /^[0-9a-f]{7,40}$/;
|
|
6532
|
+
function sniffClip(bytes) {
|
|
6533
|
+
if (bytes.byteLength < 16) return null;
|
|
6534
|
+
const b = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6535
|
+
if (b.toString("ascii", 4, 8) === "ftyp") return "video/mp4";
|
|
6536
|
+
if (b.readUInt32BE(0) === 440786851) return "video/webm";
|
|
6537
|
+
return null;
|
|
6538
|
+
}
|
|
6539
|
+
function normaliseBuildCommit(commit) {
|
|
6540
|
+
const sha = commit?.trim().toLowerCase() ?? "";
|
|
6541
|
+
return BUILD_COMMIT.test(sha) ? sha : null;
|
|
6542
|
+
}
|
|
6543
|
+
var CLIP_CLOSED = "nothing was uploaded";
|
|
6544
|
+
function clipRefusal(res, error, what) {
|
|
6545
|
+
const status2 = res.status;
|
|
6546
|
+
const wait = waitWords(res.headers.get("Retry-After"));
|
|
6547
|
+
if (status2 === 401) return "Genex refused this machine's sign-in \u2014 run `npx genex auth`, then try again.";
|
|
6548
|
+
if (error === "cover_clip_off") return `Cover clips are switched off on this Genex stand \u2014 ${CLIP_CLOSED}.`;
|
|
6549
|
+
if (error === "cover_clip_lane_closed") return `This Genex stand doesn't take cover clips from the CLI yet \u2014 ${CLIP_CLOSED}.`;
|
|
6550
|
+
if (status2 === 403) return "Only the game's owner can set its cover clip, and this machine is signed in as someone else.";
|
|
6551
|
+
if (status2 === 404) return "Genex doesn't know this game under this account \u2014 check `.genex/project.json` (or `npx genex link <slug>`).";
|
|
6552
|
+
if (error === "cover_clip_not_a_game") return "This project is an asset page, not a game \u2014 it takes no cover clip.";
|
|
6553
|
+
if (error === "cover_clip_taken_down") return "This game was taken down \u2014 it takes no cover clip.";
|
|
6554
|
+
if (error === "cover_clip_stale_build") {
|
|
6555
|
+
return "That --commit is not a build this game ships \u2014 record the build on its preview (draft) or live (published) channel and pass that build's commit.";
|
|
6556
|
+
}
|
|
6557
|
+
if (status2 === 413) return `That recording is too big for a cover clip (at most ${CLIP_MAX_BYTES / 1024 / 1024} MB).`;
|
|
6558
|
+
if (error === "cover_clip_rate_limited") {
|
|
6559
|
+
return `Too many cover clip uploads on this account for now (rate limited) \u2014 Genex takes the next one ${wait}.`;
|
|
6560
|
+
}
|
|
6561
|
+
if (error === "cover_clip_capacity") return "Today's cover clips are used up (daily cap) \u2014 Genex takes the next one after midnight UTC.";
|
|
6562
|
+
if (error === "cover_clip_busy") return `Genex is busy encoding another clip for this game \u2014 try again ${wait}.`;
|
|
6563
|
+
if (error === "cover_clip_upload_failed") return "Genex couldn't read the uploaded clip yet \u2014 try again.";
|
|
6564
|
+
if (error === "cover_clip_unavailable" || error === "cover_clip_encode_failed") {
|
|
6565
|
+
return "Genex's clip encoder is unavailable right now \u2014 try again in a few minutes.";
|
|
6566
|
+
}
|
|
6567
|
+
return `Couldn't ${what} (HTTP ${status2}) \u2014 try again.`;
|
|
6568
|
+
}
|
|
6569
|
+
async function uploadClip(args) {
|
|
6570
|
+
const { apiUrl, token, projectId, bytes, label } = args;
|
|
6571
|
+
const buildCommit = normaliseBuildCommit(args.buildCommit);
|
|
6572
|
+
if (!buildCommit) return { kind: "invalid", message: `--commit must be the build's git sha (7 to 40 hex characters).` };
|
|
6573
|
+
const type = sniffClip(bytes);
|
|
6574
|
+
if (!type) return { kind: "invalid", message: `${label} is not an MP4 or WebM video.` };
|
|
6575
|
+
if (bytes.byteLength > CLIP_MAX_BYTES) {
|
|
6576
|
+
return { kind: "invalid", message: `${label} is ${mb(bytes.byteLength)}; a cover clip recording can be at most ${CLIP_MAX_BYTES / 1024 / 1024} MB.` };
|
|
6577
|
+
}
|
|
6578
|
+
const headers = { "Content-Type": "application/json", Authorization: `Bearer ${token}` };
|
|
6579
|
+
const base = `${apiUrl}/api/projects/${encodeURIComponent(projectId)}/cover/clip`;
|
|
6580
|
+
const errorOf = async (res) => await res.json().catch(() => null);
|
|
6581
|
+
let minted;
|
|
6582
|
+
try {
|
|
6583
|
+
minted = await apiFetch(`${base}/upload-url`, { method: "POST", headers, body: JSON.stringify({ bytes: bytes.byteLength, contentType: type }) });
|
|
6584
|
+
} catch {
|
|
6585
|
+
return { kind: "failed", message: "Couldn't reach Genex to upload the cover clip." };
|
|
6586
|
+
}
|
|
6587
|
+
if (printedStructuredError(minted)) return { kind: "failed", message: null };
|
|
6588
|
+
if (!minted.ok) return { kind: "failed", message: clipRefusal(minted, (await errorOf(minted))?.error, "start the cover clip upload") };
|
|
6589
|
+
const grant = await minted.json().catch(() => null);
|
|
6590
|
+
if (!grant?.uploadUrl || !grant.key) {
|
|
6591
|
+
return { kind: "failed", message: "Genex answered the cover clip upload unexpectedly \u2014 nothing was uploaded." };
|
|
6592
|
+
}
|
|
6593
|
+
let put;
|
|
6594
|
+
try {
|
|
6595
|
+
put = await fetch(grant.uploadUrl, {
|
|
6596
|
+
method: "PUT",
|
|
6597
|
+
// Both are part of what the URL was signed for; send them back unchanged.
|
|
6598
|
+
headers: { "Content-Type": type, "Content-Length": String(bytes.byteLength) },
|
|
6599
|
+
body: new Uint8Array(bytes)
|
|
6600
|
+
});
|
|
6601
|
+
} catch {
|
|
6602
|
+
return { kind: "failed", message: "The cover clip upload didn't reach storage (network error) \u2014 try again." };
|
|
6603
|
+
}
|
|
6604
|
+
if (!put.ok) return { kind: "failed", message: `The cover clip upload failed (HTTP ${put.status}) \u2014 try again.` };
|
|
6605
|
+
let done;
|
|
6606
|
+
try {
|
|
6607
|
+
done = await apiFetch(`${base}/commit`, { method: "POST", headers, body: JSON.stringify({ key: grant.key, buildCommit }) });
|
|
6608
|
+
} catch {
|
|
6609
|
+
return { kind: "failed", message: "Couldn't reach Genex to finish the cover clip upload." };
|
|
6610
|
+
}
|
|
6611
|
+
if (printedStructuredError(done)) return { kind: "failed", message: null };
|
|
6612
|
+
const body = await done.json().catch(() => null);
|
|
6613
|
+
if (done.status === 422 && body?.error === "cover_clip_rejected") {
|
|
6614
|
+
return { kind: "rejected", reason: body.reason ?? "rejected", stats: body.stats ?? null };
|
|
6615
|
+
}
|
|
6616
|
+
if (!done.ok) return { kind: "failed", message: clipRefusal(done, body?.error, "set the cover clip") };
|
|
6617
|
+
const view = { clipUrl: body?.clipUrl ?? null, clipSource: body?.clipSource ?? null };
|
|
6618
|
+
return body?.applied === false ? { kind: "outranked", ...view } : { kind: "applied", ...view };
|
|
6619
|
+
}
|
|
6620
|
+
var CLIP_SOURCE_LABEL = {
|
|
6621
|
+
owner: "recorded by the owner",
|
|
6622
|
+
agent: "sent from the build (genex cover --clip)",
|
|
6623
|
+
owner_play: "recorded while the owner played"
|
|
6624
|
+
};
|
|
6625
|
+
function describeClipSource(source) {
|
|
6626
|
+
if (source === null) return "none";
|
|
6627
|
+
return CLIP_SOURCE_LABEL[source] ?? source;
|
|
6628
|
+
}
|
|
6629
|
+
var CLIP_REJECTION = {
|
|
6630
|
+
too_dark: "is too dark to read as the game on a card",
|
|
6631
|
+
flat: "is almost one flat colour \u2014 a loading screen or a fade reads as nothing",
|
|
6632
|
+
static: "barely moves \u2014 a frozen or paused screen is not a clip",
|
|
6633
|
+
flashing: "flashes too fast to show anyone (a strobe can hurt whoever watches it)",
|
|
6634
|
+
too_short: "is too short (record 5\u20136 s)",
|
|
6635
|
+
too_long: "is too long (at most 12 s)"
|
|
6636
|
+
};
|
|
6637
|
+
function reportClipOutcome(log, outcome2, opts) {
|
|
6638
|
+
switch (outcome2.kind) {
|
|
6639
|
+
case "applied":
|
|
6640
|
+
log.success(`Cover clip set \u2014 ${opts.label} is this game's cover clip now.`);
|
|
6641
|
+
if (outcome2.clipUrl) log.dim(` ${outcome2.clipUrl}`);
|
|
6642
|
+
return;
|
|
6643
|
+
case "outranked":
|
|
6644
|
+
log.info("Cover clip unchanged \u2014 the owner chose this game's clip themselves (or removed it), and that choice stands. Nothing to retry.");
|
|
6645
|
+
return;
|
|
6646
|
+
case "rejected":
|
|
6647
|
+
log.error(`Cover clip not used \u2014 ${opts.label} ${CLIP_REJECTION[outcome2.reason] ?? `was refused (${outcome2.reason})`}.`);
|
|
6648
|
+
log.dim(" Record 5\u20136 s of real, moving, lit gameplay and run this again \u2014 never change the game for a clip.");
|
|
6649
|
+
return;
|
|
6650
|
+
case "invalid":
|
|
6651
|
+
log.error(`Cover clip not used \u2014 ${outcome2.message}`);
|
|
6652
|
+
return;
|
|
6653
|
+
case "failed":
|
|
6654
|
+
if (outcome2.message) log.error(outcome2.message);
|
|
6655
|
+
return;
|
|
6656
|
+
}
|
|
6657
|
+
}
|
|
6528
6658
|
var pct = (n) => `${Math.round(n * 100)}%`;
|
|
6529
6659
|
function reportCoverOutcome(log, outcome2, opts) {
|
|
6530
6660
|
const fail5 = (m) => opts.asError ? log.error(m) : log.warn(m);
|
|
@@ -8824,6 +8954,11 @@ async function runCover(opts) {
|
|
|
8824
8954
|
const apiUrl = getApiUrl(opts.apiUrl ?? meta.apiUrl);
|
|
8825
8955
|
const token = await readUserToken(opts.envPath, apiUrl);
|
|
8826
8956
|
if (!token) return fail2("Not signed in. Run `genex auth` first.", opts, log);
|
|
8957
|
+
const clip = opts.clipPath?.trim();
|
|
8958
|
+
if (clip || opts.commit !== void 0) {
|
|
8959
|
+
await sendClip(apiUrl, token, meta.id, clip, opts, log);
|
|
8960
|
+
return;
|
|
8961
|
+
}
|
|
8827
8962
|
const file = opts.imagePath?.trim();
|
|
8828
8963
|
if (!file) {
|
|
8829
8964
|
await showCover(apiUrl, token, meta.slug, opts, log);
|
|
@@ -8860,8 +8995,36 @@ async function rememberNomination(bytes, file, outcome2) {
|
|
|
8860
8995
|
const stale = await fs24.readFile(scratch).catch(() => null);
|
|
8861
8996
|
if (stale) await rememberCoverUpload(stale, scratch, outcome2);
|
|
8862
8997
|
}
|
|
8998
|
+
async function sendClip(apiUrl, token, projectId, clip, opts, log) {
|
|
8999
|
+
const file = clip ?? "";
|
|
9000
|
+
const invalid = (message) => finishClip({ kind: "invalid", message }, file, opts, log);
|
|
9001
|
+
if (!clip) return invalid("--commit goes with --clip <file>: pass the recording to send.");
|
|
9002
|
+
if (opts.imagePath?.trim()) return invalid("Pass either an image (the cover) or --clip <file> (the cover clip), not both.");
|
|
9003
|
+
if (!opts.commit?.trim()) return invalid("--clip needs --commit <sha>: the git commit of the build the recording shows.");
|
|
9004
|
+
let bytes;
|
|
9005
|
+
try {
|
|
9006
|
+
const stat = await fs24.stat(clip);
|
|
9007
|
+
if (!stat.isFile()) throw new Error("not a file");
|
|
9008
|
+
if (stat.size > CLIP_MAX_BYTES) {
|
|
9009
|
+
return invalid(`${clip} is ${(stat.size / 1024 / 1024).toFixed(1)} MB; a cover clip recording can be at most ${CLIP_MAX_BYTES / 1024 / 1024} MB.`);
|
|
9010
|
+
}
|
|
9011
|
+
bytes = await fs24.readFile(clip);
|
|
9012
|
+
} catch {
|
|
9013
|
+
return invalid(`Can't read ${clip} \u2014 pass the path of an MP4 or WebM recording.`);
|
|
9014
|
+
}
|
|
9015
|
+
log.step(`Uploading ${path24.basename(clip)} as the cover clip\u2026`);
|
|
9016
|
+
const outcome2 = await uploadClip({ apiUrl, token, projectId, bytes, label: clip, buildCommit: opts.commit });
|
|
9017
|
+
finishClip(outcome2, clip, opts, log);
|
|
9018
|
+
}
|
|
9019
|
+
function finishClip(outcome2, file, opts, log) {
|
|
9020
|
+
if (opts.json) process.stdout.write(`${JSON.stringify({ file: file || null, ...outcome2 })}
|
|
9021
|
+
`);
|
|
9022
|
+
else reportClipOutcome(log, outcome2, { label: file });
|
|
9023
|
+
if (outcome2.kind !== "applied" && outcome2.kind !== "outranked") process.exitCode = 1;
|
|
9024
|
+
}
|
|
8863
9025
|
function fail2(message, opts, log) {
|
|
8864
|
-
|
|
9026
|
+
const file = opts.clipPath?.trim() || opts.imagePath?.trim() || null;
|
|
9027
|
+
if (opts.json) process.stdout.write(`${JSON.stringify({ file, kind: "failed", message })}
|
|
8865
9028
|
`);
|
|
8866
9029
|
else log.error(message);
|
|
8867
9030
|
process.exitCode = 1;
|
|
@@ -8884,7 +9047,8 @@ async function showCover(apiUrl, token, slug, opts, log) {
|
|
|
8884
9047
|
const view = await fetchOwnerProject(apiUrl, token, slug);
|
|
8885
9048
|
if (!view) return fail2("Couldn't read this game from Genex \u2014 check the sign-in (`genex doctor`) and try again.", opts, log);
|
|
8886
9049
|
if (opts.json) {
|
|
8887
|
-
|
|
9050
|
+
const clip = { clipUrl: view.clipUrl, clipSource: view.clipSource };
|
|
9051
|
+
process.stdout.write(`${JSON.stringify({ coverUrl: view.coverUrl, coverSource: view.coverSource ?? null, ...clip })}
|
|
8888
9052
|
`);
|
|
8889
9053
|
return;
|
|
8890
9054
|
}
|
|
@@ -8894,6 +9058,10 @@ async function showCover(apiUrl, token, slug, opts, log) {
|
|
|
8894
9058
|
log.plain(` cover: ${c.cyan(view.coverUrl)}`);
|
|
8895
9059
|
log.plain(` source: ${describeCoverSource(view.coverSource)}`);
|
|
8896
9060
|
}
|
|
9061
|
+
if (view.clipUrl) {
|
|
9062
|
+
log.plain(` clip: ${c.cyan(view.clipUrl)}`);
|
|
9063
|
+
log.plain(` source: ${describeClipSource(view.clipSource)}`);
|
|
9064
|
+
}
|
|
8897
9065
|
if (!view.coverUrl || coverIsStandIn(view.coverSource)) {
|
|
8898
9066
|
log.plain("");
|
|
8899
9067
|
log.dim(` A cover is ${WHAT_A_COVER_IS}.`);
|
|
@@ -23441,7 +23609,7 @@ async function runBlender(opts) {
|
|
|
23441
23609
|
return 1;
|
|
23442
23610
|
}
|
|
23443
23611
|
if (sub === "serve") {
|
|
23444
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23612
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
23445
23613
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23446
23614
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23447
23615
|
log.plain(
|
|
@@ -23450,7 +23618,7 @@ async function runBlender(opts) {
|
|
|
23450
23618
|
return serveLocalBlender({ port, log });
|
|
23451
23619
|
}
|
|
23452
23620
|
if (sub === "mcp") {
|
|
23453
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23621
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
23454
23622
|
return runBlenderMcp();
|
|
23455
23623
|
}
|
|
23456
23624
|
if (sub === "seat") {
|
|
@@ -23958,7 +24126,12 @@ import fs35 from "fs/promises";
|
|
|
23958
24126
|
import path36 from "path";
|
|
23959
24127
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23960
24128
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23961
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24129
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24130
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24131
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24132
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24133
|
+
}
|
|
24134
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23962
24135
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23963
24136
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23964
24137
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23971,7 +24144,7 @@ function serverSlotCounts(body) {
|
|
|
23971
24144
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23972
24145
|
}
|
|
23973
24146
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23974
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24147
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23975
24148
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23976
24149
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23977
24150
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23985,11 +24158,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23985
24158
|
function parseProviderKey(value) {
|
|
23986
24159
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23987
24160
|
}
|
|
24161
|
+
function bpsOrNull(value) {
|
|
24162
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24163
|
+
}
|
|
24164
|
+
function standingCreditGrantOrNull(value) {
|
|
24165
|
+
if (!value || typeof value !== "object") return null;
|
|
24166
|
+
const v = value;
|
|
24167
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24168
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24169
|
+
return {
|
|
24170
|
+
minCredits: v.minCredits,
|
|
24171
|
+
maxCredits: v.maxCredits,
|
|
24172
|
+
stepCredits: v.stepCredits,
|
|
24173
|
+
defaultCredits: v.defaultCredits,
|
|
24174
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24175
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24176
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24177
|
+
};
|
|
24178
|
+
}
|
|
23988
24179
|
async function readRuntimeLane(apiUrl, token) {
|
|
23989
24180
|
const empty = (state) => ({
|
|
23990
24181
|
state,
|
|
23991
24182
|
models: null,
|
|
23992
24183
|
recommendedDeclaredHeadroomBps: null,
|
|
24184
|
+
currency: null,
|
|
24185
|
+
feeBps: null,
|
|
24186
|
+
usdCentsPerCredit: null,
|
|
24187
|
+
standingCreditGrant: null,
|
|
23993
24188
|
externalProviders: null,
|
|
23994
24189
|
total: null,
|
|
23995
24190
|
featuredCount: null,
|
|
@@ -24008,7 +24203,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24008
24203
|
return {
|
|
24009
24204
|
state: "live",
|
|
24010
24205
|
models: body.models,
|
|
24011
|
-
recommendedDeclaredHeadroomBps:
|
|
24206
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
24207
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24208
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
24209
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24210
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24012
24211
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24013
24212
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24014
24213
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24024,18 +24223,45 @@ function percentile2(values, p) {
|
|
|
24024
24223
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24025
24224
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24026
24225
|
}
|
|
24027
|
-
function
|
|
24028
|
-
|
|
24226
|
+
function percentilePicos(values, p) {
|
|
24227
|
+
if (values.length === 0) return null;
|
|
24228
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24229
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24230
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24029
24231
|
}
|
|
24030
|
-
|
|
24031
|
-
|
|
24232
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
24233
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
24234
|
+
if (costPicos <= 0n) return 1;
|
|
24235
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24236
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24237
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24238
|
+
}
|
|
24239
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
24240
|
+
if (costsPicos.length === 0) return null;
|
|
24241
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24242
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24243
|
+
}
|
|
24244
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24245
|
+
if (costsPicos.length === 0) return null;
|
|
24246
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24247
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24248
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24249
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24250
|
+
}
|
|
24251
|
+
function formatCredits(value) {
|
|
24252
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
24253
|
+
if (value >= 100) return String(Math.round(value));
|
|
24254
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24255
|
+
}
|
|
24256
|
+
function usdFromPicos(picos) {
|
|
24257
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
24258
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24259
|
+
if (value === null) return "unknown";
|
|
24260
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24032
24261
|
}
|
|
24033
24262
|
function usdPerMillion(value) {
|
|
24034
24263
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24035
24264
|
}
|
|
24036
|
-
function usd(value) {
|
|
24037
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24038
|
-
}
|
|
24039
24265
|
async function runLlm(opts = {}) {
|
|
24040
24266
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24041
24267
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24061,16 +24287,23 @@ async function runLlm(opts = {}) {
|
|
|
24061
24287
|
return;
|
|
24062
24288
|
}
|
|
24063
24289
|
if (sub === "bench") {
|
|
24064
|
-
if (opts.maxCoins
|
|
24290
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
24291
|
+
`);
|
|
24292
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24293
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24294
|
+
return;
|
|
24295
|
+
}
|
|
24296
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24297
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24065
24298
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24066
24299
|
return;
|
|
24067
24300
|
}
|
|
24068
|
-
if (!Number.isInteger(
|
|
24069
|
-
fail5("llm bench", `--max-
|
|
24301
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
24302
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
24070
24303
|
return;
|
|
24071
24304
|
}
|
|
24072
24305
|
if (!opts.benchPrompt?.trim()) {
|
|
24073
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24306
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
24074
24307
|
return;
|
|
24075
24308
|
}
|
|
24076
24309
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24162,6 +24395,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24162
24395
|
models: null,
|
|
24163
24396
|
externalProviders: null,
|
|
24164
24397
|
recommendedDeclaredHeadroomBps: null,
|
|
24398
|
+
currency: null,
|
|
24399
|
+
feeBps: null,
|
|
24400
|
+
usdCentsPerCredit: null,
|
|
24401
|
+
standingCreditGrant: null,
|
|
24165
24402
|
total: null,
|
|
24166
24403
|
featuredCount: null,
|
|
24167
24404
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24187,6 +24424,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24187
24424
|
models: lane.models,
|
|
24188
24425
|
externalProviders: lane.externalProviders,
|
|
24189
24426
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24427
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
24428
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
24429
|
+
currency: lane.currency,
|
|
24430
|
+
feeBps: lane.feeBps,
|
|
24431
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24432
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
24190
24433
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24191
24434
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24192
24435
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24233,8 +24476,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24233
24476
|
);
|
|
24234
24477
|
for (const m of shown) {
|
|
24235
24478
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24479
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24236
24480
|
log.plain(
|
|
24237
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24481
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
24238
24482
|
);
|
|
24239
24483
|
}
|
|
24240
24484
|
if (hiddenCount > 0) {
|
|
@@ -24250,74 +24494,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24250
24494
|
}
|
|
24251
24495
|
log.plain("");
|
|
24252
24496
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24253
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24497
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
24498
|
+
} else {
|
|
24499
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24500
|
+
}
|
|
24501
|
+
if (lane.feeBps === null) {
|
|
24502
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24254
24503
|
} else {
|
|
24255
|
-
log.dim(`
|
|
24504
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24256
24505
|
}
|
|
24257
24506
|
if (toolsOnly) {
|
|
24258
24507
|
printConvertHint();
|
|
24259
24508
|
return;
|
|
24260
24509
|
}
|
|
24261
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
24262
|
-
log.dim(`
|
|
24510
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
24511
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
24263
24512
|
}
|
|
24264
24513
|
function wholeOrNull(value) {
|
|
24265
24514
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24266
24515
|
}
|
|
24516
|
+
function costPicosOf(usage) {
|
|
24517
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24518
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24519
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24520
|
+
}
|
|
24521
|
+
return null;
|
|
24522
|
+
}
|
|
24267
24523
|
function sampleFrom(id, settled) {
|
|
24268
24524
|
const usage = settled?.usage;
|
|
24525
|
+
const picos = costPicosOf(usage);
|
|
24269
24526
|
return {
|
|
24270
24527
|
id,
|
|
24271
24528
|
status: settled?.status ?? null,
|
|
24272
24529
|
billingStatus: settled?.billingStatus ?? null,
|
|
24273
|
-
|
|
24274
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24530
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
24531
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
24532
|
+
costUsdPicos: picos,
|
|
24275
24533
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24276
24534
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24277
24535
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24278
24536
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24279
24537
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24280
|
-
|
|
24538
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
24281
24539
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24282
24540
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24283
24541
|
};
|
|
24284
24542
|
}
|
|
24285
24543
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24286
|
-
function benchRecommendation(results,
|
|
24287
|
-
const settled = results.filter(
|
|
24288
|
-
|
|
24289
|
-
|
|
24290
|
-
const
|
|
24291
|
-
const
|
|
24544
|
+
function benchRecommendation(results, terms) {
|
|
24545
|
+
const settled = results.filter(
|
|
24546
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
24547
|
+
);
|
|
24548
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
24549
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
24550
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24551
|
+
const costP95 = percentilePicos(costs, 95);
|
|
24552
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24292
24553
|
const fitP95 = percentile2(fits, 95);
|
|
24293
24554
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24294
24555
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24295
24556
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24296
24557
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24297
|
-
const
|
|
24298
|
-
const
|
|
24299
|
-
const
|
|
24558
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
24559
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
24560
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
24561
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24562
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24563
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24564
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24565
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24566
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24300
24567
|
return {
|
|
24568
|
+
costs,
|
|
24301
24569
|
charged,
|
|
24302
|
-
|
|
24303
|
-
|
|
24304
|
-
|
|
24570
|
+
costP50: percentilePicos(costs, 50),
|
|
24571
|
+
costP95,
|
|
24572
|
+
costMax,
|
|
24573
|
+
chargedP50: percentile2(charged, 50),
|
|
24574
|
+
chargedP95: percentile2(charged, 95),
|
|
24575
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24305
24576
|
fits,
|
|
24306
24577
|
fitP95,
|
|
24307
24578
|
fitMax,
|
|
24308
|
-
|
|
24309
|
-
|
|
24310
|
-
|
|
24311
|
-
|
|
24579
|
+
costBasedMaxCredits: costBased,
|
|
24580
|
+
recommendedMaxCredits: recommended,
|
|
24581
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
24582
|
+
averageCreditsPerCall: average,
|
|
24583
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
24584
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24312
24585
|
cutOffSamples,
|
|
24313
24586
|
exceedsStandCap,
|
|
24314
|
-
noRecommendationReason
|
|
24587
|
+
noRecommendationReason,
|
|
24588
|
+
missingTerms
|
|
24315
24589
|
};
|
|
24316
24590
|
}
|
|
24317
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24318
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24319
|
-
function benchCutOffLine(cutOff,
|
|
24320
|
-
return `No
|
|
24591
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
24592
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24593
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
24594
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
24321
24595
|
}
|
|
24322
24596
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24323
24597
|
function rowState(row) {
|
|
@@ -24328,7 +24602,22 @@ function rowState(row) {
|
|
|
24328
24602
|
function rowHoldsSlot(row) {
|
|
24329
24603
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24330
24604
|
const state = rowState(row);
|
|
24331
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24605
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
24606
|
+
}
|
|
24607
|
+
function rowMoney(row) {
|
|
24608
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24609
|
+
if (credits) {
|
|
24610
|
+
return {
|
|
24611
|
+
unit: "credits",
|
|
24612
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24613
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24614
|
+
};
|
|
24615
|
+
}
|
|
24616
|
+
return {
|
|
24617
|
+
unit: "coin",
|
|
24618
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24619
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24620
|
+
};
|
|
24332
24621
|
}
|
|
24333
24622
|
function providerRefusalStatus(error) {
|
|
24334
24623
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24347,7 +24636,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24347
24636
|
}
|
|
24348
24637
|
async function runBench(args) {
|
|
24349
24638
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24350
|
-
const
|
|
24639
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24351
24640
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24352
24641
|
const prompt = opts.benchPrompt.trim();
|
|
24353
24642
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24369,7 +24658,7 @@ async function runBench(args) {
|
|
|
24369
24658
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24370
24659
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24371
24660
|
if (lane.state === "off") {
|
|
24372
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24661
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
24373
24662
|
else {
|
|
24374
24663
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24375
24664
|
log.dim(" Nothing was spent.");
|
|
@@ -24396,17 +24685,27 @@ async function runBench(args) {
|
|
|
24396
24685
|
);
|
|
24397
24686
|
return;
|
|
24398
24687
|
}
|
|
24399
|
-
const
|
|
24688
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24689
|
+
if (walletBefore?.emailVerified === false) {
|
|
24690
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24691
|
+
return;
|
|
24692
|
+
}
|
|
24693
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24694
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24695
|
+
return;
|
|
24696
|
+
}
|
|
24697
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24400
24698
|
if (!opts.json) {
|
|
24401
24699
|
log.plain(c.bold("genex llm bench"));
|
|
24402
24700
|
log.dim(` ${apiUrl}`);
|
|
24403
24701
|
log.plain("");
|
|
24404
24702
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24405
24703
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24406
|
-
log.plain(` Approved ${
|
|
24704
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
24407
24705
|
log.plain(
|
|
24408
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24706
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
24409
24707
|
);
|
|
24708
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24410
24709
|
log.plain("");
|
|
24411
24710
|
}
|
|
24412
24711
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24432,7 +24731,7 @@ async function runBench(args) {
|
|
|
24432
24731
|
const res = await call(base, {
|
|
24433
24732
|
method: "POST",
|
|
24434
24733
|
body: JSON.stringify({
|
|
24435
|
-
|
|
24734
|
+
maxCredits,
|
|
24436
24735
|
request: {
|
|
24437
24736
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24438
24737
|
modelId,
|
|
@@ -24462,13 +24761,14 @@ async function runBench(args) {
|
|
|
24462
24761
|
if (refusedAt !== null) providerRefused++;
|
|
24463
24762
|
if (!opts.json) {
|
|
24464
24763
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24764
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24465
24765
|
log.plain(
|
|
24466
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24766
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24467
24767
|
);
|
|
24468
24768
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24469
24769
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24470
24770
|
log.dim(
|
|
24471
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24771
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24472
24772
|
);
|
|
24473
24773
|
}
|
|
24474
24774
|
}
|
|
@@ -24476,16 +24776,19 @@ async function runBench(args) {
|
|
|
24476
24776
|
} finally {
|
|
24477
24777
|
process.removeListener("SIGINT", onSigint);
|
|
24478
24778
|
}
|
|
24479
|
-
const
|
|
24480
|
-
|
|
24481
|
-
|
|
24482
|
-
|
|
24483
|
-
|
|
24484
|
-
const
|
|
24779
|
+
const terms = {
|
|
24780
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24781
|
+
feeBps: lane.feeBps,
|
|
24782
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24783
|
+
};
|
|
24784
|
+
const rec = benchRecommendation(results, terms);
|
|
24785
|
+
const settledCount = rec.charged.length;
|
|
24485
24786
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24486
|
-
const balanceAfter = await
|
|
24787
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24788
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24487
24789
|
const record = {
|
|
24488
|
-
v:
|
|
24790
|
+
v: 2,
|
|
24791
|
+
currency: "credits",
|
|
24489
24792
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24490
24793
|
apiUrl,
|
|
24491
24794
|
projectId,
|
|
@@ -24494,28 +24797,34 @@ async function runBench(args) {
|
|
|
24494
24797
|
outputFormat,
|
|
24495
24798
|
samples,
|
|
24496
24799
|
completed: settledCount,
|
|
24497
|
-
|
|
24498
|
-
|
|
24499
|
-
|
|
24500
|
-
max,
|
|
24501
|
-
|
|
24502
|
-
|
|
24503
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24504
|
-
maxCoinsPerSample: maxCoins,
|
|
24505
|
-
fitCoins: rec.fits,
|
|
24800
|
+
maxCreditsPerSample: maxCredits,
|
|
24801
|
+
costUsdPicos: rec.costs.map(String),
|
|
24802
|
+
chargedCredits: rec.charged,
|
|
24803
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24804
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24805
|
+
fitCredits: rec.fits,
|
|
24506
24806
|
fitP95: rec.fitP95,
|
|
24507
|
-
|
|
24508
|
-
|
|
24807
|
+
fitMax: rec.fitMax,
|
|
24808
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24809
|
+
feeBps: terms.feeBps,
|
|
24810
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24811
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24812
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24813
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24814
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24815
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24816
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24509
24817
|
cutOffSamples: rec.cutOffSamples,
|
|
24510
24818
|
noRecommendationReason: rec.noRecommendationReason
|
|
24511
24819
|
};
|
|
24512
24820
|
const savedTo = await saveBench(cwd, record);
|
|
24513
24821
|
if (opts.json) {
|
|
24514
|
-
const ok =
|
|
24822
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24515
24823
|
writeJsonLine({
|
|
24516
24824
|
command: "llm bench",
|
|
24517
24825
|
status: ok ? "ok" : "failed",
|
|
24518
24826
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24827
|
+
currency: "credits",
|
|
24519
24828
|
apiUrl,
|
|
24520
24829
|
projectId,
|
|
24521
24830
|
modelId,
|
|
@@ -24524,35 +24833,54 @@ async function runBench(args) {
|
|
|
24524
24833
|
schemaPath: opts.schemaPath ?? null,
|
|
24525
24834
|
samples,
|
|
24526
24835
|
completed: settledCount,
|
|
24527
|
-
|
|
24836
|
+
maxCreditsPerSample: maxCredits,
|
|
24837
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24528
24838
|
balanceBefore,
|
|
24529
24839
|
balanceAfter,
|
|
24530
24840
|
results,
|
|
24531
24841
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24532
24842
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24533
24843
|
refusal: refusal2,
|
|
24534
|
-
|
|
24535
|
-
//
|
|
24844
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
24845
|
+
// and USD) and the whole credits charged.
|
|
24846
|
+
costUsdPicos: record.cost,
|
|
24847
|
+
costUsd: {
|
|
24848
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24849
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24850
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24851
|
+
},
|
|
24852
|
+
chargedCredits: record.charged,
|
|
24853
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24536
24854
|
// headroom already on the tokens), over the settled samples.
|
|
24537
|
-
|
|
24538
|
-
|
|
24539
|
-
|
|
24855
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
24856
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24857
|
+
feeBps: terms.feeBps,
|
|
24858
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24859
|
+
missingTerms: rec.missingTerms,
|
|
24860
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24861
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24540
24862
|
cutOffSamples: rec.cutOffSamples,
|
|
24541
24863
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24542
24864
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24543
|
-
|
|
24544
|
-
|
|
24545
|
-
|
|
24865
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24866
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24867
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24868
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24546
24869
|
savedTo
|
|
24547
24870
|
});
|
|
24548
24871
|
if (!ok) process.exitCode = 1;
|
|
24549
24872
|
return;
|
|
24550
24873
|
}
|
|
24551
24874
|
log.plain("");
|
|
24875
|
+
const printMeasured = () => {
|
|
24876
|
+
log.plain(c.bold(" Real cost per call"));
|
|
24877
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24878
|
+
log.plain(c.bold(" Charged credits"));
|
|
24879
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24880
|
+
};
|
|
24552
24881
|
if (lengthBlocked) {
|
|
24553
|
-
if (
|
|
24554
|
-
|
|
24555
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24882
|
+
if (settledCount > 0) {
|
|
24883
|
+
printMeasured();
|
|
24556
24884
|
log.plain("");
|
|
24557
24885
|
}
|
|
24558
24886
|
printRecommendation(log, record);
|
|
@@ -24560,17 +24888,16 @@ async function runBench(args) {
|
|
|
24560
24888
|
process.exitCode = 1;
|
|
24561
24889
|
return;
|
|
24562
24890
|
}
|
|
24563
|
-
if (
|
|
24891
|
+
if (settledCount === 0) {
|
|
24564
24892
|
log.error(
|
|
24565
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24893
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24566
24894
|
);
|
|
24567
24895
|
process.exitCode = 1;
|
|
24568
24896
|
return;
|
|
24569
24897
|
}
|
|
24570
|
-
|
|
24571
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24898
|
+
printMeasured();
|
|
24572
24899
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24573
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24900
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24574
24901
|
}
|
|
24575
24902
|
log.plain("");
|
|
24576
24903
|
printRecommendation(log, record);
|
|
@@ -24582,43 +24909,58 @@ function printRecommendation(log, record) {
|
|
|
24582
24909
|
return;
|
|
24583
24910
|
}
|
|
24584
24911
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24585
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24912
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24586
24913
|
return;
|
|
24587
24914
|
}
|
|
24588
|
-
|
|
24589
|
-
|
|
24915
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
24916
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
24917
|
+
log.warn(" No ceiling recommended.");
|
|
24590
24918
|
log.dim(
|
|
24591
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
24919
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24592
24920
|
);
|
|
24593
24921
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24922
|
+
printAverage(log, record, attempts);
|
|
24594
24923
|
return;
|
|
24595
24924
|
}
|
|
24596
|
-
log.plain(c.bold(` Declare
|
|
24597
|
-
if (record.
|
|
24925
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
24926
|
+
if (record.lengthRaisedCeiling) {
|
|
24598
24927
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24599
24928
|
log.dim(
|
|
24600
|
-
` = the smallest
|
|
24929
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24601
24930
|
);
|
|
24602
24931
|
log.dim(
|
|
24603
|
-
`
|
|
24932
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24604
24933
|
);
|
|
24605
24934
|
} else {
|
|
24606
24935
|
log.dim(
|
|
24607
|
-
` = p95
|
|
24936
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24608
24937
|
);
|
|
24609
|
-
log.dim(` measured over ${
|
|
24938
|
+
log.dim(` measured over ${attempts}.`);
|
|
24610
24939
|
}
|
|
24611
|
-
log.dim(" That number is a
|
|
24612
|
-
|
|
24613
|
-
|
|
24940
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
24941
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
24942
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
24943
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24614
24944
|
log.dim(
|
|
24615
|
-
record.
|
|
24945
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24616
24946
|
);
|
|
24617
|
-
|
|
24947
|
+
}
|
|
24948
|
+
printAverage(log, record, attempts);
|
|
24949
|
+
}
|
|
24950
|
+
function printAverage(log, record, attempts) {
|
|
24951
|
+
if (record.averageCreditsPerCall === null) return;
|
|
24952
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24953
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24954
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24955
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24956
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24618
24957
|
}
|
|
24619
24958
|
}
|
|
24620
24959
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24621
|
-
if (printedStructuredError(res))
|
|
24960
|
+
if (printedStructuredError(res)) {
|
|
24961
|
+
const printed = await res.json().catch(() => ({}));
|
|
24962
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24963
|
+
}
|
|
24622
24964
|
const body = await res.json().catch(() => ({}));
|
|
24623
24965
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24624
24966
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24628,6 +24970,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24628
24970
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24629
24971
|
return { stop: true, refusal: refusal2 };
|
|
24630
24972
|
}
|
|
24973
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24974
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24975
|
+
return { stop: true, refusal: refusal2 };
|
|
24976
|
+
}
|
|
24631
24977
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24632
24978
|
if (res.status === 404 && body.error === "not_found") {
|
|
24633
24979
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24643,7 +24989,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24643
24989
|
return { stop: true, refusal: refusal2 };
|
|
24644
24990
|
}
|
|
24645
24991
|
if (res.status === 402) {
|
|
24646
|
-
log.error(" Not enough
|
|
24992
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24647
24993
|
return { stop: true, refusal: refusal2 };
|
|
24648
24994
|
}
|
|
24649
24995
|
log.error(
|
|
@@ -24652,7 +24998,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24652
24998
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24653
24999
|
}
|
|
24654
25000
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24655
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
25001
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24656
25002
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24657
25003
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24658
25004
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24678,19 +25024,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24678
25024
|
}
|
|
24679
25025
|
return last;
|
|
24680
25026
|
}
|
|
24681
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24682
|
-
try {
|
|
24683
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24684
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24685
|
-
signal: AbortSignal.timeout(6e3)
|
|
24686
|
-
});
|
|
24687
|
-
if (!res.ok) return null;
|
|
24688
|
-
const body = await res.json().catch(() => null);
|
|
24689
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24690
|
-
} catch {
|
|
24691
|
-
return null;
|
|
24692
|
-
}
|
|
24693
|
-
}
|
|
24694
25027
|
async function saveBench(cwd, record) {
|
|
24695
25028
|
try {
|
|
24696
25029
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24706,9 +25039,10 @@ function benchFailed(opts, log, message) {
|
|
|
24706
25039
|
else log.error(message);
|
|
24707
25040
|
process.exitCode = 1;
|
|
24708
25041
|
}
|
|
24709
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25042
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
24710
25043
|
return {
|
|
24711
25044
|
command: "llm bench",
|
|
25045
|
+
currency: "credits",
|
|
24712
25046
|
apiUrl,
|
|
24713
25047
|
projectId,
|
|
24714
25048
|
modelId: opts.modelId ?? null,
|
|
@@ -24717,21 +25051,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
24717
25051
|
schemaPath: opts.schemaPath ?? null,
|
|
24718
25052
|
samples,
|
|
24719
25053
|
completed: 0,
|
|
24720
|
-
|
|
25054
|
+
maxCreditsPerSample: maxCredits,
|
|
24721
25055
|
balanceBefore: null,
|
|
24722
25056
|
balanceAfter: null,
|
|
24723
25057
|
results: [],
|
|
24724
25058
|
refusal: null,
|
|
24725
|
-
|
|
24726
|
-
|
|
24727
|
-
|
|
24728
|
-
|
|
25059
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
25060
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
25061
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
25062
|
+
fitCredits: { p95: null, max: null },
|
|
25063
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25064
|
+
feeBps: null,
|
|
25065
|
+
usdCentsPerCredit: null,
|
|
25066
|
+
missingTerms: null,
|
|
25067
|
+
costBasedMaxCredits: null,
|
|
25068
|
+
lengthRaisedCeiling: false,
|
|
24729
25069
|
cutOffSamples: 0,
|
|
24730
25070
|
exceedsStandCap: false,
|
|
24731
25071
|
noRecommendationReason: null,
|
|
24732
|
-
|
|
24733
|
-
|
|
24734
|
-
|
|
25072
|
+
recommendedMaxCredits: null,
|
|
25073
|
+
recommendedPerCallMaxCredits: null,
|
|
25074
|
+
averageCreditsPerCall: null,
|
|
25075
|
+
recommendedPerCallEstimateCredits: null,
|
|
24735
25076
|
savedTo: null
|
|
24736
25077
|
};
|
|
24737
25078
|
}
|
|
@@ -24794,7 +25135,8 @@ async function reportStatus(args) {
|
|
|
24794
25135
|
for (const row of rows) {
|
|
24795
25136
|
const state = rowState(row);
|
|
24796
25137
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24797
|
-
const
|
|
25138
|
+
const money2 = rowMoney(row);
|
|
25139
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24798
25140
|
log.plain(
|
|
24799
25141
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24800
25142
|
);
|
|
@@ -24866,10 +25208,11 @@ async function cancelCall(args) {
|
|
|
24866
25208
|
if (!opts.json) {
|
|
24867
25209
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
24868
25210
|
if (state === "awaiting bill") {
|
|
24869
|
-
log.dim(` Nothing to cancel:
|
|
25211
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
24870
25212
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
24871
25213
|
} else {
|
|
24872
|
-
|
|
25214
|
+
const money2 = rowMoney(row);
|
|
25215
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24873
25216
|
}
|
|
24874
25217
|
}
|
|
24875
25218
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -24891,78 +25234,126 @@ async function cancelCall(args) {
|
|
|
24891
25234
|
const after = await res.json().catch(() => null) ?? row;
|
|
24892
25235
|
if (!opts.json) {
|
|
24893
25236
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
24894
|
-
log.dim(`
|
|
25237
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
24895
25238
|
}
|
|
24896
25239
|
return done("ok", null, "cancelled", after);
|
|
24897
25240
|
}
|
|
24898
|
-
async function
|
|
24899
|
-
let
|
|
25241
|
+
async function readSavedBench(cwd) {
|
|
25242
|
+
let parsed;
|
|
24900
25243
|
try {
|
|
24901
|
-
|
|
25244
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
24902
25245
|
} catch {
|
|
24903
|
-
|
|
25246
|
+
return null;
|
|
24904
25247
|
}
|
|
24905
|
-
if (!
|
|
25248
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
25249
|
+
const v = parsed;
|
|
25250
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25251
|
+
return { kind: "coins", record: parsed };
|
|
25252
|
+
}
|
|
25253
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25254
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25255
|
+
const saved = await readSavedBench(cwd);
|
|
25256
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25257
|
+
const creditKeys = (record2) => ({
|
|
25258
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25259
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25260
|
+
fitP95: record2?.fitP95 ?? null,
|
|
25261
|
+
fitMax: record2?.fitMax ?? null,
|
|
25262
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25263
|
+
feeBps: record2?.feeBps ?? null,
|
|
25264
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25265
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25266
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25267
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25268
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25269
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25270
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25271
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25272
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25273
|
+
});
|
|
25274
|
+
if (!saved) {
|
|
24906
25275
|
if (opts.json) {
|
|
24907
25276
|
writeJsonLine({
|
|
24908
25277
|
command: "llm price",
|
|
24909
25278
|
status: "failed",
|
|
24910
25279
|
error: "no_bench",
|
|
25280
|
+
currency: null,
|
|
24911
25281
|
ranAt: null,
|
|
24912
25282
|
modelId: null,
|
|
24913
25283
|
samples: null,
|
|
24914
25284
|
completed: null,
|
|
24915
|
-
|
|
24916
|
-
|
|
24917
|
-
max: null,
|
|
24918
|
-
fitP95: null,
|
|
24919
|
-
chargedBasedEstimateCoins: null,
|
|
24920
|
-
lengthRaisedPrice: false,
|
|
24921
|
-
cutOffSamples: 0,
|
|
24922
|
-
noRecommendationReason: null,
|
|
24923
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24924
|
-
recommendedEstimateCoins: null,
|
|
24925
|
-
recommendedPerCallMaxCoins: null,
|
|
25285
|
+
...creditKeys(null),
|
|
25286
|
+
legacyCoinTerms: null,
|
|
24926
25287
|
savedTo: null
|
|
24927
25288
|
});
|
|
24928
25289
|
} else {
|
|
24929
25290
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
24930
|
-
log.dim(` Run ${c.cyan(
|
|
25291
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
24931
25292
|
}
|
|
24932
25293
|
process.exitCode = 1;
|
|
24933
25294
|
return;
|
|
24934
25295
|
}
|
|
25296
|
+
if (saved.kind === "coins") {
|
|
25297
|
+
const old = saved.record;
|
|
25298
|
+
const legacy = {
|
|
25299
|
+
p50: old.p50 ?? null,
|
|
25300
|
+
p95: old.p95 ?? null,
|
|
25301
|
+
max: old.max ?? null,
|
|
25302
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25303
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25304
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25305
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25306
|
+
};
|
|
25307
|
+
if (opts.json) {
|
|
25308
|
+
writeJsonLine({
|
|
25309
|
+
command: "llm price",
|
|
25310
|
+
status: "failed",
|
|
25311
|
+
error: "coin_terms",
|
|
25312
|
+
currency: "coins",
|
|
25313
|
+
ranAt: old.ranAt ?? null,
|
|
25314
|
+
modelId: old.modelId ?? null,
|
|
25315
|
+
samples: old.samples ?? null,
|
|
25316
|
+
completed: old.completed ?? null,
|
|
25317
|
+
...creditKeys(null),
|
|
25318
|
+
legacyCoinTerms: legacy,
|
|
25319
|
+
savedTo: LLM_BENCH_FILE
|
|
25320
|
+
});
|
|
25321
|
+
} else {
|
|
25322
|
+
log.plain(c.bold("genex llm price"));
|
|
25323
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25324
|
+
log.plain("");
|
|
25325
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25326
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25327
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25328
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25329
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25330
|
+
}
|
|
25331
|
+
process.exitCode = 1;
|
|
25332
|
+
return;
|
|
25333
|
+
}
|
|
25334
|
+
const record = saved.record;
|
|
24935
25335
|
if (opts.json) {
|
|
24936
25336
|
writeJsonLine({
|
|
24937
25337
|
command: "llm price",
|
|
24938
|
-
status: record.
|
|
24939
|
-
error: record.
|
|
25338
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
25339
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25340
|
+
currency: "credits",
|
|
24940
25341
|
ranAt: record.ranAt ?? null,
|
|
24941
25342
|
modelId: record.modelId ?? null,
|
|
24942
25343
|
samples: record.samples ?? null,
|
|
24943
25344
|
completed: record.completed ?? null,
|
|
24944
|
-
|
|
24945
|
-
|
|
24946
|
-
max: record.max ?? null,
|
|
24947
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24948
|
-
fitP95: record.fitP95 ?? null,
|
|
24949
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24950
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24951
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24952
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24953
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24954
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24955
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25345
|
+
...creditKeys(record),
|
|
25346
|
+
legacyCoinTerms: null,
|
|
24956
25347
|
savedTo: LLM_BENCH_FILE
|
|
24957
25348
|
});
|
|
24958
|
-
if (record.
|
|
25349
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24959
25350
|
return;
|
|
24960
25351
|
}
|
|
24961
25352
|
log.plain(c.bold("genex llm price"));
|
|
24962
25353
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
24963
25354
|
log.plain("");
|
|
24964
25355
|
printRecommendation(log, record);
|
|
24965
|
-
if (record.
|
|
25356
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24966
25357
|
}
|
|
24967
25358
|
function writeJsonLine(value) {
|
|
24968
25359
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25222,7 +25613,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25222
25613
|
return {
|
|
25223
25614
|
label: "Runtime",
|
|
25224
25615
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25225
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25616
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
25226
25617
|
};
|
|
25227
25618
|
}
|
|
25228
25619
|
case "off":
|
|
@@ -27178,16 +27569,16 @@ ${c.bold("Usage")}
|
|
|
27178
27569
|
status | cancel. "models" says whether this
|
|
27179
27570
|
stand serves the runtime LLM lane at all, and
|
|
27180
27571
|
at what provider rates. "bench" runs the real
|
|
27181
|
-
model on YOUR OWN
|
|
27182
|
-
attempt
|
|
27572
|
+
model on YOUR OWN credits and reports what each
|
|
27573
|
+
attempt really cost; it needs --max-credits <n>
|
|
27183
27574
|
--user-approved, like any spend the player has
|
|
27184
27575
|
to agree to. "price" reprints the last run's
|
|
27185
|
-
recommended
|
|
27576
|
+
recommended maxCredits. "status" lists this
|
|
27186
27577
|
project's open calls and the slots they hold;
|
|
27187
27578
|
"cancel <id>" stops an active one. Declare a
|
|
27188
|
-
game's
|
|
27189
|
-
a
|
|
27190
|
-
|
|
27579
|
+
game's per-call ceiling from a bench, never
|
|
27580
|
+
from a guess: the player is billed as used, and
|
|
27581
|
+
the ceiling also sizes how long an answer may be.
|
|
27191
27582
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27192
27583
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27193
27584
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27240,6 +27631,9 @@ ${c.bold("Usage")}
|
|
|
27240
27631
|
on the dashboard is never replaced from here.
|
|
27241
27632
|
'genex preview' also uploads a new
|
|
27242
27633
|
.genex/scratch/cover.png by itself.
|
|
27634
|
+
genex cover --clip <file> --commit <sha>
|
|
27635
|
+
Make a 5-6 s recording of real play (MP4/WebM,
|
|
27636
|
+
\u22648 MB) of build <sha> the game's cover clip.
|
|
27243
27637
|
genex make-remixable [options] Make THIS game remixable by everyone \u2014 migrates a
|
|
27244
27638
|
private source onto a public managed genex repo.
|
|
27245
27639
|
genex domain <sub> [host] Play this game on a domain you own:
|
|
@@ -27561,7 +27955,10 @@ ${c.bold("Options for `list`")}
|
|
|
27561
27955
|
|
|
27562
27956
|
${c.bold("Options for `cover`")}
|
|
27563
27957
|
<file> The image to make the cover (positional). Omit it to see
|
|
27564
|
-
the current cover and where
|
|
27958
|
+
the current cover and clip, and where they came from.
|
|
27959
|
+
--clip <file> A recording of real play (MP4 or WebM, \u22648 MB) to make the
|
|
27960
|
+
cover clip instead; the server keeps 5 s of it.
|
|
27961
|
+
--commit <sha> With --clip: the build the recording shows (7-40 hex).
|
|
27565
27962
|
--json One machine-readable result object.
|
|
27566
27963
|
--api-url <url> Override the API base URL.
|
|
27567
27964
|
--env <path> Token env file (default: ~/.genex/env).
|
|
@@ -27631,6 +28028,7 @@ ${c.bold("Examples")}
|
|
|
27631
28028
|
genex publish --categories games,vfx
|
|
27632
28029
|
genex cover
|
|
27633
28030
|
genex cover .genex/scratch/cover.png
|
|
28031
|
+
genex cover --clip .genex/scratch/clip.mp4 --commit 3f9c2ab
|
|
27634
28032
|
genex make-remixable
|
|
27635
28033
|
genex domain add play.mygame.com
|
|
27636
28034
|
genex domain list
|
|
@@ -27642,7 +28040,7 @@ ${c.bold("Examples")}
|
|
|
27642
28040
|
genex shop remove sku_123
|
|
27643
28041
|
genex llm models
|
|
27644
28042
|
genex llm models --all
|
|
27645
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
28043
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27646
28044
|
genex llm price
|
|
27647
28045
|
genex llm status
|
|
27648
28046
|
genex llm cancel <id>
|
|
@@ -27879,22 +28277,24 @@ ${c.bold("How the allowance works")}
|
|
|
27879
28277
|
|
|
27880
28278
|
Every number printed is live from your account; prices can change without a deploy.
|
|
27881
28279
|
`,
|
|
27882
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28280
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
27883
28281
|
|
|
27884
28282
|
${c.bold("Usage")}
|
|
27885
28283
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
27886
|
-
million tokens,
|
|
27887
|
-
|
|
27888
|
-
|
|
27889
|
-
|
|
27890
|
-
on this stand: it says so
|
|
27891
|
-
|
|
27892
|
-
|
|
27893
|
-
|
|
27894
|
-
|
|
27895
|
-
|
|
27896
|
-
|
|
27897
|
-
|
|
28284
|
+
million tokens, the platform fee a call is billed at, and
|
|
28285
|
+
the headroom it recommends over a benchmark. The catalog
|
|
28286
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
28287
|
+
printed by default and --all prints every row (--json
|
|
28288
|
+
always carries every row). Off on this stand: it says so
|
|
28289
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
28290
|
+
than promising a 404.
|
|
28291
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
28292
|
+
Run the real model through the development lane and
|
|
28293
|
+
report what each attempt really COST and was charged.
|
|
28294
|
+
Refused without both flags, before any network call:
|
|
28295
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
28296
|
+
allowance does not count. Needs a folder linked to a
|
|
28297
|
+
game you own, and a balance that covers one attempt.
|
|
27898
28298
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
27899
28299
|
Reads a file; spends nothing.
|
|
27900
28300
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -27911,29 +28311,32 @@ ${c.bold("Bench options")}
|
|
|
27911
28311
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
27912
28312
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
27913
28313
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
27914
|
-
--max-
|
|
28314
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
27915
28315
|
never a player's. The run's worst case is that number times --samples,
|
|
27916
|
-
and it is printed before anything starts.
|
|
28316
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
28317
|
+
name, read as the same number.)
|
|
27917
28318
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
27918
28319
|
--json One machine-readable object; every key is always present, null when
|
|
27919
28320
|
the CLI could not learn it.
|
|
27920
28321
|
|
|
27921
28322
|
${c.bold("Why a benchmark and not an estimate")}
|
|
27922
|
-
|
|
27923
|
-
|
|
27924
|
-
|
|
27925
|
-
|
|
27926
|
-
|
|
27927
|
-
|
|
27928
|
-
|
|
27929
|
-
|
|
27930
|
-
|
|
28323
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
28324
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
28325
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
28326
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
28327
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
28328
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
28329
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
28330
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
28331
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
28332
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
27931
28333
|
|
|
27932
|
-
|
|
27933
|
-
|
|
27934
|
-
|
|
27935
|
-
|
|
27936
|
-
|
|
28334
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
28335
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
28336
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
28337
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
28338
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
28339
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27937
28340
|
|
|
27938
28341
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
27939
28342
|
`,
|
|
@@ -28148,11 +28551,13 @@ function parseArgs(argv) {
|
|
|
28148
28551
|
"--limit",
|
|
28149
28552
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28150
28553
|
"--assets",
|
|
28151
|
-
// `genex llm bench` value flags. `--max-
|
|
28152
|
-
//
|
|
28554
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
28555
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
28556
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28153
28557
|
"--model",
|
|
28154
28558
|
"--samples",
|
|
28155
28559
|
"--schema",
|
|
28560
|
+
"--max-credits",
|
|
28156
28561
|
"--max-coins",
|
|
28157
28562
|
// `genex player install --label <name>` — what this machine is called in
|
|
28158
28563
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28172,7 +28577,10 @@ function parseArgs(argv) {
|
|
|
28172
28577
|
"--tri-band",
|
|
28173
28578
|
"--holder",
|
|
28174
28579
|
"--summary",
|
|
28175
|
-
"--tags"
|
|
28580
|
+
"--tags",
|
|
28581
|
+
// `genex cover --clip <file> --commit <sha>`: the clip and the build it shows.
|
|
28582
|
+
"--clip",
|
|
28583
|
+
"--commit"
|
|
28176
28584
|
]);
|
|
28177
28585
|
let i = 0;
|
|
28178
28586
|
while (i < argv.length) {
|
|
@@ -28667,12 +29075,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28667
29075
|
options.samples = n;
|
|
28668
29076
|
break;
|
|
28669
29077
|
}
|
|
29078
|
+
case "--max-credits":
|
|
28670
29079
|
case "--max-coins": {
|
|
28671
29080
|
const n = Number(value);
|
|
28672
29081
|
if (!Number.isInteger(n) || n < 1) {
|
|
28673
|
-
throw new Error(`Invalid
|
|
29082
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28674
29083
|
}
|
|
28675
|
-
options.
|
|
29084
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
29085
|
+
else options.maxCoins = n;
|
|
28676
29086
|
break;
|
|
28677
29087
|
}
|
|
28678
29088
|
case "--timeout": {
|
|
@@ -28855,6 +29265,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
28855
29265
|
case "--src":
|
|
28856
29266
|
options.src = value;
|
|
28857
29267
|
break;
|
|
29268
|
+
case "--clip":
|
|
29269
|
+
options.clipPath = value;
|
|
29270
|
+
break;
|
|
29271
|
+
case "--commit":
|
|
29272
|
+
options.commit = value;
|
|
29273
|
+
break;
|
|
28858
29274
|
}
|
|
28859
29275
|
}
|
|
28860
29276
|
async function main() {
|
|
@@ -28964,9 +29380,9 @@ async function main() {
|
|
|
28964
29380
|
await runBudget(parsed.options);
|
|
28965
29381
|
break;
|
|
28966
29382
|
// The in-game model lane: is it live here, what does one call really
|
|
28967
|
-
// cost, what should the game declare. `bench`
|
|
28968
|
-
//
|
|
28969
|
-
//
|
|
29383
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
29384
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
29385
|
+
// own --max-credits/--user-approved gate.
|
|
28970
29386
|
case "llm":
|
|
28971
29387
|
await runLlm(parsed.options);
|
|
28972
29388
|
break;
|