@genex-ai/cli-demo 1.36.2 → 1.36.3-dev.784
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-NRJ5FI22.js → blender-mcp-WSURPZET.js} +8 -7
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-GOP5FRSJ.js → chunk-EABIVAGT.js} +217 -12
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +600 -266
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
ASSETS_MODE_ENV,
|
|
4
|
+
FS_PERMISSION_CODES,
|
|
4
5
|
RENDER_MODES,
|
|
5
6
|
SCRATCH_DIR,
|
|
6
7
|
SHEET_FORMATS,
|
|
@@ -13,15 +14,25 @@ import {
|
|
|
13
14
|
blenderEndpoint,
|
|
14
15
|
blenderSetupHint,
|
|
15
16
|
cloneSource,
|
|
17
|
+
describeError,
|
|
16
18
|
excludeFile,
|
|
19
|
+
explainFsPermission,
|
|
17
20
|
fetchCloneGrant,
|
|
18
21
|
fetchSignedInEmail,
|
|
22
|
+
fsPermissionInfo,
|
|
23
|
+
guardStdio,
|
|
19
24
|
hostedBrowserCovers,
|
|
20
25
|
inHostedSession,
|
|
26
|
+
isBrokenPipe,
|
|
27
|
+
isExpected,
|
|
28
|
+
isNetworkFetchFailure,
|
|
21
29
|
isRenderMode,
|
|
22
30
|
isStale,
|
|
23
31
|
isTermsRefusal,
|
|
32
|
+
markExpected,
|
|
33
|
+
markPrinted,
|
|
24
34
|
mayForceOverAnotherDevice,
|
|
35
|
+
networkErrorCode,
|
|
25
36
|
printedStructuredError,
|
|
26
37
|
readProject,
|
|
27
38
|
readRemoteSource,
|
|
@@ -40,11 +51,13 @@ import {
|
|
|
40
51
|
sheetOf,
|
|
41
52
|
sourceTreeHash,
|
|
42
53
|
urlHasEmbeddedCredentials,
|
|
54
|
+
wasPrinted,
|
|
43
55
|
writeProject,
|
|
56
|
+
writeSafely,
|
|
44
57
|
writeSecretFile,
|
|
45
58
|
writeUserToken,
|
|
46
59
|
writeWorkspace
|
|
47
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
48
61
|
import {
|
|
49
62
|
CLI_CHANNEL,
|
|
50
63
|
DEFAULT_API_URL,
|
|
@@ -65,7 +78,7 @@ import {
|
|
|
65
78
|
normalizeApiOrigin,
|
|
66
79
|
originKey,
|
|
67
80
|
resolveAgentTargets
|
|
68
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
69
82
|
|
|
70
83
|
// src/instrument.ts
|
|
71
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -84,6 +97,15 @@ function redactHome(s, home = os.homedir()) {
|
|
|
84
97
|
return s.split(home).join("~");
|
|
85
98
|
}
|
|
86
99
|
var clean = (s) => redactHome(scrubString(s));
|
|
100
|
+
var OWN_PACKAGE = /@genex-ai[\\/.]cli-demo/;
|
|
101
|
+
function isOwnFrame(frame) {
|
|
102
|
+
for (const where of [frame.filename, frame.abs_path, frame.module]) {
|
|
103
|
+
if (typeof where !== "string" || where.startsWith("node:")) continue;
|
|
104
|
+
const m = OWN_PACKAGE.exec(where);
|
|
105
|
+
if (m && !where.slice(m.index + m[0].length).includes("node_modules")) return true;
|
|
106
|
+
}
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
87
109
|
function scrubEvent(event) {
|
|
88
110
|
delete event.user;
|
|
89
111
|
delete event.server_name;
|
|
@@ -91,6 +113,7 @@ function scrubEvent(event) {
|
|
|
91
113
|
for (const ex of event.exception?.values ?? []) {
|
|
92
114
|
if (typeof ex.value === "string") ex.value = clean(ex.value);
|
|
93
115
|
for (const f of ex.stacktrace?.frames ?? []) {
|
|
116
|
+
if (isOwnFrame(f)) f.in_app = true;
|
|
94
117
|
if (typeof f.filename === "string") f.filename = redactHome(f.filename);
|
|
95
118
|
if (typeof f.abs_path === "string") f.abs_path = redactHome(f.abs_path);
|
|
96
119
|
delete f.vars;
|
|
@@ -117,6 +140,55 @@ function scrubEvent(event) {
|
|
|
117
140
|
}
|
|
118
141
|
return event;
|
|
119
142
|
}
|
|
143
|
+
var EXPECTED_MESSAGES = [
|
|
144
|
+
/^That sign-in code expired\./,
|
|
145
|
+
/^Couldn't start sign-in\b/,
|
|
146
|
+
/^Sign-in was cancelled in the browser\./,
|
|
147
|
+
/^Timed out waiting for authorization\./,
|
|
148
|
+
/^Authorization is still pending\./,
|
|
149
|
+
/^That code expired\. Re-run `genex player install`/,
|
|
150
|
+
/^That request was declined in the browser\./,
|
|
151
|
+
/^Nobody approved this machine\b/
|
|
152
|
+
];
|
|
153
|
+
function lastException(event) {
|
|
154
|
+
const values = event.exception?.values;
|
|
155
|
+
return values && values.length > 0 ? values[values.length - 1] : void 0;
|
|
156
|
+
}
|
|
157
|
+
function isCliNoise(event, hint = {}) {
|
|
158
|
+
const original = hint.originalException;
|
|
159
|
+
if (isExpected(original)) return true;
|
|
160
|
+
if (isBrokenPipe(original)) return true;
|
|
161
|
+
if (isNetworkFetchFailure(original)) return true;
|
|
162
|
+
const top = lastException(event);
|
|
163
|
+
const message = original instanceof Error ? original.message : typeof top?.value === "string" ? top.value : event.message;
|
|
164
|
+
if (typeof message === "string") {
|
|
165
|
+
if (/\bEPIPE\b/.test(message)) return true;
|
|
166
|
+
if (EXPECTED_MESSAGES.some((re) => re.test(message))) return true;
|
|
167
|
+
}
|
|
168
|
+
const values = event.exception?.values ?? [];
|
|
169
|
+
if (values.some((v) => v.value === "fetch failed")) {
|
|
170
|
+
return values.some((v) => typeof v.value === "string" && networkCodeIn(v.value));
|
|
171
|
+
}
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
function networkCodeIn(text) {
|
|
175
|
+
for (const m of text.matchAll(/\b(E[A-Z_]+|UND_ERR_[A-Z_]+)\b/g)) {
|
|
176
|
+
if (networkErrorCode({ code: m[1] }) !== null) return true;
|
|
177
|
+
}
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
var ERRNO_TEXT = /^(E[A-Z]+): [^,]*, ([a-z]+)\b/;
|
|
181
|
+
function groupCliEvent(event, hint = {}) {
|
|
182
|
+
let syscall = fsPermissionInfo(hint.originalException)?.syscall;
|
|
183
|
+
if (!syscall) {
|
|
184
|
+
const m = ERRNO_TEXT.exec(lastException(event)?.value ?? "");
|
|
185
|
+
if (m && FS_PERMISSION_CODES.has(m[1])) syscall = m[2];
|
|
186
|
+
}
|
|
187
|
+
if (!syscall) return false;
|
|
188
|
+
event.fingerprint = ["cli-fs-permission", syscall];
|
|
189
|
+
event.level = "warning";
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
120
192
|
|
|
121
193
|
// src/instrument.ts
|
|
122
194
|
var GENEX_CLI_DSN = "https://43efc6f7d16c3e67cad6c60fa8175c20@o4511115493900288.ingest.us.sentry.io/4511706579599360";
|
|
@@ -151,8 +223,12 @@ if (sentryEnabled) {
|
|
|
151
223
|
// Keep @sentry/node's default onUncaughtException + onUnhandledRejection
|
|
152
224
|
// integrations (auto-registered) — they catch the fire-and-forget async
|
|
153
225
|
// paths in lib/updates.ts / lib/deploy.ts that otherwise swallow errors.
|
|
154
|
-
beforeSend(event) {
|
|
155
|
-
|
|
226
|
+
beforeSend(event, hint) {
|
|
227
|
+
const e = event;
|
|
228
|
+
const h = hint;
|
|
229
|
+
if (isCliNoise(e, h)) return null;
|
|
230
|
+
groupCliEvent(e, h);
|
|
231
|
+
scrubEvent(e);
|
|
156
232
|
return event;
|
|
157
233
|
}
|
|
158
234
|
});
|
|
@@ -234,6 +310,7 @@ var AuthPendingError = class extends Error {
|
|
|
234
310
|
this.verifyUrl = verifyUrl;
|
|
235
311
|
this.apiUrl = apiUrl;
|
|
236
312
|
this.envPath = envPath;
|
|
313
|
+
markExpected(this);
|
|
237
314
|
}
|
|
238
315
|
};
|
|
239
316
|
function formatUserCode(code) {
|
|
@@ -291,13 +368,13 @@ async function startDeviceAuth(apiBaseUrl, label) {
|
|
|
291
368
|
if (res.status === 404 || res.status === 501) throw new LegacyApiError();
|
|
292
369
|
if (res.status === 409) continue;
|
|
293
370
|
if (!res.ok) {
|
|
294
|
-
throw new Error(
|
|
371
|
+
throw markExpected(new Error(
|
|
295
372
|
`Couldn't start sign-in (HTTP ${res.status}). Check your connection and re-run this command.`
|
|
296
|
-
);
|
|
373
|
+
));
|
|
297
374
|
}
|
|
298
375
|
return await res.json();
|
|
299
376
|
}
|
|
300
|
-
throw new Error("Couldn't start sign-in \u2014 please re-run this command.");
|
|
377
|
+
throw markExpected(new Error("Couldn't start sign-in \u2014 please re-run this command."));
|
|
301
378
|
}
|
|
302
379
|
function defaultLabel() {
|
|
303
380
|
try {
|
|
@@ -339,7 +416,7 @@ async function pollForToken(apiBaseUrl, pending, opts) {
|
|
|
339
416
|
}
|
|
340
417
|
if (res.status === 410) {
|
|
341
418
|
await clearPendingAuth(apiBaseUrl, opts.envPath);
|
|
342
|
-
throw new Error("That sign-in code expired. Re-run this command for a fresh one.");
|
|
419
|
+
throw markExpected(new Error("That sign-in code expired. Re-run this command for a fresh one."));
|
|
343
420
|
}
|
|
344
421
|
if (res.ok) {
|
|
345
422
|
const body = await res.json().catch(() => null);
|
|
@@ -349,7 +426,7 @@ async function pollForToken(apiBaseUrl, pending, opts) {
|
|
|
349
426
|
}
|
|
350
427
|
if (body?.status === "denied") {
|
|
351
428
|
await clearPendingAuth(apiBaseUrl, opts.envPath);
|
|
352
|
-
throw new Error("Sign-in was cancelled in the browser. Re-run this command to try again.");
|
|
429
|
+
throw markExpected(new Error("Sign-in was cancelled in the browser. Re-run this command to try again."));
|
|
353
430
|
}
|
|
354
431
|
if (body?.interval) intervalMs = Math.max(body.interval * 1e3, MIN_POLL_INTERVAL_MS);
|
|
355
432
|
}
|
|
@@ -469,9 +546,9 @@ async function authorizeLoopback(authBaseUrl, options) {
|
|
|
469
546
|
}
|
|
470
547
|
timer = setTimeout(() => {
|
|
471
548
|
fail5(
|
|
472
|
-
new Error(
|
|
549
|
+
markExpected(new Error(
|
|
473
550
|
"Timed out waiting for authorization. Re-run the command to try again."
|
|
474
|
-
)
|
|
551
|
+
))
|
|
475
552
|
);
|
|
476
553
|
}, timeoutMs);
|
|
477
554
|
});
|
|
@@ -635,10 +712,10 @@ function tokenizeCommand(input) {
|
|
|
635
712
|
// src/utils/logger.ts
|
|
636
713
|
function createLogger(opts = {}) {
|
|
637
714
|
const out = (s) => {
|
|
638
|
-
if (!opts.quiet) process.stdout
|
|
715
|
+
if (!opts.quiet) writeSafely(process.stdout, s + "\n");
|
|
639
716
|
};
|
|
640
717
|
const err = (s) => {
|
|
641
|
-
process.stderr
|
|
718
|
+
writeSafely(process.stderr, s + "\n");
|
|
642
719
|
};
|
|
643
720
|
return {
|
|
644
721
|
info: (m) => out(`${c.cyan("i")} ${m}`),
|
|
@@ -678,7 +755,7 @@ async function runAuth(opts) {
|
|
|
678
755
|
printAuthHandoff(log, err);
|
|
679
756
|
return;
|
|
680
757
|
}
|
|
681
|
-
log.error(
|
|
758
|
+
log.error(describeError(err));
|
|
682
759
|
process.exitCode = 1;
|
|
683
760
|
return;
|
|
684
761
|
}
|
|
@@ -904,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
904
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
905
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
906
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
907
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
908
985
|
${CONTRACT_END}
|
|
909
986
|
`;
|
|
910
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -918,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
918
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
919
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
920
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
921
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
922
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
923
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
924
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -6549,16 +6626,6 @@ async function runRemix(opts, deps = {}) {
|
|
|
6549
6626
|
import fs20 from "fs/promises";
|
|
6550
6627
|
import path20 from "path";
|
|
6551
6628
|
|
|
6552
|
-
// src/lib/printed.ts
|
|
6553
|
-
var printed = /* @__PURE__ */ new WeakSet();
|
|
6554
|
-
function markPrinted(err) {
|
|
6555
|
-
if (typeof err === "object" && err !== null) printed.add(err);
|
|
6556
|
-
return err;
|
|
6557
|
-
}
|
|
6558
|
-
function wasPrinted(err) {
|
|
6559
|
-
return typeof err === "object" && err !== null && printed.has(err);
|
|
6560
|
-
}
|
|
6561
|
-
|
|
6562
6629
|
// src/lib/telemetry.ts
|
|
6563
6630
|
async function reportCliEvent(apiUrl, token, event) {
|
|
6564
6631
|
if (!token || telemetryDisabled()) return;
|
|
@@ -6654,25 +6721,41 @@ async function runInit(opts) {
|
|
|
6654
6721
|
);
|
|
6655
6722
|
let totalNew = 0;
|
|
6656
6723
|
let totalUpdated = 0;
|
|
6724
|
+
const refused = [];
|
|
6657
6725
|
for (const t of targets) {
|
|
6658
6726
|
const src = t.full ? templatesDir : path20.join(templatesDir, "skills");
|
|
6659
6727
|
const dest = t.full ? t.baseDir : path20.join(t.baseDir, "skills");
|
|
6660
6728
|
const family = remixing ? skillFamilyFilter("remix") : converting ? skillFamilyFilter("tools", { hosted: true }) : skillFamilyFilter("game");
|
|
6661
|
-
|
|
6662
|
-
|
|
6663
|
-
|
|
6664
|
-
|
|
6665
|
-
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6729
|
+
let copied;
|
|
6730
|
+
let updated;
|
|
6731
|
+
try {
|
|
6732
|
+
if (remixing) await pruneRemixWorkflow(t.baseDir);
|
|
6733
|
+
({ copied, updated } = await copyTemplates(src, dest, {
|
|
6734
|
+
force: opts.force,
|
|
6735
|
+
exclude: ["controllers", "motion", "asset-viewer", "blender-service"],
|
|
6736
|
+
filter: t.full ? family : (rel) => family(`skills/${rel}`)
|
|
6737
|
+
}));
|
|
6738
|
+
await pruneRemovedSkills(path20.join(t.baseDir, "skills"), log);
|
|
6739
|
+
await writeSkillsMarker(path20.join(t.baseDir, "skills"));
|
|
6740
|
+
} catch (err) {
|
|
6741
|
+
const refusal2 = explainFsPermission(err, dest);
|
|
6742
|
+
if (!refusal2) throw err;
|
|
6743
|
+
log.error(`${t.label}: ${refusal2.message}`);
|
|
6744
|
+
refused.push(refusal2);
|
|
6745
|
+
continue;
|
|
6746
|
+
}
|
|
6669
6747
|
const added = copied.length - updated.length;
|
|
6670
6748
|
totalNew += added;
|
|
6671
6749
|
totalUpdated += updated.length;
|
|
6672
6750
|
log.dim(` ${t.label}: ${added} added, ${updated.length} refreshed \u2192 ${dest}`);
|
|
6673
6751
|
}
|
|
6752
|
+
if (refused.length > 0 && refused.length === targets.length) {
|
|
6753
|
+
log.error("Nothing was installed. Fix that, then re-run `genex init`.");
|
|
6754
|
+
throw markPrinted(refused[0]);
|
|
6755
|
+
}
|
|
6756
|
+
const installed = targets.length - refused.length;
|
|
6674
6757
|
log.success(
|
|
6675
|
-
`Skills ready (${totalNew} added, ${totalUpdated} refreshed across ${
|
|
6758
|
+
`Skills ready (${totalNew} added, ${totalUpdated} refreshed across ${installed} agent${installed === 1 ? "" : "s"}).`
|
|
6676
6759
|
);
|
|
6677
6760
|
log.plain("");
|
|
6678
6761
|
if (remixing) await writeRemixContract(process.cwd());
|
|
@@ -6703,7 +6786,7 @@ async function runInit(opts) {
|
|
|
6703
6786
|
log.dim("Everything else is installed \u2014 `genex auth` finishes the sign-in, then re-run `genex init`.");
|
|
6704
6787
|
return;
|
|
6705
6788
|
}
|
|
6706
|
-
log.error(
|
|
6789
|
+
log.error(describeError(err));
|
|
6707
6790
|
log.dim("Your workspace files were installed. Re-run `genex init` to finish authorizing.");
|
|
6708
6791
|
throw markPrinted(err);
|
|
6709
6792
|
}
|
|
@@ -6752,7 +6835,7 @@ async function runInit(opts) {
|
|
|
6752
6835
|
log.dim("Then re-run `genex init` \u2014 your workspace files are already installed.");
|
|
6753
6836
|
return;
|
|
6754
6837
|
}
|
|
6755
|
-
log.error(
|
|
6838
|
+
log.error(describeError(err));
|
|
6756
6839
|
log.dim("Your workspace files were installed. Re-run `genex init` to finish authorizing.");
|
|
6757
6840
|
throw markPrinted(err);
|
|
6758
6841
|
}
|
|
@@ -22860,7 +22943,7 @@ async function runBlender(opts) {
|
|
|
22860
22943
|
return 1;
|
|
22861
22944
|
}
|
|
22862
22945
|
if (sub === "serve") {
|
|
22863
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
22946
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
22864
22947
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
22865
22948
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
22866
22949
|
log.plain(
|
|
@@ -22869,7 +22952,7 @@ async function runBlender(opts) {
|
|
|
22869
22952
|
return serveLocalBlender({ port, log });
|
|
22870
22953
|
}
|
|
22871
22954
|
if (sub === "mcp") {
|
|
22872
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
22955
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
22873
22956
|
return runBlenderMcp();
|
|
22874
22957
|
}
|
|
22875
22958
|
if (sub === "seat") {
|
|
@@ -23377,7 +23460,12 @@ import fs35 from "fs/promises";
|
|
|
23377
23460
|
import path36 from "path";
|
|
23378
23461
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23379
23462
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23380
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
23463
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
23464
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
23465
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
23466
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
23467
|
+
}
|
|
23468
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23381
23469
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23382
23470
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23383
23471
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23390,7 +23478,7 @@ function serverSlotCounts(body) {
|
|
|
23390
23478
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23391
23479
|
}
|
|
23392
23480
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23393
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
23481
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23394
23482
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23395
23483
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23396
23484
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23404,11 +23492,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23404
23492
|
function parseProviderKey(value) {
|
|
23405
23493
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23406
23494
|
}
|
|
23495
|
+
function bpsOrNull(value) {
|
|
23496
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23497
|
+
}
|
|
23498
|
+
function standingCreditGrantOrNull(value) {
|
|
23499
|
+
if (!value || typeof value !== "object") return null;
|
|
23500
|
+
const v = value;
|
|
23501
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
23502
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
23503
|
+
return {
|
|
23504
|
+
minCredits: v.minCredits,
|
|
23505
|
+
maxCredits: v.maxCredits,
|
|
23506
|
+
stepCredits: v.stepCredits,
|
|
23507
|
+
defaultCredits: v.defaultCredits,
|
|
23508
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
23509
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
23510
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
23511
|
+
};
|
|
23512
|
+
}
|
|
23407
23513
|
async function readRuntimeLane(apiUrl, token) {
|
|
23408
23514
|
const empty = (state) => ({
|
|
23409
23515
|
state,
|
|
23410
23516
|
models: null,
|
|
23411
23517
|
recommendedDeclaredHeadroomBps: null,
|
|
23518
|
+
currency: null,
|
|
23519
|
+
feeBps: null,
|
|
23520
|
+
usdCentsPerCredit: null,
|
|
23521
|
+
standingCreditGrant: null,
|
|
23412
23522
|
externalProviders: null,
|
|
23413
23523
|
total: null,
|
|
23414
23524
|
featuredCount: null,
|
|
@@ -23427,7 +23537,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
23427
23537
|
return {
|
|
23428
23538
|
state: "live",
|
|
23429
23539
|
models: body.models,
|
|
23430
|
-
recommendedDeclaredHeadroomBps:
|
|
23540
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
23541
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
23542
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
23543
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
23544
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
23431
23545
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
23432
23546
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
23433
23547
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -23443,18 +23557,45 @@ function percentile2(values, p) {
|
|
|
23443
23557
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23444
23558
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23445
23559
|
}
|
|
23446
|
-
function
|
|
23447
|
-
|
|
23560
|
+
function percentilePicos(values, p) {
|
|
23561
|
+
if (values.length === 0) return null;
|
|
23562
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
23563
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23564
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23448
23565
|
}
|
|
23449
|
-
|
|
23450
|
-
|
|
23566
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
23567
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
23568
|
+
if (costPicos <= 0n) return 1;
|
|
23569
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
23570
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23571
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23572
|
+
}
|
|
23573
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
23574
|
+
if (costsPicos.length === 0) return null;
|
|
23575
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23576
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
23577
|
+
}
|
|
23578
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
23579
|
+
if (costsPicos.length === 0) return null;
|
|
23580
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23581
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
23582
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23583
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23584
|
+
}
|
|
23585
|
+
function formatCredits(value) {
|
|
23586
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
23587
|
+
if (value >= 100) return String(Math.round(value));
|
|
23588
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
23589
|
+
}
|
|
23590
|
+
function usdFromPicos(picos) {
|
|
23591
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
23592
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
23593
|
+
if (value === null) return "unknown";
|
|
23594
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
23451
23595
|
}
|
|
23452
23596
|
function usdPerMillion(value) {
|
|
23453
23597
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
23454
23598
|
}
|
|
23455
|
-
function usd(value) {
|
|
23456
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
23457
|
-
}
|
|
23458
23599
|
async function runLlm(opts = {}) {
|
|
23459
23600
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
23460
23601
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -23480,16 +23621,23 @@ async function runLlm(opts = {}) {
|
|
|
23480
23621
|
return;
|
|
23481
23622
|
}
|
|
23482
23623
|
if (sub === "bench") {
|
|
23483
|
-
if (opts.maxCoins
|
|
23624
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
23625
|
+
`);
|
|
23626
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
23627
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
23628
|
+
return;
|
|
23629
|
+
}
|
|
23630
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23631
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
23484
23632
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
23485
23633
|
return;
|
|
23486
23634
|
}
|
|
23487
|
-
if (!Number.isInteger(
|
|
23488
|
-
fail5("llm bench", `--max-
|
|
23635
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
23636
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
23489
23637
|
return;
|
|
23490
23638
|
}
|
|
23491
23639
|
if (!opts.benchPrompt?.trim()) {
|
|
23492
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
23640
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
23493
23641
|
return;
|
|
23494
23642
|
}
|
|
23495
23643
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -23581,6 +23729,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23581
23729
|
models: null,
|
|
23582
23730
|
externalProviders: null,
|
|
23583
23731
|
recommendedDeclaredHeadroomBps: null,
|
|
23732
|
+
currency: null,
|
|
23733
|
+
feeBps: null,
|
|
23734
|
+
usdCentsPerCredit: null,
|
|
23735
|
+
standingCreditGrant: null,
|
|
23584
23736
|
total: null,
|
|
23585
23737
|
featuredCount: null,
|
|
23586
23738
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -23606,6 +23758,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23606
23758
|
models: lane.models,
|
|
23607
23759
|
externalProviders: lane.externalProviders,
|
|
23608
23760
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
23761
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
23762
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
23763
|
+
currency: lane.currency,
|
|
23764
|
+
feeBps: lane.feeBps,
|
|
23765
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
23766
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
23609
23767
|
total: lane.total ?? lane.models?.length ?? null,
|
|
23610
23768
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
23611
23769
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -23652,8 +23810,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23652
23810
|
);
|
|
23653
23811
|
for (const m of shown) {
|
|
23654
23812
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
23813
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
23655
23814
|
log.plain(
|
|
23656
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
23815
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
23657
23816
|
);
|
|
23658
23817
|
}
|
|
23659
23818
|
if (hiddenCount > 0) {
|
|
@@ -23669,74 +23828,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23669
23828
|
}
|
|
23670
23829
|
log.plain("");
|
|
23671
23830
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
23672
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
23831
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
23832
|
+
} else {
|
|
23833
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
23834
|
+
}
|
|
23835
|
+
if (lane.feeBps === null) {
|
|
23836
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
23673
23837
|
} else {
|
|
23674
|
-
log.dim(`
|
|
23838
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
23675
23839
|
}
|
|
23676
23840
|
if (toolsOnly) {
|
|
23677
23841
|
printConvertHint();
|
|
23678
23842
|
return;
|
|
23679
23843
|
}
|
|
23680
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
23681
|
-
log.dim(`
|
|
23844
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
23845
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
23682
23846
|
}
|
|
23683
23847
|
function wholeOrNull(value) {
|
|
23684
23848
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23685
23849
|
}
|
|
23850
|
+
function costPicosOf(usage) {
|
|
23851
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
23852
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
23853
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
23854
|
+
}
|
|
23855
|
+
return null;
|
|
23856
|
+
}
|
|
23686
23857
|
function sampleFrom(id, settled) {
|
|
23687
23858
|
const usage = settled?.usage;
|
|
23859
|
+
const picos = costPicosOf(usage);
|
|
23688
23860
|
return {
|
|
23689
23861
|
id,
|
|
23690
23862
|
status: settled?.status ?? null,
|
|
23691
23863
|
billingStatus: settled?.billingStatus ?? null,
|
|
23692
|
-
|
|
23693
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
23864
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
23865
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
23866
|
+
costUsdPicos: picos,
|
|
23694
23867
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
23695
23868
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
23696
23869
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
23697
23870
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
23698
23871
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
23699
|
-
|
|
23872
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
23700
23873
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
23701
23874
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
23702
23875
|
};
|
|
23703
23876
|
}
|
|
23704
23877
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
23705
|
-
function benchRecommendation(results,
|
|
23706
|
-
const settled = results.filter(
|
|
23707
|
-
|
|
23708
|
-
|
|
23709
|
-
const
|
|
23710
|
-
const
|
|
23878
|
+
function benchRecommendation(results, terms) {
|
|
23879
|
+
const settled = results.filter(
|
|
23880
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
23881
|
+
);
|
|
23882
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
23883
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
23884
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
23885
|
+
const costP95 = percentilePicos(costs, 95);
|
|
23886
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
23711
23887
|
const fitP95 = percentile2(fits, 95);
|
|
23712
23888
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
23713
23889
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
23714
23890
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
23715
23891
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
23716
|
-
const
|
|
23717
|
-
const
|
|
23718
|
-
const
|
|
23892
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
23893
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
23894
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
23895
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
23896
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
23897
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
23898
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
23899
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
23900
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
23719
23901
|
return {
|
|
23902
|
+
costs,
|
|
23720
23903
|
charged,
|
|
23721
|
-
|
|
23722
|
-
|
|
23723
|
-
|
|
23904
|
+
costP50: percentilePicos(costs, 50),
|
|
23905
|
+
costP95,
|
|
23906
|
+
costMax,
|
|
23907
|
+
chargedP50: percentile2(charged, 50),
|
|
23908
|
+
chargedP95: percentile2(charged, 95),
|
|
23909
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
23724
23910
|
fits,
|
|
23725
23911
|
fitP95,
|
|
23726
23912
|
fitMax,
|
|
23727
|
-
|
|
23728
|
-
|
|
23729
|
-
|
|
23730
|
-
|
|
23913
|
+
costBasedMaxCredits: costBased,
|
|
23914
|
+
recommendedMaxCredits: recommended,
|
|
23915
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
23916
|
+
averageCreditsPerCall: average,
|
|
23917
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
23918
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
23731
23919
|
cutOffSamples,
|
|
23732
23920
|
exceedsStandCap,
|
|
23733
|
-
noRecommendationReason
|
|
23921
|
+
noRecommendationReason,
|
|
23922
|
+
missingTerms
|
|
23734
23923
|
};
|
|
23735
23924
|
}
|
|
23736
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
23737
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
23738
|
-
function benchCutOffLine(cutOff,
|
|
23739
|
-
return `No
|
|
23925
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
23926
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
23927
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
23928
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
23740
23929
|
}
|
|
23741
23930
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
23742
23931
|
function rowState(row) {
|
|
@@ -23747,7 +23936,22 @@ function rowState(row) {
|
|
|
23747
23936
|
function rowHoldsSlot(row) {
|
|
23748
23937
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
23749
23938
|
const state = rowState(row);
|
|
23750
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
23939
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
23940
|
+
}
|
|
23941
|
+
function rowMoney(row) {
|
|
23942
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
23943
|
+
if (credits) {
|
|
23944
|
+
return {
|
|
23945
|
+
unit: "credits",
|
|
23946
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
23947
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
23948
|
+
};
|
|
23949
|
+
}
|
|
23950
|
+
return {
|
|
23951
|
+
unit: "coin",
|
|
23952
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
23953
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
23954
|
+
};
|
|
23751
23955
|
}
|
|
23752
23956
|
function providerRefusalStatus(error) {
|
|
23753
23957
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -23766,7 +23970,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
23766
23970
|
}
|
|
23767
23971
|
async function runBench(args) {
|
|
23768
23972
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
23769
|
-
const
|
|
23973
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23770
23974
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
23771
23975
|
const prompt = opts.benchPrompt.trim();
|
|
23772
23976
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -23788,7 +23992,7 @@ async function runBench(args) {
|
|
|
23788
23992
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
23789
23993
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
23790
23994
|
if (lane.state === "off") {
|
|
23791
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
23995
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
23792
23996
|
else {
|
|
23793
23997
|
log.plain(LLM_LANE_OFF_LINE);
|
|
23794
23998
|
log.dim(" Nothing was spent.");
|
|
@@ -23815,17 +24019,27 @@ async function runBench(args) {
|
|
|
23815
24019
|
);
|
|
23816
24020
|
return;
|
|
23817
24021
|
}
|
|
23818
|
-
const
|
|
24022
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24023
|
+
if (walletBefore?.emailVerified === false) {
|
|
24024
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24025
|
+
return;
|
|
24026
|
+
}
|
|
24027
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24028
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24029
|
+
return;
|
|
24030
|
+
}
|
|
24031
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
23819
24032
|
if (!opts.json) {
|
|
23820
24033
|
log.plain(c.bold("genex llm bench"));
|
|
23821
24034
|
log.dim(` ${apiUrl}`);
|
|
23822
24035
|
log.plain("");
|
|
23823
24036
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
23824
24037
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
23825
|
-
log.plain(` Approved ${
|
|
24038
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
23826
24039
|
log.plain(
|
|
23827
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24040
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
23828
24041
|
);
|
|
24042
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
23829
24043
|
log.plain("");
|
|
23830
24044
|
}
|
|
23831
24045
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -23851,7 +24065,7 @@ async function runBench(args) {
|
|
|
23851
24065
|
const res = await call(base, {
|
|
23852
24066
|
method: "POST",
|
|
23853
24067
|
body: JSON.stringify({
|
|
23854
|
-
|
|
24068
|
+
maxCredits,
|
|
23855
24069
|
request: {
|
|
23856
24070
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
23857
24071
|
modelId,
|
|
@@ -23881,13 +24095,14 @@ async function runBench(args) {
|
|
|
23881
24095
|
if (refusedAt !== null) providerRefused++;
|
|
23882
24096
|
if (!opts.json) {
|
|
23883
24097
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24098
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
23884
24099
|
log.plain(
|
|
23885
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24100
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
23886
24101
|
);
|
|
23887
24102
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
23888
24103
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
23889
24104
|
log.dim(
|
|
23890
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24105
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
23891
24106
|
);
|
|
23892
24107
|
}
|
|
23893
24108
|
}
|
|
@@ -23895,16 +24110,19 @@ async function runBench(args) {
|
|
|
23895
24110
|
} finally {
|
|
23896
24111
|
process.removeListener("SIGINT", onSigint);
|
|
23897
24112
|
}
|
|
23898
|
-
const
|
|
23899
|
-
|
|
23900
|
-
|
|
23901
|
-
|
|
23902
|
-
|
|
23903
|
-
const
|
|
24113
|
+
const terms = {
|
|
24114
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24115
|
+
feeBps: lane.feeBps,
|
|
24116
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24117
|
+
};
|
|
24118
|
+
const rec = benchRecommendation(results, terms);
|
|
24119
|
+
const settledCount = rec.charged.length;
|
|
23904
24120
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
23905
|
-
const balanceAfter = await
|
|
24121
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24122
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
23906
24123
|
const record = {
|
|
23907
|
-
v:
|
|
24124
|
+
v: 2,
|
|
24125
|
+
currency: "credits",
|
|
23908
24126
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
23909
24127
|
apiUrl,
|
|
23910
24128
|
projectId,
|
|
@@ -23913,28 +24131,34 @@ async function runBench(args) {
|
|
|
23913
24131
|
outputFormat,
|
|
23914
24132
|
samples,
|
|
23915
24133
|
completed: settledCount,
|
|
23916
|
-
|
|
23917
|
-
|
|
23918
|
-
|
|
23919
|
-
max,
|
|
23920
|
-
|
|
23921
|
-
|
|
23922
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
23923
|
-
maxCoinsPerSample: maxCoins,
|
|
23924
|
-
fitCoins: rec.fits,
|
|
24134
|
+
maxCreditsPerSample: maxCredits,
|
|
24135
|
+
costUsdPicos: rec.costs.map(String),
|
|
24136
|
+
chargedCredits: rec.charged,
|
|
24137
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24138
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24139
|
+
fitCredits: rec.fits,
|
|
23925
24140
|
fitP95: rec.fitP95,
|
|
23926
|
-
|
|
23927
|
-
|
|
24141
|
+
fitMax: rec.fitMax,
|
|
24142
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24143
|
+
feeBps: terms.feeBps,
|
|
24144
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24145
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24146
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24147
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24148
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24149
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24150
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
23928
24151
|
cutOffSamples: rec.cutOffSamples,
|
|
23929
24152
|
noRecommendationReason: rec.noRecommendationReason
|
|
23930
24153
|
};
|
|
23931
24154
|
const savedTo = await saveBench(cwd, record);
|
|
23932
24155
|
if (opts.json) {
|
|
23933
|
-
const ok =
|
|
24156
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
23934
24157
|
writeJsonLine({
|
|
23935
24158
|
command: "llm bench",
|
|
23936
24159
|
status: ok ? "ok" : "failed",
|
|
23937
24160
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24161
|
+
currency: "credits",
|
|
23938
24162
|
apiUrl,
|
|
23939
24163
|
projectId,
|
|
23940
24164
|
modelId,
|
|
@@ -23943,35 +24167,54 @@ async function runBench(args) {
|
|
|
23943
24167
|
schemaPath: opts.schemaPath ?? null,
|
|
23944
24168
|
samples,
|
|
23945
24169
|
completed: settledCount,
|
|
23946
|
-
|
|
24170
|
+
maxCreditsPerSample: maxCredits,
|
|
24171
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
23947
24172
|
balanceBefore,
|
|
23948
24173
|
balanceAfter,
|
|
23949
24174
|
results,
|
|
23950
24175
|
// The create-time refusal that ended the run, or null. Beside the
|
|
23951
24176
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
23952
24177
|
refusal: refusal2,
|
|
23953
|
-
|
|
23954
|
-
//
|
|
24178
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
24179
|
+
// and USD) and the whole credits charged.
|
|
24180
|
+
costUsdPicos: record.cost,
|
|
24181
|
+
costUsd: {
|
|
24182
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24183
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24184
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24185
|
+
},
|
|
24186
|
+
chargedCredits: record.charged,
|
|
24187
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
23955
24188
|
// headroom already on the tokens), over the settled samples.
|
|
23956
|
-
|
|
23957
|
-
|
|
23958
|
-
|
|
24189
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
24190
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24191
|
+
feeBps: terms.feeBps,
|
|
24192
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24193
|
+
missingTerms: rec.missingTerms,
|
|
24194
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24195
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
23959
24196
|
cutOffSamples: rec.cutOffSamples,
|
|
23960
24197
|
exceedsStandCap: rec.exceedsStandCap,
|
|
23961
24198
|
noRecommendationReason: rec.noRecommendationReason,
|
|
23962
|
-
|
|
23963
|
-
|
|
23964
|
-
|
|
24199
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24200
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24201
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24202
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
23965
24203
|
savedTo
|
|
23966
24204
|
});
|
|
23967
24205
|
if (!ok) process.exitCode = 1;
|
|
23968
24206
|
return;
|
|
23969
24207
|
}
|
|
23970
24208
|
log.plain("");
|
|
24209
|
+
const printMeasured = () => {
|
|
24210
|
+
log.plain(c.bold(" Real cost per call"));
|
|
24211
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24212
|
+
log.plain(c.bold(" Charged credits"));
|
|
24213
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24214
|
+
};
|
|
23971
24215
|
if (lengthBlocked) {
|
|
23972
|
-
if (
|
|
23973
|
-
|
|
23974
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24216
|
+
if (settledCount > 0) {
|
|
24217
|
+
printMeasured();
|
|
23975
24218
|
log.plain("");
|
|
23976
24219
|
}
|
|
23977
24220
|
printRecommendation(log, record);
|
|
@@ -23979,17 +24222,16 @@ async function runBench(args) {
|
|
|
23979
24222
|
process.exitCode = 1;
|
|
23980
24223
|
return;
|
|
23981
24224
|
}
|
|
23982
|
-
if (
|
|
24225
|
+
if (settledCount === 0) {
|
|
23983
24226
|
log.error(
|
|
23984
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24227
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
23985
24228
|
);
|
|
23986
24229
|
process.exitCode = 1;
|
|
23987
24230
|
return;
|
|
23988
24231
|
}
|
|
23989
|
-
|
|
23990
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24232
|
+
printMeasured();
|
|
23991
24233
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
23992
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24234
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
23993
24235
|
}
|
|
23994
24236
|
log.plain("");
|
|
23995
24237
|
printRecommendation(log, record);
|
|
@@ -24001,43 +24243,58 @@ function printRecommendation(log, record) {
|
|
|
24001
24243
|
return;
|
|
24002
24244
|
}
|
|
24003
24245
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24004
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24246
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24005
24247
|
return;
|
|
24006
24248
|
}
|
|
24007
|
-
|
|
24008
|
-
|
|
24249
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
24250
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
24251
|
+
log.warn(" No ceiling recommended.");
|
|
24009
24252
|
log.dim(
|
|
24010
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
24253
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24011
24254
|
);
|
|
24012
24255
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24256
|
+
printAverage(log, record, attempts);
|
|
24013
24257
|
return;
|
|
24014
24258
|
}
|
|
24015
|
-
log.plain(c.bold(` Declare
|
|
24016
|
-
if (record.
|
|
24259
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
24260
|
+
if (record.lengthRaisedCeiling) {
|
|
24017
24261
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24018
24262
|
log.dim(
|
|
24019
|
-
` = the smallest
|
|
24263
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24020
24264
|
);
|
|
24021
24265
|
log.dim(
|
|
24022
|
-
`
|
|
24266
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24023
24267
|
);
|
|
24024
24268
|
} else {
|
|
24025
24269
|
log.dim(
|
|
24026
|
-
` = p95
|
|
24270
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24027
24271
|
);
|
|
24028
|
-
log.dim(` measured over ${
|
|
24272
|
+
log.dim(` measured over ${attempts}.`);
|
|
24029
24273
|
}
|
|
24030
|
-
log.dim(" That number is a
|
|
24031
|
-
|
|
24032
|
-
|
|
24274
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
24275
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
24276
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
24277
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24033
24278
|
log.dim(
|
|
24034
|
-
record.
|
|
24279
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24035
24280
|
);
|
|
24036
|
-
|
|
24281
|
+
}
|
|
24282
|
+
printAverage(log, record, attempts);
|
|
24283
|
+
}
|
|
24284
|
+
function printAverage(log, record, attempts) {
|
|
24285
|
+
if (record.averageCreditsPerCall === null) return;
|
|
24286
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24287
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24288
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24289
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24290
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24037
24291
|
}
|
|
24038
24292
|
}
|
|
24039
24293
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24040
|
-
if (printedStructuredError(res))
|
|
24294
|
+
if (printedStructuredError(res)) {
|
|
24295
|
+
const printed = await res.json().catch(() => ({}));
|
|
24296
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24297
|
+
}
|
|
24041
24298
|
const body = await res.json().catch(() => ({}));
|
|
24042
24299
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24043
24300
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24047,6 +24304,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24047
24304
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24048
24305
|
return { stop: true, refusal: refusal2 };
|
|
24049
24306
|
}
|
|
24307
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24308
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24309
|
+
return { stop: true, refusal: refusal2 };
|
|
24310
|
+
}
|
|
24050
24311
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24051
24312
|
if (res.status === 404 && body.error === "not_found") {
|
|
24052
24313
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24062,7 +24323,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24062
24323
|
return { stop: true, refusal: refusal2 };
|
|
24063
24324
|
}
|
|
24064
24325
|
if (res.status === 402) {
|
|
24065
|
-
log.error(" Not enough
|
|
24326
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24066
24327
|
return { stop: true, refusal: refusal2 };
|
|
24067
24328
|
}
|
|
24068
24329
|
log.error(
|
|
@@ -24071,7 +24332,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24071
24332
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24072
24333
|
}
|
|
24073
24334
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24074
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
24335
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24075
24336
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24076
24337
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24077
24338
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24097,19 +24358,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24097
24358
|
}
|
|
24098
24359
|
return last;
|
|
24099
24360
|
}
|
|
24100
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24101
|
-
try {
|
|
24102
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24103
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24104
|
-
signal: AbortSignal.timeout(6e3)
|
|
24105
|
-
});
|
|
24106
|
-
if (!res.ok) return null;
|
|
24107
|
-
const body = await res.json().catch(() => null);
|
|
24108
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24109
|
-
} catch {
|
|
24110
|
-
return null;
|
|
24111
|
-
}
|
|
24112
|
-
}
|
|
24113
24361
|
async function saveBench(cwd, record) {
|
|
24114
24362
|
try {
|
|
24115
24363
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24125,9 +24373,10 @@ function benchFailed(opts, log, message) {
|
|
|
24125
24373
|
else log.error(message);
|
|
24126
24374
|
process.exitCode = 1;
|
|
24127
24375
|
}
|
|
24128
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24376
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
24129
24377
|
return {
|
|
24130
24378
|
command: "llm bench",
|
|
24379
|
+
currency: "credits",
|
|
24131
24380
|
apiUrl,
|
|
24132
24381
|
projectId,
|
|
24133
24382
|
modelId: opts.modelId ?? null,
|
|
@@ -24136,21 +24385,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
24136
24385
|
schemaPath: opts.schemaPath ?? null,
|
|
24137
24386
|
samples,
|
|
24138
24387
|
completed: 0,
|
|
24139
|
-
|
|
24388
|
+
maxCreditsPerSample: maxCredits,
|
|
24140
24389
|
balanceBefore: null,
|
|
24141
24390
|
balanceAfter: null,
|
|
24142
24391
|
results: [],
|
|
24143
24392
|
refusal: null,
|
|
24144
|
-
|
|
24145
|
-
|
|
24146
|
-
|
|
24147
|
-
|
|
24393
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
24394
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
24395
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
24396
|
+
fitCredits: { p95: null, max: null },
|
|
24397
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24398
|
+
feeBps: null,
|
|
24399
|
+
usdCentsPerCredit: null,
|
|
24400
|
+
missingTerms: null,
|
|
24401
|
+
costBasedMaxCredits: null,
|
|
24402
|
+
lengthRaisedCeiling: false,
|
|
24148
24403
|
cutOffSamples: 0,
|
|
24149
24404
|
exceedsStandCap: false,
|
|
24150
24405
|
noRecommendationReason: null,
|
|
24151
|
-
|
|
24152
|
-
|
|
24153
|
-
|
|
24406
|
+
recommendedMaxCredits: null,
|
|
24407
|
+
recommendedPerCallMaxCredits: null,
|
|
24408
|
+
averageCreditsPerCall: null,
|
|
24409
|
+
recommendedPerCallEstimateCredits: null,
|
|
24154
24410
|
savedTo: null
|
|
24155
24411
|
};
|
|
24156
24412
|
}
|
|
@@ -24213,7 +24469,8 @@ async function reportStatus(args) {
|
|
|
24213
24469
|
for (const row of rows) {
|
|
24214
24470
|
const state = rowState(row);
|
|
24215
24471
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24216
|
-
const
|
|
24472
|
+
const money2 = rowMoney(row);
|
|
24473
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24217
24474
|
log.plain(
|
|
24218
24475
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24219
24476
|
);
|
|
@@ -24285,10 +24542,11 @@ async function cancelCall(args) {
|
|
|
24285
24542
|
if (!opts.json) {
|
|
24286
24543
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
24287
24544
|
if (state === "awaiting bill") {
|
|
24288
|
-
log.dim(` Nothing to cancel:
|
|
24545
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
24289
24546
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
24290
24547
|
} else {
|
|
24291
|
-
|
|
24548
|
+
const money2 = rowMoney(row);
|
|
24549
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24292
24550
|
}
|
|
24293
24551
|
}
|
|
24294
24552
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -24310,78 +24568,126 @@ async function cancelCall(args) {
|
|
|
24310
24568
|
const after = await res.json().catch(() => null) ?? row;
|
|
24311
24569
|
if (!opts.json) {
|
|
24312
24570
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
24313
|
-
log.dim(`
|
|
24571
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
24314
24572
|
}
|
|
24315
24573
|
return done("ok", null, "cancelled", after);
|
|
24316
24574
|
}
|
|
24317
|
-
async function
|
|
24318
|
-
let
|
|
24575
|
+
async function readSavedBench(cwd) {
|
|
24576
|
+
let parsed;
|
|
24319
24577
|
try {
|
|
24320
|
-
|
|
24578
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
24321
24579
|
} catch {
|
|
24322
|
-
|
|
24580
|
+
return null;
|
|
24323
24581
|
}
|
|
24324
|
-
if (!
|
|
24582
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
24583
|
+
const v = parsed;
|
|
24584
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
24585
|
+
return { kind: "coins", record: parsed };
|
|
24586
|
+
}
|
|
24587
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
24588
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
24589
|
+
const saved = await readSavedBench(cwd);
|
|
24590
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
24591
|
+
const creditKeys = (record2) => ({
|
|
24592
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
24593
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
24594
|
+
fitP95: record2?.fitP95 ?? null,
|
|
24595
|
+
fitMax: record2?.fitMax ?? null,
|
|
24596
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
24597
|
+
feeBps: record2?.feeBps ?? null,
|
|
24598
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
24599
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
24600
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
24601
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
24602
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
24603
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
24604
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
24605
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
24606
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
24607
|
+
});
|
|
24608
|
+
if (!saved) {
|
|
24325
24609
|
if (opts.json) {
|
|
24326
24610
|
writeJsonLine({
|
|
24327
24611
|
command: "llm price",
|
|
24328
24612
|
status: "failed",
|
|
24329
24613
|
error: "no_bench",
|
|
24614
|
+
currency: null,
|
|
24330
24615
|
ranAt: null,
|
|
24331
24616
|
modelId: null,
|
|
24332
24617
|
samples: null,
|
|
24333
24618
|
completed: null,
|
|
24334
|
-
|
|
24335
|
-
|
|
24336
|
-
max: null,
|
|
24337
|
-
fitP95: null,
|
|
24338
|
-
chargedBasedEstimateCoins: null,
|
|
24339
|
-
lengthRaisedPrice: false,
|
|
24340
|
-
cutOffSamples: 0,
|
|
24341
|
-
noRecommendationReason: null,
|
|
24342
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24343
|
-
recommendedEstimateCoins: null,
|
|
24344
|
-
recommendedPerCallMaxCoins: null,
|
|
24619
|
+
...creditKeys(null),
|
|
24620
|
+
legacyCoinTerms: null,
|
|
24345
24621
|
savedTo: null
|
|
24346
24622
|
});
|
|
24347
24623
|
} else {
|
|
24348
24624
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
24349
|
-
log.dim(` Run ${c.cyan(
|
|
24625
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
24350
24626
|
}
|
|
24351
24627
|
process.exitCode = 1;
|
|
24352
24628
|
return;
|
|
24353
24629
|
}
|
|
24630
|
+
if (saved.kind === "coins") {
|
|
24631
|
+
const old = saved.record;
|
|
24632
|
+
const legacy = {
|
|
24633
|
+
p50: old.p50 ?? null,
|
|
24634
|
+
p95: old.p95 ?? null,
|
|
24635
|
+
max: old.max ?? null,
|
|
24636
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
24637
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
24638
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
24639
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
24640
|
+
};
|
|
24641
|
+
if (opts.json) {
|
|
24642
|
+
writeJsonLine({
|
|
24643
|
+
command: "llm price",
|
|
24644
|
+
status: "failed",
|
|
24645
|
+
error: "coin_terms",
|
|
24646
|
+
currency: "coins",
|
|
24647
|
+
ranAt: old.ranAt ?? null,
|
|
24648
|
+
modelId: old.modelId ?? null,
|
|
24649
|
+
samples: old.samples ?? null,
|
|
24650
|
+
completed: old.completed ?? null,
|
|
24651
|
+
...creditKeys(null),
|
|
24652
|
+
legacyCoinTerms: legacy,
|
|
24653
|
+
savedTo: LLM_BENCH_FILE
|
|
24654
|
+
});
|
|
24655
|
+
} else {
|
|
24656
|
+
log.plain(c.bold("genex llm price"));
|
|
24657
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
24658
|
+
log.plain("");
|
|
24659
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
24660
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
24661
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
24662
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
24663
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
24664
|
+
}
|
|
24665
|
+
process.exitCode = 1;
|
|
24666
|
+
return;
|
|
24667
|
+
}
|
|
24668
|
+
const record = saved.record;
|
|
24354
24669
|
if (opts.json) {
|
|
24355
24670
|
writeJsonLine({
|
|
24356
24671
|
command: "llm price",
|
|
24357
|
-
status: record.
|
|
24358
|
-
error: record.
|
|
24672
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
24673
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
24674
|
+
currency: "credits",
|
|
24359
24675
|
ranAt: record.ranAt ?? null,
|
|
24360
24676
|
modelId: record.modelId ?? null,
|
|
24361
24677
|
samples: record.samples ?? null,
|
|
24362
24678
|
completed: record.completed ?? null,
|
|
24363
|
-
|
|
24364
|
-
|
|
24365
|
-
max: record.max ?? null,
|
|
24366
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24367
|
-
fitP95: record.fitP95 ?? null,
|
|
24368
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24369
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24370
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24371
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24372
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24373
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24374
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
24679
|
+
...creditKeys(record),
|
|
24680
|
+
legacyCoinTerms: null,
|
|
24375
24681
|
savedTo: LLM_BENCH_FILE
|
|
24376
24682
|
});
|
|
24377
|
-
if (record.
|
|
24683
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24378
24684
|
return;
|
|
24379
24685
|
}
|
|
24380
24686
|
log.plain(c.bold("genex llm price"));
|
|
24381
24687
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
24382
24688
|
log.plain("");
|
|
24383
24689
|
printRecommendation(log, record);
|
|
24384
|
-
if (record.
|
|
24690
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24385
24691
|
}
|
|
24386
24692
|
function writeJsonLine(value) {
|
|
24387
24693
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -24641,7 +24947,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
24641
24947
|
return {
|
|
24642
24948
|
label: "Runtime",
|
|
24643
24949
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
24644
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
24950
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
24645
24951
|
};
|
|
24646
24952
|
}
|
|
24647
24953
|
case "off":
|
|
@@ -24751,20 +25057,36 @@ async function runTools(opts) {
|
|
|
24751
25057
|
log.step(`Installing Genex tool skills for: ${targets.map((t) => c.cyan(t.label)).join(", ")}`);
|
|
24752
25058
|
let totalNew = 0;
|
|
24753
25059
|
let totalUpdated = 0;
|
|
25060
|
+
const refused = [];
|
|
24754
25061
|
for (const t of targets) {
|
|
24755
25062
|
const dest = path38.join(t.baseDir, "skills");
|
|
24756
|
-
|
|
24757
|
-
|
|
24758
|
-
|
|
24759
|
-
|
|
24760
|
-
|
|
25063
|
+
let copied;
|
|
25064
|
+
let updated;
|
|
25065
|
+
try {
|
|
25066
|
+
({ copied, updated } = await copyTemplates(path38.join(templatesDir, "skills"), dest, {
|
|
25067
|
+
filter: (rel) => skillFamilyFilter("tools")(`skills/${rel}`)
|
|
25068
|
+
}));
|
|
25069
|
+
await pruneRemovedSkills(dest, log);
|
|
25070
|
+
await writeSkillsMarker(dest);
|
|
25071
|
+
} catch (err) {
|
|
25072
|
+
const refusal2 = explainFsPermission(err, dest);
|
|
25073
|
+
if (!refusal2) throw err;
|
|
25074
|
+
log.error(`${t.label}: ${refusal2.message}`);
|
|
25075
|
+
refused.push(refusal2);
|
|
25076
|
+
continue;
|
|
25077
|
+
}
|
|
24761
25078
|
const added = copied.length - updated.length;
|
|
24762
25079
|
totalNew += added;
|
|
24763
25080
|
totalUpdated += updated.length;
|
|
24764
25081
|
log.dim(` ${t.label}: ${added} added, ${updated.length} refreshed \u2192 ${dest}`);
|
|
24765
25082
|
}
|
|
25083
|
+
if (refused.length > 0 && refused.length === targets.length) {
|
|
25084
|
+
log.error("Nothing was installed. Fix that, then re-run `genex tools`.");
|
|
25085
|
+
throw markPrinted(refused[0]);
|
|
25086
|
+
}
|
|
25087
|
+
const installed = targets.length - refused.length;
|
|
24766
25088
|
log.success(
|
|
24767
|
-
`Tool skills ready (${totalNew} added, ${totalUpdated} refreshed across ${
|
|
25089
|
+
`Tool skills ready (${totalNew} added, ${totalUpdated} refreshed across ${installed} agent${installed === 1 ? "" : "s"}).`
|
|
24768
25090
|
);
|
|
24769
25091
|
const apiUrl = getApiUrl(opts.apiUrl);
|
|
24770
25092
|
const { path: markerPath } = await writeWorkspace({
|
|
@@ -24797,7 +25119,7 @@ async function runTools(opts) {
|
|
|
24797
25119
|
);
|
|
24798
25120
|
return;
|
|
24799
25121
|
}
|
|
24800
|
-
log.error(
|
|
25122
|
+
log.error(describeError(err));
|
|
24801
25123
|
log.dim("The workspace is set up. Re-run `genex tools` to finish authorizing.");
|
|
24802
25124
|
throw markPrinted(err);
|
|
24803
25125
|
}
|
|
@@ -25100,7 +25422,7 @@ async function authorizePlayer(apiBaseUrl, options) {
|
|
|
25100
25422
|
} catch {
|
|
25101
25423
|
res = new Response(null, { status: 599 });
|
|
25102
25424
|
}
|
|
25103
|
-
if (res.status === 410) throw new Error("That code expired. Re-run `genex player install` for a fresh one.");
|
|
25425
|
+
if (res.status === 410) throw markExpected(new Error("That code expired. Re-run `genex player install` for a fresh one."));
|
|
25104
25426
|
if (res.ok) {
|
|
25105
25427
|
const body = await res.json().catch(() => null);
|
|
25106
25428
|
if (body?.status === "approved" && body.token) {
|
|
@@ -25112,10 +25434,10 @@ async function authorizePlayer(apiBaseUrl, options) {
|
|
|
25112
25434
|
expiresAt: body.expiresAt ?? new Date(Date.now() + 30 * 24 * 36e5).toISOString()
|
|
25113
25435
|
};
|
|
25114
25436
|
}
|
|
25115
|
-
if (body?.status === "denied") throw new Error("That request was declined in the browser. Nothing was installed.");
|
|
25437
|
+
if (body?.status === "denied") throw markExpected(new Error("That request was declined in the browser. Nothing was installed."));
|
|
25116
25438
|
if (body?.interval) intervalMs = Math.max(body.interval * 1e3, MIN_POLL_INTERVAL_MS2);
|
|
25117
25439
|
}
|
|
25118
|
-
if (Date.now() + intervalMs > deadline) throw new Error(PLAYER_NOT_APPROVED);
|
|
25440
|
+
if (Date.now() + intervalMs > deadline) throw markExpected(new Error(PLAYER_NOT_APPROVED));
|
|
25119
25441
|
if (Date.now() >= nextReassureAt) {
|
|
25120
25442
|
log.dim(" still waiting for you to approve\u2026");
|
|
25121
25443
|
nextReassureAt = Date.now() + REASSURE_EVERY_MS2;
|
|
@@ -25133,12 +25455,12 @@ async function startPlayerDeviceAuth(apiUrl, label) {
|
|
|
25133
25455
|
}, { noTermsWait: true });
|
|
25134
25456
|
if (res.status === 409) continue;
|
|
25135
25457
|
if (res.status === 404 || res.status === 501) {
|
|
25136
|
-
throw new Error("This Genex server has no player sign-in \u2014 the watcher needs a current stand.");
|
|
25458
|
+
throw markExpected(new Error("This Genex server has no player sign-in \u2014 the watcher needs a current stand."));
|
|
25137
25459
|
}
|
|
25138
|
-
if (!res.ok) throw new Error(`Couldn't start sign-in (HTTP ${res.status}). Check your connection and re-run this command.`);
|
|
25460
|
+
if (!res.ok) throw markExpected(new Error(`Couldn't start sign-in (HTTP ${res.status}). Check your connection and re-run this command.`));
|
|
25139
25461
|
return await res.json();
|
|
25140
25462
|
}
|
|
25141
|
-
throw new Error("Couldn't start sign-in \u2014 please re-run this command.");
|
|
25463
|
+
throw markExpected(new Error("Couldn't start sign-in \u2014 please re-run this command."));
|
|
25142
25464
|
}
|
|
25143
25465
|
|
|
25144
25466
|
// src/lib/player/api.ts
|
|
@@ -25519,6 +25841,8 @@ async function runJob(job, options = {}) {
|
|
|
25519
25841
|
// are no groups — and `killRunner` uses taskkill there instead.
|
|
25520
25842
|
detached: platform !== "win32"
|
|
25521
25843
|
});
|
|
25844
|
+
child.stdin?.on("error", () => {
|
|
25845
|
+
});
|
|
25522
25846
|
child.stdin?.end(job.user);
|
|
25523
25847
|
const out = [];
|
|
25524
25848
|
const err = [];
|
|
@@ -25686,8 +26010,8 @@ async function installService(input, platform = servicePlatform()) {
|
|
|
25686
26010
|
} catch (error) {
|
|
25687
26011
|
return { unit: paths.unit, loaded: false, detail: `launchctl refused it: ${String(error.message).split("\n")[0]}` };
|
|
25688
26012
|
}
|
|
25689
|
-
const
|
|
25690
|
-
return { unit: paths.unit, loaded:
|
|
26013
|
+
const printed = await run2("launchctl", ["print", `${domain}/${SERVICE_LABEL}`]).then((r) => r.stdout).catch(() => "");
|
|
26014
|
+
return { unit: paths.unit, loaded: printed.includes(SERVICE_LABEL), detail: printed.includes("state = running") ? "running" : "registered" };
|
|
25691
26015
|
}
|
|
25692
26016
|
if (platform === "linux") {
|
|
25693
26017
|
await run2("systemctl", ["--user", "daemon-reload"]).catch(() => {
|
|
@@ -26579,16 +26903,16 @@ ${c.bold("Usage")}
|
|
|
26579
26903
|
status | cancel. "models" says whether this
|
|
26580
26904
|
stand serves the runtime LLM lane at all, and
|
|
26581
26905
|
at what provider rates. "bench" runs the real
|
|
26582
|
-
model on YOUR OWN
|
|
26583
|
-
attempt
|
|
26906
|
+
model on YOUR OWN credits and reports what each
|
|
26907
|
+
attempt really cost; it needs --max-credits <n>
|
|
26584
26908
|
--user-approved, like any spend the player has
|
|
26585
26909
|
to agree to. "price" reprints the last run's
|
|
26586
|
-
recommended
|
|
26910
|
+
recommended maxCredits. "status" lists this
|
|
26587
26911
|
project's open calls and the slots they hold;
|
|
26588
26912
|
"cancel <id>" stops an active one. Declare a
|
|
26589
|
-
game's
|
|
26590
|
-
a
|
|
26591
|
-
|
|
26913
|
+
game's per-call ceiling from a bench, never
|
|
26914
|
+
from a guess: the player is billed as used, and
|
|
26915
|
+
the ceiling also sizes how long an answer may be.
|
|
26592
26916
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
26593
26917
|
OWN Claude or ChatGPT subscription: install | run |
|
|
26594
26918
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27043,7 +27367,7 @@ ${c.bold("Examples")}
|
|
|
27043
27367
|
genex shop remove sku_123
|
|
27044
27368
|
genex llm models
|
|
27045
27369
|
genex llm models --all
|
|
27046
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27370
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27047
27371
|
genex llm price
|
|
27048
27372
|
genex llm status
|
|
27049
27373
|
genex llm cancel <id>
|
|
@@ -27280,22 +27604,24 @@ ${c.bold("How the allowance works")}
|
|
|
27280
27604
|
|
|
27281
27605
|
Every number printed is live from your account; prices can change without a deploy.
|
|
27282
27606
|
`,
|
|
27283
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
27607
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
27284
27608
|
|
|
27285
27609
|
${c.bold("Usage")}
|
|
27286
27610
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
27287
|
-
million tokens,
|
|
27288
|
-
|
|
27289
|
-
|
|
27290
|
-
|
|
27291
|
-
on this stand: it says so
|
|
27292
|
-
|
|
27293
|
-
|
|
27294
|
-
|
|
27295
|
-
|
|
27296
|
-
|
|
27297
|
-
|
|
27298
|
-
|
|
27611
|
+
million tokens, the platform fee a call is billed at, and
|
|
27612
|
+
the headroom it recommends over a benchmark. The catalog
|
|
27613
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
27614
|
+
printed by default and --all prints every row (--json
|
|
27615
|
+
always carries every row). Off on this stand: it says so
|
|
27616
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
27617
|
+
than promising a 404.
|
|
27618
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
27619
|
+
Run the real model through the development lane and
|
|
27620
|
+
report what each attempt really COST and was charged.
|
|
27621
|
+
Refused without both flags, before any network call:
|
|
27622
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
27623
|
+
allowance does not count. Needs a folder linked to a
|
|
27624
|
+
game you own, and a balance that covers one attempt.
|
|
27299
27625
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
27300
27626
|
Reads a file; spends nothing.
|
|
27301
27627
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -27312,29 +27638,32 @@ ${c.bold("Bench options")}
|
|
|
27312
27638
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
27313
27639
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
27314
27640
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
27315
|
-
--max-
|
|
27641
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
27316
27642
|
never a player's. The run's worst case is that number times --samples,
|
|
27317
|
-
and it is printed before anything starts.
|
|
27643
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
27644
|
+
name, read as the same number.)
|
|
27318
27645
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
27319
27646
|
--json One machine-readable object; every key is always present, null when
|
|
27320
27647
|
the CLI could not learn it.
|
|
27321
27648
|
|
|
27322
27649
|
${c.bold("Why a benchmark and not an estimate")}
|
|
27323
|
-
|
|
27324
|
-
|
|
27325
|
-
|
|
27326
|
-
|
|
27327
|
-
|
|
27328
|
-
|
|
27329
|
-
|
|
27330
|
-
|
|
27331
|
-
|
|
27650
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
27651
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
27652
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
27653
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
27654
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
27655
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
27656
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
27657
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
27658
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
27659
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
27332
27660
|
|
|
27333
|
-
|
|
27334
|
-
|
|
27335
|
-
|
|
27336
|
-
|
|
27337
|
-
|
|
27661
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
27662
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
27663
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
27664
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
27665
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
27666
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27338
27667
|
|
|
27339
27668
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
27340
27669
|
`,
|
|
@@ -27549,11 +27878,13 @@ function parseArgs(argv) {
|
|
|
27549
27878
|
"--limit",
|
|
27550
27879
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
27551
27880
|
"--assets",
|
|
27552
|
-
// `genex llm bench` value flags. `--max-
|
|
27553
|
-
//
|
|
27881
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
27882
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
27883
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
27554
27884
|
"--model",
|
|
27555
27885
|
"--samples",
|
|
27556
27886
|
"--schema",
|
|
27887
|
+
"--max-credits",
|
|
27557
27888
|
"--max-coins",
|
|
27558
27889
|
// `genex player install --label <name>` — what this machine is called in
|
|
27559
27890
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28068,12 +28399,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28068
28399
|
options.samples = n;
|
|
28069
28400
|
break;
|
|
28070
28401
|
}
|
|
28402
|
+
case "--max-credits":
|
|
28071
28403
|
case "--max-coins": {
|
|
28072
28404
|
const n = Number(value);
|
|
28073
28405
|
if (!Number.isInteger(n) || n < 1) {
|
|
28074
|
-
throw new Error(`Invalid
|
|
28406
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28075
28407
|
}
|
|
28076
|
-
options.
|
|
28408
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
28409
|
+
else options.maxCoins = n;
|
|
28077
28410
|
break;
|
|
28078
28411
|
}
|
|
28079
28412
|
case "--timeout": {
|
|
@@ -28365,9 +28698,9 @@ async function main() {
|
|
|
28365
28698
|
await runBudget(parsed.options);
|
|
28366
28699
|
break;
|
|
28367
28700
|
// The in-game model lane: is it live here, what does one call really
|
|
28368
|
-
// cost, what should the game declare. `bench`
|
|
28369
|
-
//
|
|
28370
|
-
//
|
|
28701
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
28702
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
28703
|
+
// own --max-credits/--user-approved gate.
|
|
28371
28704
|
case "llm":
|
|
28372
28705
|
await runLlm(parsed.options);
|
|
28373
28706
|
break;
|
|
@@ -28544,12 +28877,13 @@ function checkNodeVersion() {
|
|
|
28544
28877
|
process.exitCode = 1;
|
|
28545
28878
|
return false;
|
|
28546
28879
|
}
|
|
28880
|
+
guardStdio();
|
|
28547
28881
|
if (checkNodeVersion()) {
|
|
28548
28882
|
main().catch(async (err) => {
|
|
28549
|
-
Sentry2.captureException(err);
|
|
28883
|
+
if (!isExpected(err)) Sentry2.captureException(err);
|
|
28550
28884
|
if (!wasPrinted(err)) {
|
|
28551
28885
|
const log = createLogger();
|
|
28552
|
-
log.error(
|
|
28886
|
+
log.error(describeError(err));
|
|
28553
28887
|
}
|
|
28554
28888
|
await flushSentry();
|
|
28555
28889
|
process.exitCode = 1;
|