@genex-ai/cli-demo 1.35.0-dev.743 → 1.35.0-dev.746

package/dist/index.js

@@ -23592,6 +23592,64 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
   log.dim("  Those per-million rates are the PROVIDER's; what your game is charged is the platform");
   log.dim(`  tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
 }
+function wholeOrNull(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+function sampleFrom(id, settled) {
+  const usage = settled?.usage;
+  return {
+    id,
+    status: settled?.status ?? null,
+    billingStatus: settled?.billingStatus ?? null,
+    chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
+    costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
+    error: settled?.error ?? (settled ? null : "timed_out"),
+    providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
+    inputTokens: wholeOrNull(usage?.inputTokens),
+    outputTokens: wholeOrNull(usage?.outputTokens),
+    maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
+    fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
+    fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
+    truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
+  };
+}
+var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
+function benchRecommendation(results, headroomBps) {
+  const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
+  const charged = settled.map((r) => r.chargedCoins);
+  const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
+  const p95 = percentile2(charged, 95);
+  const max = charged.length ? Math.max(...charged) : null;
+  const fitP95 = percentile2(fits, 95);
+  const fitMax = fits.length ? Math.max(...fits) : null;
+  const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
+  const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
+  const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
+  const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
+  const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
+  const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
+  return {
+    charged,
+    p50: percentile2(charged, 50),
+    p95,
+    max,
+    fits,
+    fitP95,
+    fitMax,
+    chargedBasedEstimateCoins: chargedBased,
+    recommendedEstimateCoins: recommended,
+    recommendedPerCallMaxCoins: ceiling,
+    lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
+    cutOffSamples,
+    exceedsStandCap,
+    noRecommendationReason
+  };
+}
+var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
+var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
+function benchCutOffLine(cutOff, maxCoins) {
+  return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
+}
 var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
 function rowState(row) {
   if (row.status && ACTIVE_STATUSES.has(row.status)) return "active";
@@ -23723,42 +23781,39 @@ async function runBench(args) {
       }
       const started = await res.json().catch(() => ({}));
       if (!started.id) {
-        results.push({ id: null, status: null, billingStatus: null, chargedCoins: null, costUsd: null, error: "no_generation_id", providerMessage: null });
+        results.push({ ...sampleFrom("", null), id: null, error: "no_generation_id" });
         continue;
       }
       inflight = started.id;
       const settled = await pollSettled(call, `${base}/${encodeURIComponent(started.id)}`, opts.timeoutSec ?? DEFAULT_SAMPLE_TIMEOUT_SEC);
       inflight = null;
-      const row = {
-        id: started.id,
-        status: settled?.status ?? null,
-        billingStatus: settled?.billingStatus ?? null,
-        chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
-        costUsd: typeof settled?.usage?.costUsd === "number" ? settled.usage.costUsd : null,
-        error: settled?.error ?? (settled ? null : "timed_out"),
-        providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null
-      };
+      const row = sampleFrom(started.id, settled);
       results.push(row);
       const refusedAt = providerRefusalStatus(row.error);
       if (refusedAt !== null) providerRefused++;
       if (!opts.json) {
+        const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
         log.plain(
-          `  ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1}  ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${row.error ? ` \xB7 ${row.error}` : ""}`
+          `  ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1}  ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
         );
         if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
+        if (row.error === PROVIDER_TOKEN_LIMIT) {
+          log.dim(
+            row.fitExceedsCap === true ? "      The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : `      The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
+          );
+        }
       }
     }
   } finally {
     process.removeListener("SIGINT", onSigint);
   }
-  const charged = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number").map((r) => r.chargedCoins);
-  const settledCount = charged.length;
-  const p50 = percentile2(charged, 50);
-  const p95 = percentile2(charged, 95);
-  const max = charged.length ? Math.max(...charged) : null;
   const headroomBps = lane.recommendedDeclaredHeadroomBps;
-  const recommended = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
-  const ceiling = p95 !== null && max !== null && headroomBps !== null ? recommendPerCallMaxCoins(p95, max, headroomBps) : null;
+  const rec = benchRecommendation(results, headroomBps);
+  const { charged, p50, p95, max } = rec;
+  const settledCount = charged.length;
+  const recommended = rec.recommendedEstimateCoins;
+  const ceiling = rec.recommendedPerCallMaxCoins;
+  const lengthBlocked = rec.noRecommendationReason !== null;
   const balanceAfter = await readCoinBalance(apiUrl, token);
   const record = {
     v: 1,
@@ -23776,14 +23831,22 @@ async function runBench(args) {
     max,
     recommendedDeclaredHeadroomBps: headroomBps,
     recommendedEstimateCoins: recommended,
-    recommendedPerCallMaxCoins: ceiling
+    recommendedPerCallMaxCoins: ceiling,
+    maxCoinsPerSample: maxCoins,
+    fitCoins: rec.fits,
+    fitP95: rec.fitP95,
+    chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
+    lengthRaisedPrice: rec.lengthRaisedPrice,
+    cutOffSamples: rec.cutOffSamples,
+    noRecommendationReason: rec.noRecommendationReason
   };
   const savedTo = await saveBench(cwd, record);
   if (opts.json) {
+    const ok = charged.length > 0 && !lengthBlocked;
     writeJsonLine({
       command: "llm bench",
-      status: charged.length > 0 ? "ok" : "failed",
-      error: charged.length > 0 ? null : "no_sample_settled",
+      status: ok ? "ok" : "failed",
+      error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
       apiUrl,
       projectId,
       modelId,
@@ -23800,15 +23863,34 @@ async function runBench(args) {
       // samples, never among them: no attempt existed and nothing was spent.
       refusal,
       chargedCoins: { p50, p95, max },
+      // The answer's LENGTH, priced by the server per sample (`fitCoins`,
+      // headroom already on the tokens), over the settled samples.
+      fitCoins: { p95: rec.fitP95, max: rec.fitMax },
+      chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
+      lengthRaisedPrice: rec.lengthRaisedPrice,
+      cutOffSamples: rec.cutOffSamples,
+      exceedsStandCap: rec.exceedsStandCap,
+      noRecommendationReason: rec.noRecommendationReason,
       recommendedDeclaredHeadroomBps: headroomBps,
       recommendedEstimateCoins: recommended,
       recommendedPerCallMaxCoins: ceiling,
       savedTo
     });
-    if (charged.length === 0) process.exitCode = 1;
+    if (!ok) process.exitCode = 1;
     return;
   }
   log.plain("");
+  if (lengthBlocked) {
+    if (charged.length > 0) {
+      log.plain(c.bold("  Charged coins"));
+      log.plain(`    p50 ${p50}   p95 ${p95}   max ${max}   (${charged.length} of ${samples} settled)`);
+      log.plain("");
+    }
+    printRecommendation(log, record);
+    if (savedTo) log.dim(`  Saved to ${savedTo}.`);
+    process.exitCode = 1;
+    return;
+  }
   if (charged.length === 0) {
     log.error(
       providerRefused > 0 && providerRefused === results.length ? "  No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : "  No sample settled, so there is nothing to price from."
@@ -23826,6 +23908,14 @@ async function runBench(args) {
   if (savedTo) log.dim(`  Saved to ${savedTo} \u2014 re-read it any time with ${c.cyan("genex llm price")}.`);
 }
 function printRecommendation(log, record) {
+  if (record.noRecommendationReason === "answer_exceeds_stand_cap") {
+    log.warn(`  ${LLM_EXCEEDS_STAND_CAP_LINE}`);
+    return;
+  }
+  if (record.noRecommendationReason === "answer_cut_off") {
+    log.warn(`  ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
+    return;
+  }
   if (record.recommendedEstimateCoins === null || record.p95 === null) {
     log.warn("  No price recommended.");
     log.dim(
@@ -23835,14 +23925,26 @@ function printRecommendation(log, record) {
     return;
   }
   log.plain(c.bold(`  Declare estimateCoins: ${record.recommendedEstimateCoins}`));
-  log.dim(
-    `  = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
-  );
-  log.dim(`  measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
+  if (record.lengthRaisedPrice) {
+    log.plain(`  ${LLM_LENGTH_RAISED_LINE}`);
+    log.dim(
+      `  = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
+    );
+    log.dim(
+      `  charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
+    );
+  } else {
+    log.dim(
+      `  = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
+    );
+    log.dim(`  measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
+  }
   log.dim("  That number is a PRICE: a started attempt is charged in full, including one that fails.");
   if (record.recommendedPerCallMaxCoins != null) {
     log.plain(c.bold(`  Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
-    log.dim(`  = the same headroom over the worst sample (${record.max}), and never below the price above it.`);
+    log.dim(
+      record.lengthRaisedPrice ? `  = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : `  = the same headroom over the worst sample (${record.max}), and never below the price above it.`
+    );
     log.dim("  The ceiling is the price's room to be wrong; declaring one under the price is refused.");
   }
 }
@@ -23952,6 +24054,12 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
     results: [],
     refusal: null,
     chargedCoins: { p50: null, p95: null, max: null },
+    fitCoins: { p95: null, max: null },
+    chargedBasedEstimateCoins: null,
+    lengthRaisedPrice: false,
+    cutOffSamples: 0,
+    exceedsStandCap: false,
+    noRecommendationReason: null,
     recommendedDeclaredHeadroomBps: null,
     recommendedEstimateCoins: null,
     recommendedPerCallMaxCoins: null,
@@ -24138,6 +24246,11 @@ async function reportSavedPrice(cwd, opts, log) {
         p50: null,
         p95: null,
         max: null,
+        fitP95: null,
+        chargedBasedEstimateCoins: null,
+        lengthRaisedPrice: false,
+        cutOffSamples: 0,
+        noRecommendationReason: null,
         recommendedDeclaredHeadroomBps: null,
         recommendedEstimateCoins: null,
         recommendedPerCallMaxCoins: null,
@@ -24154,7 +24267,7 @@ async function reportSavedPrice(cwd, opts, log) {
     writeJsonLine({
       command: "llm price",
       status: record.recommendedEstimateCoins === null ? "failed" : "ok",
-      error: record.recommendedEstimateCoins === null ? "no_recommendation" : null,
+      error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
       ranAt: record.ranAt ?? null,
       modelId: record.modelId ?? null,
       samples: record.samples ?? null,
@@ -24162,6 +24275,12 @@ async function reportSavedPrice(cwd, opts, log) {
       p50: record.p50 ?? null,
       p95: record.p95 ?? null,
       max: record.max ?? null,
+      // Absent from a file an older CLI wrote; null / false / 0 then, never missing.
+      fitP95: record.fitP95 ?? null,
+      chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
+      lengthRaisedPrice: record.lengthRaisedPrice ?? false,
+      cutOffSamples: record.cutOffSamples ?? 0,
+      noRecommendationReason: record.noRecommendationReason ?? null,
       recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
       recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
       recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
@@ -27103,6 +27222,12 @@ ${c.bold("Why a benchmark and not an estimate")}
   that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
   invent.
 
+  The declared price also decides how LONG the answer may be: it sizes each call's output
+  allowance. So the recommendation is never below the smallest price that leaves room for
+  the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
+  (provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
+  than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
+
   Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
 `,
   player: `${c.bold("genex player")} \u2014 answer the in-game model requests you approve, on your own Claude or ChatGPT subscription.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@genex-ai/cli-demo",
-  "version": "1.35.0-dev.743",
+  "version": "1.35.0-dev.746",
   "description": "Set up your project's agent workspace (.claude/.codex/.cursor in the game folder), authorize, create a game project, generate AI assets, and publish (genex CLI).",
   "type": "module",
   "bin": {

package/templates/skills/genex-llm-in-games/SKILL.md

@@ -120,6 +120,14 @@ the price and makes `requestSpendGrant()` refuse before it reaches the network.
 loop into disclosure numbers, re-benchmarking after a prompt change — is in
 [references/pricing.md](references/pricing.md).
 
+**The declared price also decides how long the answer may be.** Each call's
+room to answer is funded from the price it declares, so a price that covers
+what an answer cost can still cut it off. The bench's recommendation already
+leaves that room — always declare what it prints, never the bare charged
+number. A sample cut off at your `--max-coins` is not a sample: re-run with a
+higher one. When the bench says the answer is longer than one call on this
+stand may produce, ask for a shorter answer.
+
 Only samples that **succeeded and settled** are priced from. A sample the
 provider refused at its door (`provider_http_<status>`) ran no inference and
 cost nothing; the bench prints the code, the provider's own message and, for a
@@ -272,7 +280,7 @@ These are source contracts, not a claim that every stand runs this lane —
 - [ ] `npx genex llm models` was run and its verdict is in the handoff
 - [ ] Model ids come from `getGenerationModels()`, never from source
 - [ ] The picker offers the featured set, labelled with the server's own `label`
-- [ ] `estimateCoins` came from `npx genex llm bench`, not from judgement
+- [ ] `estimateCoins` is the figure `npx genex llm bench` printed, verbatim — never the bare charged number
 - [ ] The schema uses only the accepted dialect (no `$ref`, `pattern`, `format`, `anyOf`, `default`)
 - [ ] A bench refused as `generation_limit` was answered with `npx genex llm status`, never a retry
 - [ ] `generate()` / `requestSpendGrant()` is the first statement of a click handler
@@ -322,6 +330,13 @@ message (`npx genex llm status` prints it under the row). A 401 or 403 is this
 stand's provider configuration refusing the model — tell the operator, and
 build nothing around it in the game.
 
+**`provider_token_limit`** — the answer was cut off: the model ran out of room
+before it finished, and the attempt is still charged. The declared price is too
+low for the answer's length — re-benchmark and declare what the bench prints,
+never the bare charged number — or the answer is longer than one call on this
+stand may produce, and the fix is a shorter answer (fewer fields, shorter
+strings, a length the prompt states).
+
 **`invalid_schema`** — the schema uses a keyword outside the accepted dialect
 (`$ref`, `pattern`, `format`, `anyOf`, `default`, `examples`, `$schema`), or an
 object without `additionalProperties: false`. Nothing was charged. Rewrite it in

package/templates/skills/genex-llm-in-games/references/pricing.md

@@ -63,6 +63,13 @@ own once its bill resolves, and stops holding a slot ten minutes after
 dispatch. Re-running the bench into the same refusal spends nothing and
 learns nothing.
 
+A sample the model had to stop writing (`provider_token_limit`) was cut off at
+your `--max-coins`: it was charged, it is not a sample, and its real length is
+unknown — so the run recommends no price and asks you to re-run with a higher
+`--max-coins`. When the stand itself cannot hold the answer, the run says the
+answer is longer than one call there may produce; no price fixes that, a
+shorter answer does.
+
 - **p50** is what a typical call costs. It is the number to reason about when
   you ask "can the game afford this loop?" — multiply it by the calls per
   minute you are about to disclose.
@@ -75,7 +82,14 @@ learns nothing.
   Fix the prompt rather than declaring a bigger number.
 
 The recommendation line already applies the **server's own recommended
-headroom** on top of p95. Declare that figure verbatim:
+headroom** on top of p95. It also covers the answer's **length**: the declared
+price decides how long each call's answer may be, because the room to answer is
+funded from it, so a price built from charged coins alone can cut the answer
+off in the game while the bench — run under a larger `--max-coins` — never saw
+it. The recommendation is never below the smallest price that leaves room for
+the benchmarked answer, and when the length is what set it the run says so in
+one sentence. Either way, declare that figure verbatim — never the bare charged
+number:
 
 ```ts
 const NPC_CALL_PRICE = <the recommended figure>;    // from `npx genex llm bench`, <date>