@letterstory/cli 0.122.0 → 0.124.0

package/lib/cli.mjs

@@ -61,13 +61,15 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.122.0";
+export const VERSION = "0.124.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value. Every `type: "bool"` flag (and its
 // `negate` spelling) in a command's ARGS map must be listed; boolean-flags.test.ts enforces it.
 export const BOOLEAN_FLAGS = new Set([
 	"json",
+	// admin models runs: only the starred ones.
+	"starred",
 	// shredder file / arena file: save the version as a candidate beside live.
 	"candidate",
 	// site tree: include children a fold-by-default family would hide; items over the whole subtree.
@@ -843,6 +845,8 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        session (a trial per arm), linked to its
                                                        ledger entry. Written by \`npm run lab --
                                                        publish\` in the kernels repo.
+  admin models runs [--model M] [--starred] | run <id> | star|unstar <id> | scoreboard --model M
+                                                       The model hub's saved runs and blind arena (models.letter.dev)
   admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month]
           | reconcile [--from D --to D] | pnl [--range day|week|month|year | --from D --to D]
           | reference add --vendor V [--key K] --metric billed_usd --value N --from D [--to D] --ref "…"
@@ -851,7 +855,8 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
           | add --vendor V --amount N --from D [--to D] --area A --category C [--basis billed|estimated] …
           | import <file.csv> [--vendor Concentrate] [--area A] [--ledger-key K] [--dry-run]
           | import <file.csv> [--vendor Concentrate] [--area A] [--keep-ledger-key] [--dry-run]
-          | usage <events.json>                       per-call costs as kernels/lettertrace push them
+          | usage <events.json>                       per-call costs as kernels/lettertrace/models push them
+          | usage list --source models --from D --to D   one source's calls back, newest first
           | remove <entry-id> | services | service <slug> [--owner …] [--monthly N|none] [--renewal D] [--units N|none]
                                                        Where Letter's money goes (spending.letter.dev):
                                                        totals, run rate, area/vendor/provider/model/

package/lib/commands/admin.mjs

@@ -19,6 +19,7 @@
 //   admin brand-detection <org-id>                                                   (fleet:read)
 //   admin brand-detection <org-id> [--term t]... [--exact t]... [--exclude p]... [--note "…"] | --clear [--note "…"]   (fleet:instrument)
 //   admin recipes list|show|vocab|export|import|save|delete …                        (recipes:read / recipes:write)
+//   admin models runs|run|star|unstar|scoreboard …                                    (models:read / models:write)
 //   admin replays [--org <id>] [--target <id>] [--status …] [--limit n]                (fleet:read)
 //   admin replay --org <id> [<response-id> | --target <id>] [--surface …]           (fleet:replay)
 //   admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
@@ -3962,6 +3963,83 @@ function spendEntryFromFlags(flags) {
 	});
 }
 
+const MODELS = "/api/superadmin/models";
+const MODELS_USAGE =
+	"Usage: admin models runs [--model ballpoint-1] [--starred] [--limit N] | run <id> | star <id> | unstar <id> | scoreboard --model ballpoint-1";
+
+/** The model hub's saved runs and blind arena (models.letter.dev), same functions as its pages. */
+async function cmdAdminModels(ctx) {
+	const { flags, positionals, io } = ctx;
+	const sub = positionals[0];
+	switch (sub) {
+		case "runs": {
+			const params = new URLSearchParams(
+				compact({
+					model: flagStr(flags.model),
+					starred: flagBool(flags.starred) ? "true" : undefined,
+					limit: flagNum(flags.limit)?.toString(),
+				})
+			);
+			const result = await staffRequest(ctx, "GET", `${MODELS}/runs${params.size ? `?${params}` : ""}`);
+			printResult(io, flags, result, (r) =>
+				(r.runs ?? [])
+					.map(
+						(x) =>
+							`${x.created_at.slice(0, 16)}  ${x.starred ? "★" : " "} ${x.model.padEnd(14)} ${String(x.outputs).padStart(3)} outputs ${String(x.votes).padStart(3)} votes  ${x.contenders.join(", ")}  ${x.id}`
+					)
+					.join("\n")
+			);
+			return 0;
+		}
+		case "run": {
+			const id = positionals[1];
+			if (!id) throw new CliError("Usage: admin models run <id>");
+			const result = await staffRequest(ctx, "GET", `${MODELS}/runs/${id}`);
+			printResult(io, flags, result, (r) =>
+				[
+					`${r.model} · ${r.created_at} · ${JSON.stringify(r.input)}`,
+					...(r.outputs ?? []).map(
+						(o) =>
+							`  [${o.slot}] ${o.contender}${o.ours ? " (ours)" : ""}: ${o.error ? `error: ${o.error}` : Object.values(o.output).join(" — ")}`
+					),
+				].join("\n")
+			);
+			return 0;
+		}
+		case "star":
+		case "unstar": {
+			const id = positionals[1];
+			if (!id) throw new CliError(`Usage: admin models ${sub} <id>`);
+			const result = await staffRequest(ctx, "PATCH", `${MODELS}/runs/${id}`, { starred: sub === "star" });
+			ok(ctx, `${sub === "star" ? "Starred" : "Unstarred"} ${result.id}`);
+			printResult(io, flags, result, () => "");
+			return 0;
+		}
+		case "scoreboard": {
+			const model = flagStr(flags.model);
+			if (!model) throw new CliError("Usage: admin models scoreboard --model ballpoint-1");
+			const result = await staffRequest(
+				ctx,
+				"GET",
+				`${MODELS}/arena/scoreboard?model=${encodeURIComponent(model)}`
+			);
+			printResult(io, flags, result, (r) =>
+				(r.opponents ?? []).length
+					? r.opponents
+							.map(
+								(o) =>
+									`${r.model} vs ${o.opponent}: ${o.wins}-${o.losses} (${o.ties} ties, ${o.both_bad} both bad) · ${o.win_rate == null ? "—" : `${Math.round(o.win_rate * 100)}% [${Math.round(o.low * 100)}–${Math.round(o.high * 100)}]`} · ${o.voters} voters`
+							)
+							.join("\n")
+					: `No votes on ${r.model} yet.`
+			);
+			return 0;
+		}
+		default:
+			throw new CliError(MODELS_USAGE);
+	}
+}
+
 async function cmdAdminSpend(ctx) {
 	const { flags, positionals, io } = ctx;
 	const sub = positionals[0];
@@ -4043,7 +4121,34 @@ async function cmdAdminSpend(ctx) {
 			return 0;
 		}
 		case "usage": {
-			// Per-call model costs, as kernels / lettertrace push them (POST /api/superadmin/spend/usage):
+			if (positionals[1] === "list") {
+				// One source's calls back, newest first (GET /api/superadmin/spend/usage).
+				const source = flagStr(flags.source);
+				const from = flagStr(flags.from);
+				const to = flagStr(flags.to);
+				if (!source || !from || !to)
+					throw new CliError(
+						"Usage: admin spend usage list --source models --from YYYY-MM-DD --to YYYY-MM-DD [--limit N]"
+					);
+				const params = new URLSearchParams(
+					compact({ source, from, to, limit: flagNum(flags.limit)?.toString() })
+				);
+				const result = await staffRequest(ctx, "GET", `${SPEND}/usage?${params}`);
+				printResult(
+					io,
+					flags,
+					result,
+					(r) =>
+						(r.events ?? [])
+							.map(
+								(e) =>
+									`${e.occurred_at}  ${e.model.padEnd(16)} ${e.operation.padEnd(10)} ${(e.outcome ?? "").padEnd(8)} ${String(e.latency_ms ?? "").padStart(7)}ms  ${usd(e.cost_usd).padStart(9)}  ${e.key_label}`
+							)
+							.join("\n") + (r.next ? "\n… more (narrow --from/--to or raise --limit)" : "")
+				);
+				return 0;
+			}
+			// Per-call model costs, as kernels / lettertrace / models push them (POST /api/superadmin/spend/usage):
 			// a JSON file holding { events: [...] } or a bare array. Idempotent on each event_id.
 			const path = positionals[1];
 			if (!path) throw new CliError("Usage: admin spend usage <events.json>   ({ events: [...] } or an array)");
@@ -4193,7 +4298,7 @@ async function cmdAdminSpend(ctx) {
 		}
 		default:
 			throw new CliError(
-				"Usage: admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month] | reconcile [--from D --to D] | pnl [--range …] | reference add … | stripe-sync --from D --to D | entries [--from D] [--to D] [--vendor V] [--area A] [--source S] | add --vendor V --amount N --from D … | import <file.csv> … | usage <events.json> | remove <entry-id> | services | service <slug> --owner … --monthly …"
+				"Usage: admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month] | reconcile [--from D --to D] | pnl [--range …] | reference add … | stripe-sync --from D --to D | entries [--from D] [--to D] [--vendor V] [--area A] [--source S] | add --vendor V --amount N --from D … | import <file.csv> … | usage <events.json> | usage list --source S --from D --to D | remove <entry-id> | services | service <slug> --owner … --monthly …"
 			);
 	}
 }
@@ -4740,6 +4845,7 @@ export async function cmdAdmin(ctx) {
 		return cmdAdminReplays({ ...ctx, positionals: ctx.positionals.slice(1) }, sub);
 	if (sub === "crm") return cmdAdminCrm({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "spend") return cmdAdminSpend({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "models") return cmdAdminModels({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "patterns") return cmdAdminPatterns({ ...ctx, positionals: ctx.positionals.slice(1) });

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.122.0",
+	"version": "0.124.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",