@letterstory/cli 0.123.0 → 0.124.0

package/lib/cli.mjs

@@ -61,13 +61,15 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.123.0";
+export const VERSION = "0.124.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value. Every `type: "bool"` flag (and its
 // `negate` spelling) in a command's ARGS map must be listed; boolean-flags.test.ts enforces it.
 export const BOOLEAN_FLAGS = new Set([
 	"json",
+	// admin models runs: only the starred ones.
+	"starred",
 	// shredder file / arena file: save the version as a candidate beside live.
 	"candidate",
 	// site tree: include children a fold-by-default family would hide; items over the whole subtree.
@@ -843,6 +845,8 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        session (a trial per arm), linked to its
                                                        ledger entry. Written by \`npm run lab --
                                                        publish\` in the kernels repo.
+  admin models runs [--model M] [--starred] | run <id> | star|unstar <id> | scoreboard --model M
+                                                       The model hub's saved runs and blind arena (models.letter.dev)
   admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month]
           | reconcile [--from D --to D] | pnl [--range day|week|month|year | --from D --to D]
           | reference add --vendor V [--key K] --metric billed_usd --value N --from D [--to D] --ref "…"

package/lib/commands/admin.mjs

@@ -19,6 +19,7 @@
 //   admin brand-detection <org-id>                                                   (fleet:read)
 //   admin brand-detection <org-id> [--term t]... [--exact t]... [--exclude p]... [--note "…"] | --clear [--note "…"]   (fleet:instrument)
 //   admin recipes list|show|vocab|export|import|save|delete …                        (recipes:read / recipes:write)
+//   admin models runs|run|star|unstar|scoreboard …                                    (models:read / models:write)
 //   admin replays [--org <id>] [--target <id>] [--status …] [--limit n]                (fleet:read)
 //   admin replay --org <id> [<response-id> | --target <id>] [--surface …]           (fleet:replay)
 //   admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
@@ -3962,6 +3963,83 @@ function spendEntryFromFlags(flags) {
 	});
 }
 
+const MODELS = "/api/superadmin/models";
+const MODELS_USAGE =
+	"Usage: admin models runs [--model ballpoint-1] [--starred] [--limit N] | run <id> | star <id> | unstar <id> | scoreboard --model ballpoint-1";
+
+/** The model hub's saved runs and blind arena (models.letter.dev), same functions as its pages. */
+async function cmdAdminModels(ctx) {
+	const { flags, positionals, io } = ctx;
+	const sub = positionals[0];
+	switch (sub) {
+		case "runs": {
+			const params = new URLSearchParams(
+				compact({
+					model: flagStr(flags.model),
+					starred: flagBool(flags.starred) ? "true" : undefined,
+					limit: flagNum(flags.limit)?.toString(),
+				})
+			);
+			const result = await staffRequest(ctx, "GET", `${MODELS}/runs${params.size ? `?${params}` : ""}`);
+			printResult(io, flags, result, (r) =>
+				(r.runs ?? [])
+					.map(
+						(x) =>
+							`${x.created_at.slice(0, 16)}  ${x.starred ? "★" : " "} ${x.model.padEnd(14)} ${String(x.outputs).padStart(3)} outputs ${String(x.votes).padStart(3)} votes  ${x.contenders.join(", ")}  ${x.id}`
+					)
+					.join("\n")
+			);
+			return 0;
+		}
+		case "run": {
+			const id = positionals[1];
+			if (!id) throw new CliError("Usage: admin models run <id>");
+			const result = await staffRequest(ctx, "GET", `${MODELS}/runs/${id}`);
+			printResult(io, flags, result, (r) =>
+				[
+					`${r.model} · ${r.created_at} · ${JSON.stringify(r.input)}`,
+					...(r.outputs ?? []).map(
+						(o) =>
+							`  [${o.slot}] ${o.contender}${o.ours ? " (ours)" : ""}: ${o.error ? `error: ${o.error}` : Object.values(o.output).join(" — ")}`
+					),
+				].join("\n")
+			);
+			return 0;
+		}
+		case "star":
+		case "unstar": {
+			const id = positionals[1];
+			if (!id) throw new CliError(`Usage: admin models ${sub} <id>`);
+			const result = await staffRequest(ctx, "PATCH", `${MODELS}/runs/${id}`, { starred: sub === "star" });
+			ok(ctx, `${sub === "star" ? "Starred" : "Unstarred"} ${result.id}`);
+			printResult(io, flags, result, () => "");
+			return 0;
+		}
+		case "scoreboard": {
+			const model = flagStr(flags.model);
+			if (!model) throw new CliError("Usage: admin models scoreboard --model ballpoint-1");
+			const result = await staffRequest(
+				ctx,
+				"GET",
+				`${MODELS}/arena/scoreboard?model=${encodeURIComponent(model)}`
+			);
+			printResult(io, flags, result, (r) =>
+				(r.opponents ?? []).length
+					? r.opponents
+							.map(
+								(o) =>
+									`${r.model} vs ${o.opponent}: ${o.wins}-${o.losses} (${o.ties} ties, ${o.both_bad} both bad) · ${o.win_rate == null ? "—" : `${Math.round(o.win_rate * 100)}% [${Math.round(o.low * 100)}–${Math.round(o.high * 100)}]`} · ${o.voters} voters`
+							)
+							.join("\n")
+					: `No votes on ${r.model} yet.`
+			);
+			return 0;
+		}
+		default:
+			throw new CliError(MODELS_USAGE);
+	}
+}
+
 async function cmdAdminSpend(ctx) {
 	const { flags, positionals, io } = ctx;
 	const sub = positionals[0];
@@ -4767,6 +4845,7 @@ export async function cmdAdmin(ctx) {
 		return cmdAdminReplays({ ...ctx, positionals: ctx.positionals.slice(1) }, sub);
 	if (sub === "crm") return cmdAdminCrm({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "spend") return cmdAdminSpend({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "models") return cmdAdminModels({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "patterns") return cmdAdminPatterns({ ...ctx, positionals: ctx.positionals.slice(1) });

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.123.0",
+	"version": "0.124.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",