@letterstory/cli 0.63.1 → 0.64.0

package/lib/cli.mjs

@@ -55,7 +55,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.63.1";
+export const VERSION = "0.64.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.

package/lib/commands/admin.mjs

@@ -452,6 +452,10 @@ async function cmdAdminTuningContext(ctx) {
  *   admin tuning arenas | arena <version-id>                   our arena's versions / one version's file and diff
  *   admin tuning arena-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"]   save a candidate
  *   admin tuning research-compare --title "…" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys …]
+ *   admin tuning research-compare --title "…" --candidates <version-id>,… | --patches '[{…},{…}]'   several
+ *                            candidates against ONE control arm (k + 1 runs per input, not 2k)
+ *   admin tuning arena-dials                                   every arena setting: help, default, evidence, live's value
+ *   admin tuning comparison-extend <comparison-id> --keys k1,k2 | --all   add inputs to a comparison on fixed inputs
  *   admin tuning research-inputs [--set v1]                    research's fixed inputs
  *   admin tuning research-split [--since-hours 72] [--at <iso>,…]   what each deploy did to research in production
  *   admin tuning research-standing                             live's level on the fixed inputs, week by week
@@ -899,12 +903,34 @@ async function cmdAdminTuning(ctx) {
 			const versionId = flagStr(flags.candidate);
 			const patch = jsonFlag(flags.patch, "--patch");
 			const nullCheck = flagBool(flags["null-check"]);
-			const named = [versionId !== undefined, patch !== undefined, nullCheck === true].filter(Boolean).length;
+			const versionIds = flagList(flags.candidates);
+			const patches = jsonFlag(flags.patches, "--patches");
+			if (patches !== undefined && !Array.isArray(patches))
+				throw new CliError("--patches is a JSON array of patches: '[{…},{…}]'");
+			const named = [
+				versionId !== undefined,
+				patch !== undefined,
+				nullCheck === true,
+				(versionIds?.length ?? 0) + (patches?.length ?? 0) > 0,
+			].filter(Boolean).length;
 			if (!title || named !== 1)
 				throw new CliError(
-					"Usage: admin tuning research-compare --title \"…\" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys k1,k2] [--concurrency 6] [--experiment <id>]"
+					"Usage: admin tuning research-compare --title \"…\" (--candidate <version-id> | --patch '{…}' | --null-check | --candidates <id>,… [--patches '[{…},…]']) [--keys k1,k2] [--concurrency 6] [--experiment <id>]"
 				);
-			const candidate = versionId ? { version_id: versionId } : patch ? { patch } : { null_check: true };
+			const several =
+				versionIds?.length || patches?.length
+					? [
+							...(versionIds ?? []).map((v) => ({ version_id: v })),
+							...(patches ?? []).map((p) => ({ patch: p })),
+						]
+					: undefined;
+			const candidate = several
+				? undefined
+				: versionId
+					? { version_id: versionId }
+					: patch
+						? { patch }
+						: { null_check: true };
 			const r = await staffRequest(
 				ctx,
 				"POST",
@@ -913,6 +939,7 @@ async function cmdAdminTuning(ctx) {
 					primitive: "research",
 					title,
 					candidate,
+					candidates: several,
 					keys: flagList(flags.keys),
 					concurrency: flagNum(flags.concurrency),
 					experiment_id: flagStr(flags.experiment),
@@ -920,6 +947,34 @@ async function cmdAdminTuning(ctx) {
 			);
 			return json ? printResult(io, flags, r) : io.log(`research comparison ${r.comparison?.id ?? ""} started`);
 		}
+		case "arena-dials": {
+			const r = await staffRequest(ctx, "GET", `${T}/arena-files/dials`);
+			if (json) return printResult(io, flags, r);
+			const valueOf = (key) => key.split(".").reduce((v, k) => (v == null ? v : v[k]), r.live.file);
+			const lines = [];
+			for (const panel of r.panels) {
+				lines.push(`${panel.title} — ${panel.question}`);
+				for (const d of r.dials.filter((x) => x.panel === panel.id))
+					lines.push(
+						`  ${d.key.padEnd(36)} live ${JSON.stringify(valueOf(d.key))}  (default ${JSON.stringify(d.fleetDefault)})  ${d.label}`
+					);
+			}
+			lines.push(
+				`providers: ${r.providers.map((p) => `${p.id}${p.configured ? "" : " (no key here)"}`).join(", ")}`
+			);
+			return io.log(lines.join("\n"));
+		}
+		case "comparison-extend": {
+			const keys = flagBool(flags.all) ? "all" : flagList(flags.keys);
+			if (!id || !keys?.length)
+				throw new CliError("Usage: admin tuning comparison-extend <comparison-id> --keys k1,k2 | --all");
+			const r = await staffRequest(ctx, "POST", `${T}/comparisons/${id}/extend`, { keys });
+			return json
+				? printResult(io, flags, r)
+				: io.log(
+						`comparison ${r.comparison.id}: ${r.comparison.added.length} input(s) added, ${r.comparison.pairs} in all`
+					);
+		}
 		case "research-inputs": {
 			const set = flagStr(flags.set);
 			return printResult(
@@ -1182,7 +1237,7 @@ async function cmdAdminTuning(ctx) {
 		}
 		default:
 			throw new CliError(
-				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
+				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
 			);
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.63.1",
+	"version": "0.64.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",