@letterstory/cli 0.47.0 → 0.48.0

package/lib/cli.mjs

@@ -48,12 +48,13 @@ import {
 	cmdMailers,
 	cmdAdmin,
 	cmdFeedback,
+	cmdEvals,
 	cmdOfferings,
 	cmdTargets,
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.47.0";
+export const VERSION = "0.48.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.
@@ -94,6 +95,8 @@ const BOOLEAN_FLAGS = new Set([
 	"no-visuals",
 	"no-enhance",
 	"no-key",
+	// `admin evals triage --overdue`
+	"overdue",
 	// `autopen start --article <id> --from-scratch`
 	"from-scratch",
 	// `collections activate <id> --no-topic-map` / `--no-headlines`
@@ -594,6 +597,16 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        org:create. onboard-plan reads the calls
                                                        (sites, who to invite, paid or not);
                                                        payment-link needs billing:write.
+  admin evals scoreboard [--scope fleet|org|collection --id <uuid>] [--days N]
+          | cases [--org <uuid>] [--status s] [--stage s] | runs [--limit N] [--mode record|full]
+          | run [--mode record|full] [--keys a,b] [--org <uuid>] [--baseline latest|<id>]
+          | triage [--owner research|kernel|links|mentions|accuracy|pipeline] [--overdue]
+          | guard <case-key> [--sha <commit>] [--fix-pr #N] | retire <case-key> --reason "…"
+                                                       The fleet-wide eval surface: scoreboard,
+                                                       every org's cases and suite runs, the
+                                                       triage queue, and the two lifecycle calls
+                                                       an org key cannot make. Reads need
+                                                       fleet:read; writes need tuning:experiment.
   admin tuning-context [--q "<question>"]              The experiment agent's context pack (JSON):
                                                        the pipeline as deployed, levers, measures,
                                                        goals, related history.
@@ -833,6 +846,18 @@ Feedback (tell us what's broken, missing, unclear, or wanted):
                   [--message <text>|--body <text>|--file <path>|--file -] [--rating 1-5]
                   [--context '<json>']
 
+Evals (how the pipeline is doing on your posts, and the regression cases captured from them):
+  evals scoreboard [--collection <id>] [--days N]      Every measure by stage: posts, failing, rate
+  evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline]
+  evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
+                (--description "…" | --body … | --file <path>) (--measure <key> | --assertion "…")
+                [--stage <s>] [--replay <stage>] [--auto-pen <id>] [--published-version N]
+                [--incident <ref>] [--link <url>] [--status open|proposed]
+                                                       Freeze the post's record as a case; exits 1 when
+                                                       it does NOT detect the issue on current code
+  evals run [--keys a,b] [--baseline latest|<run-id>]  Record-only suite; exits 1 on a regression
+  evals run-status <run-id>
+
 Anything else:
   tools list                         List every tool this server exposes, by name
   tools show <name>                  Show one tool's capability, argument schema, and return shape
@@ -907,6 +932,7 @@ const CLIENT_COMMANDS = {
 	mail: cmdMail,
 	mailers: cmdMailers,
 	feedback: cmdFeedback,
+	evals: cmdEvals,
 	// Platform-scoped (staff key) — the only commands that act BEFORE an org exists.
 	admin: cmdAdmin,
 };

package/lib/commands/admin.mjs

@@ -7,6 +7,7 @@
 //   admin experiments list|show|new|update|note|started|spend|conclude|abandon …   (tuning:experiment)
 //   admin tuning-context [--q "<question>"]                                         (tuning:experiment)
 //   admin tuning-import <brief-results.json> [--experiment <id>]                    (tuning:experiment)
+//   admin evals scoreboard|cases|runs|run|triage|guard|retire …                      (fleet:read / tuning:experiment)
 //   admin crm import|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link …  (crm:read / crm:write)
 //   admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
 //                    [--capabilities a,b] [--onboard] [--about "…"] [--sitemap <url>]
@@ -879,6 +880,125 @@ async function cmdAdminCrm(ctx) {
 	}
 }
 
+/* ------------------------------------------------------------ content evals */
+
+const EVALS = "/api/superadmin/evals";
+
+/**
+ * `admin evals …` — the staff half of the eval surface, against /api/superadmin/evals: the
+ * fleet-wide scoreboard, every org's cases and runs, the triage queue, and the two lifecycle
+ * decisions an org key cannot make (guard, retire). Reads need fleet:read; writes need
+ * tuning:experiment. The org-scoped half is `letterstory evals …` on an org key.
+ */
+async function cmdAdminEvals(ctx) {
+	const [sub, id] = ctx.positionals;
+	const { flags, io } = ctx;
+	const json = flagBool(flags.json);
+	const out = (result, line) => (json ? printResult(io, flags, result) : io.log(line(result)));
+	const query = (obj) => {
+		const params = new URLSearchParams(compact(obj));
+		const q = params.toString();
+		return q ? `?${q}` : "";
+	};
+	switch (sub) {
+		case "scoreboard": {
+			const q = query({ scope: flagStr(flags.scope), id: flagStr(flags.id), days: flagStr(flags.days) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/scoreboard${q}`),
+				(r) =>
+					(r.stages ?? [])
+						.flatMap((st) => st.measures ?? [])
+						.map(
+							(m) => `${m.stage.padEnd(9)} ${m.measure.padEnd(44)} ${m.failing}/${m.posts}  ${m.severity}`
+						)
+						.join("\n") || "(no measurements in the window)"
+			);
+		}
+		case "cases": {
+			const q = query({ org_id: flagStr(flags.org), status: flagStr(flags.status), stage: flagStr(flags.stage) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/cases${q}`),
+				(r) =>
+					(r.cases ?? [])
+						.map(
+							(c) =>
+								`${c.key.padEnd(48)} ${c.status.padEnd(9)} ${c.stage.padEnd(9)} ${c.latestResult ? c.latestResult.verdict : "never run"}`
+						)
+						.join("\n") || "(no cases)"
+			);
+		}
+		case "runs": {
+			const q = query({ limit: flagStr(flags.limit), mode: flagStr(flags.mode) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/suite/runs${q}`),
+				(r) =>
+					(r.runs ?? [])
+						.map(
+							(run) =>
+								`${run.id}  ${run.startedAt}  ${run.mode.padEnd(6)} ${run.verdict.padEnd(4)} ${run.isBaseline ? "baseline " : ""}${run.totals?.regressed ?? 0} regressed`
+						)
+						.join("\n") || "(no runs)"
+			);
+		}
+		case "run": {
+			const keys = flagStr(flags.keys);
+			const body = compact({
+				mode: flagStr(flags.mode),
+				case_keys: keys
+					? keys
+							.split(",")
+							.map((k) => k.trim())
+							.filter(Boolean)
+					: undefined,
+				org_id: flagStr(flags.org),
+				baseline: flagStr(flags.baseline),
+			});
+			const r = await staffRequest(ctx, "POST", `${EVALS}/suite/runs`, body);
+			out(
+				r,
+				(x) =>
+					`Run ${x.run?.id ?? "(queued)"} — ${String(x.run?.verdict ?? "").toUpperCase()}  ${x.run?.totals?.regressed ?? 0} regressed`
+			);
+			return r.run?.verdict === "fail" ? 1 : 0;
+		}
+		case "triage": {
+			const q = query({ owner: flagStr(flags.owner), overdue: flagBool(flags.overdue) ? "true" : undefined });
+			return out(await staffRequest(ctx, "GET", `${EVALS}/triage${q}`), (r) =>
+				[
+					...(r.findings ?? []).map(
+						(f) =>
+							`${f.id}  ${(f.owner ?? "unassigned").padEnd(10)} ${f.evalKey.padEnd(36)} ${f.overdue ? "OVERDUE " : ""}${f.title ?? f.summary ?? ""}`
+					),
+					`${r.total ?? 0} open, ${r.overdue ?? 0} overdue`,
+				].join("\n")
+			);
+		}
+		case "guard": {
+			if (!id) throw new CliError("Usage: admin evals guard <case-key> [--sha <commit>] [--fix-pr #123]");
+			const body = compact({ action: "guard", sha: flagStr(flags.sha), fix_pr: flagStr(flags["fix-pr"]) });
+			return out(await staffRequest(ctx, "PATCH", `${EVALS}/cases/${encodeURIComponent(id)}`, body), (r) =>
+				r.ok ? `${r.case.key} is now guarding (first passed ${r.case.firstPassedSha})` : `refused: ${r.reason}`
+			);
+		}
+		case "retire": {
+			if (!id) throw new CliError('Usage: admin evals retire <case-key> --reason "…"');
+			const reason = flagStr(flags.reason);
+			if (!reason) throw new CliError("Retiring a case needs --reason.");
+			return out(
+				await staffRequest(ctx, "PATCH", `${EVALS}/cases/${encodeURIComponent(id)}`, {
+					action: "retire",
+					reason,
+				}),
+				(r) => (r.ok ? `${r.case.key} retired` : `refused: ${r.reason}`)
+			);
+		}
+		default:
+			throw new CliError(
+				`Unknown admin evals subcommand: ${sub ?? "(none)"}. Try: scoreboard, cases, runs, run, triage, guard, retire`
+			);
+	}
+}
+
 export async function cmdAdmin(ctx) {
 	const sub = ctx.positionals[0];
 	if (sub === "crm") return cmdAdminCrm({ ...ctx, positionals: ctx.positionals.slice(1) });
@@ -887,7 +1007,8 @@ export async function cmdAdmin(ctx) {
 	if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-context") return cmdAdminTuningContext({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-import") return cmdAdminTuningImport({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "evals") return cmdAdminEvals({ ...ctx, positionals: ctx.positionals.slice(1) });
 	throw new CliError(
-		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin tuning-context (see --help)'
+		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin evals … | admin tuning-context (see --help)'
 	);
 }

package/lib/commands/evals.mjs

@@ -0,0 +1,198 @@
+// `evals` — the content-eval surface on an org key: how the pipeline is doing on YOUR
+// posts (the scoreboard), the regression cases captured from them, capturing a new one from
+// a bad post, and running the record-only suite.
+//
+//   evals scoreboard [--collection <id>] [--days N]
+//   evals cases [--status open|guarding|…] [--stage research|brief|…]
+//   evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
+//                 --description "…" (--measure <key> | --assertion "…") [--stage s] [--replay <stage>]
+//                 [--auto-pen <id>] [--published-version N] [--incident …] [--link …]
+//   evals run [--keys a,b] [--baseline latest|<run-id>]        exits 1 when a guarding case regressed
+//   evals run-status <run-id>
+//
+// Five commands, five tools; every input key is declared in ARGS so parity-check compares
+// them against the tools' published schemas (see shared.mjs `argsFromFlags`).
+
+import { CliError } from "../client.mjs";
+import { flagStr, readBodyInput, printResult, ok, argsFromFlags } from "./shared.mjs";
+
+const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline"];
+const STATUSES = ["proposed", "open", "guarding", "retired", "rejected"];
+
+/** `--keys a,b,c` (or repeated `--keys`) → ["a", "b", "c"]; undefined when unset. */
+function keysFrom(flags) {
+	const raw = flags.keys;
+	if (raw === undefined) return undefined;
+	const items = (Array.isArray(raw) ? raw : [raw])
+		.flatMap((v) => String(v).split(","))
+		.map((k) => k.trim())
+		.filter(Boolean);
+	return items.length ? items : undefined;
+}
+
+export const ARGS = {
+	get_eval_scoreboard: {
+		// One collection narrows the scope; otherwise the whole org (the tool's default).
+		scope: { value: (flags) => (flagStr(flags.collection) ? "collection" : undefined) },
+		collection_id: { flag: "collection" },
+		days: { flag: "days", type: "num" },
+	},
+	list_eval_cases: {
+		status: { flag: "status", enum: STATUSES },
+		stage: { flag: "stage", enum: STAGES },
+	},
+	capture_eval_case: {
+		article_id: { flag: "article" },
+		research_run_id: { flag: "research-run" },
+		kernel_job_id: { flag: "kernel-job" },
+		title: { flag: "title", required: true },
+		// --description, or the larger-blob trio --body/--file/stdin.
+		description: { value: (flags) => flagStr(flags.description) ?? readBodyInput(flags) },
+		expected: { object: { measure: { flag: "measure" }, assertion: { flag: "assertion" } } },
+		stage: { flag: "stage", enum: STAGES },
+		// Pin the incident's auto-pen run and live version when the post has been re-run since.
+		auto_pen_id: { flag: "auto-pen" },
+		published_version: { flag: "published-version", type: "num" },
+		// A record-only replay stage to re-run over the fixture before grading (evals catalog).
+		replay: "replay",
+		link: "link",
+		incident: "incident",
+		status: { flag: "status", enum: ["open", "proposed"] },
+	},
+	run_eval_suite: {
+		case_keys: { value: (flags) => keysFrom(flags) },
+		baseline: { flag: "baseline" },
+	},
+	get_eval_suite_run: {
+		run_id: { positional: 0, label: "run-id", required: true },
+	},
+};
+
+export async function cmdEvals(ctx) {
+	const sub = ctx.positionals[0];
+	const rest = { ...ctx, positionals: ctx.positionals.slice(1) };
+	switch (sub) {
+		case "scoreboard":
+			return evalsScoreboard(rest);
+		case "cases":
+			return evalsCases(rest);
+		case "capture":
+			return evalsCapture(rest);
+		case "run":
+			return evalsRun(rest);
+		case "run-status":
+			return evalsRunStatus(rest);
+		default:
+			throw new CliError(
+				`Unknown evals subcommand: ${sub ?? "(none)"}. Try: scoreboard, cases, capture, run, run-status`
+			);
+	}
+}
+
+function pct(rate) {
+	return `${Math.round((rate ?? 0) * 100)}%`;
+}
+
+function formatScoreboard(r) {
+	const lines = [`Scoreboard — ${r.scope}${r.id ? ` ${r.id}` : ""}, last ${r.days} days`];
+	for (const stage of r.stages ?? []) {
+		if (!stage.measures?.length) continue;
+		lines.push(`\n${stage.stage}`);
+		for (const m of stage.measures) {
+			lines.push(
+				`  ${m.measure.padEnd(44)} ${String(m.failing).padStart(4)}/${String(m.posts).padEnd(4)} ${pct(m.rate).padStart(4)}  ${m.severity}${m.kind === "judge" ? "  (judge)" : ""}`
+			);
+		}
+	}
+	if (r.judgeReliance) {
+		lines.push(`\njudge reliance: ${r.judgeReliance.measures} measure(s), ${r.judgeReliance.cases} case(s)`);
+	}
+	return lines.join("\n");
+}
+
+async function evalsScoreboard(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.get_eval_scoreboard, flags, ctx.positionals);
+	const result = await client.callTool("get_eval_scoreboard", args);
+	printResult(io, flags, result, formatScoreboard);
+	return 0;
+}
+
+function caseLine(c) {
+	const latest = c.latestResult ? `${c.latestResult.verdict} (${c.latestResult.gate})` : "never run";
+	const expected = c.expected?.measure ? c.expected.measure : `assertion: ${c.expected?.assertion ?? ""}`;
+	return `${c.key.padEnd(48)} ${c.status.padEnd(9)} ${c.stage.padEnd(9)} ${latest.padEnd(26)} ${expected}`;
+}
+
+async function evalsCases(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.list_eval_cases, flags, ctx.positionals);
+	const result = await client.callTool("list_eval_cases", args);
+	printResult(io, flags, result, (r) =>
+		r.cases?.length ? r.cases.map(caseLine).join("\n") : "(no cases captured from this org's posts)"
+	);
+	return 0;
+}
+
+async function evalsCapture(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.capture_eval_case, flags, ctx.positionals);
+	if (!args.article_id && !args.research_run_id && !args.kernel_job_id) {
+		throw new CliError("Name the post: --article <id>, --research-run <id> or --kernel-job <id>.");
+	}
+	if (!args.description) {
+		throw new CliError(
+			"Missing --description (or --body / --file <path> / stdin): what went wrong, in a sentence or two."
+		);
+	}
+	const expected = args.expected ?? {};
+	if ((expected.measure ? 1 : 0) + (expected.assertion ? 1 : 0) !== 1) {
+		throw new CliError('Say what must hold once fixed: exactly one of --measure <key> or --assertion "…".');
+	}
+	const result = await client.callTool("capture_eval_case", args);
+	const detects = result.detectsIssue;
+	ok(
+		ctx,
+		detects
+			? `Captured ${result.case?.key}: it FAILS on current code, so it detects the issue.`
+			: `Captured ${result.case?.key}, but it PASSES on current code — it does not detect what it claims. Tighten the assertion or pick the right measure.`
+	);
+	printResult(
+		io,
+		flags,
+		result,
+		(r) => `${r.case?.key}\n  ${r.firstResult?.verdict}: ${r.firstResult?.summary ?? ""}`
+	);
+	return detects === false ? 1 : 0;
+}
+
+function resultLine(r) {
+	const delta = r.delta && r.delta !== "no_baseline" ? ` ${r.delta}` : "";
+	return `  ${r.caseKey.padEnd(48)} ${r.verdict.padEnd(6)} ${r.gate.padEnd(16)}${delta}  ${r.summary ?? ""}`;
+}
+
+function formatRun(run) {
+	const t = run.totals ?? {};
+	const head = `Run ${run.id ?? "(not persisted)"} — ${run.verdict.toUpperCase()}  (${t.cases ?? 0} cases: ${t.pass ?? 0} pass, ${t.fail ?? 0} fail, ${t.skip ?? 0} skip, ${t.error ?? 0} error; ${t.regressed ?? 0} regressed, ${t.readyToGuard ?? 0} ready to guard)${run.gitSha ? `  @ ${run.gitSha.slice(0, 10)}` : ""}`;
+	const lines = [head, ...(run.results ?? []).map(resultLine)];
+	if (run.unresolved?.length) {
+		lines.push("", ...run.unresolved.map((u) => `  ! ${u.key}: ${u.reason}`));
+	}
+	return lines.join("\n");
+}
+
+async function evalsRun(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.run_eval_suite, flags, ctx.positionals);
+	const result = await client.callTool("run_eval_suite", args);
+	printResult(io, flags, result, (r) => formatRun(r.run));
+	return result.run?.verdict === "fail" ? 1 : 0;
+}
+
+async function evalsRunStatus(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.get_eval_suite_run, flags, ctx.positionals);
+	const result = await client.callTool("get_eval_suite_run", args);
+	printResult(io, flags, result, (r) => formatRun(r.run));
+	return 0;
+}

package/lib/commands.mjs

@@ -38,3 +38,4 @@ export * from "./commands/offerings.mjs";
 export * from "./commands/targets.mjs";
 export * from "./commands/admin.mjs";
 export * from "./commands/feedback.mjs";
+export * from "./commands/evals.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.47.0",
+	"version": "0.48.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",