@letterstory/cli 0.70.2 → 0.70.5

package/lib/cli.mjs

@@ -56,7 +56,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.70.2";
+export const VERSION = "0.70.5";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.
@@ -634,7 +634,8 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        registry the Pipeline tab draws (JSON).
   admin tuning <sub> …                                 The console's loop over a staff key: kernels,
                                                        candidate, compare, comparisons, picks, pick,
-                                                       reading-list, pair, side-by-side, promote,
+                                                       ledger, reading-list, pair, side-by-side,
+                                                       evals, eval-add, eval-set, promote,
                                                        sessions, session-new, propose, trial-add,
                                                        trial-run, trial-save, the Write input bank
                                                        (inputs, input, inputs-harvest, input-save),

package/lib/commands/admin.mjs

@@ -7,7 +7,7 @@
 //   admin experiments list|show|new|update|note|started|spend|conclude|abandon …   (tuning:experiment)
 //   admin tuning-context [--q "<question>"]                                         (tuning:experiment)
 //   admin tuning-import <brief-results.json> [--experiment <id>]                    (tuning:experiment)
-//   admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|promote …   (tuning:experiment)
+//   admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|criteria|promote …   (tuning:experiment)
 //   admin evals scoreboard|cases|runs|run|triage|guard|retire …                      (fleet:read / tuning:experiment)
 //   admin crm import|capture|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link …  (crm:read / crm:write)
 //   admin replays [--org <id>] [--target <id>] [--status …] [--limit n]                (fleet:read)
@@ -442,8 +442,13 @@ async function cmdAdminTuningContext(ctx) {
  *   admin tuning compare --title "…" --candidates Hazel:21,Aspen:20 [--experiment <id>]   start a comparison
  *                        (Hazel:23@22 reads v23 against the test version v22 instead of live; it cannot promote)
  *   admin tuning comparisons | comparison <id>                list / read one (per-measure verdicts)
- *   admin tuning picks <comparison-id>                         the blind queue (next pairs)
- *   admin tuning pick <comparison-id> --cell <key> --pick left|right|tie   record a blind pick
+ *   admin tuning picks <comparison-id> [--criterion opening]   the blind queue (next pairs); with a criterion, the pairs not yet read for it (Eye Test mode)
+ *   admin tuning pick <comparison-id> --cell <key> --pick left|right|tie [--criterion opening] [--note "why"]   record a blind pick (the note is the reader's reason; a criterion pick steers the next arm, never promotion)
+ *   admin tuning criteria                                      the Eye Test's criteria: id, label and the question each asks of a pair
+ *   admin tuning ledger <comparison-id>                        the defect ledger: per arm, what the diagnosis read named, the floor and the ceiling, the candidate's next arm
+ *   admin tuning evals                                         the eval list: every defect category with its status and the picks' agreement (n, agreement, suggestion)
+ *   admin tuning eval-add --id … --label … --how … --layer … --lever … --locality local|structural --axis … [--example "span"]   add a category (starts unvetted)
+ *   admin tuning eval-set <id> [--status confirmed|rejected|unvetted] [--example "span" [--fix "…"]]   confirm / reject (a person's call), or add an example
  *   admin tuning reading-list <comparison-id> [--n 10]          your next blind pairs as links, to read off the screen
  *   admin tuning archive <comparison-id> [--undo]               take a read comparison out of the working list (or back); the record stays
  *   admin tuning comparisons [--archived only|all]              the list; archived ones only when asked
@@ -670,24 +675,133 @@ async function cmdAdminTuning(ctx) {
 		case "comparison":
 			if (!id) throw new CliError("Usage: admin tuning comparison <id>");
 			return printResult(io, { ...flags, json: true }, await staffRequest(ctx, "GET", `${T}/comparisons/${id}`));
-		case "picks":
-			if (!id) throw new CliError("Usage: admin tuning picks <comparison-id>");
+		case "picks": {
+			if (!id) throw new CliError("Usage: admin tuning picks <comparison-id> [--criterion opening]");
+			const criterion = flagStr(flags.criterion);
+			const qs = criterion ? `?${new URLSearchParams({ criterion })}` : "";
 			return printResult(
 				io,
 				{ ...flags, json: true },
-				await staffRequest(ctx, "GET", `${T}/comparisons/${id}/picks`)
+				await staffRequest(ctx, "GET", `${T}/comparisons/${id}/picks${qs}`)
 			);
+		}
 		case "pick": {
 			const cell = flagStr(flags.cell);
 			const pick = flagStr(flags.pick);
+			const note = flagStr(flags.note);
+			const criterion = flagStr(flags.criterion);
 			if (!id || !cell || !["left", "right", "tie"].includes(pick ?? ""))
-				throw new CliError("Usage: admin tuning pick <comparison-id> --cell <key> --pick left|right|tie");
+				throw new CliError(
+					'Usage: admin tuning pick <comparison-id> --cell <key> --pick left|right|tie [--criterion opening] [--note "why"]'
+				);
 			return printResult(
 				io,
 				flags,
-				await staffRequest(ctx, "POST", `${T}/comparisons/${id}/picks`, { cell_key: cell, outcome: pick })
+				await staffRequest(ctx, "POST", `${T}/comparisons/${id}/picks`, {
+					cell_key: cell,
+					outcome: pick,
+					...(criterion ? { criterion } : {}),
+					...(note ? { note } : {}),
+				})
 			);
 		}
+		case "evals": {
+			const r = await staffRequest(ctx, "GET", `${T}/evals`);
+			if (json) return printResult(io, flags, r);
+			const rows = r.categories ?? [];
+			if (!rows.length) return io.log("(no categories)");
+			const cell = (v, w) => String(v ?? "—").padEnd(w);
+			io.log(
+				`${cell("id", 24)} ${cell("status", 10)} ${cell("layer", 10)} ${cell("lever", 30)} ${"n".padStart(4)} ${"agree".padStart(6)} ${cell("suggest", 10)} ${"ex".padStart(3)}`
+			);
+			for (const c of rows) {
+				const v = c.vetting ?? {};
+				io.log(
+					`${cell(c.id, 24)} ${cell(c.status, 10)} ${cell(c.layer, 10)} ${cell(c.lever || "—", 30)} ${String(v.n ?? 0).padStart(4)} ${String(v.agreement ?? "—").padStart(6)} ${cell(v.suggestion, 10)} ${String((c.examples ?? []).length).padStart(3)}`
+				);
+			}
+			return;
+		}
+		case "eval-add": {
+			const body = {
+				id: flagStr(flags.id),
+				label: flagStr(flags.label),
+				how: flagStr(flags.how),
+				layer: flagStr(flags.layer),
+				lever: flagStr(flags.lever) ?? "",
+				locality: flagStr(flags.locality),
+				axis: flagStr(flags.axis),
+			};
+			const example = flagStr(flags.example);
+			if (
+				!body.id ||
+				!body.label ||
+				!body.how ||
+				!body.layer ||
+				!body.axis ||
+				!["local", "structural"].includes(body.locality ?? "")
+			)
+				throw new CliError(
+					'Usage: admin tuning eval-add --id … --label … --how … --layer … --lever … --locality local|structural --axis … [--example "span"]'
+				);
+			if (example) body.examples = [{ span: example, ...(flags.fix ? { fix: flagStr(flags.fix) } : {}) }];
+			const r = await staffRequest(ctx, "POST", `${T}/evals`, body);
+			return json ? printResult(io, flags, r) : io.log(`added ${r.category.id} (${r.category.status})`);
+		}
+		case "eval-set": {
+			const status = flagStr(flags.status);
+			const example = flagStr(flags.example);
+			if (!id || (!status && !example) || (status && !["confirmed", "rejected", "unvetted"].includes(status)))
+				throw new CliError(
+					'Usage: admin tuning eval-set <id> [--status confirmed|rejected|unvetted] [--example "span" [--fix "…"]]'
+				);
+			const patch = {};
+			if (status) patch.status = status;
+			if (example) {
+				// The PATCH replaces the whole list: read what is stored and append.
+				const current = (await staffRequest(ctx, "GET", `${T}/evals`)).categories?.find((c) => c.id === id);
+				if (!current) throw new CliError(`No category ${id} in the eval list`);
+				const fix = flagStr(flags.fix);
+				patch.examples = [...(current.examples ?? []), { span: example, ...(fix ? { fix } : {}) }];
+			}
+			const r = await staffRequest(ctx, "PATCH", `${T}/evals/${encodeURIComponent(id)}`, patch);
+			return json
+				? printResult(io, flags, r)
+				: io.log(`${r.category.id}: ${r.category.status}, ${(r.category.examples ?? []).length} examples`);
+		}
+		case "criteria": {
+			const r = await staffRequest(ctx, "GET", `${T}/comparisons/criteria`);
+			if (json) return printResult(io, flags, r);
+			for (const c of r.criteria ?? []) io.log(`${c.id.padEnd(12)} ${c.label.padEnd(12)} ${c.question}`);
+			return;
+		}
+		case "ledger": {
+			if (!id) throw new CliError("Usage: admin tuning ledger <comparison-id>");
+			const read = await staffRequest(ctx, "GET", `${T}/comparisons/${id}`);
+			const ledger = read.ledger;
+			if (!ledger)
+				throw new CliError(
+					"This comparison has no ledger (not a Write comparison, or read before the diagnosis read existed)."
+				);
+			if (json) return printResult(io, flags, ledger);
+			const arm = (a) =>
+				`${a.arm}: ${a.read} read · floor ${a.floor ?? "—"} · ceiling ${a.ceiling ?? "—"} · local/pair ${a.local_per_pair ?? "—"} · structural/pair ${a.structural_per_pair ?? "—"}`;
+			io.log(arm(ledger.control));
+			io.log(arm(ledger.candidate));
+			io.log(`Δ floor ${ledger.delta.floor ?? "—"} · Δ ceiling ${ledger.delta.ceiling ?? "—"}`);
+			for (const a of [ledger.control, ledger.candidate]) {
+				io.log(`\n${a.arm} — by category (cost, pairs, lever):`);
+				for (const c of a.categories.slice(0, 12))
+					io.log(
+						`  ${c.category}${c.locality === "structural" ? " (structural)" : ""}  cost ${c.cost}  on ${c.pairs} pairs  → ${c.lever ?? "—"}${c.example ? `  e.g. “${String(c.example).slice(0, 90)}”` : ""}`
+					);
+			}
+			io.log("\nnext arm:");
+			if (!ledger.next_arm.length) io.log("  nothing local recurs on two pairs or more");
+			for (const n of ledger.next_arm)
+				io.log(`  ${n.instruction}${n.patch ? `  patch ${JSON.stringify(n.patch)}` : ""}`);
+			return;
+		}
 		case "reading-list": {
 			if (!id) throw new CliError("Usage: admin tuning reading-list <comparison-id> [--n 10]");
 			const n = flagStr(flags.n);
@@ -1456,7 +1570,7 @@ async function cmdAdminTuning(ctx) {
 		}
 		default:
 			throw new CliError(
-				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
+				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|ledger|evals|eval-add|eval-set|reading-list|pair|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
 			);
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.70.2",
+	"version": "0.70.5",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",