@letterstory/cli 0.72.1 → 0.73.0

package/README.md

@@ -368,6 +368,8 @@ letterstory admin tuning bank-judge <eval-id> --prompt-file judge.md --mode scor
 letterstory admin tuning bank-score <eval-id> --version 2         # score it over the pool; prints agreement on held-out labels
 letterstory admin tuning bank-try <eval-id> --version 2           # the judge as a trial rule in candidates, and a comparison against live
 letterstory admin tuning invite --email editor@example.com        # invite an editor (Pick and Eval bank only); `people`, `revoke <id>`
+letterstory admin tuning assign --pick <comparison-id> --instructions "Read for the opening"   # a task for every editor (--to <email> for one)
+letterstory admin tuning assign --label bank-draft-x-not-y --target 50                      # editors cut or keep 50 of this eval's sentences
 letterstory admin tuning side-by-side <id>,<id>,<id> [--cell <key>]   # several writers as one board: each one's measures against live; with --cell, every writer's text on that input
 letterstory admin tuning candidates --kernels all --patch '{"writer":"echo/echo"}' --note "Echo"   # the same change as a candidate of each kernel
 letterstory admin tuning session-new --title "Echo round 1" --kernel Hazel --input-file brief.md --kind outline [--version 22] [--versions 23,29]   # --versions: saved candidates, a trial each

package/lib/cli.mjs

@@ -56,7 +56,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.72.1";
+export const VERSION = "0.73.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.
@@ -646,7 +646,9 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        (inputs, input, inputs-harvest, input-save),
                                                        screens (screen-new, screens, screen,
                                                        screen-advance), the bake-off (bakeoff,
-                                                       bakeoff-pick, bakeoff-merge),
+                                                       bakeoff-pick, bakeoff-merge), programmes
+                                                       (programmes, programme, programme-new,
+                                                       programme-stop, programme-advance),
                                                        and the arena, shredder
                                                        and enhancer twins (every flag is listed in
                                                        cli/lib/commands/admin.mjs).

package/lib/commands/admin.mjs

@@ -473,6 +473,8 @@ async function cmdAdminTuningContext(ctx) {
  *   admin tuning bank-score <eval-id> --version N             score a judge version over the pool (call again for more than 600)
  *   admin tuning bank-try <eval-id> --version N [--kernels Hazel,Aspen]   the judge as a trial rule in candidates, and a comparison against live
  *   admin tuning people | invite --email e [--note "…"] | revoke <id>   who an admin invited into the console as an editor
+ *   admin tuning assignments | assign --pick <comparison-id>|--label <eval-id> [--to e] [--instructions "…"] [--target N] | unassign <id>   what editors are asked to do
+ *   admin tuning tasks                                         your own open tasks, with progress
  *   admin tuning side-by-side <id>,<id>[,…] [--cell <key>]     several comparisons as one board: each writer's measures against live, and one input's texts
  *   admin tuning candidates --kernels Hazel,Aspen|all --patch '{…}' [--base N] [--note "…"]   the same changes as a candidate of each kernel
  *   admin tuning promote <experiment-id> --versions <service-kernel-id>:<version>,…   promote from a "ship" decision
@@ -514,6 +516,15 @@ async function cmdAdminTuningContext(ctx) {
  *   admin tuning bakeoff-pick <screen-id> --input <key> --item <category:id|axis:id> --winner <version|none>   a person's grade on one input and item (replaces the machine's)
  *   admin tuning bakeoff-merge <screen-id> [--resolve item=version,…] [--note "…"] [--compare 10]   save the merged file as a candidate of the base kernel (never live); --compare starts a comparison against live on that many pairs
  *
+ * Programmes: the bake-off run by the machine, generation after generation, on the cron, with a run
+ * budget and stop rules (the budget; N flat generations; a finalist that won its picks). Each
+ * generation screens the base against its challengers, merges the winners, compares the merge
+ * against live; a merge that cleared the gates and lost no craft becomes the base. People pick.
+ *   admin tuning programmes | programme <id>                   list / one programme: every generation, its table, merge and comparison, and what happens next
+ *   admin tuning programme-new --title "…" --kernel <kernel-id|service-kernel-id> --seeds 21,22,23 [--inputs id,…|--n 8] [--filter "…"] [--pairs 10] --budget 200 [--flat 2]
+ *   admin tuning programme-stop <id> [--reason "…"]            stop it by hand
+ *   admin tuning programme-advance <id>                        one manual tick (the cron takes the same step every minute)
+ *
  * Research (Phase 3): our arena's versions as files, research comparisons, the deploy split.
  *   admin tuning arenas | arena <version-id>                   our arena's versions / one version's file and diff
  *   admin tuning arena-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"]   save a candidate
@@ -573,6 +584,49 @@ function tagFilterParams(raw) {
  * A session as an agent reads it: each trial's changes and runs with their headline numbers, and
  * (with `texts`) the latest finished text per trial. The full session (traces, records) is `--json`.
  */
+/** A programme as a person reads it: the row, then one block per generation, then what happens next. */
+function printProgramme(io, r) {
+	const p = r.programme;
+	io.log(
+		`programme ${p.title} (${p.id}): ${p.status}${p.stop_reason ? ` — ${p.stop_reason}` : ""} · generation ${p.generation} · base ${p.base_version === null ? "live" : `v${p.base_version}`} · ${p.runs_used}/${p.budget_runs} runs · seeds v${p.seed_versions.join(", v")} · ${p.pairs} pairs/generation · stop after ${p.max_flat_generations} flat`
+	);
+	for (const g of r.generations) {
+		const sm = g.summary ?? {};
+		io.log("");
+		io.log(
+			`generation ${g.generation}: ${g.status} · screen ${g.screen_id} · v${g.versions.join(", v")}${g.merged_version !== null ? ` · merged v${g.merged_version}` : ""}${g.comparison_id ? ` · comparison ${g.comparison_id}` : ""}`
+		);
+		if (sm.from)
+			io.log(
+				`  from: base ${sm.from.base === null ? "live" : `v${sm.from.base}`}${sm.from.seeds ? ` · seeds v${sm.from.seeds.join(", v")}` : ""}${sm.from.strongest?.length ? ` · strongest v${sm.from.strongest.join(", v")}` : ""}${sm.from.merged != null ? ` · merged v${sm.from.merged}` : ""}${sm.from.mutations?.length ? ` · mutations ${sm.from.mutations.map((m) => `${m.category} → v${m.version}`).join(", ")}` : ""}`
+			);
+		if (sm.leader !== undefined)
+			io.log(
+				`  leader ${sm.leader === null ? "none" : `v${sm.leader}`}${sm.flat ? " · flat (nothing moved off the base)" : ""}`
+			);
+		for (const row of sm.table ?? [])
+			if (row.leader !== null && row.leader !== sm.base)
+				io.log(
+					`    ${row.item.id.padEnd(32)} ${row.trusted ? "trusted  " : "unvetted "} v${row.leader}  ${Object.entries(
+						row.wins
+					)
+						.map(([v, n]) => `v${v}:${n}`)
+						.join(" ")}`
+				);
+		if (sm.merge)
+			io.log(
+				`  merge: from v${sm.merge.base} → v${sm.merge.version}; applied ${sm.merge.applied.map((a) => `${a.item} from v${a.from}`).join(", ") || "nothing"}${sm.merge.conflicts.length ? `; conflicts kept the base: ${sm.merge.conflicts.map((c) => c.path).join(", ")}` : ""}`
+			);
+		if (sm.comparison)
+			io.log(
+				`  comparison: quality ${sm.comparison.quality ?? "unread"}${sm.comparison.guards_worse.length ? ` · guards worse: ${sm.comparison.guards_worse.join(", ")}` : " · gates clean"} · ${sm.comparison.finalist ? "finalist" : "not a finalist"}${sm.comparison.picks ? ` · picks ${sm.comparison.picks}` : ""}${sm.comparison.delta?.ceiling != null ? ` · ceiling ${sm.comparison.delta.ceiling > 0 ? "+" : ""}${sm.comparison.delta.ceiling}` : ""}${sm.comparison.next_arm.length ? ` · next arm: ${sm.comparison.next_arm.map((a) => `${a.category}${a.patch ? " (patch)" : ""}`).join(", ")}` : ""}`
+			);
+		if (sm.error) io.log(`  error: ${sm.error}`);
+	}
+	io.log("");
+	io.log(`next: ${r.next}`);
+}
+
 function sessionSummary(session, texts) {
 	const runsOf = (trialId) => session.runs.filter((r) => r.trialId === trialId);
 	return {
@@ -1135,6 +1189,48 @@ async function cmdAdminTuning(ctx) {
 				`${ctx.client.url}/superadmin/tuning?${new URLSearchParams({ tab: "compare", comparison: r.comparison_id })}`
 			);
 		}
+		case "assignments": {
+			const r = await staffRequest(ctx, "GET", `${T}/assignments${flagBool(flags.closed) ? "?closed=1" : ""}`);
+			if (json) return printResult(io, flags, r);
+			for (const a of r.assignments)
+				io.log(
+					`${a.id} ${a.kind} "${a.title}" → ${a.assignee_email ?? "every editor"}${a.target ? `, ${a.target} asked` : ""}${a.closed_at ? " (closed)" : ""}`
+				);
+			return;
+		}
+		case "assign": {
+			const pick = flagStr(flags.pick);
+			const label = flagStr(flags.label);
+			if (!pick === !label)
+				throw new CliError(
+					'Usage: admin tuning assign --pick <comparison-id> | --label <eval-id> [--to email] [--instructions "…"] [--title "…"] [--target N]'
+				);
+			const body = compact({
+				...(pick ? { kind: "pick", comparison_id: pick } : { kind: "label", eval_id: label }),
+				assignee_email: flagStr(flags.to),
+				instructions: flagStr(flags.instructions),
+				title: flagStr(flags.title),
+				target: flagNum(flags.target),
+			});
+			const r = await staffRequest(ctx, "POST", `${T}/assignments`, body);
+			if (json) return printResult(io, flags, r);
+			return io.log(
+				`assigned ${r.assignment.kind} task "${r.assignment.title}" to ${r.assignment.assignee_email ?? "every editor"} (${r.assignment.id})`
+			);
+		}
+		case "unassign": {
+			if (!id) throw new CliError("Usage: admin tuning unassign <assignment-id>");
+			return printResult(io, flags, await staffRequest(ctx, "DELETE", `${T}/assignments/${id}`));
+		}
+		case "tasks": {
+			const r = await staffRequest(ctx, "GET", `${T}/editor/tasks`);
+			if (json) return printResult(io, flags, r);
+			for (const t of r.tasks)
+				io.log(
+					`${t.id} ${t.kind} "${t.title}": ${t.done} done, ${t.remaining} left${t.complete ? " (complete)" : ""}`
+				);
+			return;
+		}
 		case "people": {
 			const r = await staffRequest(ctx, "GET", `${T}/people`);
 			if (json) return printResult(io, flags, r);
@@ -1635,6 +1731,95 @@ async function cmdAdminTuning(ctx) {
 				);
 			return;
 		}
+		// --- Programmes: the bake-off run by the machine over generations ---
+		case "programmes": {
+			const r = await staffRequest(ctx, "GET", `${T}/programmes?limit=${flagNum(flags.limit) ?? 30}`);
+			if (json) return printResult(io, flags, r);
+			return io.log(
+				r.programmes
+					.map(
+						(x) =>
+							`${x.id}  ${x.status.padEnd(7)} ${x.title} · generation ${x.generation} · base ${x.base_version === null ? "live" : `v${x.base_version}`} · ${x.runs_used}/${x.budget_runs} runs${x.stop_reason ? ` · ${x.stop_reason}` : ""}`
+					)
+					.join("\n") || "(no programmes)"
+			);
+		}
+		case "programme": {
+			if (!id) throw new CliError("Usage: admin tuning programme <programme-id>");
+			const r = await staffRequest(ctx, "GET", `${T}/programmes/${id}`);
+			if (json) return printResult(io, flags, r);
+			printProgramme(io, r);
+			return;
+		}
+		case "programme-new": {
+			const title = flagStr(flags.title);
+			const kernel = flagStr(flags.kernel);
+			const seeds = flagList(flags.seeds)
+				.flatMap((x) => x.split(","))
+				.map((x) => Number(x.trim()))
+				.filter((x) => Number.isFinite(x));
+			const budget = flagNum(flags.budget);
+			if (!title || !kernel || !seeds.length || budget === undefined)
+				throw new CliError(
+					'Usage: admin tuning programme-new --title "…" --kernel <kernel-id|service-kernel-id> --seeds 21,22,23 [--inputs id,…|--n 8] [--filter "…"] [--pairs 10] --budget 200 [--flat 2]'
+				);
+			const filter = Object.fromEntries(
+				Object.entries(tagFilterParams(flagStr(flags.filter))).map(([k, v]) => [
+					k,
+					v
+						.split(",")
+						.map((x) => x.trim())
+						.filter(Boolean),
+				])
+			);
+			const inputIds = flagList(flags.inputs).flatMap((x) =>
+				x
+					.split(",")
+					.map((y) => y.trim())
+					.filter(Boolean)
+			);
+			const r = await staffRequest(
+				ctx,
+				"POST",
+				`${T}/programmes`,
+				compact({
+					title,
+					base_kernel_id: kernel,
+					seed_versions: seeds,
+					input_ids: inputIds.length ? inputIds : undefined,
+					n: flagNum(flags.n),
+					filter: Object.keys(filter).length ? filter : undefined,
+					pairs: flagNum(flags.pairs),
+					budget_runs: budget,
+					max_flat_generations: flagNum(flags.flat),
+				})
+			);
+			if (json) return printResult(io, flags, r);
+			return io.log(
+				`programme ${r.programme.id}: generation 1 screening v${r.programme.seed_versions.join(", v")} against live; budget ${r.programme.budget_runs} runs · ${r.next}`
+			);
+		}
+		case "programme-stop": {
+			if (!id) throw new CliError('Usage: admin tuning programme-stop <programme-id> [--reason "…"]');
+			const r = await staffRequest(
+				ctx,
+				"POST",
+				`${T}/programmes/${id}/stop`,
+				compact({ reason: flagStr(flags.reason) })
+			);
+			if (json) return printResult(io, flags, r);
+			return io.log(`programme ${r.programme.id} ${r.programme.status}: ${r.programme.stop_reason}`);
+		}
+		case "programme-advance": {
+			if (!id) throw new CliError("Usage: admin tuning programme-advance <programme-id>");
+			const r = await staffRequest(ctx, "POST", `${T}/programmes/${id}/advance`, {});
+			if (json) return printResult(io, flags, r);
+			if (!r.transitions.length) io.log("nothing to do yet");
+			for (const t of r.transitions)
+				io.log(`generation ${t.generation}: ${t.from} → ${t.to}${t.note ? ` (${t.note})` : ""}`);
+			if (r.read) io.log(`next: ${r.read.next}`);
+			return;
+		}
 		// --- Research: our arena's versions, research comparisons, the deploy split ---
 		case "arenas": {
 			const r = await staffRequest(ctx, "GET", `${T}/arena-files`);
@@ -2003,7 +2188,7 @@ async function cmdAdminTuning(ctx) {
 		}
 		default:
 			throw new CliError(
-				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|loop|ledger|evals|eval-add|eval-set|proposals|propose-eval|capture-eval|reject-eval|reading-list|finalists|finalists-queue|pair|bank|bank-eval|bank-draft|bank-shape|bank-pool|bank-judge|bank-score|bank-try|people|invite|revoke|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
+				"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|loop|ledger|evals|eval-add|eval-set|proposals|propose-eval|capture-eval|reject-eval|reading-list|finalists|finalists-queue|pair|bank|bank-eval|bank-draft|bank-shape|bank-pool|bank-judge|bank-score|bank-try|people|invite|revoke|assignments|assign|unassign|tasks|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
 			);
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.72.1",
+	"version": "0.73.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",