@letterstory/cli 0.46.0 → 0.48.0

package/README.md

@@ -670,6 +670,50 @@ letterstory insights bank [--source phantoms|marketing|all] [--phantom <uuid>]..
 letterstory insights access [--period 14d|30d|90d] [--collection <uuid>]
 ```
 
+## Outbound email — mailers, sequences, prospects
+
+The cold-email stack: sending **mailers** (real Google mailboxes on domains we own), **sequences**
+(multi-step flows sent from them, with a review queue) and **prospects** (who you write to). Writes
+here can SEND email or spend money, so they refuse unknown flags instead of ignoring them. Scopes:
+`mailer:read` / `mailer:manage` / `mailer:provision`, `sequence:read` / `sequence:write`,
+`prospect:read` / `prospect:write` (the write scopes are admin-only keys).
+
+```bash
+# Mailers
+letterstory mailers list [--domain acme.com]                 # status + daily send limit
+letterstory mailers domain <owned-domain-id>                 # provisioning stage, DNS, mailers on it
+letterstory mailers messages <mailer-id> [--folder inbox|sent|all] [--limit 25]   # envelopes
+letterstory mailers message <mailer-id> <uid> --folder inbox # one message, plain-text body
+letterstory mailers forward <mailer-id> --to me@acme.com    # or --clear
+letterstory mailers warmup start|pause <mailer-id>           # 10 days to ready
+letterstory mailers limit <mailer-id> 40                     # 0..500; refused while warming
+letterstory mailers retire <mailer-id>                       # disable; history kept
+letterstory mailers retire <mailer-id> --release-mailbox --yes   # also delete the hosted mailbox (irreversible)
+letterstory mailers suppressions [list] [--reason bounce] [--limit 100]
+letterstory mailers suppressions add person@x.com --reason manual --note "asked to stop"
+letterstory mailers suppressions remove person@x.com         # only 'manual' entries; opt-outs stay
+
+# Launching buys a domain (up to $30) and the mailboxes, so it is a tool call, not a command:
+letterstory call launch_mailers --args '{"domain":"acme-mail.com","mailers":[{"local_part":"hello"}],"consent_attested":true,"mail_from_postal_address":"…"}'
+
+# Org-level mail settings
+letterstory mail connections                                 # connected mailboxes, key + postal address status
+letterstory mail postal-address "123 Main St, Portland, OR 97201"   # CAN-SPAM footer
+
+# Sequences (approving or enrolling SENDS)
+letterstory sequences list | get <id> | create | update <id> | pause <id> | resume <id>
+letterstory sequences enroll <id> --list <list-id>           # or --email a@x.com ...
+letterstory sequences review list | approve <item-id> | skip <item-id> | edit <item-id> --subject "…"
+letterstory sequences links create --days 2                  # no-login phone review link, shown once
+
+# Prospects
+letterstory prospects lists | ls --list <list-id> | get <id> | import --csv-file leads.csv --list-name "Q4"
+letterstory prospects source --description "heads of growth at seed-stage fintech" --list-name "Fintech"
+```
+
+Connecting a bring-your-own Gmail mailbox needs a browser (Google consent), so it is done in the
+app; `letterstory mail connections` shows what is connected.
+
 ## Anything else
 
 Every Letterstory tool is reachable, not just the ones with a dedicated command group:

package/lib/cli.mjs

@@ -45,14 +45,16 @@ import {
 	cmdSequences,
 	cmdProspects,
 	cmdMail,
+	cmdMailers,
 	cmdAdmin,
 	cmdFeedback,
+	cmdEvals,
 	cmdOfferings,
 	cmdTargets,
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.46.0";
+export const VERSION = "0.48.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.
@@ -93,6 +95,8 @@ const BOOLEAN_FLAGS = new Set([
 	"no-visuals",
 	"no-enhance",
 	"no-key",
+	// `admin evals triage --overdue`
+	"overdue",
 	// `autopen start --article <id> --from-scratch`
 	"from-scratch",
 	// `collections activate <id> --no-topic-map` / `--no-headlines`
@@ -593,6 +597,16 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        org:create. onboard-plan reads the calls
                                                        (sites, who to invite, paid or not);
                                                        payment-link needs billing:write.
+  admin evals scoreboard [--scope fleet|org|collection --id <uuid>] [--days N]
+          | cases [--org <uuid>] [--status s] [--stage s] | runs [--limit N] [--mode record|full]
+          | run [--mode record|full] [--keys a,b] [--org <uuid>] [--baseline latest|<id>]
+          | triage [--owner research|kernel|links|mentions|accuracy|pipeline] [--overdue]
+          | guard <case-key> [--sha <commit>] [--fix-pr #N] | retire <case-key> --reason "…"
+                                                       The fleet-wide eval surface: scoreboard,
+                                                       every org's cases and suite runs, the
+                                                       triage queue, and the two lifecycle calls
+                                                       an org key cannot make. Reads need
+                                                       fleet:read; writes need tuning:experiment.
   admin tuning-context [--q "<question>"]              The experiment agent's context pack (JSON):
                                                        the pipeline as deployed, levers, measures,
                                                        goals, related history.
@@ -798,6 +812,22 @@ Prospects (who you write to):
            [--first-name] [--last-name] [--title] [--company] [--website] [--linkedin] [--school]
            [--city] [--country] [--years] [--notes] [--custom key=value]...
 
+Mailers (the sending mailboxes; launching one buys a domain, so use: call launch_mailers):
+  mailers list [--domain <apex>]                       Your mailers + status + daily send limit
+  mailers domain <owned-domain-id>                     Provisioning stage, DNS state, mailers on a domain
+  mailers messages <mailer-id> [--folder all|inbox|sent] [--limit <n>]
+                                                       Recent mail (envelopes)
+  mailers message <mailer-id> <uid> --folder inbox|sent
+                                                       One message in full (plain-text body)
+  mailers forward <mailer-id> (--to <email> | --clear) Copy inbound mail to an address
+  mailers warmup start|pause <mailer-id>               Start / pause warming (10 days to ready)
+  mailers limit <mailer-id> <per-day>                  Daily send limit, 0..500 (not while warming)
+  mailers retire <mailer-id> [--release-mailbox --yes] Disable a mailer; --release-mailbox also deletes
+                                                       the hosted mailbox (stops billing, irreversible)
+  mailers suppressions [list] [--reason <r>] [--limit <n>]
+  mailers suppressions add <address> [--reason unsubscribe|bounce|complaint|manual] [--note <t>]
+  mailers suppressions remove <address>                Only 'manual' entries can be removed
+
 Mail:
   mail connections                                     Connected mailboxes, Google connect, Vercel key + mailing address status
   mail postal-address "<address>"                      Set the org's CAN-SPAM mailing address (email footers)
@@ -816,6 +846,18 @@ Feedback (tell us what's broken, missing, unclear, or wanted):
                   [--message <text>|--body <text>|--file <path>|--file -] [--rating 1-5]
                   [--context '<json>']
 
+Evals (how the pipeline is doing on your posts, and the regression cases captured from them):
+  evals scoreboard [--collection <id>] [--days N]      Every measure by stage: posts, failing, rate
+  evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline]
+  evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
+                (--description "…" | --body … | --file <path>) (--measure <key> | --assertion "…")
+                [--stage <s>] [--replay <stage>] [--auto-pen <id>] [--published-version N]
+                [--incident <ref>] [--link <url>] [--status open|proposed]
+                                                       Freeze the post's record as a case; exits 1 when
+                                                       it does NOT detect the issue on current code
+  evals run [--keys a,b] [--baseline latest|<run-id>]  Record-only suite; exits 1 on a regression
+  evals run-status <run-id>
+
 Anything else:
   tools list                         List every tool this server exposes, by name
   tools show <name>                  Show one tool's capability, argument schema, and return shape
@@ -888,7 +930,9 @@ const CLIENT_COMMANDS = {
 	sequences: cmdSequences,
 	prospects: cmdProspects,
 	mail: cmdMail,
+	mailers: cmdMailers,
 	feedback: cmdFeedback,
+	evals: cmdEvals,
 	// Platform-scoped (staff key) — the only commands that act BEFORE an org exists.
 	admin: cmdAdmin,
 };

package/lib/commands/admin.mjs

@@ -7,6 +7,7 @@
 //   admin experiments list|show|new|update|note|started|spend|conclude|abandon …   (tuning:experiment)
 //   admin tuning-context [--q "<question>"]                                         (tuning:experiment)
 //   admin tuning-import <brief-results.json> [--experiment <id>]                    (tuning:experiment)
+//   admin evals scoreboard|cases|runs|run|triage|guard|retire …                      (fleet:read / tuning:experiment)
 //   admin crm import|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link …  (crm:read / crm:write)
 //   admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
 //                    [--capabilities a,b] [--onboard] [--about "…"] [--sitemap <url>]
@@ -879,6 +880,125 @@ async function cmdAdminCrm(ctx) {
 	}
 }
 
+/* ------------------------------------------------------------ content evals */
+
+const EVALS = "/api/superadmin/evals";
+
+/**
+ * `admin evals …` — the staff half of the eval surface, against /api/superadmin/evals: the
+ * fleet-wide scoreboard, every org's cases and runs, the triage queue, and the two lifecycle
+ * decisions an org key cannot make (guard, retire). Reads need fleet:read; writes need
+ * tuning:experiment. The org-scoped half is `letterstory evals …` on an org key.
+ */
+async function cmdAdminEvals(ctx) {
+	const [sub, id] = ctx.positionals;
+	const { flags, io } = ctx;
+	const json = flagBool(flags.json);
+	const out = (result, line) => (json ? printResult(io, flags, result) : io.log(line(result)));
+	const query = (obj) => {
+		const params = new URLSearchParams(compact(obj));
+		const q = params.toString();
+		return q ? `?${q}` : "";
+	};
+	switch (sub) {
+		case "scoreboard": {
+			const q = query({ scope: flagStr(flags.scope), id: flagStr(flags.id), days: flagStr(flags.days) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/scoreboard${q}`),
+				(r) =>
+					(r.stages ?? [])
+						.flatMap((st) => st.measures ?? [])
+						.map(
+							(m) => `${m.stage.padEnd(9)} ${m.measure.padEnd(44)} ${m.failing}/${m.posts}  ${m.severity}`
+						)
+						.join("\n") || "(no measurements in the window)"
+			);
+		}
+		case "cases": {
+			const q = query({ org_id: flagStr(flags.org), status: flagStr(flags.status), stage: flagStr(flags.stage) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/cases${q}`),
+				(r) =>
+					(r.cases ?? [])
+						.map(
+							(c) =>
+								`${c.key.padEnd(48)} ${c.status.padEnd(9)} ${c.stage.padEnd(9)} ${c.latestResult ? c.latestResult.verdict : "never run"}`
+						)
+						.join("\n") || "(no cases)"
+			);
+		}
+		case "runs": {
+			const q = query({ limit: flagStr(flags.limit), mode: flagStr(flags.mode) });
+			return out(
+				await staffRequest(ctx, "GET", `${EVALS}/suite/runs${q}`),
+				(r) =>
+					(r.runs ?? [])
+						.map(
+							(run) =>
+								`${run.id}  ${run.startedAt}  ${run.mode.padEnd(6)} ${run.verdict.padEnd(4)} ${run.isBaseline ? "baseline " : ""}${run.totals?.regressed ?? 0} regressed`
+						)
+						.join("\n") || "(no runs)"
+			);
+		}
+		case "run": {
+			const keys = flagStr(flags.keys);
+			const body = compact({
+				mode: flagStr(flags.mode),
+				case_keys: keys
+					? keys
+							.split(",")
+							.map((k) => k.trim())
+							.filter(Boolean)
+					: undefined,
+				org_id: flagStr(flags.org),
+				baseline: flagStr(flags.baseline),
+			});
+			const r = await staffRequest(ctx, "POST", `${EVALS}/suite/runs`, body);
+			out(
+				r,
+				(x) =>
+					`Run ${x.run?.id ?? "(queued)"} — ${String(x.run?.verdict ?? "").toUpperCase()}  ${x.run?.totals?.regressed ?? 0} regressed`
+			);
+			return r.run?.verdict === "fail" ? 1 : 0;
+		}
+		case "triage": {
+			const q = query({ owner: flagStr(flags.owner), overdue: flagBool(flags.overdue) ? "true" : undefined });
+			return out(await staffRequest(ctx, "GET", `${EVALS}/triage${q}`), (r) =>
+				[
+					...(r.findings ?? []).map(
+						(f) =>
+							`${f.id}  ${(f.owner ?? "unassigned").padEnd(10)} ${f.evalKey.padEnd(36)} ${f.overdue ? "OVERDUE " : ""}${f.title ?? f.summary ?? ""}`
+					),
+					`${r.total ?? 0} open, ${r.overdue ?? 0} overdue`,
+				].join("\n")
+			);
+		}
+		case "guard": {
+			if (!id) throw new CliError("Usage: admin evals guard <case-key> [--sha <commit>] [--fix-pr #123]");
+			const body = compact({ action: "guard", sha: flagStr(flags.sha), fix_pr: flagStr(flags["fix-pr"]) });
+			return out(await staffRequest(ctx, "PATCH", `${EVALS}/cases/${encodeURIComponent(id)}`, body), (r) =>
+				r.ok ? `${r.case.key} is now guarding (first passed ${r.case.firstPassedSha})` : `refused: ${r.reason}`
+			);
+		}
+		case "retire": {
+			if (!id) throw new CliError('Usage: admin evals retire <case-key> --reason "…"');
+			const reason = flagStr(flags.reason);
+			if (!reason) throw new CliError("Retiring a case needs --reason.");
+			return out(
+				await staffRequest(ctx, "PATCH", `${EVALS}/cases/${encodeURIComponent(id)}`, {
+					action: "retire",
+					reason,
+				}),
+				(r) => (r.ok ? `${r.case.key} retired` : `refused: ${r.reason}`)
+			);
+		}
+		default:
+			throw new CliError(
+				`Unknown admin evals subcommand: ${sub ?? "(none)"}. Try: scoreboard, cases, runs, run, triage, guard, retire`
+			);
+	}
+}
+
 export async function cmdAdmin(ctx) {
 	const sub = ctx.positionals[0];
 	if (sub === "crm") return cmdAdminCrm({ ...ctx, positionals: ctx.positionals.slice(1) });
@@ -887,7 +1007,8 @@ export async function cmdAdmin(ctx) {
 	if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-context") return cmdAdminTuningContext({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-import") return cmdAdminTuningImport({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "evals") return cmdAdminEvals({ ...ctx, positionals: ctx.positionals.slice(1) });
 	throw new CliError(
-		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin tuning-context (see --help)'
+		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin evals … | admin tuning-context (see --help)'
 	);
 }

package/lib/commands/evals.mjs

@@ -0,0 +1,198 @@
+// `evals` — the content-eval surface on an org key: how the pipeline is doing on YOUR
+// posts (the scoreboard), the regression cases captured from them, capturing a new one from
+// a bad post, and running the record-only suite.
+//
+//   evals scoreboard [--collection <id>] [--days N]
+//   evals cases [--status open|guarding|…] [--stage research|brief|…]
+//   evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
+//                 --description "…" (--measure <key> | --assertion "…") [--stage s] [--replay <stage>]
+//                 [--auto-pen <id>] [--published-version N] [--incident …] [--link …]
+//   evals run [--keys a,b] [--baseline latest|<run-id>]        exits 1 when a guarding case regressed
+//   evals run-status <run-id>
+//
+// Five commands, five tools; every input key is declared in ARGS so parity-check compares
+// them against the tools' published schemas (see shared.mjs `argsFromFlags`).
+
+import { CliError } from "../client.mjs";
+import { flagStr, readBodyInput, printResult, ok, argsFromFlags } from "./shared.mjs";
+
+const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline"];
+const STATUSES = ["proposed", "open", "guarding", "retired", "rejected"];
+
+/** `--keys a,b,c` (or repeated `--keys`) → ["a", "b", "c"]; undefined when unset. */
+function keysFrom(flags) {
+	const raw = flags.keys;
+	if (raw === undefined) return undefined;
+	const items = (Array.isArray(raw) ? raw : [raw])
+		.flatMap((v) => String(v).split(","))
+		.map((k) => k.trim())
+		.filter(Boolean);
+	return items.length ? items : undefined;
+}
+
+export const ARGS = {
+	get_eval_scoreboard: {
+		// One collection narrows the scope; otherwise the whole org (the tool's default).
+		scope: { value: (flags) => (flagStr(flags.collection) ? "collection" : undefined) },
+		collection_id: { flag: "collection" },
+		days: { flag: "days", type: "num" },
+	},
+	list_eval_cases: {
+		status: { flag: "status", enum: STATUSES },
+		stage: { flag: "stage", enum: STAGES },
+	},
+	capture_eval_case: {
+		article_id: { flag: "article" },
+		research_run_id: { flag: "research-run" },
+		kernel_job_id: { flag: "kernel-job" },
+		title: { flag: "title", required: true },
+		// --description, or the larger-blob trio --body/--file/stdin.
+		description: { value: (flags) => flagStr(flags.description) ?? readBodyInput(flags) },
+		expected: { object: { measure: { flag: "measure" }, assertion: { flag: "assertion" } } },
+		stage: { flag: "stage", enum: STAGES },
+		// Pin the incident's auto-pen run and live version when the post has been re-run since.
+		auto_pen_id: { flag: "auto-pen" },
+		published_version: { flag: "published-version", type: "num" },
+		// A record-only replay stage to re-run over the fixture before grading (evals catalog).
+		replay: "replay",
+		link: "link",
+		incident: "incident",
+		status: { flag: "status", enum: ["open", "proposed"] },
+	},
+	run_eval_suite: {
+		case_keys: { value: (flags) => keysFrom(flags) },
+		baseline: { flag: "baseline" },
+	},
+	get_eval_suite_run: {
+		run_id: { positional: 0, label: "run-id", required: true },
+	},
+};
+
+export async function cmdEvals(ctx) {
+	const sub = ctx.positionals[0];
+	const rest = { ...ctx, positionals: ctx.positionals.slice(1) };
+	switch (sub) {
+		case "scoreboard":
+			return evalsScoreboard(rest);
+		case "cases":
+			return evalsCases(rest);
+		case "capture":
+			return evalsCapture(rest);
+		case "run":
+			return evalsRun(rest);
+		case "run-status":
+			return evalsRunStatus(rest);
+		default:
+			throw new CliError(
+				`Unknown evals subcommand: ${sub ?? "(none)"}. Try: scoreboard, cases, capture, run, run-status`
+			);
+	}
+}
+
+function pct(rate) {
+	return `${Math.round((rate ?? 0) * 100)}%`;
+}
+
+function formatScoreboard(r) {
+	const lines = [`Scoreboard — ${r.scope}${r.id ? ` ${r.id}` : ""}, last ${r.days} days`];
+	for (const stage of r.stages ?? []) {
+		if (!stage.measures?.length) continue;
+		lines.push(`\n${stage.stage}`);
+		for (const m of stage.measures) {
+			lines.push(
+				`  ${m.measure.padEnd(44)} ${String(m.failing).padStart(4)}/${String(m.posts).padEnd(4)} ${pct(m.rate).padStart(4)}  ${m.severity}${m.kind === "judge" ? "  (judge)" : ""}`
+			);
+		}
+	}
+	if (r.judgeReliance) {
+		lines.push(`\njudge reliance: ${r.judgeReliance.measures} measure(s), ${r.judgeReliance.cases} case(s)`);
+	}
+	return lines.join("\n");
+}
+
+async function evalsScoreboard(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.get_eval_scoreboard, flags, ctx.positionals);
+	const result = await client.callTool("get_eval_scoreboard", args);
+	printResult(io, flags, result, formatScoreboard);
+	return 0;
+}
+
+function caseLine(c) {
+	const latest = c.latestResult ? `${c.latestResult.verdict} (${c.latestResult.gate})` : "never run";
+	const expected = c.expected?.measure ? c.expected.measure : `assertion: ${c.expected?.assertion ?? ""}`;
+	return `${c.key.padEnd(48)} ${c.status.padEnd(9)} ${c.stage.padEnd(9)} ${latest.padEnd(26)} ${expected}`;
+}
+
+async function evalsCases(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.list_eval_cases, flags, ctx.positionals);
+	const result = await client.callTool("list_eval_cases", args);
+	printResult(io, flags, result, (r) =>
+		r.cases?.length ? r.cases.map(caseLine).join("\n") : "(no cases captured from this org's posts)"
+	);
+	return 0;
+}
+
+async function evalsCapture(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.capture_eval_case, flags, ctx.positionals);
+	if (!args.article_id && !args.research_run_id && !args.kernel_job_id) {
+		throw new CliError("Name the post: --article <id>, --research-run <id> or --kernel-job <id>.");
+	}
+	if (!args.description) {
+		throw new CliError(
+			"Missing --description (or --body / --file <path> / stdin): what went wrong, in a sentence or two."
+		);
+	}
+	const expected = args.expected ?? {};
+	if ((expected.measure ? 1 : 0) + (expected.assertion ? 1 : 0) !== 1) {
+		throw new CliError('Say what must hold once fixed: exactly one of --measure <key> or --assertion "…".');
+	}
+	const result = await client.callTool("capture_eval_case", args);
+	const detects = result.detectsIssue;
+	ok(
+		ctx,
+		detects
+			? `Captured ${result.case?.key}: it FAILS on current code, so it detects the issue.`
+			: `Captured ${result.case?.key}, but it PASSES on current code — it does not detect what it claims. Tighten the assertion or pick the right measure.`
+	);
+	printResult(
+		io,
+		flags,
+		result,
+		(r) => `${r.case?.key}\n  ${r.firstResult?.verdict}: ${r.firstResult?.summary ?? ""}`
+	);
+	return detects === false ? 1 : 0;
+}
+
+function resultLine(r) {
+	const delta = r.delta && r.delta !== "no_baseline" ? ` ${r.delta}` : "";
+	return `  ${r.caseKey.padEnd(48)} ${r.verdict.padEnd(6)} ${r.gate.padEnd(16)}${delta}  ${r.summary ?? ""}`;
+}
+
+function formatRun(run) {
+	const t = run.totals ?? {};
+	const head = `Run ${run.id ?? "(not persisted)"} — ${run.verdict.toUpperCase()}  (${t.cases ?? 0} cases: ${t.pass ?? 0} pass, ${t.fail ?? 0} fail, ${t.skip ?? 0} skip, ${t.error ?? 0} error; ${t.regressed ?? 0} regressed, ${t.readyToGuard ?? 0} ready to guard)${run.gitSha ? `  @ ${run.gitSha.slice(0, 10)}` : ""}`;
+	const lines = [head, ...(run.results ?? []).map(resultLine)];
+	if (run.unresolved?.length) {
+		lines.push("", ...run.unresolved.map((u) => `  ! ${u.key}: ${u.reason}`));
+	}
+	return lines.join("\n");
+}
+
+async function evalsRun(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.run_eval_suite, flags, ctx.positionals);
+	const result = await client.callTool("run_eval_suite", args);
+	printResult(io, flags, result, (r) => formatRun(r.run));
+	return result.run?.verdict === "fail" ? 1 : 0;
+}
+
+async function evalsRunStatus(ctx) {
+	const { client, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.get_eval_suite_run, flags, ctx.positionals);
+	const result = await client.callTool("get_eval_suite_run", args);
+	printResult(io, flags, result, (r) => formatRun(r.run));
+	return 0;
+}

package/lib/commands/outbound.mjs

@@ -1,4 +1,4 @@
-// `sequences`, `prospects` and `mail` — the outbound email stack, as thin wrappers
+// `sequences`, `prospects`, `mailers` and `mail` — the outbound email stack, as thin wrappers
 // over the MCP tools, which call the same lib functions the app's /api/sequences
 // and /api/prospects routes do. Every call is built from the ARGS map below, so
 // `scripts/parity-check.ts` can hold each command to its tool's input schema.
@@ -58,6 +58,22 @@ const mailerIds = {
 	},
 };
 
+// A required positional that must be a whole number (the server schema wants a number, not a string).
+function intPositional(index, label) {
+	return {
+		value: (_flags, positionals) => {
+			const raw = positionals[index];
+			if (raw === undefined) throw new CliError(`Missing required <${label}>`);
+			const n = Number(raw);
+			if (!Number.isInteger(n)) throw new CliError(`<${label}> must be a whole number (got ${raw})`);
+			return n;
+		},
+	};
+}
+
+const SUPPRESSION_REASONS = ["unsubscribe", "bounce", "complaint", "manual"];
+const MAILER_FOLDERS = ["inbox", "sent"];
+
 // How every tool this module calls gets its arguments — see `argsFromFlags` in shared.mjs.
 export const ARGS = {
 	list_sequences: {},
@@ -112,6 +128,9 @@ export const ARGS = {
 	get_prospect_list: {
 		list_id: { positional: 0, label: "list-id", required: true },
 	},
+	delete_prospect_list: {
+		list_id: { positional: 0, label: "list-id", required: true },
+	},
 	list_prospects: {
 		list_id: "list",
 		status: { flag: "status", enum: PROSPECT_STATUSES },
@@ -166,6 +185,59 @@ export const ARGS = {
 		notes: "notes",
 		custom: { value: readCustom },
 	},
+	list_mailers: {
+		domain: "domain",
+	},
+	get_mailer_domain_status: {
+		owned_domain_id: { positional: 0, label: "owned-domain-id", required: true },
+	},
+	list_mailer_messages: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+		folder: { flag: "folder", enum: ["all", ...MAILER_FOLDERS] },
+		limit: { flag: "limit", type: "num" },
+	},
+	get_mailer_message: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+		uid: intPositional(1, "uid"),
+		folder: { flag: "folder", required: true, enum: MAILER_FOLDERS },
+	},
+	set_mailer_forwarding: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+		forward_to: {
+			value: (flags) => {
+				if (flags.clear === true) return null;
+				const to = flagStr(flags.to);
+				if (to === undefined) throw new CliError("Pass --to <email> (or --clear to stop forwarding)");
+				return to;
+			},
+		},
+	},
+	start_mailer_warmup: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+	},
+	pause_mailer_warmup: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+	},
+	set_mailer_daily_send_limit: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+		daily_send_limit: intPositional(1, "per-day"),
+	},
+	retire_mailer: {
+		mailer_id: { positional: 0, label: "mailer-id", required: true },
+		release_mailbox: { flag: "release-mailbox", type: "bool" },
+	},
+	add_email_suppression: {
+		address: { positional: 0, label: "address", required: true },
+		reason: { flag: "reason", enum: SUPPRESSION_REASONS },
+		note: "note",
+	},
+	list_email_suppressions: {
+		reason: { flag: "reason", enum: SUPPRESSION_REASONS },
+		limit: { flag: "limit", type: "num" },
+	},
+	remove_email_suppression: {
+		address: { positional: 0, label: "address", required: true },
+	},
 	get_mail_connections: {},
 	set_mail_postal_address: {
 		mail_postal_address: { positional: 0, label: "address", required: true },
@@ -180,6 +252,7 @@ const VALUE_FLAGS = {
 	update_review_item: ["body", "file"],
 	import_prospects_csv: ["csv", "csv-file"],
 	update_prospect: ["custom"],
+	set_mailer_forwarding: ["to", "clear"],
 };
 
 async function run(ctx, tool, message) {
@@ -280,7 +353,7 @@ async function sequencesLinks(ctx, sub) {
 	}
 }
 
-const PROSPECTS_USAGE = `Usage: prospects <lists|list|ls|get|create-list|import|source|enrich|update>`;
+const PROSPECTS_USAGE = `Usage: prospects <lists|list|ls|get|create-list|delete-list|import|source|enrich|update>`;
 
 export async function cmdProspects(ctx) {
 	const sub = ctx.positionals[0];
@@ -297,6 +370,8 @@ export async function cmdProspects(ctx) {
 			return run(rest, "get_prospect");
 		case "create-list":
 			return run(rest, "create_prospect_list", "List ready.");
+		case "delete-list":
+			return run(rest, "delete_prospect_list", "List deleted (its prospects stay).");
 		case "import":
 			return run(rest, "import_prospects_csv", "Imported.");
 		case "source":
@@ -314,6 +389,66 @@ export async function cmdProspects(ctx) {
 	}
 }
 
+const MAILERS_USAGE = `Usage: mailers <list|domain|messages|message|forward|warmup|limit|retire|suppressions>`;
+
+export async function cmdMailers(ctx) {
+	const sub = ctx.positionals[0];
+	const rest = { ...ctx, positionals: ctx.positionals.slice(1) };
+	switch (sub) {
+		case "list":
+		case "ls":
+			return run(rest, "list_mailers");
+		case "domain":
+			return run(rest, "get_mailer_domain_status");
+		case "messages":
+			return run(rest, "list_mailer_messages");
+		case "message":
+		case "show":
+			return run(rest, "get_mailer_message");
+		case "forward":
+			return run(rest, "set_mailer_forwarding", "Forwarding updated.");
+		case "warmup": {
+			const action = rest.positionals[0];
+			const inner = { ...rest, positionals: rest.positionals.slice(1) };
+			if (action === "start") return run(inner, "start_mailer_warmup", "Warmup started.");
+			if (action === "pause") return run(inner, "pause_mailer_warmup", "Warmup paused.");
+			throw new CliError(`Unknown warmup action: ${action ?? "(none)"}. Try: start, pause`);
+		}
+		case "limit":
+			return run(rest, "set_mailer_daily_send_limit", "Daily send limit set.");
+		case "retire": {
+			if (rest.flags["release-mailbox"] && !rest.flags.yes) {
+				const id = flagStr(rest.positionals[0]) ?? "<mailer-id>";
+				ctx.io.error(
+					`--release-mailbox deletes the hosted mailbox at the provider and cannot be undone. Re-run with --yes to confirm: ${ctx.bin} mailers retire ${id} --release-mailbox --yes`
+				);
+				return 1;
+			}
+			return run(rest, "retire_mailer", "Mailer retired: nothing further sends from it.");
+		}
+		case "suppressions":
+			return mailersSuppressions(rest, rest.positionals[0]);
+		default:
+			throw new CliError(`Unknown mailers subcommand: ${sub ?? "(none)"}. ${MAILERS_USAGE}`);
+	}
+}
+
+async function mailersSuppressions(ctx, sub) {
+	const inner = { ...ctx, positionals: ctx.positionals.slice(1) };
+	switch (sub) {
+		case "list":
+		case "ls":
+		case undefined:
+			return run(sub === undefined ? ctx : inner, "list_email_suppressions");
+		case "add":
+			return run(inner, "add_email_suppression", "Added to the do-not-send list.");
+		case "remove":
+			return run(inner, "remove_email_suppression", "Removed from the do-not-send list.");
+		default:
+			throw new CliError(`Unknown suppressions subcommand: ${sub}. Try: list, add, remove`);
+	}
+}
+
 export async function cmdMail(ctx) {
 	const sub = ctx.positionals[0];
 	const rest = { ...ctx, positionals: ctx.positionals.slice(1) };

package/lib/commands.mjs

@@ -38,3 +38,4 @@ export * from "./commands/offerings.mjs";
 export * from "./commands/targets.mjs";
 export * from "./commands/admin.mjs";
 export * from "./commands/feedback.mjs";
+export * from "./commands/evals.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.46.0",
+	"version": "0.48.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",