@letterstory/cli 0.82.3 → 0.82.4

package/lib/cli.mjs

@@ -57,7 +57,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.82.3";
+export const VERSION = "0.82.4";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value. Every `type: "bool"` flag (and its
@@ -639,6 +639,21 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        Replay a breakthrough live now (background; poll
                                                        admin replays). With neither, the client's newest
                                                        breakthrough. Needs fleet:replay.
+  admin fleet-experiments list                         Fleet experiments (sites, collections, single posts)
+                                                       with each arm's rate per AI, before vs after.
+          | articles [--experiment <id>] [--org <uuid>] [--filter all|assigned|unassigned] [--arm <arm>] [--q …]
+                                                       Posts on independent sites that name the client,
+                                                       with the paragraphs that name it and the live URL.
+          | mark <experiment-id> <article-id…> --arm <arm> | unmark <experiment-id> <article-id…>
+          | new --template sponsor_disclosure_v1|sponsor_disclosure_clients_v1|article_disclosure_v1|mention_degree_v1|citation_shape_v1
+          | new --spec-file spec.json                  A custom experiment ({key, hypothesis, unit, arms, response_var}).
+          | assign <experiment-id> <unit-id…> | assign-eligible <experiment-id> | apply <experiment-id>
+          | assign-orgs <experiment-id> <org-domain-or-id…>
+                                                       Site/collection arms (safe mode: our own sites, or one
+                                                       site per named client). A running experiment applies
+                                                       each new unit's arm at once; start applies them all.
+          | start|stop <experiment-id> | decide <experiment-id> --decision "…"
+                                                       Reads need fleet:read; writes fleet:experiment.
   admin experiments list [--q colons] [--status proposed] [--primitive write] [--stage rarity] | show <id>
           | new "<what you are investigating>" [--primitive write|research|shred|enhance] [--stages a,b]
                 [--hypothesis …] [--plan-file p.json] [--tags a,b] [--as "<agent>"]

package/lib/commands/admin.mjs

@@ -12,6 +12,7 @@
 //     (feedback-add --fix "…" files the reader's rewrite; the diff is classified and a disposition suggested.
 //      The old names — evals, eval-add, eval-set, proposals, propose-eval, capture-eval, reject-eval — still work.)
 //   admin evals scoreboard|cases|runs|run|triage|guard|retire …                      (fleet:read / tuning:experiment)
+//   admin fleet-experiments list|articles|mark|unmark|new|assign|assign-eligible|assign-orgs|apply|start|stop|decide …  (fleet:read / fleet:experiment)
 //   admin crm import|capture|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link …  (crm:read / crm:write)
 //   admin spend summary|reconcile|pnl|entries|add|import|usage|remove|reference|stripe-sync|services|service …  (spend:read / spend:write)
 //   admin replays [--org <id>] [--target <id>] [--status …] [--limit n]                (fleet:read)
@@ -3346,6 +3347,176 @@ async function cmdAdminSpend(ctx) {
 	}
 }
 
+const FLEET_EXPERIMENTS = "/api/superadmin/experiments";
+
+function pct(rate) {
+	return rate && rate.asks > 0 ? `${Math.round(rate.point * 100)}% (${rate.hits}/${rate.asks})` : "no answers yet";
+}
+
+function signedPoints(value) {
+	if (value === null || value === undefined) return "not enough yet";
+	const r = Number((value * 100).toFixed(1));
+	return `${r > 0 ? "+" : r < 0 ? "-" : ""}${Math.abs(r).toFixed(1)} pts`;
+}
+
+const READOUT_OUTCOMES = [
+	["cited", "cited when probed"],
+	["namedWhenCited", "client named when cited"],
+	["reads", "read by the AI"],
+];
+
+function fleetExperimentLines(e) {
+	const lines = [`${e.id}  ${e.key}  [${e.status}]  per ${e.unit} → ${e.responseVar}`, `  ${e.hypothesis}`];
+	for (const arm of e.armStats ?? []) {
+		const live = arm.live !== undefined ? `, ${arm.live} live` : "";
+		lines.push(`  ${arm.arm}: ${arm.units} unit${arm.units === 1 ? "" : "s"}${live} · after ${pct(arm.rate)}`);
+	}
+	const r = e.articleReadout;
+	if (r) {
+		for (const s of r.surfaces) {
+			lines.push(`  ${s.label}:`);
+			for (const [key, title] of READOUT_OUTCOMES) {
+				const arms = s.arms.map((a) => `${a.arm} ${pct(a[key].before)} → ${pct(a[key].after)}`).join("; ");
+				const effects = s.effects
+					.map((x) => `${x.arm} vs ${r.referenceArm} ${signedPoints(x[key])}`)
+					.join("; ");
+				lines.push(`    ${title}: ${arms}${effects ? ` | ${effects}` : ""}`);
+			}
+		}
+	}
+	if (e.decision) lines.push(`  decision: ${e.decision}`);
+	return lines.join("\n");
+}
+
+function mentionedArticleLines(a) {
+	const state = a.assignment
+		? a.assignment.changedSinceApplied
+			? ", republished mid-test"
+			: a.assignment.appliedAt
+				? `, live since ${a.assignment.appliedAt.slice(0, 10)}`
+				: ", starts with the experiment"
+		: "";
+	const arm = a.assignment
+		? `  [${a.assignment.arm}${state}]`
+		: a.ineligible
+			? `  [can't join: ${a.ineligible}]`
+			: "";
+	const lines = [
+		`${a.articleId}  ${a.orgName} · ${a.title}${arm}`,
+		`  ${a.url ?? "(no live URL)"}  · ${a.mentionCount} mention${a.mentionCount === 1 ? "" : "s"} · cited in ${a.citedAnswers} answer${a.citedAnswers === 1 ? "" : "s"}${a.probed ? " · page-probed" : ""}`,
+	];
+	for (const p of a.passages) lines.push(`  > ${p.text}`);
+	if (a.passageTotal > a.passages.length) lines.push(`  … ${a.passageTotal - a.passages.length} more`);
+	return lines.join("\n");
+}
+
+/**
+ * `admin fleet-experiments …` — the fleet experiment registry and the client-mention
+ * worklist: the headless half of /superadmin/experiments. Reads need `fleet:read` on the
+ * staff key, writes `fleet:experiment`. (`admin experiments` is the kernel tuning ledger.)
+ */
+async function cmdAdminFleetExperiments(ctx) {
+	const [sub, id, ...rest] = ctx.positionals;
+	const { flags, io } = ctx;
+	const json = flagBool(flags.json);
+	const out = (result, line) => (json ? printResult(io, flags, result) : io.log(line(result)));
+	const action = (body) => staffRequest(ctx, "POST", FLEET_EXPERIMENTS, body);
+	switch (sub) {
+		case "list":
+			return out(await staffRequest(ctx, "GET", FLEET_EXPERIMENTS), (r) =>
+				r.experiments.length ? r.experiments.map(fleetExperimentLines).join("\n\n") : "(no experiments)"
+			);
+		case "articles": {
+			const params = new URLSearchParams(
+				compact({
+					experiment_id: flagStr(flags.experiment),
+					org_id: flagStr(flags.org),
+					filter: flagStr(flags.filter),
+					arm: flagStr(flags.arm),
+					q: flagStr(flags.q),
+					limit: flagStr(flags.limit),
+					offset: flagStr(flags.offset),
+				})
+			);
+			return out(await staffRequest(ctx, "GET", `${FLEET_EXPERIMENTS}/articles?${params}`), (r) => {
+				const head = `${r.total} post${r.total === 1 ? "" : "s"} name the client (showing ${r.articles.length} from ${r.offset})${r.unaudited ? `; ${r.unaudited} unaudited` : ""}`;
+				return [head, ...r.articles.map(mentionedArticleLines)].join("\n\n");
+			});
+		}
+		case "mark":
+		case "unmark": {
+			const articleIds = rest.filter(Boolean);
+			const arm = sub === "unmark" ? null : flagStr(flags.arm);
+			if (!id || articleIds.length === 0 || (sub === "mark" && !arm))
+				throw new CliError(
+					`Usage: admin fleet-experiments ${sub} <experiment-id> <article-id> [<article-id> …]${sub === "mark" ? " --arm <arm>" : ""}`
+				);
+			const result = await staffRequest(ctx, "PATCH", `${FLEET_EXPERIMENTS}/articles`, {
+				experiment_id: id,
+				article_ids: articleIds,
+				arm,
+			});
+			return out(
+				result,
+				(r) => `assigned ${r.assigned}, moved ${r.moved}, unchanged ${r.unchanged}, cleared ${r.cleared}`
+			);
+		}
+		case "new": {
+			const template = flagStr(flags.template);
+			const specFile = flagStr(flags["spec-file"]);
+			if (!template && !specFile)
+				throw new CliError(
+					"Usage: admin fleet-experiments new --template sponsor_disclosure_v1|sponsor_disclosure_clients_v1|article_disclosure_v1|mention_degree_v1|citation_shape_v1 | --spec-file spec.json"
+				);
+			const body = template
+				? { action: "create_template", template }
+				: { action: "create", experiment: JSON.parse(readFileSync(specFile, "utf8")) };
+			return out(await action(body), (r) => `created ${r.experiment_id}`);
+		}
+		case "assign":
+		case "assign-eligible":
+		case "assign-orgs":
+		case "apply": {
+			const ids = rest.filter(Boolean);
+			if (!id || ((sub === "assign" || sub === "assign-orgs") && ids.length === 0))
+				throw new CliError(
+					`Usage: admin fleet-experiments ${sub} <experiment-id>${sub === "assign" ? " <unit-id…>" : sub === "assign-orgs" ? " <org-domain-or-id…>" : ""}`
+				);
+			const body =
+				sub === "assign"
+					? { action: "assign", experiment_id: id, unit_ids: ids }
+					: sub === "assign-orgs"
+						? { action: "assign_orgs", experiment_id: id, orgs: ids }
+						: { action: sub === "apply" ? "apply" : "assign_eligible", experiment_id: id };
+			return out(await action(body), (r) => {
+				const lines = [];
+				if (r.assigned !== undefined) lines.push(`assigned ${r.assigned}, already in ${r.skipped}`);
+				if (r.applied)
+					lines.push(
+						`applied ${r.applied.applied}, unchanged ${r.applied.unchanged}, failed ${r.applied.failed.length}`
+					);
+				for (const f of r.applied?.failed ?? []) lines.push(`  ${f.unitId} (${f.arm}): ${f.message}`);
+				for (const k of r.skippedOrgs ?? []) lines.push(`  skipped ${k.orgId}: ${k.reason}`);
+				return lines.join("\n") || "ok";
+			});
+		}
+		case "start":
+		case "stop":
+			if (!id) throw new CliError(`Usage: admin fleet-experiments ${sub} <experiment-id>`);
+			return out(await action({ action: sub, experiment_id: id }), (r) => `${id} is ${r.status}`);
+		case "decide": {
+			const decision = flagStr(flags.decision);
+			if (!id || !decision)
+				throw new CliError('Usage: admin fleet-experiments decide <experiment-id> --decision "<readout>"');
+			return out(await action({ action: "decide", experiment_id: id, decision }), (r) => `${id} is ${r.status}`);
+		}
+		default:
+			throw new CliError(
+				'Usage: admin fleet-experiments list | articles [--experiment <id>] [--org <id>] [--filter all|assigned|unassigned] [--arm <arm>] [--q …] [--limit n] [--offset n] | mark <experiment-id> <article-id…> --arm <arm> | unmark <experiment-id> <article-id…> | new --template <key> | start|stop <experiment-id> | decide <experiment-id> --decision "…"'
+			);
+	}
+}
+
 export async function cmdAdmin(ctx) {
 	const sub = ctx.positionals[0];
 	if (sub === "replays" || sub === "replay")
@@ -3355,11 +3526,12 @@ export async function cmdAdmin(ctx) {
 	if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "fleet-experiments") return cmdAdminFleetExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-context") return cmdAdminTuningContext({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-import") return cmdAdminTuningImport({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning") return cmdAdminTuning({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "evals") return cmdAdminEvals({ ...ctx, positionals: ctx.positionals.slice(1) });
 	throw new CliError(
-		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin spend … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
+		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin fleet-experiments … | admin crm … | admin spend … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
 	);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.82.3",
+	"version": "0.82.4",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",