@letterstory/cli 0.82.3 → 0.83.0

package/lib/cli.mjs

@@ -57,7 +57,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.82.3";
+export const VERSION = "0.83.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value. Every `type: "bool"` flag (and its
@@ -333,16 +333,20 @@ Collections:
   collections autopen <id> [--template <key>|none] [--kernel-family <name>|org]
                            [--mention off|subtle|woven] [--outline-depth barebones|fleshed_out]
                            [--direction-gate true|false] [--auto-start true|false]
-                           [--prefer <site> …|none]
+                           [--prefer <site> …|none] [--client-one-in <n>]
       The Auto-Pen settings the app's panel writes. --template binds a content
       template (from 'templates list') as this collection's default, which is how
       a template you author here actually gets used. A patch: unset flags are
       left alone. --prefer REPLACES the preferred sources: SITES research
       searches for every post's topic and reaches for first when a figure fits
       (a URL is saved as its site) — your own site included; 'none' clears them.
-  collections cite-preferred <id> [--limit <n>] [--dry-run]
-      Bring published posts up to the preferred sources: each eligible post gets
-      at most one preferred figure, linked on the figure, then republishes.
+  collections preview-preferred <id>
+      Step 1: check every published post with a Sources list against the
+      preferred sites — dry, nothing written. Proposals are judged by fit.
+  collections preferred-preview <id>
+      Step 2: the proposals ranked, what the ceiling chooses, and the 1-in-N.
+  collections cite-preferred <id>
+      Step 3: apply exactly the chosen sentences, then republish.
 
 Authors (the per-collection author bank — recurring bylines):
   authors list <collection-id>                        List the bank + byline distribution
@@ -639,6 +643,21 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        Replay a breakthrough live now (background; poll
                                                        admin replays). With neither, the client's newest
                                                        breakthrough. Needs fleet:replay.
+  admin fleet-experiments list                         Fleet experiments (sites, collections, single posts)
+                                                       with each arm's rate per AI, before vs after.
+          | articles [--experiment <id>] [--org <uuid>] [--filter all|assigned|unassigned] [--arm <arm>] [--q …]
+                                                       Posts on independent sites that name the client,
+                                                       with the paragraphs that name it and the live URL.
+          | mark <experiment-id> <article-id…> --arm <arm> | unmark <experiment-id> <article-id…>
+          | new --template sponsor_disclosure_v1|sponsor_disclosure_clients_v1|article_disclosure_v1|mention_degree_v1|citation_shape_v1
+          | new --spec-file spec.json                  A custom experiment ({key, hypothesis, unit, arms, response_var}).
+          | assign <experiment-id> <unit-id…> | assign-eligible <experiment-id> | apply <experiment-id>
+          | assign-orgs <experiment-id> <org-domain-or-id…>
+                                                       Site/collection arms (safe mode: our own sites, or one
+                                                       site per named client). A running experiment applies
+                                                       each new unit's arm at once; start applies them all.
+          | start|stop <experiment-id> | decide <experiment-id> --decision "…"
+                                                       Reads need fleet:read; writes fleet:experiment.
   admin experiments list [--q colons] [--status proposed] [--primitive write] [--stage rarity] | show <id>
           | new "<what you are investigating>" [--primitive write|research|shred|enhance] [--stages a,b]
                 [--hypothesis …] [--plan-file p.json] [--tags a,b] [--as "<agent>"]

package/lib/commands/admin.mjs

@@ -12,6 +12,7 @@
 //     (feedback-add --fix "…" files the reader's rewrite; the diff is classified and a disposition suggested.
 //      The old names — evals, eval-add, eval-set, proposals, propose-eval, capture-eval, reject-eval — still work.)
 //   admin evals scoreboard|cases|runs|run|triage|guard|retire …                      (fleet:read / tuning:experiment)
+//   admin fleet-experiments list|articles|mark|unmark|new|assign|assign-eligible|assign-orgs|apply|start|stop|decide …  (fleet:read / fleet:experiment)
 //   admin crm import|capture|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link …  (crm:read / crm:write)
 //   admin spend summary|reconcile|pnl|entries|add|import|usage|remove|reference|stripe-sync|services|service …  (spend:read / spend:write)
 //   admin replays [--org <id>] [--target <id>] [--status …] [--limit n]                (fleet:read)
@@ -3346,6 +3347,176 @@ async function cmdAdminSpend(ctx) {
 	}
 }
 
+const FLEET_EXPERIMENTS = "/api/superadmin/experiments";
+
+function pct(rate) {
+	return rate && rate.asks > 0 ? `${Math.round(rate.point * 100)}% (${rate.hits}/${rate.asks})` : "no answers yet";
+}
+
+function signedPoints(value) {
+	if (value === null || value === undefined) return "not enough yet";
+	const r = Number((value * 100).toFixed(1));
+	return `${r > 0 ? "+" : r < 0 ? "-" : ""}${Math.abs(r).toFixed(1)} pts`;
+}
+
+const READOUT_OUTCOMES = [
+	["cited", "cited when probed"],
+	["namedWhenCited", "client named when cited"],
+	["reads", "read by the AI"],
+];
+
+function fleetExperimentLines(e) {
+	const lines = [`${e.id}  ${e.key}  [${e.status}]  per ${e.unit} → ${e.responseVar}`, `  ${e.hypothesis}`];
+	for (const arm of e.armStats ?? []) {
+		const live = arm.live !== undefined ? `, ${arm.live} live` : "";
+		lines.push(`  ${arm.arm}: ${arm.units} unit${arm.units === 1 ? "" : "s"}${live} · after ${pct(arm.rate)}`);
+	}
+	const r = e.articleReadout;
+	if (r) {
+		for (const s of r.surfaces) {
+			lines.push(`  ${s.label}:`);
+			for (const [key, title] of READOUT_OUTCOMES) {
+				const arms = s.arms.map((a) => `${a.arm} ${pct(a[key].before)} → ${pct(a[key].after)}`).join("; ");
+				const effects = s.effects
+					.map((x) => `${x.arm} vs ${r.referenceArm} ${signedPoints(x[key])}`)
+					.join("; ");
+				lines.push(`    ${title}: ${arms}${effects ? ` | ${effects}` : ""}`);
+			}
+		}
+	}
+	if (e.decision) lines.push(`  decision: ${e.decision}`);
+	return lines.join("\n");
+}
+
+function mentionedArticleLines(a) {
+	const state = a.assignment
+		? a.assignment.changedSinceApplied
+			? ", republished mid-test"
+			: a.assignment.appliedAt
+				? `, live since ${a.assignment.appliedAt.slice(0, 10)}`
+				: ", starts with the experiment"
+		: "";
+	const arm = a.assignment
+		? `  [${a.assignment.arm}${state}]`
+		: a.ineligible
+			? `  [can't join: ${a.ineligible}]`
+			: "";
+	const lines = [
+		`${a.articleId}  ${a.orgName} · ${a.title}${arm}`,
+		`  ${a.url ?? "(no live URL)"}  · ${a.mentionCount} mention${a.mentionCount === 1 ? "" : "s"} · cited in ${a.citedAnswers} answer${a.citedAnswers === 1 ? "" : "s"}${a.probed ? " · page-probed" : ""}`,
+	];
+	for (const p of a.passages) lines.push(`  > ${p.text}`);
+	if (a.passageTotal > a.passages.length) lines.push(`  … ${a.passageTotal - a.passages.length} more`);
+	return lines.join("\n");
+}
+
+/**
+ * `admin fleet-experiments …` — the fleet experiment registry and the client-mention
+ * worklist: the headless half of /superadmin/experiments. Reads need `fleet:read` on the
+ * staff key, writes `fleet:experiment`. (`admin experiments` is the kernel tuning ledger.)
+ */
+async function cmdAdminFleetExperiments(ctx) {
+	const [sub, id, ...rest] = ctx.positionals;
+	const { flags, io } = ctx;
+	const json = flagBool(flags.json);
+	const out = (result, line) => (json ? printResult(io, flags, result) : io.log(line(result)));
+	const action = (body) => staffRequest(ctx, "POST", FLEET_EXPERIMENTS, body);
+	switch (sub) {
+		case "list":
+			return out(await staffRequest(ctx, "GET", FLEET_EXPERIMENTS), (r) =>
+				r.experiments.length ? r.experiments.map(fleetExperimentLines).join("\n\n") : "(no experiments)"
+			);
+		case "articles": {
+			const params = new URLSearchParams(
+				compact({
+					experiment_id: flagStr(flags.experiment),
+					org_id: flagStr(flags.org),
+					filter: flagStr(flags.filter),
+					arm: flagStr(flags.arm),
+					q: flagStr(flags.q),
+					limit: flagStr(flags.limit),
+					offset: flagStr(flags.offset),
+				})
+			);
+			return out(await staffRequest(ctx, "GET", `${FLEET_EXPERIMENTS}/articles?${params}`), (r) => {
+				const head = `${r.total} post${r.total === 1 ? "" : "s"} name the client (showing ${r.articles.length} from ${r.offset})${r.unaudited ? `; ${r.unaudited} unaudited` : ""}`;
+				return [head, ...r.articles.map(mentionedArticleLines)].join("\n\n");
+			});
+		}
+		case "mark":
+		case "unmark": {
+			const articleIds = rest.filter(Boolean);
+			const arm = sub === "unmark" ? null : flagStr(flags.arm);
+			if (!id || articleIds.length === 0 || (sub === "mark" && !arm))
+				throw new CliError(
+					`Usage: admin fleet-experiments ${sub} <experiment-id> <article-id> [<article-id> …]${sub === "mark" ? " --arm <arm>" : ""}`
+				);
+			const result = await staffRequest(ctx, "PATCH", `${FLEET_EXPERIMENTS}/articles`, {
+				experiment_id: id,
+				article_ids: articleIds,
+				arm,
+			});
+			return out(
+				result,
+				(r) => `assigned ${r.assigned}, moved ${r.moved}, unchanged ${r.unchanged}, cleared ${r.cleared}`
+			);
+		}
+		case "new": {
+			const template = flagStr(flags.template);
+			const specFile = flagStr(flags["spec-file"]);
+			if (!template && !specFile)
+				throw new CliError(
+					"Usage: admin fleet-experiments new --template sponsor_disclosure_v1|sponsor_disclosure_clients_v1|article_disclosure_v1|mention_degree_v1|citation_shape_v1 | --spec-file spec.json"
+				);
+			const body = template
+				? { action: "create_template", template }
+				: { action: "create", experiment: JSON.parse(readFileSync(specFile, "utf8")) };
+			return out(await action(body), (r) => `created ${r.experiment_id}`);
+		}
+		case "assign":
+		case "assign-eligible":
+		case "assign-orgs":
+		case "apply": {
+			const ids = rest.filter(Boolean);
+			if (!id || ((sub === "assign" || sub === "assign-orgs") && ids.length === 0))
+				throw new CliError(
+					`Usage: admin fleet-experiments ${sub} <experiment-id>${sub === "assign" ? " <unit-id…>" : sub === "assign-orgs" ? " <org-domain-or-id…>" : ""}`
+				);
+			const body =
+				sub === "assign"
+					? { action: "assign", experiment_id: id, unit_ids: ids }
+					: sub === "assign-orgs"
+						? { action: "assign_orgs", experiment_id: id, orgs: ids }
+						: { action: sub === "apply" ? "apply" : "assign_eligible", experiment_id: id };
+			return out(await action(body), (r) => {
+				const lines = [];
+				if (r.assigned !== undefined) lines.push(`assigned ${r.assigned}, already in ${r.skipped}`);
+				if (r.applied)
+					lines.push(
+						`applied ${r.applied.applied}, unchanged ${r.applied.unchanged}, failed ${r.applied.failed.length}`
+					);
+				for (const f of r.applied?.failed ?? []) lines.push(`  ${f.unitId} (${f.arm}): ${f.message}`);
+				for (const k of r.skippedOrgs ?? []) lines.push(`  skipped ${k.orgId}: ${k.reason}`);
+				return lines.join("\n") || "ok";
+			});
+		}
+		case "start":
+		case "stop":
+			if (!id) throw new CliError(`Usage: admin fleet-experiments ${sub} <experiment-id>`);
+			return out(await action({ action: sub, experiment_id: id }), (r) => `${id} is ${r.status}`);
+		case "decide": {
+			const decision = flagStr(flags.decision);
+			if (!id || !decision)
+				throw new CliError('Usage: admin fleet-experiments decide <experiment-id> --decision "<readout>"');
+			return out(await action({ action: "decide", experiment_id: id, decision }), (r) => `${id} is ${r.status}`);
+		}
+		default:
+			throw new CliError(
+				'Usage: admin fleet-experiments list | articles [--experiment <id>] [--org <id>] [--filter all|assigned|unassigned] [--arm <arm>] [--q …] [--limit n] [--offset n] | mark <experiment-id> <article-id…> --arm <arm> | unmark <experiment-id> <article-id…> | new --template <key> | start|stop <experiment-id> | decide <experiment-id> --decision "…"'
+			);
+	}
+}
+
 export async function cmdAdmin(ctx) {
 	const sub = ctx.positionals[0];
 	if (sub === "replays" || sub === "replay")
@@ -3355,11 +3526,12 @@ export async function cmdAdmin(ctx) {
 	if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "fleet-experiments") return cmdAdminFleetExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-context") return cmdAdminTuningContext({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning-import") return cmdAdminTuningImport({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "tuning") return cmdAdminTuning({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "evals") return cmdAdminEvals({ ...ctx, positionals: ctx.positionals.slice(1) });
 	throw new CliError(
-		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin spend … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
+		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin fleet-experiments … | admin crm … | admin spend … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
 	);
 }

package/lib/commands/collections.mjs

@@ -47,10 +47,14 @@ export const ARGS = {
 		regenerate_topic_map: { value: (flags) => (flags["no-topic-map"] === undefined ? undefined : false) },
 		stock_headlines: { value: (flags) => (flags["no-headlines"] === undefined ? undefined : false) },
 	},
+	preview_preferred_sources: {
+		collection_id: { positional: 0, label: "collection-id", required: true },
+	},
+	get_preferred_sources_preview: {
+		collection_id: { positional: 0, label: "collection-id", required: true },
+	},
 	cite_preferred_sources: {
 		collection_id: { positional: 0, label: "collection-id", required: true },
-		limit: { flag: "limit", type: "num" },
-		dry_run: { value: (flags) => (flags["dry-run"] === true ? true : undefined) },
 	},
 	update_auto_pen_config: {
 		collection_id: { positional: 0, label: "collection-id", required: true },
@@ -76,6 +80,8 @@ export const ARGS = {
 		auto_start_on_curate: stringBool("auto-start"),
 		// Replaces the collection's preferred sources; `--prefer none` clears them.
 		preferred_sources: { value: (flags) => preferredSourcesFlag(flags.prefer) },
+		// The ceiling on the client's own site: at most one in N published posts cites it.
+		client_one_in: { flag: "client-one-in", type: "num" },
 	},
 };
 
@@ -106,11 +112,15 @@ export async function cmdCollections(ctx) {
 			return collectionsAutoPen(rest);
 		case "activate":
 			return collectionsActivate(rest);
+		case "preview-preferred":
+			return collectionsPreviewPreferred(rest);
+		case "preferred-preview":
+			return collectionsReadPreferredPreview(rest);
 		case "cite-preferred":
 			return collectionsCitePreferred(rest);
 		default:
 			throw new CliError(
-				`Unknown collections subcommand: ${sub ?? "(none)"}. Try: list, new, update, delete, assign, autopen, activate, cite-preferred`
+				`Unknown collections subcommand: ${sub ?? "(none)"}. Try: list, new, update, delete, assign, autopen, activate, preview-preferred, preferred-preview, cite-preferred`
 			);
 	}
 }
@@ -139,10 +149,36 @@ async function collectionsActivate(ctx) {
 }
 
 /**
- * `collections cite-preferred <id> [--limit <n>] [--dry-run]` — bring the collection's published
- * posts up to its preferred sources: each eligible post places at most one preferred figure and
- * republishes. The same function as the app's "Apply to published posts".
+ * `collections preview-preferred <id>` — step 1: queue a dry proposal per eligible published post
+ * (nothing is written to a post). The same preview as the app's "Preview published posts".
  */
+async function collectionsPreviewPreferred(ctx) {
+	const { client, positionals, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.preview_preferred_sources, flags, positionals);
+	const result = await client.callTool("preview_preferred_sources", args);
+	ok(
+		ctx,
+		`Queued ${result?.queued ?? 0} post${result?.queued === 1 ? "" : "s"} for a proposal. Read it with \`collections preferred-preview ${args.collection_id}\`.`
+	);
+	printResult(io, flags, result);
+	return 0;
+}
+
+/** `collections preferred-preview <id>` — step 2: the latest preview, ranked, and the 1-in-N it projects. */
+async function collectionsReadPreferredPreview(ctx) {
+	const { client, positionals, flags, io } = ctx;
+	const args = argsFromFlags(ARGS.get_preferred_sources_preview, flags, positionals);
+	const r = await client.callTool("get_preferred_sources_preview", args);
+	const oneIn = r?.projected?.one_in ? `1 in ${r.projected.one_in}` : "none";
+	ok(
+		ctx,
+		`Preview ${r?.status}: ${r?.checked ?? 0}/${r?.total ?? 0} checked; ${r?.chosen?.length ?? 0} chosen, ${r?.held_by_ceiling?.length ?? 0} held by the ceiling, ${r?.below_bar?.length ?? 0} below the bar. Client cited in ${r?.projected?.citing_client ?? 0} of ${r?.projected?.of ?? 0} (${oneIn}).`
+	);
+	printResult(io, flags, r);
+	return 0;
+}
+
+/** `collections cite-preferred <id>` — step 3: apply exactly the latest finished preview's chosen sentences. */
 async function collectionsCitePreferred(ctx) {
 	const { client, positionals, flags, io } = ctx;
 	const args = argsFromFlags(ARGS.cite_preferred_sources, flags, positionals);
@@ -150,9 +186,7 @@ async function collectionsCitePreferred(ctx) {
 	const n = result?.started?.length ?? 0;
 	ok(
 		ctx,
-		result?.dry_run
-			? `Would run ${n} post${n === 1 ? "" : "s"}; ${result?.skipped?.length ?? 0} skipped.`
-			: `Started ${n} post${n === 1 ? "" : "s"}${result?.republishes === false ? " (changes wait on the draft for approval)" : ""}; ${result?.skipped?.length ?? 0} skipped.`
+		`Started ${n} post${n === 1 ? "" : "s"}${result?.republishes === false ? " (changes wait on the draft for approval)" : ""}; ${result?.skipped?.length ?? 0} skipped.`
 	);
 	printResult(io, flags, result);
 	return 0;
@@ -195,7 +229,7 @@ async function collectionsAutoPen(ctx) {
 	const patch = argsFromFlags(ARGS.update_auto_pen_config, flags, positionals);
 	if (Object.keys(patch).length === 1) {
 		throw new CliError(
-			"Nothing to change. Pass at least one of --template, --kernel-family, --mention, --outline-depth, --direction-gate, --auto-start, --prefer."
+			"Nothing to change. Pass at least one of --template, --kernel-family, --mention, --outline-depth, --direction-gate, --auto-start, --prefer, --client-one-in."
 		);
 	}
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.82.3",
+	"version": "0.83.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",