@letterstory/cli 0.36.1 → 0.37.0

package/README.md

@@ -324,6 +324,29 @@ letterstory admin ask "which clients published nothing in the last 30 days?" --s
 This is the headless half of `/superadmin/ask` — same function, same schema — so a
 long-running agent can watch the fleet on a schedule without a browser.
 
+### Staff: kernel experiments
+
+```bash
+# Needs a STAFF key with the tuning:experiment capability. A key always acts as an
+# agent: it can propose, plan, note, spend and conclude, never approve a plan.
+export LETTERSTORY_STAFF_API_KEY=<your staff key with tuning:experiment>
+
+letterstory admin tuning-context --q "colons"          # what the agent knows (JSON)
+letterstory admin experiments list --q colons          # what we've tried
+letterstory admin experiments new "We use too many colons" --as "claude-code"
+letterstory admin experiments update <id> --plan-file plan.json --budget 120
+# ...a person approves the plan in Kernel Tuning -> Experiments...
+letterstory admin experiments spend <id> 54
+letterstory admin tuning-import lab/results/colons/publish/<brief>.json --experiment <id>
+letterstory admin experiments conclude <id> --decision drop \
+  --conclusion "Colons fell 60%; P(machine) did not move; quality held." \
+  --evidence lab:"kernels lab/results/colons/report.md"
+```
+
+The ledger behind Kernel Tuning's Experiments tab
+(docs/architecture/kernel-experiment-agent.md): every question asked of the kernels,
+how it was tested, and what we learned and why.
+
 ### Propose, then onboard a phantom
 
 The Fleet wizard's Plan and Name screens as two commands: pick the beat, then the name,

package/lib/cli.mjs

@@ -47,7 +47,7 @@ import {
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.36.1";
+export const VERSION = "0.37.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value.
@@ -506,6 +506,24 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
                                                        read-only SQL across every org; --sql prints
                                                        the queries behind the answer. Needs the
                                                        fleet:query capability on the staff key.
+  admin experiments list [--q colons] [--status proposed] | show <id>
+          | new "<question>" [--hypothesis …] [--plan-file p.json] [--tags a,b] [--as "<agent>"]
+          | update <id> [--plan-file …] [--result-file …] [--evidence kind:ref,…] [--budget N]
+          | note|started <id> "<text>" | spend <id> <runs>
+          | conclude <id> --decision ship|drop|follow-up|inconclusive --conclusion "…" [--evidence …]
+          | abandon <id> --reason "…"
+                                                       The kernel experiment ledger: what we tried,
+                                                       how, what it found and why. Approving a plan
+                                                       stays with a signed-in person. Needs the
+                                                       tuning:experiment capability.
+  admin tuning-import <brief-results.json> [--experiment <id>]
+                                                       One brief's kernels lab results as a console
+                                                       session (a trial per arm), linked to its
+                                                       ledger entry. Written by \`npm run lab --
+                                                       publish\` in the kernels repo.
+  admin tuning-context [--q "<question>"]              The experiment agent's context pack (JSON):
+                                                       the pipeline as deployed, levers, measures,
+                                                       goals, related history.
   admin create-org --domain <bare-domain> [--name "Acme"] [--test] [--no-key]
           [--key-name N] [--capabilities a,b] [--onboard] [--watch] [+ the onboard domain flags]
                                                        The key exchange: create the client org from

package/lib/commands/admin.mjs

@@ -4,6 +4,9 @@
 // routes, because they exist precisely to act BEFORE an org exists.
 //
 //   admin ask "which kernels are hooked up to the most clients?" [--sql] [--json]
+//   admin experiments list|show|new|update|note|started|spend|conclude|abandon …   (tuning:experiment)
+//   admin tuning-context [--q "<question>"]                                         (tuning:experiment)
+//   admin tuning-import <brief-results.json> [--experiment <id>]                    (tuning:experiment)
 //   admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
 //                    [--capabilities a,b] [--onboard] [--about "…"] [--sitemap <url>]
 //                    [--cms <provider>] [--posts N] [--seers suggest|skip] [--no-visuals]
@@ -26,6 +29,7 @@
 // The staff key comes from --staff-key or LETTERSTORY_STAFF_API_KEY; the base URL from the
 // usual --url / LETTERSTORY_API_URL / config resolution.
 
+import { readFileSync } from "node:fs";
 import { CliError, LetterstoryClient } from "../client.mjs";
 import { flagStr, flagNum, flagBool, flagList, printResult, compact, ok } from "./shared.mjs";
 import { watchDomainJob } from "./onboard.mjs";
@@ -46,12 +50,17 @@ function staffKeyFrom(flags) {
 
 /** POST a superadmin route with the staff key; surfaces the server's error text on non-2xx. */
 async function staffPost(ctx, path, body) {
+	return staffRequest(ctx, "POST", path, body);
+}
+
+/** Any method against a superadmin route with the staff key. */
+async function staffRequest(ctx, method, path, body) {
 	const { client, flags } = ctx;
 	const url = `${client.url}${path}`;
 	const res = await client.fetch(url, {
-		method: "POST",
+		method,
 		headers: { "content-type": "application/json", "x-staff-api-key": staffKeyFrom(flags) },
-		body: JSON.stringify(body),
+		...(body === undefined ? {} : { body: JSON.stringify(body) }),
 	});
 	let payload = null;
 	try {
@@ -220,11 +229,195 @@ async function cmdAdminAsk(ctx) {
 	}
 }
 
+/* ------------------------------------------------------------ kernel experiments */
+
+const EXPERIMENTS = "/api/superadmin/tuning/experiments";
+
+/** `--as "<agent name>"` records the actor as that agent; a staff key always acts as an agent. */
+function actorFrom(flags) {
+	const as = flagStr(flags.as);
+	return as ? { kind: "agent", label: as } : undefined;
+}
+
+/** A JSON object from `--<name>-file <path>` (or `-` for stdin is not supported: pass a file). */
+function jsonFile(flags, name) {
+	const path = flagStr(flags[`${name}-file`]);
+	if (!path) return undefined;
+	try {
+		return JSON.parse(readFileSync(path, "utf8"));
+	} catch (err) {
+		throw new CliError(`--${name}-file ${path}: ${err.message}`);
+	}
+}
+
+/** `--evidence kind:ref[,kind:ref…]`, e.g. `lab:kernels lab/results/colons/report.md,session:<uuid>`. */
+function evidenceFrom(flags) {
+	const items = flagList(flags.evidence);
+	if (!items?.length) return undefined;
+	return items.map((item) => {
+		const at = item.indexOf(":");
+		if (at < 1) throw new CliError(`--evidence items are kind:ref, got "${item}"`);
+		return { kind: item.slice(0, at), ref: item.slice(at + 1) };
+	});
+}
+
+function experimentLine(e) {
+	const decision = e.decision ? ` → ${e.decision}` : "";
+	return `${e.id}  [${e.status}${decision}]  ${e.question}${e.gist ? `\n    ${e.gist}` : ""}`;
+}
+
+/**
+ * `admin experiments …` — the experiment ledger (docs/architecture/kernel-experiment-agent.md),
+ * the headless half of the tuning console's Experiments tab. Needs `tuning:experiment` on the key.
+ */
+async function cmdAdminExperiments(ctx) {
+	const [sub, id, ...rest] = ctx.positionals;
+	const { flags, io } = ctx;
+	const json = flagBool(flags.json);
+	const out = (result, line) => (json ? printResult(io, flags, result) : io.log(line(result)));
+	const one = (r) => {
+		const e = r.experiment;
+		const lines = [experimentLine(e)];
+		if (e.restated) lines.push(`  hypothesis: ${e.restated}`);
+		if (e.plan) lines.push(`  plan: ${JSON.stringify(e.plan)}`);
+		lines.push(`  runs: ${e.runsSpent} of ${e.budgetRuns}`);
+		for (const ev of e.events ?? [])
+			lines.push(`  ${ev.at}  ${ev.kind}  ${ev.actorLabel ?? ev.actorKind}${ev.text ? `: ${ev.text}` : ""}`);
+		return lines.join("\n");
+	};
+	switch (sub) {
+		case "list": {
+			const params = new URLSearchParams(
+				compact({ q: flagStr(flags.q), status: flagStr(flags.status), limit: flagStr(flags.limit) })
+			);
+			const result = await staffRequest(ctx, "GET", `${EXPERIMENTS}?${params}`);
+			return out(result, (r) =>
+				r.experiments.length ? r.experiments.map(experimentLine).join("\n") : "(no experiments)"
+			);
+		}
+		case "show":
+			if (!id) throw new CliError("Usage: admin experiments show <id>");
+			return out(await staffRequest(ctx, "GET", `${EXPERIMENTS}/${id}`), one);
+		case "new": {
+			const question = [id, ...rest].filter(Boolean).join(" ").trim();
+			if (question.length < 3)
+				throw new CliError('Usage: admin experiments new "<question>" [--plan-file plan.json] [--tags a,b]');
+			const body = compact({
+				question,
+				restated: flagStr(flags.hypothesis),
+				plan: jsonFile(flags, "plan"),
+				method: flagStr(flags.method),
+				tags: flagList(flags.tags),
+				builds_on: flagList(flags["builds-on"]),
+				evidence: evidenceFrom(flags),
+				actor: actorFrom(flags),
+			});
+			return out(await staffRequest(ctx, "POST", EXPERIMENTS, body), one);
+		}
+		case "update": {
+			if (!id)
+				throw new CliError(
+					"Usage: admin experiments update <id> [--hypothesis …] [--plan-file …] [--result-file …] [--evidence kind:ref,…]"
+				);
+			const body = compact({
+				restated: flagStr(flags.hypothesis),
+				plan: jsonFile(flags, "plan"),
+				measures: jsonFile(flags, "measures"),
+				result: jsonFile(flags, "result"),
+				method: flagStr(flags.method),
+				tags: flagList(flags.tags),
+				budget_runs: flagNum(flags.budget),
+				engine_build: flagStr(flags.build),
+				add_evidence: evidenceFrom(flags),
+				actor: actorFrom(flags),
+			});
+			return out(await staffRequest(ctx, "PATCH", `${EXPERIMENTS}/${id}`, body), one);
+		}
+		case "note":
+		case "started": {
+			const text = rest.join(" ").trim();
+			if (!id || (sub === "note" && !text)) throw new CliError(`Usage: admin experiments ${sub} <id> "<text>"`);
+			const body = compact({ kind: sub, text: text || undefined, actor: actorFrom(flags) });
+			return out(await staffRequest(ctx, "POST", `${EXPERIMENTS}/${id}/events`, body), one);
+		}
+		case "spend": {
+			const runs = Number(rest[0]);
+			if (!id || !Number.isInteger(runs) || runs < 1)
+				throw new CliError("Usage: admin experiments spend <id> <runs> [--text …]");
+			const body = compact({ kind: "spend", runs, text: flagStr(flags.text), actor: actorFrom(flags) });
+			return out(await staffRequest(ctx, "POST", `${EXPERIMENTS}/${id}/events`, body), one);
+		}
+		case "conclude": {
+			const decision = flagStr(flags.decision);
+			const conclusion = flagStr(flags.conclusion);
+			if (!id || !decision || !conclusion)
+				throw new CliError(
+					'Usage: admin experiments conclude <id> --decision ship|drop|follow-up|inconclusive --conclusion "<what we learned and why>" [--evidence kind:ref,…] [--result-file r.json]'
+				);
+			const body = compact({
+				action: "conclude",
+				decision,
+				conclusion,
+				result: jsonFile(flags, "result"),
+				add_evidence: evidenceFrom(flags),
+				actor: actorFrom(flags),
+			});
+			return out(await staffRequest(ctx, "POST", `${EXPERIMENTS}/${id}/decide`, body), one);
+		}
+		case "abandon": {
+			const reason = flagStr(flags.reason);
+			if (!id || !reason) throw new CliError('Usage: admin experiments abandon <id> --reason "<why>"');
+			const body = compact({ action: "abandon", reason, actor: actorFrom(flags) });
+			return out(await staffRequest(ctx, "POST", `${EXPERIMENTS}/${id}/decide`, body), one);
+		}
+		default:
+			throw new CliError(
+				"Usage: admin experiments list|show|new|update|note|started|spend|conclude|abandon (see --help)"
+			);
+	}
+}
+
+/**
+ * `admin tuning-import <file.json>` — one brief's kernels lab results into a console session
+ * (the lab's `publish` writes these files; the body is POST /api/superadmin/tuning/lab-import's).
+ */
+async function cmdAdminTuningImport(ctx) {
+	const path = ctx.positionals[0];
+	if (!path) throw new CliError("Usage: admin tuning-import <brief-results.json> [--experiment <id>]");
+	let body;
+	try {
+		body = JSON.parse(readFileSync(path, "utf8"));
+	} catch (err) {
+		throw new CliError(`${path}: ${err.message}`);
+	}
+	const experiment = flagStr(ctx.flags.experiment);
+	if (experiment) body.experiment_id = experiment;
+	const result = await staffRequest(ctx, "POST", "/api/superadmin/tuning/lab-import", body);
+	if (flagBool(ctx.flags.json)) return printResult(ctx.io, ctx.flags, result);
+	ctx.io.log(
+		`${result.trials} trials, ${result.runs} runs → ${ctx.client.url}/superadmin/tuning/${result.sessionId}`
+	);
+}
+
+/** `admin tuning-context [--q "<question>"]` — the experiment agent's context pack, as JSON. */
+async function cmdAdminTuningContext(ctx) {
+	const q = flagStr(ctx.flags.q) || ctx.positionals.join(" ").trim() || undefined;
+	const params = q ? `?${new URLSearchParams({ q })}` : "";
+	printResult(
+		ctx.io,
+		{ ...ctx.flags, json: true },
+		await staffRequest(ctx, "GET", `/api/superadmin/tuning/context${params}`)
+	);
+}
+
 export async function cmdAdmin(ctx) {
 	const sub = ctx.positionals[0];
 	if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
 	if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "tuning-context") return cmdAdminTuningContext({ ...ctx, positionals: ctx.positionals.slice(1) });
+	if (sub === "tuning-import") return cmdAdminTuningImport({ ...ctx, positionals: ctx.positionals.slice(1) });
 	throw new CliError(
-		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" (see --help)'
+		'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin tuning-context (see --help)'
 	);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.36.1",
+	"version": "0.37.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",