@letterstory/cli 0.138.0 → 0.139.0

package/lib/cli.mjs

@@ -60,10 +60,11 @@ import {
 	cmdCampaigns,
 	cmdTargets,
 	cmdVisits,
+	cmdSim,
 } from "./commands.mjs";
 
 // Keep in sync with cli/package.json.
-export const VERSION = "0.138.0";
+export const VERSION = "0.139.0";
 
 // Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
 // can't accidentally swallow the id as --json's value. Every `type: "bool"` flag (and its
@@ -71,6 +72,9 @@ export const VERSION = "0.138.0";
 export const BOOLEAN_FLAGS = new Set([
 	"remap",
 	"json",
+	// sim seed: make the new seed active; collections it doesn't cover start empty.
+	"activate",
+	"empty",
 	// admin models runs: only the starred ones.
 	"starred",
 	// shredder file / arena file: save the version as a candidate beside live.
@@ -1440,6 +1444,17 @@ Visits (Visitors: AI agents reading your site, and AI answers sending people to
   visits pages [--since 28d] [--path '/docs/*'] [--agent <id>] [--limit <n>] [--site <id>]
                                                        People and AI agents per page
 
+Sim (Simulations: a free, simulated copy of your API for agents, and what they did with it):
+  sim [list]                                           Your simulations and their URLs
+  sim create --name <n> (--spec-url <url>|--spec-file <path>) [--slug <s>] [--upstream-url <url> --upstream-credential <c>]
+  sim show <id>                                        Resources, their serving rules, and who set them
+  sim rule <id> <resource> synthesize|forward|sink|refuse|inferred [--note <why>]
+  sim pause <id> | sim resume <id>
+  sim key <id> [--agent <a>] [--model <m>] [--goal <g>] Mint a key for your own agents or CI
+  sim sessions <id> [--limit N]                        Agent sessions, newest first
+  sim session <session-id>                             One session, call by call
+  sim report <id> [--since 30d]                        Funnel, missing operations, argument errors, loops, by agent
+
 Evals (how the pipeline is doing on your posts, and the regression cases captured from them):
   evals scoreboard [--collection <id>] [--days N]      Every measure by stage: posts, failing, rate
   evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline|shred]
@@ -1537,6 +1552,7 @@ const CLIENT_COMMANDS = {
 	site: cmdSite,
 	evals: cmdEvals,
 	visits: cmdVisits,
+	sim: cmdSim,
 	// Platform-scoped (staff key) — the only commands that act BEFORE an org exists.
 	admin: cmdAdmin,
 };

package/lib/commands/sim.mjs

@@ -0,0 +1,285 @@
+// `sim` — Simulations from a terminal: a free, simulated copy of your API that agents can
+// use, and what they did with it. Thin wrappers over the simulation tools (create_simulation,
+// get_simulation, update_simulation, mint_simulation_key, list_simulation_sessions,
+// get_simulation_session, get_simulation_report), the same lib functions the app calls.
+//
+//   letterstory sim create --name "Attio API" --spec-url https://api.attio.com/openapi/api
+//   letterstory sim show <id>
+//   letterstory sim rule <id> /v2/webhooks sink --note "never call real endpoints"
+//   letterstory sim create --name "Letterstory" --tools-file tools.json --tool-route "/api/integrations/tools/{name}"
+//   letterstory sim key <id> --agent our-ci --goal "nightly integration test"
+//   letterstory sim seed <id> --prompt "a 3-person content team with two blogs" --activate
+//   letterstory sim seed <id> --use <seed-id|none>
+//   letterstory sim report <id> --since 7d
+//   letterstory sim sessions <id>
+//   letterstory sim session <session-id>
+
+import { readFileSync } from "node:fs";
+import { CliError } from "../client.mjs";
+import { flagStr, printResult, argsFromFlags, refuseUnknownFlags } from "./shared.mjs";
+import { sinceDays } from "./visits.mjs";
+
+const RULES = ["synthesize", "forward", "sink", "refuse", "inferred"];
+const simulationId = { positional: 0, label: "simulation-id", required: true };
+
+export const ARGS = {
+	create_simulation: {
+		name: { flag: "name", required: true },
+		slug: "slug",
+		spec_url: "spec-url",
+		spec: {
+			value: (flags) => {
+				const path = flagStr(flags["spec-file"]);
+				if (!path) return undefined;
+				try {
+					return JSON.parse(readFileSync(path, "utf8"));
+				} catch {
+					throw new CliError(`--spec-file must be a JSON OpenAPI document (could not parse ${path})`);
+				}
+			},
+		},
+		offering_id: "offering",
+		upstream_base_url: "upstream-url",
+		upstream_credential: "upstream-credential",
+		mcp_tools: {
+			value: (flags) => {
+				const path = flagStr(flags["tools-file"]);
+				if (!path) return undefined;
+				let parsed;
+				try {
+					parsed = JSON.parse(readFileSync(path, "utf8"));
+				} catch {
+					throw new CliError(
+						`--tools-file must be JSON: a tools/list result or its tools array (could not parse ${path})`
+					);
+				}
+				return Array.isArray(parsed) ? parsed : (parsed?.result?.tools ?? parsed?.tools);
+			},
+		},
+		tool_route: "tool-route",
+		auth_header: "auth-header",
+	},
+	list_simulations: {},
+	get_simulation: { simulation_id: simulationId },
+	update_simulation: {
+		simulation_id: simulationId,
+		status: { flag: "status", enum: ["live", "paused"] },
+		active_seed_id: {
+			value: (flags) => {
+				const v = flagStr(flags.use);
+				return v === undefined ? undefined : v === "none" ? null : v;
+			},
+		},
+		resource_rules: {
+			value: (flags, positionals) => {
+				if (positionals.length < 3) return undefined;
+				const rule = positionals[2];
+				if (!RULES.includes(rule)) throw new CliError(`rule must be one of: ${RULES.join(", ")}`);
+				return [
+					{ resource: positionals[1], rule: rule === "inferred" ? null : rule, note: flagStr(flags.note) },
+				];
+			},
+		},
+	},
+	generate_simulation_seed: {
+		simulation_id: simulationId,
+		prompt: { flag: "prompt", required: true },
+		resources: { flag: "resources", type: "list" },
+		items_per_collection: { flag: "items", type: "num" },
+		empty_workspace: { flag: "empty", type: "bool" },
+		activate: { flag: "activate", type: "bool" },
+	},
+	mint_simulation_key: { simulation_id: simulationId, agent: "agent", model: "model", goal: "goal" },
+	list_simulation_sessions: { simulation_id: simulationId, limit: { flag: "limit", type: "num" } },
+	get_simulation_session: { session_id: { positional: 0, label: "session-id", required: true } },
+	get_simulation_report: { simulation_id: simulationId, days: { value: (flags) => sinceDays(flags.since) } },
+};
+
+function call(ctx, tool) {
+	return ctx.client.callTool(tool, argsFromFlags(ARGS[tool], ctx.flags, ctx.positionals));
+}
+
+export async function cmdSim(ctx) {
+	const sub = ctx.positionals[0];
+	const rest = { ...ctx, positionals: ctx.positionals.slice(1) };
+	switch (sub) {
+		case undefined:
+		case "list":
+		case "ls":
+			return simList(rest);
+		case "create":
+			refuseUnknownFlags(rest, ARGS.create_simulation, ["spec-file", "tools-file"]);
+			return simShow(rest, await call(rest, "create_simulation"));
+		case "show":
+			return simShow(rest, await call(rest, "get_simulation"));
+		case "rule":
+		case "pause":
+		case "resume": {
+			refuseUnknownFlags(rest, ARGS.update_simulation, ["note"]);
+			if (sub !== "rule") rest.flags = { ...rest.flags, status: sub === "pause" ? "paused" : "live" };
+			return simShow(rest, await call(rest, "update_simulation"));
+		}
+		case "seed":
+			return simSeed(rest);
+		case "key":
+			return simKey(rest);
+		case "sessions":
+			return simSessions(rest);
+		case "session":
+			return simSession(rest);
+		case "report":
+			return simReport(rest);
+		default:
+			throw new CliError("Usage: sim [list|create|show|rule|pause|resume|seed|key|sessions|session|report]");
+	}
+}
+
+async function simList(ctx) {
+	const result = await call(ctx, "list_simulations");
+	printResult(ctx.io, ctx.flags, result, (r) => {
+		if (!r.simulations?.length)
+			return "No simulations yet. Create one: sim create --name <name> --spec-url <openapi url>";
+		return r.simulations
+			.map(
+				(s) =>
+					`${s.id}  ${s.name.padEnd(24)} ${s.status.padEnd(7)} ${String(s.sessions_7d).padStart(4)} sessions/7d  ${s.urls.base}`
+			)
+			.join("\n");
+	});
+	return 0;
+}
+
+function simShow(ctx, result) {
+	printResult(ctx.io, ctx.flags, result, (r) => {
+		const s = r.simulation;
+		const lines = [
+			`${s.name} (${s.title} v${s.version}) — ${s.status}, ${s.operations} operations`,
+			`  REST  ${s.urls.base}`,
+			`  MCP   ${s.urls.mcp}`,
+			`  Docs  ${s.urls.llms}`,
+			"",
+			"Resources (rule · side effect · who set it):",
+		];
+		for (const res of r.resources ?? []) {
+			lines.push(`  ${res.rule.padEnd(10)} ${res.side_effect.padEnd(9)} ${res.set_by.padEnd(8)} ${res.resource}`);
+		}
+		return lines.join("\n");
+	});
+	return 0;
+}
+
+/** `sim seed <id> --prompt …` writes a seed; `sim seed <id> --use <seed-id|none>` switches the active one. */
+async function simSeed(ctx) {
+	if (ctx.flags.use !== undefined) {
+		refuseUnknownFlags(ctx, ARGS.update_simulation, ["use"]);
+		return simShow(ctx, await call(ctx, "update_simulation"));
+	}
+	const result = await call(ctx, "generate_simulation_seed");
+	printResult(ctx.io, ctx.flags, result, (r) => {
+		const s = r.seed;
+		const rep = s.report;
+		return [
+			`Seed v${s.version} (${s.id})${s.active ? " — active" : ""}, by ${s.model ?? "upload"}`,
+			...Object.entries(s.items).map(([k, n]) => `  ${k}: ${n} items`),
+			`Kept ${rep.kept}, completed ${rep.filled}, dropped ${rep.dropped}, dangling ids ${rep.dangling.length}`,
+			s.active ? "" : `Make it active: sim seed <simulation-id> --use ${s.id}`,
+		]
+			.filter(Boolean)
+			.join("\n");
+	});
+	return 0;
+}
+
+async function simKey(ctx) {
+	const result = await call(ctx, "mint_simulation_key");
+	printResult(ctx.io, ctx.flags, result, (r) =>
+		[
+			`Key (shown once): ${r.key}`,
+			`Expires: ${r.expires_at}`,
+			`Base URL: ${r.urls.base}`,
+			`Session: ${r.session_id}`,
+		].join("\n")
+	);
+	return 0;
+}
+
+async function simSessions(ctx) {
+	const result = await call(ctx, "list_simulation_sessions");
+	printResult(ctx.io, ctx.flags, result, (r) => {
+		if (!r.sessions?.length) return "No sessions yet.";
+		return r.sessions
+			.map(
+				(s) =>
+					`${s.id}  ${String(s.calls).padStart(4)} calls  ${(s.agent ?? s.client ?? s.requester_agent ?? "unknown").padEnd(22)} ${s.goal ?? ""}`
+			)
+			.join("\n");
+	});
+	return 0;
+}
+
+async function simSession(ctx) {
+	const result = await call(ctx, "get_simulation_session");
+	printResult(ctx.io, ctx.flags, result, (r) => {
+		const lines = [
+			`Session ${r.session.id} — ${r.session.agent ?? r.session.client ?? "unknown agent"}, ${r.calls.length} events`,
+		];
+		for (const c of r.calls) {
+			if (c.kind === "feedback") {
+				lines.push(`  ${String(c.seq).padStart(3)}  feedback: ${c.note}`);
+				continue;
+			}
+			const errs =
+				Array.isArray(c.field_errors) && c.field_errors.length
+					? `  ${c.field_errors.map((e) => `${e.path || "body"}:${e.code}`).join(", ")}`
+					: "";
+			lines.push(
+				`  ${String(c.seq).padStart(3)}  ${String(c.status).padEnd(4)}${(c.served ?? "").padEnd(12)} ${c.method} ${c.path}${errs}`
+			);
+		}
+		return lines.join("\n");
+	});
+	return 0;
+}
+
+async function simReport(ctx) {
+	const result = await call(ctx, "get_simulation_report");
+	printResult(ctx.io, ctx.flags, result, ({ report: r }) => {
+		const f = r.funnel;
+		const lines = [
+			`${r.sessions} sessions, ${r.calls} calls in ${r.window_days} days`,
+			`Funnel: ${f.minted} keys → ${f.called} called → ${f.succeeded} succeeded → ${f.wrote} wrote`,
+			`First success: median ${r.first_success.median_calls ?? "–"} calls, ${r.first_success.median_seconds ?? "–"}s`,
+		];
+		if (r.hallucinated.length) {
+			lines.push("", "Expected but missing:");
+			for (const h of r.hallucinated.slice(0, 10))
+				lines.push(
+					`  ${String(h.sessions).padStart(3)} sessions  ${h.method} ${h.path}${h.nearest_operation ? `  (nearest: ${h.nearest_operation})` : ""}`
+				);
+		}
+		if (r.field_errors.length) {
+			lines.push("", "Argument errors:");
+			for (const e of r.field_errors.slice(0, 10))
+				lines.push(`  ${String(e.count).padStart(3)}×  ${e.operation_id}  ${e.field} (${e.code})`);
+		}
+		if (r.error_loops.length) {
+			lines.push("", "Error loops:");
+			for (const l of r.error_loops)
+				lines.push(`  ${l.operation_id}: ${l.sessions} sessions, up to ${l.longest} in a row`);
+		}
+		if (r.by_agent.length) {
+			lines.push("", "By agent:");
+			for (const a of r.by_agent)
+				lines.push(
+					`  ${a.agent.padEnd(22)} ${a.sessions} sessions, ${a.calls} calls, ${Math.round(a.success_rate * 100)}% ok`
+				);
+		}
+		if (r.feedback.length) {
+			lines.push("", "Feedback:");
+			for (const fb of r.feedback.slice(-5)) lines.push(`  “${fb.note.slice(0, 200)}”`);
+		}
+		lines.push("", `Unused: ${r.unused.count} of ${r.unused.total} operations`);
+		return lines.join("\n");
+	});
+	return 0;
+}

package/lib/commands.mjs

@@ -47,3 +47,4 @@ export * from "./commands/repo.mjs";
 export * from "./commands/site.mjs";
 export * from "./commands/evals.mjs";
 export * from "./commands/visits.mjs";
+export * from "./commands/sim.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@letterstory/cli",
-	"version": "0.138.0",
+	"version": "0.139.0",
 	"description": "Spin up and manage Letterstory phantom blogs from your terminal.",
 	"type": "module",
 	"license": "UNLICENSED",