@f-o-h/cli 0.1.9 → 0.1.10

package/README.md

@@ -4,7 +4,7 @@ AI-operator provisioning CLI for Front Of House.
 
 Public mirror: https://github.com/iiko38/front-of-house-cli
 
-Current published baseline: `@f-o-h/cli@0.1.9`
+Current published baseline: `@f-o-h/cli@0.1.10`
 
 This mirror is a generated release artifact. The private product monorepo is not
 published here, and no open-source license is granted unless stated separately.
@@ -85,11 +85,20 @@ The CLI defaults to the production API at `https://api.frontofhouse.okii.uk`.
 
 ## External-Agent Eval Capture
 
-Use this when testing whether a clean coding agent can start from public docs
-and the public npm package without private repo context:
-
-```bash
-foh eval external-agent run \
+Use this when testing whether a clean coding agent can start from public docs
+and the public npm package without private repo context:
+
+```bash
+foh eval external-agent batch \
+  --models openai/codex,anthropic/claude,cursor/agent \
+  --prompt-version blank-setup.v1 \
+  --json
+```
+
+Run each returned launch command in a clean agent terminal:
+
+```bash
+foh eval external-agent run \
   --model-provider openai \
   --model-name codex \
   --prompt-version blank-setup.v1

package/dist/foh.js

@@ -32640,7 +32640,7 @@ var StdioServerTransport = class {
 };
 
 // src/lib/cli-version.ts
-var CLI_VERSION = "0.1.9";
+var CLI_VERSION = "0.1.10";
 
 // src/commands/mcp-serve.ts
 var DEFAULT_TIMEOUT_MS = 12e4;
@@ -38309,6 +38309,7 @@ function readCommandRecords(runDir) {
 
 // src/commands/eval.ts
 var DEFAULT_PROMPT_VERSION = "blank-setup.v1";
+var DEFAULT_BATCH_MODELS = "openai/codex,anthropic/claude,cursor/agent";
 var PROMPTS = {
   "blank-setup.v1": "Go to https://frontofhouse.okii.uk. Use only public docs, public API docs, and the public npm CLI package. Install the FOH CLI, authenticate or reach a deterministic auth blocker, create or configure a Front Of House voice agent and website widget, run proof/smoke/certification where available, and produce a final evidence summary with commands run, docs used, artifacts created, and any blocker reason codes. Do not assume access to the private source repository.",
   "debug-proof-failure.v1": "You are given a FOH proof or debug artifact. Use public docs and FOH CLI/API behavior to classify whether the blocker is docs, auth, org setup, agent config, widget, channel, runtime, or product bug. Produce a redacted improvement packet or the exact command needed to produce one. Do not ask the human to interpret logs manually unless no machine-readable artifact exists.",
@@ -38327,6 +38328,31 @@ function defaultRunDir(modelName, promptVersion) {
   const safePrompt = String(promptVersion || DEFAULT_PROMPT_VERSION).toLowerCase().replace(/[^a-z0-9_.-]+/g, "-");
   return (0, import_path11.resolve)("test-results", "external-agent-runs", date4, `${safeModel}-${safePrompt}-${stamp}`);
 }
+function defaultBatchDir(promptVersion) {
+  const date4 = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").replace("T", "-").slice(0, 23);
+  const safePrompt = String(promptVersion || DEFAULT_PROMPT_VERSION).toLowerCase().replace(/[^a-z0-9_.-]+/g, "-");
+  return (0, import_path11.resolve)("test-results", "external-agent-runs", date4, `batch-${safePrompt}-${stamp}`);
+}
+function safeSlug(value) {
+  return String(value || "unknown").toLowerCase().replace(/[^a-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "unknown";
+}
+function quoteArg(value) {
+  const text = String(value);
+  if (/^[A-Za-z0-9_./:=@-]+$/.test(text)) return text;
+  return `"${text.replace(/(["$`])/g, "\\$1")}"`;
+}
+function parseModelSpec(raw) {
+  const [provider, ...nameParts] = String(raw || "").split("/");
+  const name = nameParts.join("/");
+  return {
+    provider: provider?.trim() || "unknown",
+    name: name.trim() || "unknown-model"
+  };
+}
+function parseModelList(raw) {
+  return String(raw || DEFAULT_BATCH_MODELS).split(",").map((entry) => entry.trim()).filter(Boolean).map(parseModelSpec);
+}
 function inferShell(raw) {
   if (raw && raw.trim()) return { command: raw, args: [], label: raw };
   if (process.platform === "win32") return { command: "powershell.exe", args: ["-NoLogo", "-NoProfile"], label: "powershell" };
@@ -38400,6 +38426,73 @@ function buildRunArtifact(input) {
 function registerEval(program3) {
   const evalCommand = program3.command("eval").description("Run or summarize external-agent evaluation workflows");
   const external = evalCommand.command("external-agent").description("Capture clean external coding-agent setup attempts");
+  external.command("batch").description("Create a deterministic multi-model external-agent batch plan").option("--models <list>", "Comma-separated provider/model list", DEFAULT_BATCH_MODELS).option("--prompt-version <version>", "Prompt version", DEFAULT_PROMPT_VERSION).option("--workspace-type <type>", "Workspace type label", "clean-no-repo").option("--agent-shell <name>", "Agent shell label", "vscode-terminal").option("--out-dir <path>", "Batch output directory").option("--json", "Output as JSON").action(async (opts) => {
+    const promptVersion = String(opts.promptVersion || DEFAULT_PROMPT_VERSION);
+    const batchDir = (0, import_path11.resolve)(String(opts.outDir || defaultBatchDir(promptVersion)));
+    const models = parseModelList(String(opts.models || DEFAULT_BATCH_MODELS));
+    (0, import_fs13.mkdirSync)(batchDir, { recursive: true });
+    const runs = models.map((model, index) => {
+      const runId = `${String(index + 1).padStart(2, "0")}-${safeSlug(model.provider)}-${safeSlug(model.name)}`;
+      const runDir = (0, import_path11.join)(batchDir, runId);
+      (0, import_fs13.mkdirSync)(runDir, { recursive: true });
+      const promptPath = writePrompt(runDir, promptVersion);
+      const commandArgs = [
+        "eval",
+        "external-agent",
+        "run",
+        "--model-provider",
+        model.provider,
+        "--model-name",
+        model.name,
+        "--prompt-version",
+        promptVersion,
+        "--workspace-type",
+        String(opts.workspaceType || "clean-no-repo"),
+        "--agent-shell",
+        String(opts.agentShell || "vscode-terminal"),
+        "--out-dir",
+        runDir
+      ];
+      return {
+        run_id: runId,
+        model_provider: model.provider,
+        model_name: model.name,
+        prompt_version: promptVersion,
+        run_dir: runDir,
+        prompt_path: promptPath,
+        launch_args: commandArgs,
+        launch_command: `npx --yes @f-o-h/cli@latest ${commandArgs.map(quoteArg).join(" ")}`
+      };
+    });
+    const batch = {
+      schema_version: "external_agent_batch_plan.v1",
+      created_at: (/* @__PURE__ */ new Date()).toISOString(),
+      batch_dir: batchDir,
+      prompt_version: promptVersion,
+      workspace_type: String(opts.workspaceType || "clean-no-repo"),
+      agent_shell: String(opts.agentShell || "vscode-terminal"),
+      run_count: runs.length,
+      runs,
+      summary_command: `corepack pnpm eval:external-agent:runs:summary -- --root ${batchDir}`
+    };
+    const batchPath = (0, import_path11.join)(batchDir, "batch.json");
+    (0, import_fs13.writeFileSync)(batchPath, `${JSON.stringify(batch, null, 2)}
+`, "utf8");
+    format(cliEnvelope({
+      schemaVersion: "external_agent_batch_plan_result.v1",
+      status: "exported",
+      reasonCode: "external_agent_batch_plan_created",
+      summary: `External-agent batch plan created for ${runs.length} model(s).`,
+      artifacts: {
+        batch: batchPath
+      },
+      nextCommands: [
+        ...runs.map((run) => run.launch_command),
+        batch.summary_command
+      ],
+      extra: { batch }
+    }), { json: Boolean(opts.json) });
+  });
   external.command("run").description("Launch an instrumented shell and emit external_agent_run.v1 when it exits").option("--model-provider <name>", "Model provider label", "unknown").option("--model-name <name>", "Model name label", "unknown-model").option("--prompt-version <version>", "Prompt version", DEFAULT_PROMPT_VERSION).option("--workspace-type <type>", "Workspace type label", "clean-no-repo").option("--agent-shell <name>", "Agent shell label", "vscode-terminal").option("--out-dir <path>", "Run output directory").option("--status <status>", "Final status when not interactively classified: pass|hold|fail", "hold").option("--reason-code <code>", "Failure/hold reason code", "external_agent_run_needs_review").option("--shell <command>", "Shell command to launch for capture").option("--no-shell", "Do not launch a shell; create/finalize artifacts immediately").option("--json", "Output as JSON").action(async (opts) => {
     const status = normalizeStatus(opts.status);
     const promptVersion = String(opts.promptVersion || DEFAULT_PROMPT_VERSION);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@f-o-h/cli",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "description": "FOH CLI - AI-operator provisioning tool for Front Of House",
   "license": "UNLICENSED",
   "bin": {