@f-o-h/cli 0.1.6 → 0.1.8

package/README.md

@@ -4,7 +4,7 @@ AI-operator provisioning CLI for Front Of House.
 
 Public mirror: https://github.com/iiko38/front-of-house-cli
 
-Current published baseline: `@f-o-h/cli@0.1.5`
+Current published baseline: `@f-o-h/cli@0.1.8`
 
 This mirror is a generated release artifact. The private product monorepo is not
 published here, and no open-source license is granted unless stated separately.
@@ -15,10 +15,12 @@ Package-local examples and schemas ship with the npm artifact:
 - `examples/proof-report.example.json`
 - `examples/transcript-export.example.json`
 - `examples/improvement-packet.example.json`
+- `examples/external-agent-run.example.json`
 - `schemas/cli-envelope.schema.json`
 - `schemas/scenario-suite.schema.json`
 - `schemas/transcript-export.schema.json`
 - `schemas/improvement-packet.schema.json`
+- `schemas/external-agent-run.schema.json`
 
 ## Install
 
@@ -81,6 +83,22 @@ automation when holds should fail the command, and `--mission voice` or
 
 The CLI defaults to the production API at `https://api.frontofhouse.okii.uk`.
 
+## External-Agent Eval Capture
+
+Use this when testing whether a clean coding agent can start from public docs
+and the public npm package without private repo context:
+
+```bash
+foh eval external-agent run \
+  --model-provider openai \
+  --model-name codex \
+  --prompt-version blank-setup.v1
+```
+
+The command writes a versioned prompt, launches an instrumented shell, captures
+FOH CLI commands into `commands.ndjson`, and finalizes `run.json` as an
+`external_agent_run.v1` artifact when the shell exits.
+
 ## Local Scenario Suites
 
 `foh test run --suite <file>` runs deterministic widget-runtime checks for a

package/dist/foh.js

@@ -6046,7 +6046,7 @@ var require_compile = __commonJS({
       const schOrFunc = root.refs[ref];
       if (schOrFunc)
         return schOrFunc;
-      let _sch = resolve8.call(this, root, ref);
+      let _sch = resolve10.call(this, root, ref);
       if (_sch === void 0) {
         const schema2 = (_a2 = root.localRefs) === null || _a2 === void 0 ? void 0 : _a2[ref];
         const { schemaId } = this.opts;
@@ -6073,7 +6073,7 @@ var require_compile = __commonJS({
     function sameSchemaEnv(s1, s2) {
       return s1.schema === s2.schema && s1.root === s2.root && s1.baseId === s2.baseId;
     }
-    function resolve8(root, ref) {
+    function resolve10(root, ref) {
       let sch;
       while (typeof (sch = this.refs[ref]) == "string")
         ref = sch;
@@ -6648,7 +6648,7 @@ var require_fast_uri = __commonJS({
       }
       return uri;
     }
-    function resolve8(baseURI, relativeURI, options) {
+    function resolve10(baseURI, relativeURI, options) {
       const schemelessOptions = options ? Object.assign({ scheme: "null" }, options) : { scheme: "null" };
       const resolved = resolveComponent(parse3(baseURI, schemelessOptions), parse3(relativeURI, schemelessOptions), schemelessOptions, true);
       schemelessOptions.skipEscape = true;
@@ -6875,7 +6875,7 @@ var require_fast_uri = __commonJS({
     var fastUri = {
       SCHEMES,
       normalize,
-      resolve: resolve8,
+      resolve: resolve10,
       resolveComponent,
       equal,
       serialize,
@@ -10105,21 +10105,21 @@ async function promptLine(label, {
   allowEmpty = false,
   defaultValue
 } = {}) {
-  return await new Promise((resolve8) => {
+  return await new Promise((resolve10) => {
     const suffix = defaultValue ? ` [${defaultValue}]` : "";
     const rl = (0, import_readline.createInterface)({ input: process.stdin, output: process.stdout, terminal: true });
     rl.question(`${label}${suffix}: `, (answer) => {
       rl.close();
       const value = String(answer ?? "").trim();
       if (!value && typeof defaultValue === "string") {
-        resolve8(defaultValue);
+        resolve10(defaultValue);
         return;
       }
       if (!value && !allowEmpty) {
-        resolve8("");
+        resolve10("");
         return;
       }
-      resolve8(value);
+      resolve10(value);
     });
   });
 }
@@ -10127,7 +10127,7 @@ async function promptSecret(label) {
   if (!process.stdin.isTTY || !process.stdout.isTTY || typeof process.stdin.setRawMode !== "function") {
     return await promptLine(label);
   }
-  return await new Promise((resolve8) => {
+  return await new Promise((resolve10) => {
     const stdin = process.stdin;
     const stdout = process.stdout;
     const wasRaw = Boolean(stdin.isRaw);
@@ -10141,7 +10141,7 @@ async function promptSecret(label) {
     const finish = () => {
       cleanup();
       stdout.write("\n");
-      resolve8(value);
+      resolve10(value);
     };
     const onData = (chunk) => {
       const text = typeof chunk === "string" ? chunk : chunk.toString("utf8");
@@ -10150,7 +10150,7 @@ async function promptSecret(label) {
           cleanup();
           process.exitCode = 130;
           stdout.write("\n");
-          return resolve8("");
+          return resolve10("");
         }
         if (char === "\r" || char === "\n") {
           finish();
@@ -10419,7 +10419,7 @@ async function storeAuthenticatedSession(params) {
   return output;
 }
 function sleep(ms) {
-  return new Promise((resolve8) => setTimeout(resolve8, ms));
+  return new Promise((resolve10) => setTimeout(resolve10, ms));
 }
 async function runDeviceLogin(opts) {
   const jsonMode = Boolean(opts.json);
@@ -10957,7 +10957,7 @@ async function pollUntil(check2, opts) {
   }
 }
 function sleep2(ms) {
-  return new Promise((resolve8) => setTimeout(resolve8, ms));
+  return new Promise((resolve10) => setTimeout(resolve10, ms));
 }
 
 // src/commands/compliance.ts
@@ -13995,8 +13995,8 @@ function registerAgentGuardrailCommands(agent) {
     try {
       rule = JSON.parse(opts.rule);
     } catch {
-      const { readFileSync: readFileSync9 } = await import("fs");
-      rule = JSON.parse(readFileSync9(opts.rule, "utf-8"));
+      const { readFileSync: readFileSync10 } = await import("fs");
+      rule = JSON.parse(readFileSync10(opts.rule, "utf-8"));
     }
     const data = await apiFetch(`/v1/console/agents/${opts.agent}/guardrails`, {
       method: "POST",
@@ -14596,9 +14596,9 @@ function registerAgent(program3) {
       process.stdout.write(yaml);
       return;
     }
-    const { writeFileSync: writeFileSync6 } = await import("fs");
+    const { writeFileSync: writeFileSync7 } = await import("fs");
     const outputPath = opts.output ?? "tenant.yaml";
-    writeFileSync6(
+    writeFileSync7(
       outputPath,
       `# tenant.yaml - Front Of House agent manifest
 # Edit this file and run: foh plan tenant.yaml
@@ -16033,11 +16033,11 @@ function registerVoice(program3) {
     }
     const outputPath = String(opts.out || `foh-voice-preview-${provider}-${voiceId}.mp3`).trim();
     const audio = Buffer.from(await res.arrayBuffer());
-    const { mkdirSync: mkdirSync4, writeFileSync: writeFileSync6 } = await import("fs");
-    const { dirname: dirname5, resolve: resolve8 } = await import("path");
-    const absolutePath = resolve8(outputPath);
-    mkdirSync4(dirname5(absolutePath), { recursive: true });
-    writeFileSync6(absolutePath, audio);
+    const { mkdirSync: mkdirSync6, writeFileSync: writeFileSync7 } = await import("fs");
+    const { dirname: dirname5, resolve: resolve10 } = await import("path");
+    const absolutePath = resolve10(outputPath);
+    mkdirSync6(dirname5(absolutePath), { recursive: true });
+    writeFileSync7(absolutePath, audio);
     format({
       status: "ok",
       provider,
@@ -30518,7 +30518,7 @@ var Protocol = class {
           return;
         }
         const pollInterval = task2.pollInterval ?? this._options?.defaultTaskPollInterval ?? 1e3;
-        await new Promise((resolve8) => setTimeout(resolve8, pollInterval));
+        await new Promise((resolve10) => setTimeout(resolve10, pollInterval));
         options?.signal?.throwIfAborted();
       }
     } catch (error2) {
@@ -30535,7 +30535,7 @@ var Protocol = class {
    */
   request(request, resultSchema, options) {
     const { relatedRequestId, resumptionToken, onresumptiontoken, task, relatedTask } = options ?? {};
-    return new Promise((resolve8, reject) => {
+    return new Promise((resolve10, reject) => {
       const earlyReject = (error2) => {
         reject(error2);
       };
@@ -30613,7 +30613,7 @@ var Protocol = class {
           if (!parseResult.success) {
             reject(parseResult.error);
           } else {
-            resolve8(parseResult.data);
+            resolve10(parseResult.data);
           }
         } catch (error2) {
           reject(error2);
@@ -30874,12 +30874,12 @@ var Protocol = class {
       }
     } catch {
     }
-    return new Promise((resolve8, reject) => {
+    return new Promise((resolve10, reject) => {
       if (signal.aborted) {
         reject(new McpError(ErrorCode.InvalidRequest, "Request cancelled"));
         return;
       }
-      const timeoutId = setTimeout(resolve8, interval);
+      const timeoutId = setTimeout(resolve10, interval);
       signal.addEventListener("abort", () => {
         clearTimeout(timeoutId);
         reject(new McpError(ErrorCode.InvalidRequest, "Request cancelled"));
@@ -31979,7 +31979,7 @@ var McpServer = class {
     let task = createTaskResult.task;
     const pollInterval = task.pollInterval ?? 5e3;
     while (task.status !== "completed" && task.status !== "failed" && task.status !== "cancelled") {
-      await new Promise((resolve8) => setTimeout(resolve8, pollInterval));
+      await new Promise((resolve10) => setTimeout(resolve10, pollInterval));
       const updatedTask = await extra.taskStore.getTask(taskId);
       if (!updatedTask) {
         throw new McpError(ErrorCode.InternalError, `Task ${taskId} not found during polling`);
@@ -32628,19 +32628,19 @@ var StdioServerTransport = class {
     this.onclose?.();
   }
   send(message) {
-    return new Promise((resolve8) => {
+    return new Promise((resolve10) => {
       const json3 = serializeMessage(message);
       if (this._stdout.write(json3)) {
-        resolve8();
+        resolve10();
       } else {
-        this._stdout.once("drain", resolve8);
+        this._stdout.once("drain", resolve10);
       }
     });
   }
 };
 
 // src/lib/cli-version.ts
-var CLI_VERSION = "0.1.6";
+var CLI_VERSION = "0.1.8";
 
 // src/commands/mcp-serve.ts
 var DEFAULT_TIMEOUT_MS = 12e4;
@@ -32825,7 +32825,7 @@ async function runFohCli(params) {
     effectiveArgv.push("--json");
   }
   const command = `foh ${effectiveArgv.join(" ")}`;
-  return await new Promise((resolve8) => {
+  return await new Promise((resolve10) => {
     const child = (0, import_node_child_process.spawn)(process.execPath, [cliEntry, ...effectiveArgv], {
       stdio: ["ignore", "pipe", "pipe"],
       env: {
@@ -32850,7 +32850,7 @@ async function runFohCli(params) {
     });
     child.once("error", (error2) => {
       clearTimeout(timeoutHandle);
-      resolve8({
+      resolve10({
         ok: false,
         command,
         argv: effectiveArgv,
@@ -32866,7 +32866,7 @@ async function runFohCli(params) {
       const stderrText = finalizeBoundedText(stderrBuffer);
       const exitCode = Number.isFinite(code ?? NaN) ? Number(code) : 1;
       const stdoutJson = tryParseJson(stdoutText);
-      resolve8({
+      resolve10({
         ok: !timedOut && exitCode === 0,
         command,
         argv: effectiveArgv,
@@ -34775,8 +34775,8 @@ function registerSetup(program3) {
         }
         try {
           const manifest = await agentExport(resolvedAgentId, { apiUrlOverride: opts.apiUrl });
-          const { writeFileSync: writeFileSync6 } = await import("fs");
-          writeFileSync6(
+          const { writeFileSync: writeFileSync7 } = await import("fs");
+          writeFileSync7(
             "tenant.yaml",
             `# tenant.yaml - Front Of House agent manifest
 # Edit this file and run: foh plan tenant.yaml
@@ -34944,8 +34944,8 @@ function registerSim(program3) {
       }
       const cert = response.certificate;
       if (opts.out) {
-        const { writeFileSync: writeFileSync6 } = await import("fs");
-        writeFileSync6(opts.out, JSON.stringify(cert, null, 2) + "\n", "utf-8");
+        const { writeFileSync: writeFileSync7 } = await import("fs");
+        writeFileSync7(opts.out, JSON.stringify(cert, null, 2) + "\n", "utf-8");
         process.stderr.write(`  Certificate written to ${opts.out}
 `);
       }
@@ -34995,8 +34995,8 @@ function registerSim(program3) {
         });
       }
       if (opts.out) {
-        const { writeFileSync: writeFileSync6 } = await import("fs");
-        writeFileSync6(opts.out, JSON.stringify(response.certificate, null, 2) + "\n", "utf-8");
+        const { writeFileSync: writeFileSync7 } = await import("fs");
+        writeFileSync7(opts.out, JSON.stringify(response.certificate, null, 2) + "\n", "utf-8");
         process.stderr.write(`  Final certificate written to ${opts.out}
 `);
       }
@@ -36362,6 +36362,7 @@ var import_path8 = require("path");
 var import_fs9 = require("fs");
 var import_path7 = require("path");
 var IMPROVEMENT_SOURCE_TYPES = [
+  "external_agent_run",
   "setup_failure",
   "proof_failure",
   "replay_failure",
@@ -36373,6 +36374,8 @@ var IMPROVEMENT_DECISIONS = [
   "ignore",
   "fix_docs",
   "fix_config",
+  "fix_cli",
+  "fix_api",
   "fix_runtime",
   "add_test"
 ];
@@ -36408,6 +36411,7 @@ function parseEnum(raw, allowed, label) {
 function inferSourceType(artifact) {
   const schema2 = nonEmpty2(getPath2(artifact, "schema_version")) || "";
   const status = nonEmpty2(getPath2(artifact, "status")) || "";
+  if (schema2 === "external_agent_run.v1") return "external_agent_run";
   if (schema2.includes("knowledge_query") || nonEmpty2(getPath2(artifact, "failure_packet.schema_version"))?.includes("knowledge_query")) {
     return "knowledge_miss";
   }
@@ -36419,6 +36423,8 @@ function inferSourceType(artifact) {
 function inferReasonCode(artifact) {
   const direct = nonEmpty2(getPath2(artifact, "reason_code"));
   if (direct) return direct;
+  const failureReason = nonEmpty2(getPath2(artifact, "failure_reason_code"));
+  if (failureReason) return failureReason;
   const nested = nonEmpty2(getPath2(artifact, "failure_packet.reason_code"));
   if (nested) return nested;
   const checks = getPath2(artifact, "checks");
@@ -36432,6 +36438,7 @@ function inferReasonCode(artifact) {
   return nonEmpty2(getPath2(artifact, "status"));
 }
 function inferPromotionDecision(sourceType) {
+  if (sourceType === "external_agent_run") return "fix_docs";
   if (sourceType === "knowledge_miss") return "fix_docs";
   if (sourceType === "setup_failure" || sourceType === "proof_failure" || sourceType === "live_proof_failure") return "fix_config";
   if (sourceType === "replay_failure" || sourceType === "runtime_miss") return "add_test";
@@ -36473,6 +36480,9 @@ function defaultNextCommands(input) {
   if (input.sourceType === "knowledge_miss" && input.ids.agent_id) {
     commands.push(`foh knowledge query --agent ${input.ids.agent_id} --text "<question>" --explain --json`);
   }
+  if (input.sourceType === "external_agent_run" && input.sourceArtifactPath) {
+    commands.push(`foh bug improve --from external-agent-run --file ${input.sourceArtifactPath} --json`);
+  }
   if (input.sourceArtifactPath) {
     commands.push(`foh bug report --out test-results/bug-report.from-improvement.json --command "investigate ${input.reasonCode}" --request-url https://front-of-house-api.stldocs.app/api --response-status 500 --next-check "Review ${(0, import_path7.basename)(input.sourceArtifactPath)}" --json`);
   }
@@ -36835,18 +36845,20 @@ function registerBug(program3) {
       throw e;
     }
   });
-  bug.command("improve").description("Write a redacted failure-to-improvement packet from a setup/proof/replay/knowledge/runtime artifact").option("--from-file <path>", "Source JSON artifact to convert into an improvement packet").option("--out <path>", "Output JSON file path for improvement packet").option("--source-type <type>", "setup_failure|proof_failure|replay_failure|knowledge_miss|runtime_miss|live_proof_failure").option("--reason-code <code>", "Deterministic failure reason code").option("--recommendation <decision>", "ignore|fix_docs|fix_config|fix_runtime|add_test").option("--evidence-summary <text>", "Compact redacted evidence summary").option("--source-command <text>", "Command or operation that produced the source artifact").option("--org <id>", "Org ID for boundary check and packet IDs").option("--agent <id>", "Agent ID to attach").option("--conversation <id>", "Conversation ID to attach").option("--trace <id>", "Trace ID to attach").option("--correlation <id>", "Correlation ID to attach").option("--proof-artifact <path>", "Proof artifact path to attach").option(
+  bug.command("improve").description("Write a redacted failure-to-improvement packet from an external-agent/setup/proof/replay/knowledge/runtime artifact").option("--from <type>", "Source artifact type alias, for example external-agent-run").option("--file <path>", "Source JSON artifact alias for --from-file").option("--from-file <path>", "Source JSON artifact to convert into an improvement packet").option("--out <path>", "Output JSON file path for improvement packet").option("--source-type <type>", "external_agent_run|setup_failure|proof_failure|replay_failure|knowledge_miss|runtime_miss|live_proof_failure").option("--reason-code <code>", "Deterministic failure reason code").option("--recommendation <decision>", "ignore|fix_docs|fix_config|fix_cli|fix_api|fix_runtime|add_test").option("--evidence-summary <text>", "Compact redacted evidence summary").option("--source-command <text>", "Command or operation that produced the source artifact").option("--org <id>", "Org ID for boundary check and packet IDs").option("--agent <id>", "Agent ID to attach").option("--conversation <id>", "Conversation ID to attach").option("--trace <id>", "Trace ID to attach").option("--correlation <id>", "Correlation ID to attach").option("--proof-artifact <path>", "Proof artifact path to attach").option(
     "--next-command <text>",
     "Deterministic next command (repeat this flag for multiple commands)",
     (value, previous = []) => [...previous, value],
     []
   ).option("--json", "Output as JSON").action(async (opts) => withCommandErrorHandling(async () => {
-    const sourceArtifact = readSourceArtifact(opts.fromFile);
+    const sourcePath = opts.fromFile ?? opts.file;
+    const sourceType = opts.sourceType ?? (typeof opts.from === "string" ? opts.from.replaceAll("-", "_") : void 0);
+    const sourceArtifact = readSourceArtifact(sourcePath);
     const outPath = String(opts.out || defaultImprovementArtifactPath());
     const packet = buildImprovementPacket({
       sourceArtifact,
-      sourceArtifactPath: opts.fromFile,
-      sourceType: opts.sourceType,
+      sourceArtifactPath: sourcePath,
+      sourceType,
       reasonCode: opts.reasonCode,
       evidenceSummary: opts.evidenceSummary,
       promotionDecision: opts.recommendation,
@@ -37698,7 +37710,7 @@ async function runSelf(args, apiUrlOverride) {
   if (apiUrlOverride && !spawnArgs.includes("--api-url")) {
     spawnArgs.push("--api-url", apiUrlOverride);
   }
-  return await new Promise((resolve8, reject) => {
+  return await new Promise((resolve10, reject) => {
     const child = (0, import_child_process2.spawn)(process.execPath, [process.argv[1], ...spawnArgs], {
       stdio: "inherit",
       env: {
@@ -37708,7 +37720,7 @@ async function runSelf(args, apiUrlOverride) {
       }
     });
     child.once("error", reject);
-    child.once("close", (code) => resolve8(typeof code === "number" ? code : 1));
+    child.once("close", (code) => resolve10(typeof code === "number" ? code : 1));
   });
 }
 function shouldUseInteractiveHome(argv) {
@@ -38086,17 +38098,17 @@ function detectUpdateAvailability(currentVersion, cwd = process.cwd()) {
 async function applyRepoUpdate(repoRoot) {
   const scriptPath = (0, import_path9.join)(repoRoot, "scripts", "Install-FohCli.ps1");
   if (process.platform === "win32") {
-    return await new Promise((resolve8, reject) => {
+    return await new Promise((resolve10, reject) => {
       const child = (0, import_child_process3.spawn)(
         "powershell",
         ["-ExecutionPolicy", "Bypass", "-File", scriptPath],
         { stdio: "inherit" }
       );
       child.once("error", reject);
-      child.once("close", (code) => resolve8(typeof code === "number" ? code : 1));
+      child.once("close", (code) => resolve10(typeof code === "number" ? code : 1));
     });
   }
-  return await new Promise((resolve8, reject) => {
+  return await new Promise((resolve10, reject) => {
     const child = (0, import_child_process3.spawn)(
       "corepack",
       ["pnpm", "cli:install:global"],
@@ -38106,7 +38118,7 @@ async function applyRepoUpdate(repoRoot) {
       }
     );
     child.once("error", reject);
-    child.once("close", (code) => resolve8(typeof code === "number" ? code : 1));
+    child.once("close", (code) => resolve10(typeof code === "number" ? code : 1));
   });
 }
 function shouldShowUpdateNotice(argv = process.argv) {
@@ -38241,6 +38253,214 @@ function registerUpdate(program3) {
   });
 }
 
+// src/commands/eval.ts
+var import_fs13 = require("fs");
+var import_path11 = require("path");
+var import_child_process4 = require("child_process");
+
+// src/lib/external-agent-capture.ts
+var import_fs12 = require("fs");
+var import_path10 = require("path");
+var EXTERNAL_AGENT_RUN_DIR_ENV = "FOH_EXTERNAL_AGENT_RUN_DIR";
+var EXTERNAL_AGENT_PROMPT_VERSION_ENV = "FOH_EXTERNAL_AGENT_PROMPT_VERSION";
+var SECRET_RE2 = /\b(?:Bearer\s+)?(?:sk|pk|xai|whsec|EAAN|ghp|gho|github_pat|npm_)[A-Za-z0-9_\-.]{12,}\b/gi;
+function redactText(value) {
+  return value.replace(SECRET_RE2, "[redacted_secret]");
+}
+function safeJsonLine(value) {
+  return JSON.stringify(value).replace(/\r?\n/g, " ") + "\n";
+}
+function getExternalAgentRunDir() {
+  const raw = process.env[EXTERNAL_AGENT_RUN_DIR_ENV];
+  if (!raw || !raw.trim()) return null;
+  return (0, import_path10.resolve)(raw);
+}
+function recordExternalAgentCliInvocation(input) {
+  const runDir = getExternalAgentRunDir();
+  if (!runDir) return;
+  try {
+    (0, import_fs12.mkdirSync)(runDir, { recursive: true });
+    const args = input.argv.slice(2).map((arg) => redactText(String(arg)));
+    const record2 = {
+      schema_version: "external_agent_cli_command.v1",
+      recorded_at: (/* @__PURE__ */ new Date()).toISOString(),
+      cli_version: input.cliVersion,
+      cwd: redactText(process.cwd()),
+      argv: args,
+      command: args.join(" "),
+      json_requested: args.includes("--json"),
+      prompt_version: process.env[EXTERNAL_AGENT_PROMPT_VERSION_ENV] || null
+    };
+    (0, import_fs12.appendFileSync)((0, import_path10.join)(runDir, "commands.ndjson"), safeJsonLine(record2), "utf8");
+  } catch {
+  }
+}
+function readCommandRecords(runDir) {
+  const commandLogPath = (0, import_path10.join)(runDir, "commands.ndjson");
+  if (!(0, import_fs12.existsSync)(commandLogPath)) return [];
+  return (0, import_fs12.readFileSync)(commandLogPath, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
+}
+
+// src/commands/eval.ts
+var DEFAULT_PROMPT_VERSION = "blank-setup.v1";
+var PROMPTS = {
+  "blank-setup.v1": "Go to https://frontofhouse.okii.uk. Use only public docs, public API docs, and the public npm CLI package. Install the FOH CLI, authenticate or reach a deterministic auth blocker, create or configure a Front Of House voice agent and website widget, run proof/smoke/certification where available, and produce a final evidence summary with commands run, docs used, artifacts created, and any blocker reason codes. Do not assume access to the private source repository.",
+  "debug-proof-failure.v1": "You are given a FOH proof or debug artifact. Use public docs and FOH CLI/API behavior to classify whether the blocker is docs, auth, org setup, agent config, widget, channel, runtime, or product bug. Produce a redacted improvement packet or the exact command needed to produce one. Do not ask the human to interpret logs manually unless no machine-readable artifact exists.",
+  "knowledge-miss.v1": "A FOH agent failed to answer a business question. Use CLI/API/docs to determine whether this is a knowledge-ingestion issue, retrieval issue, config issue, prompt/behavior issue, or runtime issue. Prefer foh knowledge query, transcript export, replay, and foh bug improve artifacts over screenshots.",
+  "replay-failure.v1": "You are given a FOH transcript or replay artifact. Use CLI/API/docs to replay or inspect the failed interaction, identify expected vs actual behavior, and produce a scenario-test or improvement-packet candidate."
+};
+function normalizeStatus(raw) {
+  const value = String(raw ?? "hold").trim().toLowerCase();
+  if (value === "pass" || value === "hold" || value === "fail") return value;
+  throw new Error("Invalid --status. Use pass, hold, or fail.");
+}
+function defaultRunDir(modelName, promptVersion) {
+  const date4 = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").replace("T", "-").slice(0, 23);
+  const safeModel = String(modelName || "unknown-model").toLowerCase().replace(/[^a-z0-9_-]+/g, "-");
+  const safePrompt = String(promptVersion || DEFAULT_PROMPT_VERSION).toLowerCase().replace(/[^a-z0-9_.-]+/g, "-");
+  return (0, import_path11.resolve)("test-results", "external-agent-runs", date4, `${safeModel}-${safePrompt}-${stamp}`);
+}
+function inferShell(raw) {
+  if (raw && raw.trim()) return { command: raw, args: [], label: raw };
+  if (process.platform === "win32") return { command: "powershell.exe", args: ["-NoLogo", "-NoProfile"], label: "powershell" };
+  return { command: process.env.SHELL || "sh", args: [], label: process.env.SHELL || "sh" };
+}
+function writePrompt(runDir, promptVersion) {
+  const prompt = PROMPTS[promptVersion] ?? PROMPTS[DEFAULT_PROMPT_VERSION];
+  const path2 = (0, import_path11.join)(runDir, "prompt.txt");
+  (0, import_fs13.writeFileSync)(path2, `${prompt}
+`, "utf8");
+  return path2;
+}
+function writeSession(runDir, session) {
+  const path2 = (0, import_path11.join)(runDir, "session.json");
+  (0, import_fs13.writeFileSync)(path2, `${JSON.stringify(session, null, 2)}
+`, "utf8");
+  return path2;
+}
+function buildRunArtifact(input) {
+  const commands = readCommandRecords(input.runDir);
+  const startedAt = String(input.session.started_at);
+  const endedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const status = input.status;
+  const reasonCode = status === "pass" ? null : String(input.reasonCode || "external_agent_run_needs_review");
+  return {
+    schema_version: "external_agent_run.v1",
+    run_id: input.session.run_id,
+    status,
+    failure_reason_code: reasonCode,
+    model_provider: input.session.model_provider,
+    model_name: input.session.model_name,
+    agent_shell: input.session.agent_shell,
+    workspace_type: input.session.workspace_type,
+    prompt_version: input.session.prompt_version,
+    prompt_path: "prompt.txt",
+    started_at: startedAt,
+    ended_at: endedAt,
+    manual_intervention_count: Number(input.session.manual_intervention_count || 0),
+    manual_interventions: [],
+    environment: {
+      os: process.platform,
+      node_version: process.version,
+      npm_version: null,
+      foh_cli_version: CLI_VERSION,
+      shell_exit_code: input.shellExitCode ?? null
+    },
+    public_entrypoints: [
+      "https://frontofhouse.okii.uk",
+      "https://frontofhouse.okii.uk/llms.txt",
+      "https://frontofhouse.okii.uk/openapi.yaml",
+      "npx --yes @f-o-h/cli@latest"
+    ],
+    commands_run: commands.map((command) => command.command),
+    docs_pages_used: [],
+    artifacts: {
+      terminal_transcript: null,
+      command_log: "commands.ndjson",
+      proof_bundle: null,
+      replay_packet: null,
+      knowledge_packet: null,
+      improvement_packet: status === "pass" ? null : "improvement-packet.json",
+      notes: "notes.md"
+    },
+    summary: status === "pass" ? "External-agent capture session completed and was marked pass." : `External-agent capture session completed with ${commands.length} captured FOH command(s); classify and improve reason ${reasonCode}.`,
+    next_commands: status === "pass" ? ["corepack pnpm eval:external-agent:runs:summary"] : [
+      `foh bug improve --from external-agent-run --file ${(0, import_path11.join)(input.runDir, "run.json")} --out ${(0, import_path11.join)(input.runDir, "improvement-packet.json")} --json`,
+      "corepack pnpm eval:external-agent:runs:summary"
+    ]
+  };
+}
+function registerEval(program3) {
+  const evalCommand = program3.command("eval").description("Run or summarize external-agent evaluation workflows");
+  const external = evalCommand.command("external-agent").description("Capture clean external coding-agent setup attempts");
+  external.command("run").description("Launch an instrumented shell and emit external_agent_run.v1 when it exits").option("--model-provider <name>", "Model provider label", "unknown").option("--model-name <name>", "Model name label", "unknown-model").option("--prompt-version <version>", "Prompt version", DEFAULT_PROMPT_VERSION).option("--workspace-type <type>", "Workspace type label", "clean-no-repo").option("--agent-shell <name>", "Agent shell label", "vscode-terminal").option("--out-dir <path>", "Run output directory").option("--status <status>", "Final status when not interactively classified: pass|hold|fail", "hold").option("--reason-code <code>", "Failure/hold reason code", "external_agent_run_needs_review").option("--shell <command>", "Shell command to launch for capture").option("--no-shell", "Do not launch a shell; create/finalize artifacts immediately").option("--json", "Output as JSON").action(async (opts) => {
+    const status = normalizeStatus(opts.status);
+    const promptVersion = String(opts.promptVersion || DEFAULT_PROMPT_VERSION);
+    const runDir = (0, import_path11.resolve)(String(opts.outDir || defaultRunDir(opts.modelName, promptVersion)));
+    (0, import_fs13.mkdirSync)(runDir, { recursive: true });
+    const runId = runDir.split(/[\\/]/).filter(Boolean).slice(-1)[0];
+    const promptPath = writePrompt(runDir, promptVersion);
+    const shell = inferShell(opts.shell);
+    const session = {
+      schema_version: "external_agent_capture_session.v1",
+      run_id: runId,
+      started_at: (/* @__PURE__ */ new Date()).toISOString(),
+      model_provider: String(opts.modelProvider || "unknown"),
+      model_name: String(opts.modelName || "unknown-model"),
+      prompt_version: promptVersion,
+      workspace_type: String(opts.workspaceType || "clean-no-repo"),
+      agent_shell: String(opts.agentShell || shell.label),
+      manual_intervention_count: 0,
+      run_dir: runDir,
+      prompt_path: promptPath,
+      capture_env: {
+        [EXTERNAL_AGENT_RUN_DIR_ENV]: runDir,
+        [EXTERNAL_AGENT_PROMPT_VERSION_ENV]: promptVersion
+      }
+    };
+    writeSession(runDir, session);
+    (0, import_fs13.writeFileSync)((0, import_path11.join)(runDir, "notes.md"), "# External Agent Run Notes\n\n", "utf8");
+    let shellExitCode = null;
+    if (opts.shell !== false) {
+      process.stdout.write(`
+External-agent capture is active.
+Run dir: ${runDir}
+Prompt: ${promptPath}
+Exit the shell to finalize run.json.
+
+`);
+      const result = (0, import_child_process4.spawnSync)(shell.command, shell.args, {
+        stdio: "inherit",
+        env: {
+          ...process.env,
+          [EXTERNAL_AGENT_RUN_DIR_ENV]: runDir,
+          [EXTERNAL_AGENT_PROMPT_VERSION_ENV]: promptVersion,
+          FOH_CLI_SUPPRESS_BANNER: process.env.FOH_CLI_SUPPRESS_BANNER ?? "1"
+        }
+      });
+      shellExitCode = typeof result.status === "number" ? result.status : null;
+    }
+    const artifact = buildRunArtifact({ runDir, session, status, reasonCode: opts.reasonCode, shellExitCode });
+    const runPath = (0, import_path11.join)(runDir, "run.json");
+    (0, import_fs13.writeFileSync)(runPath, `${JSON.stringify(artifact, null, 2)}
+`, "utf8");
+    format(cliEnvelope({
+      schemaVersion: "external_agent_capture_result.v1",
+      status: "exported",
+      reasonCode: "external_agent_run_created",
+      summary: "External-agent run artifact created.",
+      artifacts: {
+        run: runPath,
+        prompt: promptPath,
+        commands: (0, import_path11.join)(runDir, "commands.ndjson")
+      },
+      nextCommands: artifact.next_commands,
+      extra: { run: artifact }
+    }), { json: Boolean(opts.json) });
+  });
+}
+
 // src/lib/banner.ts
 var import_picocolors6 = __toESM(require_picocolors());
 var BANNER_LINES = [
@@ -38311,6 +38531,7 @@ function installSoftExitTrap() {
 
 // src/index.ts
 installSoftExitTrap();
+recordExternalAgentCliInvocation({ argv: process.argv, cliVersion: CLI_VERSION });
 var program2 = new Command();
 var BUG_REPORT_URL = process.env.FOH_BUG_REPORT_URL ?? "https://github.com/iiko38/front-of-house/issues";
 function shouldRenderBanner2(argv) {
@@ -38392,6 +38613,7 @@ registerSim(program2);
 registerDiag(program2);
 registerBug(program2);
 registerProve(program2);
+registerEval(program2);
 registerUpdate(program2);
 registerHome(program2);
 hideInternalApiUrlOptions(program2);

package/examples/external-agent-run.example.json

@@ -0,0 +1,53 @@
+{
+  "schema_version": "external_agent_run.v1",
+  "run_id": "2026-05-04-codex-001",
+  "status": "fail",
+  "failure_reason_code": "docs_widget_proof_path_unclear",
+  "model_provider": "openai",
+  "model_name": "codex",
+  "agent_shell": "vscode-terminal",
+  "workspace_type": "clean-no-repo",
+  "prompt_version": "blank-setup.v1",
+  "prompt_path": "Docs/operations/external-agent-feedback-loop.md#blank-setupv1",
+  "started_at": "2026-05-04T09:00:00.000Z",
+  "ended_at": "2026-05-04T09:30:00.000Z",
+  "manual_intervention_count": 1,
+  "manual_interventions": [
+    {
+      "at": "2026-05-04T09:12:00.000Z",
+      "summary": "Human pointed the agent from generic docs to the CLI install guide."
+    }
+  ],
+  "environment": {
+    "os": "windows",
+    "node_version": "v24.11.1",
+    "npm_version": "10.x",
+    "foh_cli_version": "0.1.6"
+  },
+  "public_entrypoints": [
+    "https://frontofhouse.okii.uk",
+    "https://frontofhouse.okii.uk/llms.txt",
+    "npx --yes @f-o-h/cli@latest"
+  ],
+  "commands_run": [
+    "npx --yes @f-o-h/cli@latest --version",
+    "npx --yes @f-o-h/cli@latest setup --help",
+    "npx --yes @f-o-h/cli@latest prove --help"
+  ],
+  "docs_pages_used": [
+    "https://frontofhouse.okii.uk/guides/cli-install-and-upgrade",
+    "https://frontofhouse.okii.uk/guides/error-handling-and-debugging"
+  ],
+  "artifacts": {
+    "terminal_transcript": "terminal-transcript.txt",
+    "proof_bundle": null,
+    "replay_packet": null,
+    "knowledge_packet": null,
+    "improvement_packet": "improvement-packet.json",
+    "notes": "notes.md"
+  },
+  "summary": "Agent installed the CLI but could not identify the shortest widget proof path without human direction.",
+  "next_commands": [
+    "foh bug improve --from external-agent-run --file run.json --out improvement-packet.json --json"
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@f-o-h/cli",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "FOH CLI - AI-operator provisioning tool for Front Of House",
   "license": "UNLICENSED",
   "bin": {

package/schemas/external-agent-run.schema.json

@@ -0,0 +1,95 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://frontofhouse.okii.uk/schemas/external-agent-run.schema.json",
+  "title": "FOH External Agent Run",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "run_id",
+    "status",
+    "model_provider",
+    "model_name",
+    "prompt_version",
+    "started_at",
+    "manual_intervention_count",
+    "environment",
+    "public_entrypoints",
+    "commands_run",
+    "docs_pages_used",
+    "artifacts"
+  ],
+  "properties": {
+    "schema_version": { "const": "external_agent_run.v1" },
+    "run_id": { "type": "string", "minLength": 1 },
+    "status": { "type": "string", "enum": ["pass", "hold", "fail"] },
+    "failure_reason_code": { "type": ["string", "null"] },
+    "model_provider": { "type": "string", "minLength": 1 },
+    "model_name": { "type": "string", "minLength": 1 },
+    "agent_shell": { "type": "string" },
+    "workspace_type": { "type": "string" },
+    "prompt_version": { "type": "string", "minLength": 1 },
+    "prompt_path": { "type": "string" },
+    "started_at": { "type": "string" },
+    "ended_at": { "type": ["string", "null"] },
+    "manual_intervention_count": { "type": "integer", "minimum": 0 },
+    "manual_interventions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "at": { "type": "string" },
+          "summary": { "type": "string" }
+        },
+        "additionalProperties": true
+      }
+    },
+    "environment": {
+      "type": "object",
+      "required": ["os", "foh_cli_version"],
+      "properties": {
+        "os": { "type": "string", "minLength": 1 },
+        "node_version": { "type": ["string", "null"] },
+        "npm_version": { "type": ["string", "null"] },
+        "foh_cli_version": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": true
+    },
+    "public_entrypoints": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "commands_run": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "docs_pages_used": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "artifacts": {
+      "type": "object",
+      "properties": {
+        "terminal_transcript": { "type": ["string", "null"] },
+        "proof_bundle": { "type": ["string", "null"] },
+        "replay_packet": { "type": ["string", "null"] },
+        "knowledge_packet": { "type": ["string", "null"] },
+        "improvement_packet": { "type": ["string", "null"] },
+        "notes": { "type": ["string", "null"] }
+      },
+      "additionalProperties": true
+    },
+    "summary": { "type": "string" },
+    "next_commands": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "additionalProperties": true,
+  "allOf": [
+    {
+      "if": { "properties": { "status": { "enum": ["hold", "fail"] } }, "required": ["status"] },
+      "then": { "required": ["failure_reason_code"] }
+    }
+  ]
+}

package/schemas/improvement-packet.schema.json

@@ -1,76 +1,77 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://frontofhouse.okii.uk/schemas/improvement-packet.schema.json",
-  "title": "FOH Improvement Packet",
-  "type": "object",
-  "required": [
-    "schema_version",
-    "created_at",
-    "source_type",
-    "reason_code",
-    "promotion_decision",
-    "ids",
-    "evidence",
-    "next_commands"
-  ],
-  "properties": {
-    "schema_version": { "const": "foh_improvement_packet.v1" },
-    "created_at": { "type": "string" },
-    "source_type": {
-      "type": "string",
-      "enum": [
-        "setup_failure",
-        "proof_failure",
-        "replay_failure",
-        "knowledge_miss",
-        "runtime_miss",
-        "live_proof_failure"
-      ]
-    },
-    "reason_code": { "type": "string", "minLength": 1 },
-    "promotion_decision": {
-      "type": "string",
-      "enum": ["ignore", "fix_docs", "fix_config", "fix_runtime", "add_test"]
-    },
-    "ids": {
-      "type": "object",
-      "properties": {
-        "org_id": { "type": "string" },
-        "agent_id": { "type": "string" },
-        "conversation_id": { "type": "string" },
-        "trace_id": { "type": "string" },
-        "correlation_id": { "type": "string" },
-        "proof_artifact": { "type": "string" }
-      },
-      "additionalProperties": false
-    },
-    "evidence": {
-      "type": "object",
-      "required": ["summary", "redaction"],
-      "properties": {
-        "summary": { "type": "string", "minLength": 1 },
-        "source_artifact_path": { "type": ["string", "null"] },
-        "source_command": { "type": ["string", "null"] },
-        "redaction": {
-          "type": "object",
-          "required": ["enabled", "fields"],
-          "properties": {
-            "enabled": { "const": true },
-            "fields": {
-              "type": "array",
-              "items": { "type": "string" }
-            }
-          },
-          "additionalProperties": false
-        },
-        "redacted_source": {}
-      },
-      "additionalProperties": false
-    },
-    "next_commands": {
-      "type": "array",
-      "items": { "type": "string", "minLength": 1 }
-    }
-  },
-  "additionalProperties": false
-}
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://frontofhouse.okii.uk/schemas/improvement-packet.schema.json",
+  "title": "FOH Improvement Packet",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "created_at",
+    "source_type",
+    "reason_code",
+    "promotion_decision",
+    "ids",
+    "evidence",
+    "next_commands"
+  ],
+  "properties": {
+    "schema_version": { "const": "foh_improvement_packet.v1" },
+    "created_at": { "type": "string" },
+    "source_type": {
+      "type": "string",
+      "enum": [
+        "external_agent_run",
+        "setup_failure",
+        "proof_failure",
+        "replay_failure",
+        "knowledge_miss",
+        "runtime_miss",
+        "live_proof_failure"
+      ]
+    },
+    "reason_code": { "type": "string", "minLength": 1 },
+    "promotion_decision": {
+      "type": "string",
+      "enum": ["ignore", "fix_docs", "fix_config", "fix_cli", "fix_api", "fix_runtime", "add_test"]
+    },
+    "ids": {
+      "type": "object",
+      "properties": {
+        "org_id": { "type": "string" },
+        "agent_id": { "type": "string" },
+        "conversation_id": { "type": "string" },
+        "trace_id": { "type": "string" },
+        "correlation_id": { "type": "string" },
+        "proof_artifact": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "evidence": {
+      "type": "object",
+      "required": ["summary", "redaction"],
+      "properties": {
+        "summary": { "type": "string", "minLength": 1 },
+        "source_artifact_path": { "type": ["string", "null"] },
+        "source_command": { "type": ["string", "null"] },
+        "redaction": {
+          "type": "object",
+          "required": ["enabled", "fields"],
+          "properties": {
+            "enabled": { "const": true },
+            "fields": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          },
+          "additionalProperties": false
+        },
+        "redacted_source": {}
+      },
+      "additionalProperties": false
+    },
+    "next_commands": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    }
+  },
+  "additionalProperties": false
+}