@f-o-h/cli 0.1.36 → 0.1.38

package/README.md

@@ -4,7 +4,7 @@ AI-operator provisioning CLI for Front Of House.
 
 Public mirror: https://github.com/iiko38/front-of-house-cli
 
-Current published baseline: `@f-o-h/cli@0.1.36`
+Current published baseline: `@f-o-h/cli@0.1.38`
 
 This mirror is a generated release artifact. The private product monorepo is not
 published here, and no open-source license is granted unless stated separately.
@@ -87,6 +87,9 @@ read-only by default; pass `--mutation-mode ensure` or `--repair` only when you
 explicitly want proof to ensure missing widget state. Use `--strict` in
 automation when holds should fail the command, and `--mission voice` or
 `--require-phone` when a voice/contact number is mandatory for the demo.
+Use `--contact-path byon` when the proof is meant to validate a
+customer-owned/BYON phone route; missing BYON config then reports
+`byon_voice_number_not_configured` instead of suggesting FOH number purchase.
 
 For mass AI-agent evals and repeated demo rehearsals, keep setup on the free
 scaffold lane:
@@ -99,6 +102,10 @@ FOH_CLI_SPEND_POLICY=no_spend foh setup --org <org-id> --agent-template <templat
 buying one. `--phone-mode skip` bypasses the phone step. `--phone-mode purchase`
 is the explicit paid contact path and is fail-closed when
 `FOH_CLI_SPEND_POLICY=no_spend` is set.
+
+If managed-number provisioning is blocked by account/provider capacity or empty
+reserve inventory, proof reports `provider_capacity_blocked`; fix capacity or
+switch to BYON rather than retrying blindly.
 
 The CLI defaults to the production API at `https://api.frontofhouse.okii.uk`.
 
@@ -123,9 +130,13 @@ foh eval external-agent run \
   --prompt-version blank-setup.v1
 ```
 
-The command writes a versioned prompt, launches an instrumented shell, captures
-FOH CLI commands into `commands.ndjson`, and finalizes `run.json` as an
-`external_agent_run.v1` artifact when the shell exits.
+The command writes a versioned prompt, launches an instrumented shell, captures
+FOH CLI commands into `commands.ndjson`, and finalizes `run.json` as an
+`external_agent_run.v1` artifact when the shell exits.
+
+Run artifacts include `eval_state` so repeated benchmark runs make reuse
+explicit: org, agent, and widget reuse are expected; fresh paid phone-number
+creation is not expected.
 
 For guarded programmable-runner planning:
 

package/dist/foh.js

@@ -32755,7 +32755,7 @@ var StdioServerTransport = class {
 };
 
 // src/lib/cli-version.ts
-var CLI_VERSION = "0.1.36";
+var CLI_VERSION = "0.1.38";
 
 // src/commands/mcp-serve.ts
 var DEFAULT_TIMEOUT_MS = 12e4;
@@ -37173,6 +37173,11 @@ function normalizeMutationMode(raw, repair) {
   const value = String(raw || "read-only").trim().toLowerCase();
   return value === "ensure" ? "ensure" : "read-only";
 }
+function normalizeContactPath(raw) {
+  const value = String(raw || "auto").trim().toLowerCase();
+  if (value === "managed" || value === "byon") return value;
+  return "auto";
+}
 function agentIdFromList(response) {
   const agents = Array.isArray(response.agents) ? response.agents : [];
   const usable = agents.filter((agent) => typeof agent.id === "string" && agent.id.trim());
@@ -37186,10 +37191,41 @@ function firstUsableOrgId(response) {
   const usable = orgs.map((org) => org && typeof org === "object" ? org : {}).map((org) => String(org.org_id ?? org.id ?? "").trim()).filter(Boolean);
   return { orgId: usable.length === 1 ? usable[0] : void 0, count: usable.length };
 }
+function readStringField(record2, keys) {
+  for (const key of keys) {
+    const value = record2[key];
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return "";
+}
+function isProviderCapacityBlocked(onboarding) {
+  const code = readStringField(onboarding, [
+    "provisioning_reason_code",
+    "provisioning_error_code",
+    "reason_code",
+    "code"
+  ]).toLowerCase();
+  if ([
+    "provider_capacity_blocked",
+    "twilio_subaccount_limit_reached",
+    "reserve_pool_exhausted",
+    "global_safety_limit_reached"
+  ].includes(code)) {
+    return true;
+  }
+  const message = readStringField(onboarding, [
+    "provisioning_error",
+    "provisioning_message",
+    "error",
+    "message"
+  ]).toLowerCase();
+  return /maximum number of subaccounts|subaccount limit|reserve[- ]number pool|reserve pool exhausted|global safety limit/.test(message);
+}
 function registerProve(program3) {
-  program3.command("prove").description("Produce one setup/runtime proof bundle for an agent").option("--agent <id>", "Agent ID to prove").option("--org <id>", "Org ID (default: stored org from foh org use)").option("--cert-mode <m>", "Simulation cert mode: quick, full, stress", "quick").option("--cert-adaptive-runs <n>", "Adaptive runs for full/stress certification", "30").option("--cert-max-improvement-rounds <n>", "Max prompt improvement rounds in cert loop (0-5)", "1").option("--mission <mission>", "Proof mission: setup, widget, voice, publish", "setup").option("--mutation-mode <mode>", "Proof mutation mode: read-only or ensure", "read-only").option("--repair", "Alias for --mutation-mode ensure").option("--require-phone", "Hold proof if no phone/contact number is provisioned").option("--skip-cert", "Skip simulation certification check").option("--skip-smoke", "Skip widget runtime smoke check").option("--skip-voice-health", "Skip realtime voice provider health check").option("--out <path>", "Write signed proof report JSON to this path").option("--strict", "Exit non-zero unless all non-skipped checks pass").option("--api-url <url>", "API base URL override").option("--json", "Output as JSON").action(async (opts) => withCommandErrorHandling(async () => {
+  program3.command("prove").description("Produce one setup/runtime proof bundle for an agent").option("--agent <id>", "Agent ID to prove").option("--org <id>", "Org ID (default: stored org from foh org use)").option("--cert-mode <m>", "Simulation cert mode: quick, full, stress", "quick").option("--cert-adaptive-runs <n>", "Adaptive runs for full/stress certification", "30").option("--cert-max-improvement-rounds <n>", "Max prompt improvement rounds in cert loop (0-5)", "1").option("--mission <mission>", "Proof mission: setup, widget, voice, publish", "setup").option("--contact-path <mode>", "Voice contact path: auto, managed, or byon", "auto").option("--mutation-mode <mode>", "Proof mutation mode: read-only or ensure", "read-only").option("--repair", "Alias for --mutation-mode ensure").option("--require-phone", "Hold proof if no phone/contact number is provisioned").option("--skip-cert", "Skip simulation certification check").option("--skip-smoke", "Skip widget runtime smoke check").option("--skip-voice-health", "Skip realtime voice provider health check").option("--out <path>", "Write signed proof report JSON to this path").option("--strict", "Exit non-zero unless all non-skipped checks pass").option("--api-url <url>", "API base URL override").option("--json", "Output as JSON").action(async (opts) => withCommandErrorHandling(async () => {
     const checks = [];
     const mission = normalizeMission(opts.mission);
+    const contactPath = normalizeContactPath(opts.contactPath);
     const mutationMode = normalizeMutationMode(opts.mutationMode, Boolean(opts.repair));
     const ctx = {
       tokenPresent: false,
@@ -37291,18 +37327,41 @@ function registerProve(program3) {
           if (phoneNumber) {
             checks.push(pass("contact_channel", "Contact phone number is provisioned.", {
               phone_number_present: true,
-              provisioning_status: provisioningStatus
+              provisioning_status: provisioningStatus,
+              contact_path: contactPath
+            }));
+          } else if (contactPath === "byon" && (opts.requirePhone || mission === "voice")) {
+            checks.push(hold("contact_channel", "byon_voice_number_not_configured", "BYON/customer-owned voice contact path was requested, but no contact phone is configured for this org.", `foh provision status --org ${ctx.orgId} --json`, {
+              provisioning_status: provisioningStatus,
+              mission,
+              contact_path: contactPath,
+              spend_policy: resolveCliSpendPolicy(),
+              spend_class: "customer_owned",
+              safe_to_retry: false,
+              operator_note: "Attach or configure the customer-owned voice number, then rerun proof. This path should not buy a FOH-owned number."
+            }));
+          } else if (provisioningStatus === "failed" && isProviderCapacityBlocked(onboarding) && (opts.requirePhone || mission === "voice")) {
+            checks.push(hold("contact_channel", "provider_capacity_blocked", "Phone/contact provisioning is blocked by provider/account capacity or empty reserve inventory.", `foh provision status --org ${ctx.orgId} --json`, {
+              provisioning_status: provisioningStatus,
+              mission,
+              contact_path: contactPath,
+              spend_policy: resolveCliSpendPolicy(),
+              spend_class: "paid_foh",
+              safe_to_retry: false,
+              operator_note: "Do not retry blindly. Resolve provider/account capacity or use BYON/customer-owned contact path."
             }));
           } else if (provisioningStatus === "failed" && (opts.requirePhone || mission === "voice")) {
             checks.push(hold("contact_channel", "contact_phone_provisioning_failed", "Phone/contact provisioning failed for this org.", `foh provision status --org ${ctx.orgId} --json`, {
               provisioning_status: provisioningStatus,
               mission,
+              contact_path: contactPath,
               spend_policy: resolveCliSpendPolicy()
             }));
           } else if (isNoSpendPolicy() && (opts.requirePhone || mission === "voice")) {
             checks.push(hold("contact_channel", "voice_contact_expected_no_spend_hold", "No phone/contact number is provisioned; this is expected in no-spend eval mode unless a BYON/customer-approved phone path exists.", `foh provision status --org ${ctx.orgId} --json`, {
               provisioning_status: provisioningStatus,
               mission,
+              contact_path: contactPath,
               spend_policy: resolveCliSpendPolicy(),
               spend_class: "free",
               safe_to_retry: true,
@@ -37312,6 +37371,7 @@ function registerProve(program3) {
             checks.push(hold("contact_channel", "contact_phone_missing", "No phone/contact number is provisioned for this org.", `foh provision buy --org ${ctx.orgId} --json`, {
               provisioning_status: provisioningStatus,
               mission,
+              contact_path: contactPath,
               spend_policy: resolveCliSpendPolicy()
             }));
           } else {
@@ -37464,6 +37524,7 @@ function registerProve(program3) {
         org_id: ctx.orgId ?? null,
         agent_id: ctx.agentId ?? null,
         mission,
+        contact_path: contactPath,
         mutation_mode: mutationMode,
         widget_public_key_present: Boolean(ctx.widgetPublicKey),
         conversation_id: ctx.conversationId ?? null,
@@ -39196,6 +39257,12 @@ function classifyRun(input) {
   if (hasCommandReason(new RegExp(PAID_RESOURCE_BLOCKED_REASON_CODE, "i"))) {
     return { status: "hold", reasonCode: PAID_RESOURCE_BLOCKED_REASON_CODE };
   }
+  if (hasCommandReason(/provider_capacity_blocked/i)) {
+    return { status: "hold", reasonCode: "provider_capacity_blocked" };
+  }
+  if (hasCommandReason(/byon_voice_number_not_configured/i)) {
+    return { status: "hold", reasonCode: "byon_voice_number_not_configured" };
+  }
   if (hasCommandReason(/contact_phone_provisioning_failed/i)) {
     return { status: "hold", reasonCode: "voice_contact_phone_provisioning_failed" };
   }
@@ -39230,12 +39297,18 @@ ${stderr}`;
   if (/ENV_NETWORK_DNS_BLOCK|Could not resolve host|npm ping.*timeout|NO_EXECUTABLE_INSTALL/i.test(combined)) {
     return { status: "hold", reasonCode: "codex_network_dns_blocked" };
   }
-  if (/contact_phone_provisioning_failed/i.test(combined)) {
-    return { status: "hold", reasonCode: "voice_contact_phone_provisioning_failed" };
-  }
   if (new RegExp(PAID_RESOURCE_BLOCKED_REASON_CODE, "i").test(combined)) {
     return { status: "hold", reasonCode: PAID_RESOURCE_BLOCKED_REASON_CODE };
   }
+  if (/provider_capacity_blocked/i.test(combined)) {
+    return { status: "hold", reasonCode: "provider_capacity_blocked" };
+  }
+  if (/byon_voice_number_not_configured/i.test(combined)) {
+    return { status: "hold", reasonCode: "byon_voice_number_not_configured" };
+  }
+  if (/contact_phone_provisioning_failed/i.test(combined)) {
+    return { status: "hold", reasonCode: "voice_contact_phone_provisioning_failed" };
+  }
   if (/voice_contact_expected_no_spend_hold/i.test(combined)) {
     return { status: "hold", reasonCode: "voice_contact_expected_no_spend_hold" };
   }
@@ -39292,6 +39365,17 @@ function buildExecutedRunArtifact(input) {
     ],
     commands_run: commands.map((command) => command.command),
     docs_pages_used: [],
+    eval_state: {
+      org_reuse_expected: true,
+      agent_reuse_expected: true,
+      widget_reuse_expected: true,
+      fresh_org_expected: false,
+      fresh_agent_expected: false,
+      phone_purchase_expected: false,
+      paid_resource_creation_expected: false,
+      spend_policy_expected: NO_SPEND_POLICY,
+      rationale: "Mass external-agent evals benchmark public docs/CLI/API clarity; reuse avoids paid phone and Twilio inventory churn."
+    },
     artifacts: {
       terminal_transcript: relativeArtifactName(input.run.outputs.jsonl),
       command_log: (0, import_fs14.existsSync)((0, import_path12.join)(input.run.run_dir, "commands.ndjson")) ? "commands.ndjson" : null,
@@ -39509,6 +39593,19 @@ function writeSession(runDir, session) {
 `, "utf8");
   return path2;
 }
+function buildDefaultEvalState() {
+  return {
+    org_reuse_expected: true,
+    agent_reuse_expected: true,
+    widget_reuse_expected: true,
+    fresh_org_expected: false,
+    fresh_agent_expected: false,
+    phone_purchase_expected: false,
+    paid_resource_creation_expected: false,
+    spend_policy_expected: "no_spend",
+    rationale: "Mass external-agent evals benchmark public docs/CLI/API clarity; reuse avoids paid phone and Twilio inventory churn."
+  };
+}
 function buildRunArtifact(input) {
   const commands = readCommandRecords(input.runDir);
   const startedAt = String(input.session.started_at);
@@ -39545,6 +39642,7 @@ function buildRunArtifact(input) {
     ],
     commands_run: commands.map((command) => command.command),
     docs_pages_used: [],
+    eval_state: buildDefaultEvalState(),
     artifacts: {
       terminal_transcript: null,
       command_log: "commands.ndjson",
@@ -39651,6 +39749,7 @@ function registerEval(program3) {
       manual_intervention_count: 0,
       run_dir: runDir,
       prompt_path: promptPath,
+      eval_state: buildDefaultEvalState(),
       capture_env: {
         [EXTERNAL_AGENT_RUN_DIR_ENV]: runDir,
         [EXTERNAL_AGENT_PROMPT_VERSION_ENV]: promptVersion

package/examples/external-agent-run.example.json

@@ -38,6 +38,17 @@
     "https://frontofhouse.okii.uk/guides/cli-install-and-upgrade",
     "https://frontofhouse.okii.uk/guides/error-handling-and-debugging"
   ],
+  "eval_state": {
+    "org_reuse_expected": true,
+    "agent_reuse_expected": true,
+    "widget_reuse_expected": true,
+    "fresh_org_expected": false,
+    "fresh_agent_expected": false,
+    "phone_purchase_expected": false,
+    "paid_resource_creation_expected": false,
+    "spend_policy_expected": "no_spend",
+    "rationale": "Mass external-agent evals benchmark public docs/CLI/API clarity; reuse avoids paid phone and Twilio inventory churn."
+  },
   "artifacts": {
     "terminal_transcript": "terminal-transcript.txt",
     "proof_bundle": null,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@f-o-h/cli",
-  "version": "0.1.36",
+  "version": "0.1.38",
   "description": "FOH CLI - AI-operator provisioning tool for Front Of House",
   "license": "UNLICENSED",
   "bin": {

package/schemas/external-agent-run.schema.json

@@ -67,6 +67,21 @@
       "type": "array",
       "items": { "type": "string" }
     },
+    "eval_state": {
+      "type": "object",
+      "properties": {
+        "org_reuse_expected": { "type": "boolean" },
+        "agent_reuse_expected": { "type": "boolean" },
+        "widget_reuse_expected": { "type": "boolean" },
+        "fresh_org_expected": { "type": "boolean" },
+        "fresh_agent_expected": { "type": "boolean" },
+        "phone_purchase_expected": { "type": "boolean" },
+        "paid_resource_creation_expected": { "type": "boolean" },
+        "spend_policy_expected": { "type": "string" },
+        "rationale": { "type": "string" }
+      },
+      "additionalProperties": true
+    },
     "artifacts": {
       "type": "object",
       "properties": {