@chanl-ai/cli 2.0.1

package/dist/commands/scenarios.js

@@ -0,0 +1,591 @@
+import { Command } from "commander";
+import ora from "ora";
+import chalk from "chalk";
+import { createSdk } from "../utils/sdk-factory.js";
+import {
+  printError,
+  printInfo,
+  printBlank,
+  printSimpleTable,
+  printLabel,
+  isJsonOutput,
+  printJson,
+  formatDate
+} from "../utils/output.js";
+import {
+  formatElapsed,
+  printSegment,
+  displayScorecard
+} from "../utils/interactive.js";
+function createScenariosCommand() {
+  const scenarios = new Command("scenarios").description("Manage and run test scenarios").addHelpText(
+    "after",
+    `
+What are Scenarios?
+  Scenarios are test cases that run against your AI agents. Each scenario
+  defines a conversation flow, expected responses, and scoring criteria.
+
+Quick Start:
+  $ chanl scenarios list                    # See all scenarios
+  $ chanl scenarios run <id>                # Execute a scenario
+  $ chanl scenarios run <id> --watch        # Execute and watch progress
+  $ chanl scenarios history <id>            # View past runs
+
+Execution Workflow:
+  1. List scenarios to find the one you want to test
+  2. Run the scenario (optionally with --watch)
+  3. Check execution status with 'executions' or 'execution <id>'
+  4. Review history with 'history <scenarioId>'`
+  );
+  scenarios.command("list").description("List all scenarios").option("-s, --status <status>", "Filter by status (draft, active, paused, completed, archived)").option("-c, --category <category>", "Filter by category").option("-l, --limit <number>", "Number of items per page", "20").option("-p, --page <number>", "Page number", "1").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios list                    # List all scenarios
+  $ chanl scenarios list --status active    # Only active scenarios
+  $ chanl scenarios list --category sales   # Filter by category
+  $ chanl scenarios list --json             # Output as JSON`
+  ).action(handleScenariosList);
+  scenarios.command("get <id>").description("Get a scenario by ID").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios get abc123              # Get scenario details
+  $ chanl scenarios get abc123 --json       # Output as JSON`
+  ).action(handleScenariosGet);
+  scenarios.command("run <id>").description("Execute a scenario").option("-a, --agent <agentId>", "Specific agent to run against").option("-m, --mode <mode>", "Simulation mode: text, websocket, phone", "websocket").option("-p, --phone <number>", "Phone number for phone mode (E.164 format)").option("-w, --watch", "Watch execution progress until completion").option("--dry-run", "Validate without actually executing").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios run abc123                      # Run with defaults
+  $ chanl scenarios run abc123 --watch              # Run and watch progress
+  $ chanl scenarios run abc123 --agent agent_xyz   # Run against specific agent
+  $ chanl scenarios run abc123 --mode phone --phone +14155551234
+  $ chanl scenarios run abc123 --dry-run            # Validate only
+
+Simulation Modes:
+  text       Free, instant text-based testing
+  websocket  Real WebSocket testing (requires provider API key)
+  phone      Real phone call (requires provider + phone number)`
+  ).action(handleScenariosRun);
+  scenarios.command("executions").description("List scenario executions").option("-s, --scenario <scenarioId>", "Filter by scenario ID").option("--status <status>", "Filter by status (queued, running, completed, failed, timeout, cancelled)").option("-l, --limit <number>", "Number of items", "10").option("-p, --page <number>", "Page number", "1").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios executions                          # List recent executions
+  $ chanl scenarios executions --scenario abc123        # For specific scenario
+  $ chanl scenarios executions --status completed       # Only completed
+  $ chanl scenarios executions --json                   # Output as JSON`
+  ).action(handleScenariosExecutions);
+  scenarios.command("execution <executionId>").description("Get execution details").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios execution exec_abc123       # Get execution details
+  $ chanl scenarios execution exec_abc123 --json`
+  ).action(handleScenariosExecution);
+  scenarios.command("history <scenarioId>").description("Get simulation history for a scenario").option("-l, --limit <number>", "Number of simulations to show", "10").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios history abc123              # Last 10 runs
+  $ chanl scenarios history abc123 --limit 5    # Last 5 runs
+  $ chanl scenarios history abc123 --json       # Output as JSON`
+  ).action(handleScenariosHistory);
+  scenarios.command("run-all").description("Run all scenarios for an agent and show aggregate results").requiredOption("-a, --agent <agentId>", "Agent ID to run scenarios against").option("--min-score <score>", "Minimum passing score (0-100)", "0").option("--parallel <count>", "Max concurrent executions", "3").option("-m, --mode <mode>", "Simulation mode: text, phone", "text").addHelpText(
+    "after",
+    `
+Examples:
+  $ chanl scenarios run-all --agent agent_abc                     # Run all with defaults
+  $ chanl scenarios run-all --agent agent_abc --min-score 80      # Fail if any score < 80
+  $ chanl scenarios run-all --agent agent_abc --parallel 5        # Run 5 at a time
+  $ chanl scenarios run-all --agent agent_abc --json              # Output as JSON
+
+Exit Codes:
+  0  All scenarios passed (score >= min-score)
+  1  One or more scenarios failed`
+  ).action(handleScenariosRunAll);
+  return scenarios;
+}
+async function handleScenariosList(options) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Fetching scenarios...").start();
+  try {
+    const response = await sdk.scenarios.list({
+      status: options.status,
+      category: options.category,
+      limit: parseInt(options.limit, 10),
+      page: parseInt(options.page, 10)
+    });
+    spinner.stop();
+    if (!response.success || !response.data) {
+      printError("Failed to fetch scenarios", response.message);
+      process.exitCode = 1;
+      return;
+    }
+    const { scenarios, pagination } = response.data;
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    if (scenarios.length === 0) {
+      printInfo("No scenarios found");
+      return;
+    }
+    printBlank();
+    printSimpleTable(
+      ["Name", "Status", "Category", "Difficulty", "Created"],
+      scenarios.map((s) => [
+        s.name,
+        formatStatus(s.status),
+        s.category || "-",
+        s.difficulty || "-",
+        formatDate(s.createdAt)
+      ])
+    );
+    printBlank();
+    printInfo(`Total: ${pagination.total} scenarios (Page ${pagination.page} of ${pagination.pages})`);
+  } catch (error) {
+    spinner.fail("Failed to fetch scenarios");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+async function handleScenariosGet(id) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Fetching scenario...").start();
+  try {
+    const response = await sdk.scenarios.get(id);
+    spinner.stop();
+    if (!response.success || !response.data) {
+      printError("Failed to fetch scenario", response.message);
+      process.exitCode = 1;
+      return;
+    }
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    const { scenario } = response.data;
+    printBlank();
+    console.log(chalk.bold("Scenario Details:"));
+    console.log(`  Name:        ${scenario.name}`);
+    console.log(`  ID:          ${scenario.id}`);
+    console.log(`  Status:      ${formatStatus(scenario.status)}`);
+    console.log(`  Category:    ${scenario.category || "-"}`);
+    console.log(`  Difficulty:  ${scenario.difficulty || "-"}`);
+    console.log(`  Created:     ${formatDate(scenario.createdAt)}`);
+    if (scenario.description) {
+      console.log(`  Description: ${scenario.description}`);
+    }
+    printBlank();
+  } catch (error) {
+    spinner.fail("Failed to fetch scenario");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+function formatStatus(status) {
+  switch (status) {
+    case "active":
+      return chalk.green("active");
+    case "paused":
+      return chalk.yellow("paused");
+    case "completed":
+      return chalk.blue("completed");
+    case "archived":
+      return chalk.gray("archived");
+    case "draft":
+    default:
+      return chalk.gray(status || "draft");
+  }
+}
+function formatExecutionStatus(status) {
+  switch (status) {
+    case "completed":
+      return chalk.green("completed");
+    case "running":
+      return chalk.cyan("running");
+    case "queued":
+      return chalk.blue("queued");
+    case "failed":
+      return chalk.red("failed");
+    case "timeout":
+      return chalk.yellow("timeout");
+    case "cancelled":
+      return chalk.gray("cancelled");
+    default:
+      return chalk.gray(status || "unknown");
+  }
+}
+function formatScore(score) {
+  if (score === void 0 || score === null) return "-";
+  if (score >= 80) return chalk.green(`${score}%`);
+  if (score >= 60) return chalk.yellow(`${score}%`);
+  return chalk.red(`${score}%`);
+}
+function formatDuration(seconds) {
+  if (!seconds) return "-";
+  if (seconds < 60) return `${seconds.toFixed(1)}s`;
+  const mins = Math.floor(seconds / 60);
+  const secs = seconds % 60;
+  return `${mins}m ${secs.toFixed(0)}s`;
+}
+async function handleScenariosRun(id, options) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Starting scenario execution...").start();
+  try {
+    const response = await sdk.scenarios.run(id, {
+      agentId: options.agent,
+      simulationMode: options.mode,
+      phoneNumber: options.phone,
+      dryRun: options.dryRun
+    });
+    if (!response.success || !response.data) {
+      spinner.fail("Failed to start execution");
+      printError("Error", response.message || "Unknown error");
+      process.exitCode = 1;
+      return;
+    }
+    const { execution, executionId } = response.data;
+    const execId = executionId || execution?.id || execution?.executionId;
+    if (options.dryRun) {
+      spinner.succeed("Dry run validation passed");
+      printInfo("Scenario is valid and ready to execute");
+      return;
+    }
+    spinner.succeed(`Execution started: ${execId}`);
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    if (!options.watch) {
+      printBlank();
+      printLabel("Execution ID", execId || "N/A");
+      printLabel("Status", formatExecutionStatus(execution?.status));
+      printInfo(`Run 'chanl scenarios execution ${execId}' to check status`);
+      printBlank();
+      return;
+    }
+    await watchExecution(sdk, execId, options.mode === "phone");
+  } catch (error) {
+    spinner.fail("Failed to execute scenario");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+async function watchExecution(sdk, executionId, isPhoneMode = false) {
+  const startTime = /* @__PURE__ */ new Date();
+  let spinner = ora(
+    isPhoneMode ? `${chalk.red("LIVE")} \u2014 ${formatElapsed(startTime)} | Watching voice execution...` : "Watching execution..."
+  ).start();
+  const live = sdk.scenarios.watchExecution(executionId);
+  let finalExecution;
+  let scorecardReceived = false;
+  live.on("status", (status) => {
+    if (isPhoneMode) {
+      spinner.text = `${chalk.red("LIVE")} \u2014 ${formatElapsed(startTime)} | Status: ${status}`;
+    } else {
+      spinner.text = `Status: ${status}`;
+    }
+  });
+  live.on("step", (step) => {
+    if (!isPhoneMode) {
+      spinner.stop();
+      const stepStatus = step.status === "completed" ? chalk.green("\u2713") : chalk.red("\u2717");
+      console.log(`  ${stepStatus} Step ${step.stepId}: ${step.response?.slice(0, 80) || "-"}`);
+      if (step.toolCalls.length > 0) {
+        for (const tc of step.toolCalls) {
+          const tcIcon = tc.status === "success" ? chalk.green("\u2713") : chalk.red("\u2717");
+          const dur = tc.durationMs ? chalk.dim(` (${tc.durationMs}ms)`) : "";
+          console.log(`      ${tcIcon} ${chalk.yellow(tc.name)}${dur}`);
+        }
+      }
+      spinner.start();
+    }
+  });
+  live.on("transcript", (seg) => {
+    spinner.stop();
+    printSegment(seg);
+    spinner.start();
+  });
+  live.on("completed", (execution) => {
+    finalExecution = execution;
+    const status = execution.status;
+    if (status === "completed") {
+      spinner.succeed(
+        isPhoneMode ? `Execution complete \u2014 ${formatElapsed(startTime)}` : "Execution completed"
+      );
+    } else if (status === "failed") {
+      spinner.fail(
+        isPhoneMode ? `Execution failed \u2014 ${formatElapsed(startTime)}` : "Execution failed"
+      );
+    } else {
+      spinner.warn(`Execution ${status}`);
+    }
+    printBlank();
+    printExecutionDetails(execution);
+    if (isPhoneMode && execution.callDetails?.callId) {
+      spinner = ora("Checking for scorecard...").start();
+    }
+  });
+  live.on("scorecard", (results) => {
+    scorecardReceived = true;
+    spinner.stop();
+    displayScorecard(results);
+    printBlank();
+  });
+  live.on("timeout", () => {
+    spinner.warn("Timeout waiting for execution to complete");
+    printInfo(`Run 'chanl scenarios execution ${executionId}' to check status`);
+  });
+  live.on("error", (err) => {
+    spinner.fail("Error while watching");
+    printError("Error", err.message);
+    process.exitCode = 1;
+  });
+  await live.completed;
+  if (finalExecution && isPhoneMode && finalExecution.callDetails?.callId && !scorecardReceived) {
+    spinner.stop();
+  }
+}
+function printExecutionDetails(execution) {
+  console.log(chalk.bold("Execution Results:"));
+  console.log(`  Execution ID:  ${execution.id || execution.executionId}`);
+  console.log(`  Scenario ID:   ${execution.scenarioId}`);
+  console.log(`  Status:        ${formatExecutionStatus(execution.status)}`);
+  console.log(`  Score:         ${formatScore(execution.overallScore)}`);
+  console.log(`  Duration:      ${formatDuration(execution.duration)}`);
+  if (execution.metrics) {
+    console.log(chalk.bold("\n  Metrics:"));
+    if (execution.metrics.responseTime !== void 0) {
+      console.log(`    Response Time: ${execution.metrics.responseTime.toFixed(0)}ms`);
+    }
+    if (execution.metrics.accuracy !== void 0) {
+      console.log(`    Accuracy:      ${formatScore(execution.metrics.accuracy)}`);
+    }
+    if (execution.metrics.completion !== void 0) {
+      console.log(`    Completion:    ${formatScore(execution.metrics.completion)}`);
+    }
+  }
+  if (execution.callDetails?.callId) {
+    console.log(chalk.bold("\n  Call Details:"));
+    console.log(`    Call ID:       ${execution.callDetails.callId}`);
+    if (execution.callDetails.phoneNumber) {
+      console.log(`    Phone:         ${execution.callDetails.phoneNumber}`);
+    }
+  }
+  const allToolCalls = (execution.stepResults ?? []).flatMap((s) => s.toolCalls ?? []);
+  if (allToolCalls.length > 0) {
+    console.log(chalk.bold("\n  Tool Calls:"));
+    printScenarioToolCalls(allToolCalls);
+  }
+  if (execution.errorMessages && execution.errorMessages.length > 0) {
+    console.log(chalk.bold("\n  Errors:"));
+    execution.errorMessages.forEach((msg) => {
+      console.log(chalk.red(`    - ${msg}`));
+    });
+  }
+  printBlank();
+}
+function printScenarioToolCalls(toolCalls) {
+  for (const tc of toolCalls) {
+    const statusIcon = tc.status === "success" ? chalk.green("\u2713") : tc.status === "error" ? chalk.red("\u2717") : tc.status === "timeout" ? chalk.yellow("\u23F1") : chalk.dim("\u2026");
+    const duration = tc.durationMs ? chalk.dim(` (${tc.durationMs}ms)`) : "";
+    const params = Object.keys(tc.parameters).length ? chalk.dim(` ${JSON.stringify(tc.parameters)}`) : "";
+    console.log(`    ${statusIcon} ${chalk.yellow(tc.name)}${params}${duration}`);
+    if (tc.status === "error" && tc.error) {
+      console.log(`      ${chalk.red(tc.error.message)}`);
+    }
+  }
+}
+async function handleScenariosExecutions(options) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Fetching executions...").start();
+  try {
+    const response = await sdk.scenarios.getExecutions({
+      scenarioId: options.scenario,
+      status: options.status,
+      limit: parseInt(options.limit, 10),
+      page: parseInt(options.page, 10)
+    });
+    spinner.stop();
+    if (!response.success || !response.data) {
+      printError("Failed to fetch executions", response.message);
+      process.exitCode = 1;
+      return;
+    }
+    const { executions, total } = response.data;
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    if (executions.length === 0) {
+      printInfo("No executions found");
+      return;
+    }
+    printBlank();
+    printSimpleTable(
+      ["Execution ID", "Scenario", "Status", "Score", "Duration", "Started"],
+      executions.map((e) => [
+        (e.id || e.executionId || "").slice(-12),
+        e.scenarioId?.slice(-8) || "-",
+        formatExecutionStatus(e.status),
+        formatScore(e.overallScore),
+        formatDuration(e.duration),
+        e.startTime ? formatDate(e.startTime) : "-"
+      ])
+    );
+    printBlank();
+    printInfo(`Total: ${total} executions`);
+  } catch (error) {
+    spinner.fail("Failed to fetch executions");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+async function handleScenariosExecution(executionId) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Fetching execution...").start();
+  try {
+    const response = await sdk.scenarios.getExecution(executionId);
+    spinner.stop();
+    if (!response.success || !response.data) {
+      printError("Failed to fetch execution", response.message);
+      process.exitCode = 1;
+      return;
+    }
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    const { execution } = response.data;
+    printBlank();
+    printExecutionDetails(execution);
+  } catch (error) {
+    spinner.fail("Failed to fetch execution");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+async function handleScenariosHistory(scenarioId, options) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const spinner = ora("Fetching simulation history...").start();
+  try {
+    const response = await sdk.scenarios.getSimulations(
+      scenarioId,
+      parseInt(options.limit, 10)
+    );
+    spinner.stop();
+    if (!response.success || !response.data) {
+      printError("Failed to fetch history", response.message);
+      process.exitCode = 1;
+      return;
+    }
+    const { simulations } = response.data;
+    if (isJsonOutput()) {
+      printJson(response.data);
+      return;
+    }
+    if (simulations.length === 0) {
+      printInfo("No simulation history found for this scenario");
+      return;
+    }
+    printBlank();
+    console.log(chalk.bold(`Simulation History for ${scenarioId}`));
+    console.log(chalk.dim("\u2500".repeat(60)));
+    printSimpleTable(
+      ["#", "Status", "Score", "Duration", "Started"],
+      simulations.map((s, i) => [
+        (i + 1).toString(),
+        formatExecutionStatus(s.status),
+        formatScore(s.overallScore),
+        formatDuration(s.duration),
+        s.startTime ? formatDate(s.startTime) : "-"
+      ])
+    );
+    printBlank();
+    printInfo(`Showing last ${simulations.length} simulations`);
+  } catch (error) {
+    spinner.fail("Failed to fetch history");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+async function handleScenariosRunAll(options) {
+  const sdk = createSdk();
+  if (!sdk) return;
+  const minScore = parseInt(options.minScore, 10);
+  const parallel = parseInt(options.parallel, 10);
+  const spinner = ora(`Running all scenarios for agent ${options.agent}...`).start();
+  try {
+    const result = await sdk.scenarios.runAll({
+      agentId: options.agent,
+      minScore,
+      parallel,
+      mode: options.mode
+    });
+    spinner.stop();
+    if (isJsonOutput()) {
+      printJson(result);
+      if (!result.allPassed) {
+        process.exitCode = 1;
+      }
+      return;
+    }
+    if (result.totalScenarios === 0) {
+      printInfo("No active scenarios found for this agent");
+      return;
+    }
+    printBlank();
+    console.log(chalk.bold("Batch Scenario Results"));
+    console.log(chalk.dim("\u2500".repeat(60)));
+    printSimpleTable(
+      ["Scenario", "Status", "Score", "Duration", "Tools", "Result"],
+      result.results.map((r) => [
+        r.scenarioName,
+        formatExecutionStatus(r.status),
+        formatScore(r.score),
+        formatDuration(r.duration),
+        r.toolCallCount.toString(),
+        r.passed ? chalk.green("PASS") : chalk.red("FAIL")
+      ])
+    );
+    printBlank();
+    console.log(chalk.bold("Summary"));
+    console.log(`  Total:         ${result.totalScenarios}`);
+    console.log(`  Passed:        ${chalk.green(result.passed.toString())}`);
+    console.log(`  Failed:        ${result.failed > 0 ? chalk.red(result.failed.toString()) : "0"}`);
+    console.log(`  Average Score: ${formatScore(result.averageScore)}`);
+    console.log(`  Min Score:     ${result.minScore}`);
+    printBlank();
+    if (result.allPassed) {
+      console.log(chalk.green(`All ${result.totalScenarios} scenarios passed`));
+    } else {
+      console.log(chalk.red(`${result.failed} of ${result.totalScenarios} scenarios failed`));
+      process.exitCode = 1;
+    }
+    printBlank();
+  } catch (error) {
+    spinner.fail("Failed to run scenarios");
+    const message = error instanceof Error ? error.message : "Unknown error";
+    printError("Error", message);
+    process.exitCode = 1;
+  }
+}
+export {
+  createScenariosCommand
+};
+//# sourceMappingURL=scenarios.js.map