@microsoft/m365agentsplayground-cli 0.2.26 → 0.2.27-alpha.20260518-462fbee.0

package/src/runConversation.ts

@@ -1,382 +0,0 @@
-/**
- * Conversation execution logic for the test harness.
- *
- * Runs a sequence of turns against a bot through a fresh TestClient
- * and returns the bot's responses. No eval logic — just execution.
- */
-
-import * as http from "http";
-import * as https from "https";
-import { TestClient } from "./testClient";
-import { sendNotificationAndWait } from "./notificationSender";
-import { validateAdaptiveCard } from "./cardValidator";
-import type {
-  E2EConfig,
-  PersonaConfig,
-  ConversationInput,
-  Turn,
-  TurnResult,
-  TurnAttachment,
-  ConversationResult,
-} from "./conversationTypes";
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Logging (stderr only — stdout is reserved for structured output)
-// ─────────────────────────────────────────────────────────────────────────────
-
-export function log(msg: string): void {
-  process.stderr.write(`[agents-simulator] ${msg}\n`);
-}
-
-export function logError(msg: string): void {
-  process.stderr.write(`[agents-simulator][ERROR] ${msg}\n`);
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Persona resolution
-// ─────────────────────────────────────────────────────────────────────────────
-
-function resolvePersona(
-  turn: Turn,
-  input: ConversationInput,
-  config: E2EConfig
-): PersonaConfig | undefined {
-  const personaName = turn.persona ?? input.persona;
-  if (!personaName || !config.personas) return undefined;
-
-  const persona = config.personas[personaName];
-  if (!persona) {
-    log(`Warning: persona "${personaName}" not found in config, using default`);
-    return undefined;
-  }
-  return persona;
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Bot auth preflight check
-// ─────────────────────────────────────────────────────────────────────────────
-
-/**
- * Send a bare probe request to the bot endpoint (no Authorization header).
- * Returns the HTTP status code, or throws if the bot is unreachable.
- *
- * Bot Framework SDK behaviour:
- *   - MicrosoftAppId is empty → anonymous mode → accepts any request → 200/202
- *   - MicrosoftAppId is set   → validates JWT → rejects bare request → 401
- */
-async function probeBot(botEndpoint: string): Promise<number> {
-  return new Promise((resolve, reject) => {
-    const url = new URL(botEndpoint);
-    const lib = url.protocol === "https:" ? https : http;
-
-    // Use type:"typing" — the Bot Framework adapter acks it immediately without
-    // invoking the AI planner (no LLM call, no sendActivity, instant response).
-    // Bot returns 200/202 for unauthenticated bots, 401 for authenticated ones.
-    const body = JSON.stringify({ type: "typing" });
-    const req = lib.request(
-      {
-        hostname: url.hostname,
-        port: url.port || (url.protocol === "https:" ? 443 : 80),
-        path: url.pathname,
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "Content-Length": Buffer.byteLength(body),
-        },
-      },
-      (res) => resolve(res.statusCode ?? 0)
-    );
-
-    req.on("error", reject);
-    req.setTimeout(5000, () => {
-      req.destroy(new Error(`Connection to bot timed out: ${botEndpoint}`));
-    });
-    req.write(body);
-    req.end();
-  });
-}
-
-/**
- * Verify the bot is reachable and accepting unauthenticated requests.
- * Throws a descriptive error when the bot returns 401 or is unreachable.
- */
-async function checkBotReachable(botEndpoint: string): Promise<void> {
-  let statusCode: number;
-  try {
-    statusCode = await probeBot(botEndpoint);
-  } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err);
-    throw new Error(
-      `Cannot reach bot at ${botEndpoint}: ${msg}\n` +
-        `  → Make sure the bot is running before starting a conversation.`
-    );
-  }
-
-  if (statusCode === 401) {
-    throw new Error(
-      `Bot at ${botEndpoint} requires authentication (HTTP 401).\n` +
-        `  → For local testing, start the bot with BOT_ID="" BOT_PASSWORD="" so it\n` +
-        `    accepts requests without a Bot Framework JWT token.\n` +
-        `  → Example: BOT_ID= BOT_PASSWORD= node ./src/index.ts`
-    );
-  }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Conversation execution
-// ─────────────────────────────────────────────────────────────────────────────
-
-/**
- * Extract and validate Adaptive Card attachments from bot responses.
- * Only processes "application/vnd.microsoft.card.adaptive" attachments.
- */
-function extractAttachments(
-  responses: { attachments?: Array<{ contentType?: string; content?: unknown }> }[]
-): TurnAttachment[] {
-  const result: TurnAttachment[] = [];
-  for (const r of responses) {
-    for (const att of r.attachments ?? []) {
-      if (att.contentType === "application/vnd.microsoft.card.adaptive" && att.content) {
-        const content = att.content as Record<string, unknown>;
-        const card_errors = validateAdaptiveCard(content);
-        result.push({ contentType: att.contentType, content, card_errors });
-      }
-    }
-  }
-  return result;
-}
-
-/**
- * Run a single conversation through a fresh TestClient.
- *
- * Creates an isolated TestClient, sends each turn sequentially, and
- * collects the bot's responses. On failure, remaining turns are skipped.
- */
-export async function runConversation(
-  config: E2EConfig,
-  scenario: string,
-  input: ConversationInput
-): Promise<ConversationResult> {
-  const results: TurnResult[] = [];
-  const convStart = Date.now();
-  const timeout = config.timeout ?? 120000;
-
-  const turns = input.turns || [];
-  if (turns.length === 0) {
-    return {
-      type: "conversation_result",
-      scenario,
-      status: "Errored",
-      duration_seconds: 0,
-      turns: [],
-    };
-  }
-
-  // Preflight: verify the bot is reachable and not requiring auth before
-  // spending timeout budget on the first real turn.
-  try {
-    await checkBotReachable(config.botEndpoint);
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logError(message);
-    return {
-      type: "conversation_result",
-      scenario,
-      status: "Errored",
-      duration_seconds: (Date.now() - convStart) / 1000,
-      turns: turns.map((t) => ({
-        test_id: t.test_id,
-        prompt: t.prompt,
-        actual_response: null,
-        attachments: [],
-        status: "Errored" as const,
-        error_message: message,
-        duration_seconds: 0,
-      })),
-    };
-  }
-
-  const client = new TestClient({
-    botEndpoint: config.botEndpoint,
-    timeout,
-    bot: config.bot,
-    deliveryMode: config.deliveryMode ?? "expectReplies",
-    chatType: config.chatType ?? "personal",
-    streamingSettleDelayMs: config.streamingSettleDelayMs ?? 0,
-  });
-
-  try {
-    log(`Starting client for: ${scenario}`);
-    await client.start();
-    log(`Client started on port ${client.getPort()}`);
-
-    // Maps turn test_id → messageId of the bot's first card response in that turn
-    const cardMessageIds = new Map<string, string>();
-
-    for (let i = 0; i < turns.length; i++) {
-      const turn = turns[i];
-      const turnType = turn.turn_type ?? "chat";
-      const turnStart = Date.now();
-
-      log(
-        `  Turn ${i + 1}/${turns.length} [${turn.test_id}] (${turnType}): ${turn.prompt.substring(
-          0,
-          80
-        )}...`
-      );
-
-      try {
-        let responseText: string;
-        let attachments: TurnAttachment[] = [];
-
-        if (turnType === "chat") {
-          const responses = await client.sendMessage(turn.prompt);
-          responseText = responses
-            .map((r) => r.text ?? "")
-            .filter((t) => t.length > 0)
-            .join("\n\n");
-          attachments = extractAttachments(responses);
-          // Track the first card-containing message ID for card_action turns
-          const firstCardMsg = responses.find((r) =>
-            r.attachments?.some((a) => a.contentType === "application/vnd.microsoft.card.adaptive")
-          );
-          if (firstCardMsg?.messageId) {
-            cardMessageIds.set(turn.test_id, firstCardMsg.messageId);
-          }
-        } else if (turnType === "card_action") {
-          const cardAction = turn.card_action;
-          if (!cardAction) {
-            throw new Error(
-              `Turn "${turn.test_id}" has turn_type "card_action" but no card_action field.`
-            );
-          }
-          if (!cardAction.verb) {
-            throw new Error(`Turn "${turn.test_id}" card_action is missing required "verb" field.`);
-          }
-          // Resolve the target message ID from the referenced turn
-          let replyToId: string | undefined;
-          if (cardAction.reply_to_turn != null) {
-            replyToId = cardMessageIds.get(cardAction.reply_to_turn);
-            if (!replyToId) {
-              throw new Error(
-                `card_action.reply_to_turn "${cardAction.reply_to_turn}" did not produce a card message. ` +
-                  `Ensure that turn sends a card before using card_action.`
-              );
-            }
-          } else {
-            // Use the most recently stored card messageId
-            const entries = [...cardMessageIds.values()];
-            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-            replyToId = entries[entries.length - 1];
-          }
-          if (!replyToId) {
-            throw new Error(
-              `card_action for turn "${turn.test_id}" could not find any card message to target. ` +
-                `Ensure a prior turn produces an Adaptive Card.`
-            );
-          }
-          if (cardAction.action_type === "Action.Submit") {
-            const invokeResponse = await client.submitCardForm(replyToId, cardAction.data ?? {});
-            responseText = JSON.stringify(invokeResponse);
-          } else {
-            const invokeResponse = await client.clickCardButton(
-              replyToId,
-              cardAction.verb,
-              cardAction.data ?? {}
-            );
-            responseText = JSON.stringify(invokeResponse);
-          }
-        } else {
-          const persona = resolvePersona(turn, input, config);
-          responseText = await sendNotificationAndWait(
-            client.getConversationId(),
-            turn,
-            persona,
-            timeout
-          );
-        }
-
-        const duration = (Date.now() - turnStart) / 1000;
-        log(`  [OK] (${duration.toFixed(1)}s): ${responseText.substring(0, 100)}...`);
-
-        results.push({
-          test_id: turn.test_id,
-          prompt: turn.prompt,
-          actual_response: responseText || null,
-          attachments,
-          status: "Completed",
-          duration_seconds: duration,
-        });
-      } catch (err) {
-        const duration = (Date.now() - turnStart) / 1000;
-        const message = err instanceof Error ? err.message : String(err);
-        const isTimeout = message.includes("Timeout");
-
-        logError(`  Turn ${turn.test_id} failed: ${message}`);
-
-        results.push({
-          test_id: turn.test_id,
-          prompt: turn.prompt,
-          actual_response: null,
-          attachments: [],
-          status: isTimeout ? "TimedOut" : "Errored",
-          error_message: message,
-          duration_seconds: duration,
-        });
-
-        for (let j = i + 1; j < turns.length; j++) {
-          results.push({
-            test_id: turns[j].test_id,
-            prompt: turns[j].prompt,
-            actual_response: null,
-            attachments: [],
-            status: "Skipped",
-            duration_seconds: 0,
-          });
-        }
-        break;
-      }
-    }
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logError(`Client startup failed for ${scenario}: ${message}`);
-
-    for (const turn of turns) {
-      results.push({
-        test_id: turn.test_id,
-        prompt: turn.prompt,
-        actual_response: null,
-        attachments: [],
-        status: "Errored",
-        error_message: message,
-        duration_seconds: 0,
-      });
-    }
-  } finally {
-    try {
-      await client.stop();
-    } catch {
-      // Ignore stop errors.
-    }
-  }
-
-  const convDuration = (Date.now() - convStart) / 1000;
-
-  const statuses = new Set(results.map((t) => t.status));
-  let overallStatus: "Completed" | "TimedOut" | "Errored";
-  if (statuses.has("TimedOut")) {
-    overallStatus = "TimedOut";
-  } else if (statuses.has("Errored")) {
-    overallStatus = "Errored";
-  } else {
-    overallStatus = "Completed";
-  }
-
-  return {
-    type: "conversation_result",
-    scenario,
-    status: overallStatus,
-    duration_seconds: convDuration,
-    turns: results,
-  };
-}

package/src/serverManager.ts

@@ -1,172 +0,0 @@
-import express, { Express } from "express";
-import { Server } from "http";
-import { AddressInfo } from "net";
-import { mountBackend, Factory, ConfigManager } from "server";
-import { DeliveryMode } from "schema";
-import { ResponseCapture } from "./responseCapture";
-import { TestClientConfig } from "./types";
-
-/**
- * Manages the shared Express server lifecycle for the test harness.
- *
- * The server package's Factory is a process-wide singleton — calling
- * mountBackend() more than once replaces ConfigManager while leaving
- * every other cached manager/service stale. ServerManager therefore
- * starts the Express server exactly once and reuses it for all
- * TestClient instances in the same process.
- */
-
-// Fixed conversation ID used only for the initial ConfigManager bootstrap.
-// Individual TestClients register their own unique conversations in
-// ConversationManager (see TestClient.registerUniqueConversation).
-const BOOTSTRAP_CONVERSATION_ID = "playground-cli-bootstrap";
-
-export class ServerManager {
-  // ── Shared process-level state ───────────────────────────────────────
-  private static server: Server | null = null;
-  private static app: Express | null = null;
-  private static _port = 0;
-  private static _serviceUrl = "";
-  private static _started = false;
-  private static _responseCapture: ResponseCapture | null = null;
-
-  /**
-   * Start the shared test server (no-op if already running).
-   *
-   * The first call boots Express + mountBackend. Subsequent calls
-   * return immediately, reusing the existing server.
-   */
-  async start(config: TestClientConfig): Promise<void> {
-    if (ServerManager._started) {
-      return;
-    }
-
-    const app = express();
-    ServerManager.app = app;
-
-    // Create HTTP server first
-    ServerManager.server = app.listen(config.port ?? 0);
-
-    // Wait for server to be listening
-    await new Promise<void>((resolve, reject) => {
-      ServerManager.server?.once("listening", resolve);
-      ServerManager.server?.once("error", reject);
-    });
-
-    // Get the assigned port
-    const address = ServerManager.server?.address() as AddressInfo;
-    ServerManager._port = address.port;
-    ServerManager._serviceUrl = `http://localhost:${ServerManager._port}/_connector`;
-
-    // Get default config and merge bot config if provided
-    const defaultConfig = ConfigManager.getDefaultConfig({
-      path: "",
-      conversationId: BOOTSTRAP_CONVERSATION_ID,
-    });
-    if (config.bot) {
-      defaultConfig.bot = {
-        ...defaultConfig.bot,
-        ...config.bot,
-      };
-      // Also update the root tenantId if bot.tenantId is provided
-      if (config.bot.tenantId) {
-        defaultConfig.tenantId = config.bot.tenantId;
-      }
-    }
-
-    if (!ServerManager.server) {
-      throw new Error("Server failed to start");
-    }
-
-    // Map delivery mode string to enum value
-    const deliveryMode =
-      config.deliveryMode === "expectReplies"
-        ? DeliveryMode.ExpectReplies
-        : config.deliveryMode === "default"
-        ? DeliveryMode.Default
-        : undefined;
-
-    mountBackend(app, ServerManager.server, {
-      configFileOptions: { configFile: defaultConfig },
-      appConfig: {
-        endpoint: config.botEndpoint,
-      },
-      debugConfig: {
-        eventsRecordingEnabled: false,
-        deliveryMode,
-      },
-    });
-
-    // Add error-handling middleware for connector routes.
-    app.use(
-      (
-        err: Error,
-        _req: express.Request,
-        res: express.Response,
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        _next: express.NextFunction
-      ) => {
-        console.error("[agents-simulator] Unhandled error in connector:", err);
-        if (!res.headersSent) {
-          res.status(500).json({
-            error: {
-              code: "InternalError",
-              message: err.message || "Unknown error",
-            },
-          });
-        }
-      }
-    );
-
-    // Hook the shared ResponseCapture into BotConnectorService once.
-    const capture = ServerManager.getResponseCapture();
-    capture.hookIntoBotConnectorService(Factory.getBotConnectorService());
-
-    ServerManager._started = true;
-  }
-
-  /**
-   * No-op — the shared server stays running for the process lifetime.
-   *
-   * Per-client cleanup (WebSockets, conversation messages) is handled
-   * by TestClient.stop().
-   *
-   * TODO(server-package): Add a Factory.reset() so the harness can
-   * fully tear down between test suites if needed.
-   */
-  async stop(): Promise<void> {
-    // Intentionally empty.
-  }
-
-  /**
-   * Get the port the server is running on
-   */
-  get port(): number {
-    return ServerManager._port;
-  }
-
-  /**
-   * Get the service URL for the connector API
-   */
-  get serviceUrl(): string {
-    return ServerManager._serviceUrl;
-  }
-
-  /**
-   * Get the Factory instance for accessing services
-   */
-  getFactory(): typeof Factory {
-    return Factory;
-  }
-
-  /**
-   * Shared ResponseCapture instance — keyed by conversationId, so
-   * concurrent TestClients don't interfere with each other.
-   */
-  static getResponseCapture(): ResponseCapture {
-    if (!ServerManager._responseCapture) {
-      ServerManager._responseCapture = new ResponseCapture();
-    }
-    return ServerManager._responseCapture;
-  }
-}

package/src/start-server.ts

@@ -1,26 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Starts the Conversation Server.
- *
- * Usage:
- *   node build/start-server.js [--port <port>]
- */
-
-import { createConversationServer } from "./conversationServer";
-
-function parsePort(): number {
-  const args = process.argv.slice(2);
-  for (let i = 0; i < args.length; i++) {
-    if (args[i] === "--port" && args[i + 1]) {
-      return parseInt(args[++i], 10);
-    }
-  }
-  return 0;
-}
-
-const port = parsePort();
-
-void createConversationServer({ port }).then((server) => {
-  process.stdout.write(JSON.stringify({ port: server.port }) + "\n");
-});