@use-lattice/litmus 0.121.3

package/dist/src/chatkit-swAIVuea.js

@@ -0,0 +1,1157 @@
+import { r as logger } from "./logger-C40ZGil9.js";
+import { t as OpenAiGenericProvider } from "./openai-DumO8WQn.js";
+import { t as providerRegistry } from "./providerRegistry-4yjhaEM8.js";
+import * as http$1 from "http";
+import { chromium } from "playwright";
+//#region src/providers/openai/chatkit-pool.ts
+/**
+* ChatKit Browser Pool
+*
+* Manages a pool of browser contexts for concurrent ChatKit evaluations.
+* This significantly reduces resource usage compared to spawning separate
+* browsers for each test.
+*
+* Architecture:
+*   - Single browser process (shared across all tests)
+*   - Multiple browser contexts (isolated like incognito windows)
+*   - Shared HTTP server with per-workflow template routing
+*   - Pages are workflow-specific (different workflows get different pages)
+*/
+const CHATKIT_READY_TIMEOUT_MS$1 = 6e4;
+const PAGE_REFRESH_TIMEOUT_MS = 6e4;
+const PAGE_ACQUIRE_TIMEOUT_MS = 12e4;
+const IDLE_SHUTDOWN_DELAY_MS = 5e3;
+/**
+* Singleton browser pool for ChatKit evaluations.
+* Supports high concurrency by reusing browser contexts.
+* Each workflow gets its own isolated pages via template routing.
+*/
+var ChatKitBrowserPool = class ChatKitBrowserPool {
+	static instance = null;
+	static cleanupRegistered = false;
+	browser = null;
+	server = null;
+	serverPort = 0;
+	pages = [];
+	waitQueue = [];
+	config;
+	templates = /* @__PURE__ */ new Map();
+	initialized = false;
+	initPromise = null;
+	idleTimer = null;
+	constructor(config) {
+		this.config = config;
+	}
+	/**
+	* Register process exit handlers to clean up browser resources
+	*/
+	static registerCleanupHandlers() {
+		if (ChatKitBrowserPool.cleanupRegistered) return;
+		ChatKitBrowserPool.cleanupRegistered = true;
+		const cleanup = () => {
+			if (ChatKitBrowserPool.instance) {
+				ChatKitBrowserPool.instance.shutdown().catch(() => {});
+				ChatKitBrowserPool.instance = null;
+			}
+		};
+		process.on("beforeExit", () => {
+			if (ChatKitBrowserPool.instance) {
+				ChatKitBrowserPool.instance.shutdown().catch(() => {});
+				ChatKitBrowserPool.instance = null;
+			}
+		});
+		process.on("exit", cleanup);
+	}
+	/**
+	* Get the singleton pool instance
+	*/
+	static getInstance(config) {
+		if (!ChatKitBrowserPool.instance) {
+			ChatKitBrowserPool.instance = new ChatKitBrowserPool({
+				maxConcurrency: config?.maxConcurrency ?? 4,
+				headless: config?.headless ?? true,
+				serverPort: config?.serverPort ?? 0
+			});
+			ChatKitBrowserPool.registerCleanupHandlers();
+			const instance = ChatKitBrowserPool.instance;
+			providerRegistry.register({ async shutdown() {
+				if (instance) {
+					await instance.shutdown();
+					ChatKitBrowserPool.instance = null;
+				}
+			} });
+		} else if (config) {
+			const existing = ChatKitBrowserPool.instance.config;
+			if (config.maxConcurrency !== void 0 && config.maxConcurrency !== existing.maxConcurrency || config.headless !== void 0 && config.headless !== existing.headless) logger.warn("[ChatKitPool] Pool already exists with different config, ignoring new config", {
+				existing: {
+					maxConcurrency: existing.maxConcurrency,
+					headless: existing.headless
+				},
+				requested: {
+					maxConcurrency: config.maxConcurrency,
+					headless: config.headless
+				}
+			});
+		}
+		return ChatKitBrowserPool.instance;
+	}
+	/**
+	* Reset the singleton (for testing)
+	*/
+	static resetInstance() {
+		if (ChatKitBrowserPool.instance) {
+			ChatKitBrowserPool.instance.shutdown().catch((err) => {
+				logger.debug("[ChatKitPool] Error during shutdown:", { error: String(err) });
+			});
+			ChatKitBrowserPool.instance = null;
+		}
+	}
+	/**
+	* Generate a template key from workflow configuration.
+	* This ensures different workflows get isolated pages.
+	*/
+	static generateTemplateKey(workflowId, version, userId) {
+		return `${workflowId}:${version || "default"}:${userId || "default"}`;
+	}
+	/**
+	* Register a template for a workflow configuration
+	*/
+	setTemplate(templateKey, html) {
+		if (this.templates.get(templateKey) !== html) {
+			this.templates.set(templateKey, html);
+			logger.debug("[ChatKitPool] Registered template", { templateKey });
+			for (const page of this.pages) if (page.templateKey === templateKey) page.ready = false;
+		}
+	}
+	/**
+	* Initialize the pool - launches browser and creates server
+	*/
+	async initialize() {
+		if (this.initialized) return;
+		if (this.initPromise != null) return this.initPromise;
+		this.initPromise = this.doInitialize();
+		await this.initPromise;
+		this.initPromise = null;
+	}
+	async doInitialize() {
+		logger.debug("[ChatKitPool] Initializing browser pool", { maxConcurrency: this.config.maxConcurrency });
+		this.server = http$1.createServer((req, res) => {
+			const pathParts = new URL(req.url || "/", `http://localhost`).pathname.split("/").filter(Boolean);
+			if (pathParts[0] === "template" && pathParts[1]) {
+				const templateKey = decodeURIComponent(pathParts[1]);
+				const template = this.templates.get(templateKey);
+				if (template) {
+					res.writeHead(200, { "Content-Type": "text/html" });
+					res.end(template);
+					return;
+				}
+			}
+			res.writeHead(404, { "Content-Type": "text/plain" });
+			res.end("Template not found");
+		});
+		await new Promise((resolve, reject) => {
+			this.server.once("error", (err) => {
+				reject(/* @__PURE__ */ new Error(`Failed to start ChatKit pool server: ${err.message}`));
+			});
+			this.server.listen(this.config.serverPort, () => {
+				const address = this.server.address();
+				this.serverPort = typeof address === "object" ? address?.port || 0 : 0;
+				logger.debug("[ChatKitPool] Server started", { port: this.serverPort });
+				resolve();
+			});
+		});
+		try {
+			this.browser = await chromium.launch({ headless: this.config.headless });
+		} catch (error) {
+			if ((error instanceof Error ? error.message : String(error)).includes("Executable doesn't exist")) throw new Error("Playwright browser not installed. Run: npx playwright install chromium");
+			throw error;
+		}
+		this.initialized = true;
+		logger.debug("[ChatKitPool] Browser pool initialized");
+	}
+	/**
+	* Acquire a page from the pool for a specific template.
+	* Only returns pages configured for the requested template.
+	* Blocks if all pages are in use.
+	*/
+	async acquirePage(templateKey) {
+		this.cancelIdleTimer();
+		await this.initialize();
+		if (!this.templates.has(templateKey)) throw new Error(`Template not registered: ${templateKey}. Call setTemplate first.`);
+		const available = this.pages.find((p) => !p.inUse && p.ready && p.templateKey === templateKey);
+		if (available) {
+			available.inUse = true;
+			logger.debug("[ChatKitPool] Acquired existing page", {
+				templateKey,
+				poolSize: this.pages.length
+			});
+			return available;
+		}
+		const needsRefresh = this.pages.find((p) => !p.inUse && !p.ready && p.templateKey === templateKey);
+		if (needsRefresh) {
+			await this.refreshPooledPage(needsRefresh);
+			needsRefresh.inUse = true;
+			logger.debug("[ChatKitPool] Acquired and refreshed page", {
+				templateKey,
+				poolSize: this.pages.length
+			});
+			return needsRefresh;
+		}
+		if (this.pages.length < this.config.maxConcurrency) {
+			const pooledPage = await this.createPooledPage(templateKey);
+			pooledPage.inUse = true;
+			this.pages.push(pooledPage);
+			logger.debug("[ChatKitPool] Created new page", {
+				templateKey,
+				poolSize: this.pages.length
+			});
+			return pooledPage;
+		}
+		logger.debug("[ChatKitPool] Waiting for available page", {
+			templateKey,
+			poolSize: this.pages.length,
+			waiting: this.waitQueue.length + 1
+		});
+		return new Promise((resolve, reject) => {
+			const timeoutId = setTimeout(() => {
+				const index = this.waitQueue.findIndex((w) => w.resolve === wrappedResolve);
+				if (index >= 0) this.waitQueue.splice(index, 1);
+				reject(/* @__PURE__ */ new Error(`Timeout waiting for available page after ${PAGE_ACQUIRE_TIMEOUT_MS}ms. Pool has ${this.pages.length} pages, ${this.pages.filter((p) => p.inUse).length} in use.`));
+			}, PAGE_ACQUIRE_TIMEOUT_MS);
+			const wrappedResolve = (page) => {
+				clearTimeout(timeoutId);
+				resolve(page);
+			};
+			this.waitQueue.push({
+				templateKey,
+				resolve: wrappedResolve
+			});
+		});
+	}
+	/**
+	* Release a page back to the pool
+	*/
+	async releasePage(pooledPage) {
+		const originalTemplateKey = pooledPage.templateKey;
+		try {
+			await this.refreshPooledPage(pooledPage);
+		} catch (error) {
+			logger.warn("[ChatKitPool] Failed to reset page, recreating", { error });
+			const index = this.pages.indexOf(pooledPage);
+			if (index >= 0) this.pages.splice(index, 1);
+			try {
+				await pooledPage.context.close();
+			} catch {}
+			try {
+				const newPage = await this.createPooledPage(originalTemplateKey);
+				this.pages.push(newPage);
+				pooledPage = newPage;
+			} catch (createError) {
+				logger.warn("[ChatKitPool] Failed to create replacement page", { error: createError });
+				await this.tryServeWaiters();
+				this.scheduleIdleShutdown();
+				return;
+			}
+		}
+		const waiterIndex = this.waitQueue.findIndex((w) => w.templateKey === pooledPage.templateKey);
+		if (waiterIndex >= 0) {
+			const waiter = this.waitQueue.splice(waiterIndex, 1)[0];
+			pooledPage.inUse = true;
+			waiter.resolve(pooledPage);
+			this.cancelIdleTimer();
+		} else {
+			pooledPage.inUse = false;
+			await this.tryServeWaiters();
+			this.scheduleIdleShutdown();
+		}
+	}
+	/**
+	* Try to serve waiting requests by creating new pages if we have capacity
+	*/
+	async tryServeWaiters() {
+		while (this.waitQueue.length > 0 && this.pages.length < this.config.maxConcurrency) {
+			const waiter = this.waitQueue.shift();
+			if (!waiter) break;
+			try {
+				const newPage = await this.createPooledPage(waiter.templateKey);
+				newPage.inUse = true;
+				this.pages.push(newPage);
+				waiter.resolve(newPage);
+				logger.debug("[ChatKitPool] Created page for waiting request", {
+					templateKey: waiter.templateKey,
+					poolSize: this.pages.length,
+					remainingWaiters: this.waitQueue.length
+				});
+			} catch (error) {
+				logger.warn("[ChatKitPool] Failed to create page for waiter", {
+					templateKey: waiter.templateKey,
+					error
+				});
+				this.waitQueue.unshift(waiter);
+				break;
+			}
+		}
+	}
+	/**
+	* Schedule automatic shutdown if pool remains idle
+	*/
+	scheduleIdleShutdown() {
+		this.cancelIdleTimer();
+		if (this.pages.filter((p) => p.inUse).length === 0 && this.waitQueue.length === 0 && this.pages.length > 0) {
+			logger.debug("[ChatKitPool] Pool idle, scheduling shutdown", { delay: IDLE_SHUTDOWN_DELAY_MS });
+			this.idleTimer = setTimeout(() => {
+				if (this.pages.filter((p) => p.inUse).length === 0 && this.waitQueue.length === 0) {
+					logger.debug("[ChatKitPool] Auto-shutting down idle pool");
+					this.shutdown().catch((err) => {
+						logger.debug("[ChatKitPool] Error during idle shutdown", { error: String(err) });
+					});
+					ChatKitBrowserPool.instance = null;
+				}
+			}, IDLE_SHUTDOWN_DELAY_MS);
+			if (this.idleTimer.unref) this.idleTimer.unref();
+		}
+	}
+	/**
+	* Cancel scheduled idle shutdown
+	*/
+	cancelIdleTimer() {
+		if (this.idleTimer) {
+			clearTimeout(this.idleTimer);
+			this.idleTimer = null;
+		}
+	}
+	/**
+	* Create a new pooled page with ChatKit initialized for a specific template
+	*/
+	async createPooledPage(templateKey) {
+		if (!this.browser) throw new Error("Browser not initialized");
+		const context = await this.browser.newContext({ viewport: {
+			width: 800,
+			height: 600
+		} });
+		context.setDefaultTimeout(12e4);
+		try {
+			const page = await context.newPage();
+			const templateUrl = `http://localhost:${this.serverPort}/template/${encodeURIComponent(templateKey)}`;
+			await page.goto(templateUrl, { waitUntil: "domcontentloaded" });
+			await page.waitForFunction(() => window.__state?.ready === true, { timeout: CHATKIT_READY_TIMEOUT_MS$1 });
+			return {
+				context,
+				page,
+				ready: true,
+				inUse: false,
+				templateKey
+			};
+		} catch (error) {
+			try {
+				await context.close();
+			} catch {}
+			throw error;
+		}
+	}
+	async refreshPooledPage(pooledPage) {
+		logger.debug("[ChatKitPool] Refreshing page", { timeout: PAGE_REFRESH_TIMEOUT_MS });
+		await pooledPage.page.reload({ waitUntil: "domcontentloaded" });
+		await pooledPage.page.waitForFunction(() => window.__state?.ready === true, { timeout: PAGE_REFRESH_TIMEOUT_MS });
+		pooledPage.ready = true;
+	}
+	/**
+	* Get pool statistics
+	*/
+	getStats() {
+		return {
+			total: this.pages.length,
+			inUse: this.pages.filter((p) => p.inUse).length,
+			waiting: this.waitQueue.length,
+			templates: this.templates.size
+		};
+	}
+	/**
+	* Shutdown the pool and release all resources
+	*/
+	async shutdown() {
+		logger.debug("[ChatKitPool] Shutting down");
+		this.cancelIdleTimer();
+		if (this.waitQueue.length > 0) {
+			logger.debug("[ChatKitPool] Clearing pending waiters", { count: this.waitQueue.length });
+			this.waitQueue = [];
+		}
+		for (const pooledPage of this.pages) try {
+			await pooledPage.context.close();
+		} catch {}
+		this.pages = [];
+		if (this.browser) {
+			try {
+				await this.browser.close();
+			} catch {}
+			this.browser = null;
+		}
+		if (this.server) {
+			this.server.close();
+			this.server = null;
+		}
+		this.initialized = false;
+		this.templates.clear();
+		logger.debug("[ChatKitPool] Shutdown complete");
+	}
+};
+//#endregion
+//#region src/providers/openai/chatkit.ts
+/**
+* OpenAI ChatKit Provider
+*
+* Evaluates ChatKit workflows deployed via Agent Builder using Playwright
+* to interact with the ChatKit web component.
+*
+* ChatKit workflows created in OpenAI's Agent Builder don't expose a direct
+* REST API for sending messages. Instead, they require interaction through
+* the ChatKit web component, which this provider automates using Playwright.
+*
+* Prerequisites:
+*   - Playwright installed: npm install playwright && npx playwright install chromium
+*   - OPENAI_API_KEY environment variable set
+*
+* Usage:
+*   providers:
+*     - id: openai:chatkit:wf_68ffb83dbfc88190a38103c2bb9f421003f913035dbdb131
+*       config:
+*         version: '3'           # Optional: workflow version
+*         timeout: 120000        # Optional: response timeout in ms (default: 120000)
+*         headless: true         # Optional: run browser headless (default: true)
+*
+* Performance Notes:
+*   - Each evaluation spawns a browser instance, so it's slower than REST APIs
+*   - For reliable results, use --max-concurrency 1 to avoid resource contention
+*   - First test may be slower due to browser launch and ChatKit initialization
+*
+* Troubleshooting:
+*   - "Playwright not found": Run `npx playwright install chromium`
+*   - Timeout errors: Increase timeout config or use --max-concurrency 1
+*   - Empty responses: The workflow may not generate text for some inputs
+*/
+const DEFAULT_TIMEOUT_MS = 12e4;
+const DEFAULT_MAX_APPROVALS = 5;
+const DEFAULT_POOL_SIZE = 4;
+const CHATKIT_READY_TIMEOUT_MS = 6e4;
+const DOM_SETTLE_DELAY_MS = 2e3;
+const APPROVAL_PROCESS_DELAY_MS = 500;
+const APPROVAL_CLICK_DELAY_MS = 1e3;
+const RESPONSE_EXTRACT_RETRY_DELAY_MS = 500;
+const CONTENT_STABILIZATION_MS = 1e4;
+const CONTENT_POLL_MS = 500;
+const MIN_WORKFLOW_WAIT_MS = 6e4;
+const SHORT_RESPONSE_THRESHOLD = 100;
+/**
+* Check if a URL is from OpenAI's CDN by parsing the hostname.
+* This is more secure than substring matching which could be bypassed.
+*/
+function isOpenAICdnUrl(url) {
+	try {
+		return new URL(url).hostname === "cdn.platform.openai.com";
+	} catch {
+		return false;
+	}
+}
+/**
+* Validate workflowId format to prevent script injection
+*/
+function validateWorkflowId(workflowId) {
+	if (!workflowId || !/^wf_[a-zA-Z0-9]+$/.test(workflowId)) throw new Error(`Invalid workflowId format: ${workflowId}. Expected format: wf_<alphanumeric>`);
+}
+/**
+* Validate version format to prevent script injection
+*/
+function validateVersion(version) {
+	if (!/^[a-zA-Z0-9._-]+$/.test(version)) throw new Error(`Invalid version format: ${version}. Only alphanumeric, dot, dash, and underscore allowed.`);
+}
+/**
+* Validate userId format to prevent script injection
+*/
+function validateUserId(userId) {
+	if (!/^[a-zA-Z0-9._@-]+$/.test(userId)) throw new Error(`Invalid userId format: ${userId}. Only alphanumeric, dot, dash, underscore, and @ allowed.`);
+}
+/**
+* Clean up assistant response text by removing noise and artifacts.
+* This includes Cloudflare scripts, approval UI text, user echo, and JSON classification prefixes.
+*/
+function cleanAssistantResponse(text) {
+	if (!text) return "";
+	let cleaned = text.replace(/\(function\(\)\{.*?\}\)\(\);?/gs, "").trim();
+	cleaned = cleaned.replace(/\n?Approval required\n?Does this work for you\?\n?Approve\n?Reject$/gi, "").replace(/\n?Approval required[\s\n]+Does this work for you\?[\s\n]+Approve[\s\n]+Reject$/gi, "").trim();
+	if (/^You said:/i.test(cleaned)) cleaned = "";
+	else cleaned = cleaned.replace(/You said:[\s\S]*/gi, "").trim();
+	const jsonMatch = cleaned.match(/^(\{[^}]+\})\s+(.+)/s);
+	if (jsonMatch && jsonMatch[2].trim().length > 50) cleaned = jsonMatch[2].trim();
+	return cleaned;
+}
+/**
+* Generate the HTML page that hosts the ChatKit component
+*/
+function generateChatKitHTML(apiKey, workflowId, version, userId) {
+	validateWorkflowId(workflowId);
+	if (version) validateVersion(version);
+	if (!userId) throw new Error("userId is required for ChatKit HTML generation");
+	validateUserId(userId);
+	return `<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>ChatKit Eval</title>
+</head>
+<body>
+  <openai-chatkit id="chatkit"></openai-chatkit>
+
+  <script src="https://cdn.platform.openai.com/deployments/chatkit/chatkit.js"><\/script>
+
+  <script>
+    window.__state = { ready: false, responses: [], threadId: null, error: null, responding: false };
+
+    async function init() {
+      const chatkit = document.getElementById('chatkit');
+
+      // Wait for element to be ready
+      let attempts = 0;
+      while (typeof chatkit.setOptions !== 'function' && attempts < 100) {
+        await new Promise(r => setTimeout(r, 100));
+        attempts++;
+      }
+
+      if (typeof chatkit.setOptions !== 'function') {
+        window.__state.error = 'ChatKit component failed to initialize';
+        return;
+      }
+
+      let cachedSecret = null;
+
+      chatkit.setOptions({
+        api: {
+          getClientSecret: async (existing) => {
+            if (existing) return existing;
+            if (cachedSecret) return cachedSecret;
+
+            const res = await fetch('https://api.openai.com/v1/chatkit/sessions', {
+              method: 'POST',
+              headers: {
+                'Authorization': 'Bearer ${apiKey}',
+                'Content-Type': 'application/json',
+                'OpenAI-Beta': 'chatkit_beta=v1'
+              },
+              body: JSON.stringify({
+                workflow: { id: '${workflowId}'${version ? `, version: '${version}'` : ""} },
+                user: '${userId}'
+              })
+            });
+
+            if (!res.ok) {
+              const text = await res.text();
+              throw new Error('Session failed: ' + res.status + ' ' + text);
+            }
+
+            const data = await res.json();
+            cachedSecret = data.client_secret;
+            return cachedSecret;
+          }
+        },
+        header: { enabled: false },
+        history: { enabled: false },
+      });
+
+      chatkit.addEventListener('chatkit.ready', () => {
+        window.__state.ready = true;
+      });
+
+      chatkit.addEventListener('chatkit.error', (e) => {
+        window.__state.error = e.detail.error?.message || 'Unknown error';
+      });
+
+      chatkit.addEventListener('chatkit.thread.change', (e) => {
+        window.__state.threadId = e.detail.threadId;
+      });
+
+      chatkit.addEventListener('chatkit.response.start', () => {
+        window.__state.responding = true;
+      });
+
+      chatkit.addEventListener('chatkit.response.end', () => {
+        window.__state.responding = false;
+        window.__state.responses.push({ timestamp: Date.now() });
+      });
+
+      window.__chatkit = chatkit;
+    }
+
+    init().catch(e => {
+      window.__state.error = e.message;
+    });
+  <\/script>
+</body>
+</html>`;
+}
+/**
+* Extract assistant response text from the ChatKit iframe
+* Uses retry logic since DOM may still be updating after response.end event
+*/
+async function extractResponseFromFrame(page, maxRetries = 3) {
+	for (let attempt = 0; attempt < maxRetries; attempt++) {
+		const frames = page.frames();
+		for (const frame of frames) {
+			const url = frame.url();
+			if (isOpenAICdnUrl(url)) try {
+				const result = await frame.evaluate(() => {
+					const isUserMessage = (el) => {
+						const className = el.className?.toString().toLowerCase() || "";
+						const role = el.getAttribute("data-role") || "";
+						const testId = el.getAttribute("data-testid") || "";
+						return className.includes("user") || role === "user" || testId.includes("user");
+					};
+					const isAssistantMessage = (el) => {
+						const className = el.className?.toString().toLowerCase() || "";
+						const role = el.getAttribute("data-role") || "";
+						const testId = el.getAttribute("data-testid") || "";
+						return className.includes("assistant") || role === "assistant" || testId.includes("assistant");
+					};
+					for (const sel of [
+						"[data-thread-item=\"assistant-message\"]",
+						"[data-testid=\"assistant-message\"]",
+						"[data-role=\"assistant\"]",
+						"[class*=\"assistant\"]:not([class*=\"user\"])"
+					]) {
+						const els = document.querySelectorAll(sel);
+						if (els.length > 0) {
+							const text = els[els.length - 1].textContent?.trim() || "";
+							if (text.length > 0) return {
+								text,
+								source: sel,
+								isAssistant: true
+							};
+						}
+					}
+					const allMessages = document.querySelectorAll("[class*=\"message\"]");
+					const messages = [];
+					allMessages.forEach((msg) => {
+						const text = msg.textContent?.trim() || "";
+						if (text.length > 0) messages.push({
+							text,
+							isUser: isUserMessage(msg),
+							isAssistant: isAssistantMessage(msg)
+						});
+					});
+					for (let i = messages.length - 1; i >= 0; i--) if (!messages[i].isUser && messages[i].text.length > 0) return {
+						text: messages[i].text,
+						source: "last-non-user",
+						isAssistant: true
+					};
+					const markdown = document.querySelectorAll(".markdown, [class*=\"markdown\"]");
+					if (markdown.length > 0) for (let i = markdown.length - 1; i >= 0; i--) {
+						const el = markdown[i];
+						let parent = el.parentElement;
+						let inUserArea = false;
+						while (parent && parent !== document.body) {
+							if (isUserMessage(parent)) {
+								inUserArea = true;
+								break;
+							}
+							parent = parent.parentElement;
+						}
+						if (!inUserArea) {
+							const text = el.textContent?.trim() || "";
+							if (text.length > 0) return {
+								text,
+								source: "markdown",
+								isAssistant: true
+							};
+						}
+					}
+					const responseContainers = document.querySelectorAll("[class*=\"response\"], [class*=\"reply\"], [class*=\"answer\"]");
+					for (let i = responseContainers.length - 1; i >= 0; i--) {
+						const container = responseContainers[i];
+						if (!isUserMessage(container)) {
+							const text = container.textContent?.trim() || "";
+							if (text.length > 0) return {
+								text,
+								source: "response-container",
+								isAssistant: true
+							};
+						}
+					}
+					const divs = Array.from(document.querySelectorAll("div"));
+					const candidateDivs = [];
+					for (const div of divs) {
+						const text = div.textContent?.trim() || "";
+						if (text.length > 0 && text.length < 5e3 && !isUserMessage(div)) {
+							let parent = div.parentElement;
+							let inUserArea = false;
+							while (parent && parent !== document.body) {
+								if (isUserMessage(parent)) {
+									inUserArea = true;
+									break;
+								}
+								parent = parent.parentElement;
+							}
+							if (!inUserArea) candidateDivs.push({
+								text,
+								el: div
+							});
+						}
+					}
+					if (candidateDivs.length > 0) {
+						const leafDivs = candidateDivs.filter((d) => d.el.querySelectorAll("[class*=\"message\"]").length === 0);
+						if (leafDivs.length > 0) return {
+							text: leafDivs[leafDivs.length - 1].text,
+							source: "leaf-div"
+						};
+						return {
+							text: candidateDivs[candidateDivs.length - 1].text,
+							source: "fallback-div"
+						};
+					}
+					return {
+						text: document.body?.textContent?.trim() || "",
+						source: "body"
+					};
+				});
+				if (result.text && result.text.length > 0) {
+					const trimmed = result.text.trim();
+					if (trimmed === "ApproveReject" || trimmed === "Approve" || trimmed === "Reject") {
+						logger.debug("[ChatKitProvider] Skipping approval button text", { text: trimmed });
+						continue;
+					}
+					const cleaned = cleanAssistantResponse(result.text);
+					if (cleaned.length > 0) {
+						logger.debug("[ChatKitProvider] Extracted response", {
+							source: result.source,
+							length: cleaned.length,
+							preview: cleaned.substring(0, 100)
+						});
+						return cleaned;
+					}
+					logger.debug("[ChatKitProvider] No assistant content found after cleaning", {
+						originalLength: result.text.length,
+						source: result.source
+					});
+				}
+			} catch (e) {
+				logger.debug("[ChatKitProvider] Could not access frame", {
+					url,
+					error: e,
+					attempt
+				});
+			}
+		}
+		if (attempt < maxRetries - 1) await page.waitForTimeout(RESPONSE_EXTRACT_RETRY_DELAY_MS);
+	}
+	return "";
+}
+/**
+* Get the current visible text content from the ChatKit iframe.
+* Returns the text content or null if iframe not accessible.
+*/
+async function getIframeContent(page) {
+	const frames = page.frames();
+	logger.debug("[ChatKitProvider] Checking frames", {
+		frameCount: frames.length,
+		frameUrls: frames.map((f) => f.url())
+	});
+	for (const frame of frames) if (isOpenAICdnUrl(frame.url())) try {
+		return await frame.evaluate(() => {
+			return document.body?.innerText || "";
+		});
+	} catch {}
+	return null;
+}
+async function waitForContentStabilization(page, timeout, startTime) {
+	let lastContent = "";
+	let lastChangeTime = Date.now();
+	const pollStartTime = Date.now();
+	let capturedAssistantResponse = "";
+	logger.debug("[ChatKitProvider] Starting content stabilization polling");
+	while (Date.now() - pollStartTime < timeout) {
+		const state = await page.evaluate(() => window.__state);
+		const pollElapsed = Date.now() - pollStartTime;
+		if (pollElapsed % 5e3 < CONTENT_POLL_MS) logger.debug("[ChatKitProvider] Polling state", {
+			pollElapsedMs: pollElapsed,
+			responding: state.responding,
+			responseCount: state.responses?.length,
+			error: state.error,
+			threadId: state.threadId
+		});
+		const currentContent = await getIframeContent(page) || "";
+		if (currentContent !== lastContent) {
+			logger.debug("[ChatKitProvider] Content changed", {
+				previousLength: lastContent.length,
+				newLength: currentContent.length,
+				preview: currentContent.substring(Math.max(0, currentContent.length - 200))
+			});
+			lastContent = currentContent;
+			lastChangeTime = Date.now();
+		}
+		const timeSinceStart = Date.now() - startTime;
+		const timeSinceLastChange = Date.now() - lastChangeTime;
+		const assistantMatch = currentContent.match(/The assistant said:\s*\n*([\s\S]*)/i);
+		const assistantResponse = assistantMatch ? assistantMatch[1].trim() : currentContent;
+		if (!assistantMatch && currentContent.length > 0) logger.debug("[ChatKitProvider] Assistant pattern not found, using full content for length check", { contentLength: currentContent.length });
+		capturedAssistantResponse = assistantResponse;
+		const isShortResponse = assistantResponse.length < SHORT_RESPONSE_THRESHOLD;
+		const effectiveStabilizationMs = isShortResponse ? CONTENT_STABILIZATION_MS * 2 : CONTENT_STABILIZATION_MS;
+		const effectiveMinWaitMs = isShortResponse ? MIN_WORKFLOW_WAIT_MS * 2 : MIN_WORKFLOW_WAIT_MS;
+		if (!state.responding && timeSinceLastChange >= effectiveStabilizationMs && timeSinceStart >= effectiveMinWaitMs) {
+			logger.debug("[ChatKitProvider] Content stabilized", {
+				timeSinceStart,
+				timeSinceLastChange,
+				contentLength: currentContent.length,
+				assistantResponseLength: assistantResponse.length,
+				isShortResponse,
+				responseCount: state.responses?.length
+			});
+			return {
+				assistantResponse: capturedAssistantResponse,
+				fullContent: currentContent
+			};
+		}
+		await page.waitForTimeout(CONTENT_POLL_MS);
+	}
+	logger.debug("[ChatKitProvider] Content stabilization timeout reached");
+	return {
+		assistantResponse: capturedAssistantResponse,
+		fullContent: lastContent
+	};
+}
+/**
+* Handle workflow approval steps by clicking approve/reject buttons.
+* Returns true if an approval was handled, false if no approval found.
+*/
+async function handleApproval(page, action) {
+	const frames = page.frames();
+	for (const frame of frames) if (isOpenAICdnUrl(frame.url())) try {
+		const buttonText = action === "auto-approve" ? "Approve" : "Reject";
+		const buttonSelectors = [
+			`button:has-text("${buttonText}")`,
+			`[role="button"]:has-text("${buttonText}")`,
+			`[data-testid="${buttonText.toLowerCase()}-button"]`
+		];
+		for (const selector of buttonSelectors) {
+			const button = await frame.$(selector);
+			if (button) {
+				if (await button.isVisible()) {
+					logger.debug("[ChatKitProvider] Found approval button, clicking", {
+						action,
+						selector
+					});
+					await button.click();
+					await page.waitForTimeout(APPROVAL_CLICK_DELAY_MS);
+					return true;
+				}
+			}
+		}
+		if (await frame.evaluate((btnText) => {
+			const approveBtn = Array.from(document.querySelectorAll("button, [role=\"button\"]")).find((b) => b.textContent?.toLowerCase().includes(btnText.toLowerCase()));
+			if (approveBtn && approveBtn instanceof HTMLElement) {
+				approveBtn.click();
+				return true;
+			}
+			return false;
+		}, buttonText)) {
+			logger.debug("[ChatKitProvider] Clicked approval button via evaluate", { action });
+			await page.waitForTimeout(APPROVAL_CLICK_DELAY_MS);
+			return true;
+		}
+	} catch (e) {
+		logger.debug("[ChatKitProvider] Error checking for approval buttons", { error: e });
+	}
+	return false;
+}
+/**
+* Process approvals until none remain or max reached.
+* Returns the number of approvals processed.
+*/
+async function processApprovals(page, approvalHandling, maxApprovals, timeout) {
+	if (approvalHandling === "skip") return 0;
+	let approvalCount = 0;
+	while (approvalCount < maxApprovals) {
+		await page.waitForTimeout(APPROVAL_PROCESS_DELAY_MS);
+		if (!await handleApproval(page, approvalHandling)) break;
+		approvalCount++;
+		logger.debug("[ChatKitProvider] Processed approval", {
+			count: approvalCount,
+			max: maxApprovals
+		});
+		try {
+			await page.waitForFunction((prevCount) => window.__state?.responses?.length > prevCount, approvalCount, { timeout: timeout / 2 });
+			await page.waitForTimeout(DOM_SETTLE_DELAY_MS);
+		} catch {
+			break;
+		}
+	}
+	return approvalCount;
+}
+var OpenAiChatKitProvider = class OpenAiChatKitProvider extends OpenAiGenericProvider {
+	chatKitConfig;
+	browser = null;
+	context = null;
+	page = null;
+	server = null;
+	serverPort = 0;
+	initialized = false;
+	static defaultUserId = null;
+	static getDefaultUserId() {
+		if (!OpenAiChatKitProvider.defaultUserId) OpenAiChatKitProvider.defaultUserId = `promptfoo-eval-${Date.now()}`;
+		return OpenAiChatKitProvider.defaultUserId;
+	}
+	constructor(workflowId, options = {}) {
+		super(workflowId, options);
+		const envPoolSize = process.env.PROMPTFOO_MAX_CONCURRENCY ? parseInt(process.env.PROMPTFOO_MAX_CONCURRENCY, 10) : NaN;
+		const defaultPoolSize = Number.isNaN(envPoolSize) ? DEFAULT_POOL_SIZE : envPoolSize;
+		this.chatKitConfig = {
+			workflowId: options.config?.workflowId || workflowId,
+			version: options.config?.version,
+			userId: options.config?.userId || OpenAiChatKitProvider.getDefaultUserId(),
+			timeout: options.config?.timeout || DEFAULT_TIMEOUT_MS,
+			headless: options.config?.headless ?? true,
+			serverPort: options.config?.serverPort || 0,
+			usePool: options.config?.usePool ?? true,
+			poolSize: options.config?.poolSize ?? defaultPoolSize,
+			approvalHandling: options.config?.approvalHandling ?? "auto-approve",
+			maxApprovals: options.config?.maxApprovals ?? DEFAULT_MAX_APPROVALS,
+			stateful: options.config?.stateful ?? false
+		};
+	}
+	id() {
+		const version = this.chatKitConfig.version ? `:${this.chatKitConfig.version}` : "";
+		return `openai:chatkit:${this.chatKitConfig.workflowId}${version}`;
+	}
+	toString() {
+		return `[OpenAI ChatKit Provider ${this.chatKitConfig.workflowId}]`;
+	}
+	/**
+	* Initialize the browser and ChatKit page
+	*/
+	async initialize() {
+		if (this.initialized) return;
+		const apiKey = this.getApiKey();
+		if (!apiKey) throw new Error("OpenAI API key is required for ChatKit provider");
+		const workflowId = this.chatKitConfig.workflowId;
+		if (!workflowId) throw new Error("ChatKit workflowId is required");
+		logger.debug("[ChatKitProvider] Initializing", {
+			workflowId,
+			version: this.chatKitConfig.version
+		});
+		const html = generateChatKitHTML(apiKey, workflowId, this.chatKitConfig.version, this.chatKitConfig.userId);
+		this.server = http$1.createServer((_req, res) => {
+			res.writeHead(200, { "Content-Type": "text/html" });
+			res.end(html);
+		});
+		await new Promise((resolve, reject) => {
+			this.server.once("error", (err) => {
+				reject(/* @__PURE__ */ new Error(`Failed to start ChatKit server: ${err.message}`));
+			});
+			this.server.listen(this.chatKitConfig.serverPort, () => {
+				const address = this.server.address();
+				this.serverPort = typeof address === "object" ? address?.port || 0 : 0;
+				logger.debug("[ChatKitProvider] Server started", { port: this.serverPort });
+				resolve();
+			});
+		});
+		try {
+			this.browser = await chromium.launch({ headless: this.chatKitConfig.headless });
+		} catch (launchError) {
+			const errorMessage = launchError instanceof Error ? launchError.message : String(launchError);
+			if (errorMessage.includes("Executable doesn't exist") || errorMessage.includes("browserType.launch")) throw new Error(`Playwright browser not installed. Run: npx playwright install chromium
+Original error: ${errorMessage}`);
+			throw launchError;
+		}
+		this.context = await this.browser.newContext({ viewport: {
+			width: 800,
+			height: 600
+		} });
+		this.page = await this.context.newPage();
+		this.page.on("console", (msg) => {
+			const type = msg.type();
+			if (type === "error" || type === "warning") logger.debug("[ChatKitProvider] Browser console", {
+				type,
+				text: msg.text()
+			});
+		});
+		await this.page.goto(`http://localhost:${this.serverPort}`, { waitUntil: "domcontentloaded" });
+		logger.debug("[ChatKitProvider] Waiting for ChatKit ready");
+		await this.page.waitForFunction(() => window.__state?.ready === true, { timeout: CHATKIT_READY_TIMEOUT_MS });
+		this.initialized = true;
+		if (!this.chatKitConfig.usePool) providerRegistry.register(this);
+		logger.debug("[ChatKitProvider] Initialized successfully");
+	}
+	/**
+	* Shutdown method for providerRegistry cleanup
+	*/
+	async shutdown() {
+		await this.cleanup();
+	}
+	/**
+	* Clean up browser resources
+	*/
+	async cleanup() {
+		if (this.context) {
+			await this.context.close();
+			this.context = null;
+			this.page = null;
+		}
+		if (this.browser) {
+			await this.browser.close();
+			this.browser = null;
+		}
+		if (this.server) {
+			this.server.close();
+			this.server = null;
+		}
+		this.initialized = false;
+	}
+	/**
+	* Call the ChatKit workflow with the given prompt
+	*/
+	async callApi(prompt, _context, _callApiOptions) {
+		const usePool = this.chatKitConfig.usePool && !this.chatKitConfig.stateful;
+		logger.debug("[ChatKitProvider] Starting call", {
+			prompt: prompt.substring(0, 100),
+			workflowId: this.chatKitConfig.workflowId,
+			usePool,
+			stateful: this.chatKitConfig.stateful
+		});
+		if (usePool) return this.callApiWithPool(prompt);
+		const startTime = Date.now();
+		try {
+			await this.initialize();
+			if (!this.page) throw new Error("Browser page not initialized");
+			if (!this.chatKitConfig.stateful) {
+				await this.page.reload({ waitUntil: "domcontentloaded" });
+				await this.page.waitForFunction(() => window.__state?.ready === true, { timeout: CHATKIT_READY_TIMEOUT_MS });
+			}
+			const responseCount = await this.page.evaluate(() => window.__state?.responses?.length || 0);
+			const isFollowUp = this.chatKitConfig.stateful && responseCount > 0;
+			logger.debug("[ChatKitProvider] Sending message", {
+				stateful: this.chatKitConfig.stateful,
+				isFollowUp,
+				responseCount
+			});
+			await this.page.evaluate(({ text, newThread }) => {
+				return window.__chatkit.sendUserMessage({
+					text,
+					newThread
+				});
+			}, {
+				text: prompt,
+				newThread: !isFollowUp
+			});
+			logger.debug("[ChatKitProvider] Waiting for response");
+			const expectedResponseCount = responseCount + 1;
+			await this.page.waitForFunction((expected) => window.__state?.responses?.length >= expected, expectedResponseCount, { timeout: this.chatKitConfig.timeout });
+			const stabilizationResult = await waitForContentStabilization(this.page, this.chatKitConfig.timeout ?? DEFAULT_TIMEOUT_MS, startTime);
+			const approvalsHandled = await processApprovals(this.page, this.chatKitConfig.approvalHandling ?? "auto-approve", this.chatKitConfig.maxApprovals ?? DEFAULT_MAX_APPROVALS, this.chatKitConfig.timeout ?? DEFAULT_TIMEOUT_MS);
+			if (approvalsHandled > 0) logger.debug("[ChatKitProvider] Processed approvals", { count: approvalsHandled });
+			let responseText = await extractResponseFromFrame(this.page);
+			if (!responseText && stabilizationResult.assistantResponse) {
+				logger.debug("[ChatKitProvider] Using fallback content from stabilization", { fallbackLength: stabilizationResult.assistantResponse.length });
+				responseText = cleanAssistantResponse(stabilizationResult.assistantResponse);
+			}
+			const threadId = await this.page.evaluate(() => window.__state.threadId);
+			const finalResponseCount = await this.page.evaluate(() => window.__state?.responses?.length || 0);
+			const latencyMs = Date.now() - startTime;
+			logger.debug("[ChatKitProvider] Response received", {
+				threadId,
+				textLength: responseText.length,
+				turnNumber: finalResponseCount,
+				latencyMs
+			});
+			return {
+				output: responseText,
+				cached: false,
+				latencyMs,
+				sessionId: threadId,
+				tokenUsage: { numRequests: 1 },
+				metadata: {
+					workflowId: this.chatKitConfig.workflowId,
+					version: this.chatKitConfig.version,
+					stateful: this.chatKitConfig.stateful,
+					turnNumber: finalResponseCount
+				}
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			logger.error("[ChatKitProvider] Call failed", { error: errorMessage });
+			if (this.page) try {
+				const stateError = await this.page.evaluate(() => window.__state?.error);
+				if (stateError) return { error: `ChatKit workflow error: ${stateError}` };
+			} catch {}
+			if (errorMessage.includes("Timeout") || errorMessage.includes("timeout")) return { error: `ChatKit response timeout after ${this.chatKitConfig.timeout}ms. Try increasing timeout in config or use --max-concurrency 1 for more reliable results.` };
+			if (errorMessage.includes("API key")) return { error: "OpenAI API key is required. Set OPENAI_API_KEY environment variable." };
+			if (errorMessage.includes("Playwright") || errorMessage.includes("browser")) return { error: `Browser error: ${errorMessage}. Ensure Playwright is installed: npx playwright install chromium` };
+			return { error: `ChatKit provider error: ${errorMessage}` };
+		}
+	}
+	/**
+	* Pool-based callApi for better concurrency support.
+	* Uses a shared browser with multiple contexts instead of separate browsers.
+	*/
+	async callApiWithPool(prompt) {
+		const apiKey = this.getApiKey();
+		if (!apiKey) return { error: "OpenAI API key is required. Set OPENAI_API_KEY environment variable." };
+		const workflowId = this.chatKitConfig.workflowId;
+		if (!workflowId) return { error: "ChatKit workflowId is required" };
+		const pool = ChatKitBrowserPool.getInstance({
+			maxConcurrency: this.chatKitConfig.poolSize,
+			headless: this.chatKitConfig.headless
+		});
+		const templateKey = ChatKitBrowserPool.generateTemplateKey(workflowId, this.chatKitConfig.version, this.chatKitConfig.userId);
+		const html = generateChatKitHTML(apiKey, workflowId, this.chatKitConfig.version, this.chatKitConfig.userId);
+		pool.setTemplate(templateKey, html);
+		let pooledPage = null;
+		const startTime = Date.now();
+		try {
+			pooledPage = await pool.acquirePage(templateKey);
+			const page = pooledPage.page;
+			logger.debug("[ChatKitProvider] Acquired page from pool", { stats: pool.getStats() });
+			await page.evaluate((text) => {
+				return window.__chatkit.sendUserMessage({
+					text,
+					newThread: true
+				});
+			}, prompt);
+			await page.waitForFunction(() => window.__state?.responses?.length > 0, { timeout: this.chatKitConfig.timeout });
+			const stabilizationResult = await waitForContentStabilization(page, this.chatKitConfig.timeout ?? DEFAULT_TIMEOUT_MS, startTime);
+			const approvalsHandled = await processApprovals(page, this.chatKitConfig.approvalHandling ?? "auto-approve", this.chatKitConfig.maxApprovals ?? DEFAULT_MAX_APPROVALS, this.chatKitConfig.timeout ?? DEFAULT_TIMEOUT_MS);
+			if (approvalsHandled > 0) logger.debug("[ChatKitProvider] Pool processed approvals", { count: approvalsHandled });
+			let responseText = await extractResponseFromFrame(page);
+			if (!responseText && stabilizationResult.assistantResponse) {
+				logger.debug("[ChatKitProvider] Pool using fallback content from stabilization", { fallbackLength: stabilizationResult.assistantResponse.length });
+				responseText = cleanAssistantResponse(stabilizationResult.assistantResponse);
+			}
+			const threadId = await page.evaluate(() => window.__state.threadId);
+			const latencyMs = Date.now() - startTime;
+			logger.debug("[ChatKitProvider] Pool response received", {
+				threadId,
+				textLength: responseText.length,
+				latencyMs
+			});
+			return {
+				output: responseText,
+				cached: false,
+				latencyMs,
+				sessionId: threadId,
+				tokenUsage: { numRequests: 1 },
+				metadata: {
+					workflowId: this.chatKitConfig.workflowId,
+					version: this.chatKitConfig.version,
+					poolMode: true
+				}
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			logger.error("[ChatKitProvider] Pool call failed", { error: errorMessage });
+			if (errorMessage.includes("Timeout") || errorMessage.includes("timeout")) return { error: `ChatKit response timeout after ${this.chatKitConfig.timeout}ms. Try increasing timeout or reducing concurrency.` };
+			return { error: `ChatKit provider error: ${errorMessage}` };
+		} finally {
+			if (pooledPage) await pool.releasePage(pooledPage);
+		}
+	}
+};
+//#endregion
+export { OpenAiChatKitProvider };
+
+//# sourceMappingURL=chatkit-swAIVuea.js.map