@evalstudio/core 0.2.0

package/dist/run-processor.js

@@ -0,0 +1,495 @@
+import { getEval } from "./eval.js";
+import { invokeConnector, } from "./connector.js";
+import { getLLMProvider } from "./llm-provider.js";
+import { getPersona } from "./persona.js";
+import { getProject } from "./project.js";
+import { getScenario } from "./scenario.js";
+import { getRun, listRuns, updateRun, } from "./run.js";
+import { buildTestAgentSystemPrompt } from "./prompt.js";
+import { evaluateCriteria } from "./evaluator.js";
+import { generatePersonaMessage } from "./persona-generator.js";
+/**
+ * Background processor for executing queued evaluation runs.
+ *
+ * The RunProcessor polls for runs with status "queued" and executes them
+ * via the configured connector. It supports concurrent execution and provides
+ * callbacks for monitoring status changes.
+ *
+ * Works from both CLI and API contexts - the same processor logic can be used
+ * with different status update mechanisms (terminal output vs WebSocket).
+ *
+ * @example
+ * ```typescript
+ * const processor = new RunProcessor({
+ *   pollIntervalMs: 5000,
+ *   maxConcurrent: 3,
+ *   onStatusChange: (runId, status, run) => {
+ *     console.log(`Run ${runId} is now ${status}`);
+ *   },
+ * });
+ *
+ * processor.start();
+ *
+ * // Later: graceful shutdown
+ * await processor.stop();
+ * ```
+ */
+export class RunProcessor {
+    running = false;
+    intervalId = null;
+    activeRuns = new Map();
+    options;
+    constructor(options = {}) {
+        this.options = {
+            pollIntervalMs: options.pollIntervalMs ?? 5000,
+            maxConcurrent: options.maxConcurrent ?? 3,
+            projectId: options.projectId,
+            onStatusChange: options.onStatusChange,
+            onRunStart: options.onRunStart,
+            onRunComplete: options.onRunComplete,
+            onRunError: options.onRunError,
+        };
+    }
+    /**
+     * Starts the processor loop.
+     * Call this on server/CLI startup.
+     */
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        // Reset any "running" runs to "queued" (recovery from crash)
+        this.recoverStuckRuns();
+        // Start polling loop
+        this.intervalId = setInterval(() => this.tick(), this.options.pollIntervalMs);
+        // Immediate first tick
+        this.tick();
+    }
+    /**
+     * Stops the processor gracefully.
+     * Waits for active runs to complete.
+     */
+    async stop() {
+        this.running = false;
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        // Wait for active runs to complete
+        await Promise.all(this.activeRuns.values());
+    }
+    /**
+     * Process a single tick (useful for testing or one-shot processing).
+     * Returns the number of runs started and waits for them to complete.
+     */
+    async processOnce() {
+        return this.tick(true);
+    }
+    /**
+     * Returns true if the processor is currently running.
+     */
+    isRunning() {
+        return this.running;
+    }
+    /**
+     * Returns the number of currently active runs.
+     */
+    getActiveRunCount() {
+        return this.activeRuns.size;
+    }
+    /**
+     * Main processing tick - picks up queued runs and executes them.
+     * Returns the number of runs started.
+     * @param oneShot If true, waits for runs to complete before returning
+     */
+    async tick(oneShot = false) {
+        // Skip if not running and no active runs (unless one-shot mode)
+        if (!oneShot && !this.running && this.activeRuns.size === 0)
+            return 0;
+        // Calculate available slots
+        const availableSlots = this.options.maxConcurrent - this.activeRuns.size;
+        if (availableSlots <= 0)
+            return 0;
+        // Get queued runs (filtered by project if specified)
+        const queuedRuns = listRuns({
+            status: "queued",
+            projectId: this.options.projectId,
+            limit: availableSlots,
+        });
+        let started = 0;
+        const promises = [];
+        for (const run of queuedRuns) {
+            if (this.activeRuns.has(run.id))
+                continue;
+            // Attempt to claim the run atomically
+            if (!this.claimRun(run.id)) {
+                continue; // Already claimed by another processor
+            }
+            // Start execution
+            const promise = this.executeRun(run);
+            this.activeRuns.set(run.id, promise);
+            started++;
+            if (oneShot) {
+                promises.push(promise);
+            }
+            // Clean up when done
+            promise.finally(() => this.activeRuns.delete(run.id));
+        }
+        // In one-shot mode, wait for all runs to complete
+        if (oneShot && promises.length > 0) {
+            await Promise.all(promises);
+        }
+        return started;
+    }
+    /**
+     * Atomically claims a run for processing.
+     * Returns true if successful, false if already claimed.
+     */
+    claimRun(runId) {
+        const run = getRun(runId);
+        if (!run || run.status !== "queued") {
+            return false; // Already claimed or doesn't exist
+        }
+        // Update status atomically
+        const updated = updateRun(runId, {
+            status: "running",
+            startedAt: new Date().toISOString(),
+        });
+        return updated !== undefined;
+    }
+    /**
+     * Executes a single run with the evaluation loop.
+     *
+     * The loop:
+     * 1. Sends conversation to tested agent
+     * 2. Evaluates agent response against success/failure criteria
+     * 3. If success criteria met or failure criteria met, finish the run
+     * 4. If max messages reached, finish the run
+     * 5. Otherwise, generate a new persona message and continue
+     */
+    async executeRun(run) {
+        // Re-fetch run to get updated state after claim
+        let currentRun = getRun(run.id);
+        if (!currentRun) {
+            return;
+        }
+        try {
+            // Notify start
+            this.options.onStatusChange?.(currentRun.id, "running", currentRun);
+            this.options.onRunStart?.(currentRun);
+            // Determine connector ID
+            let connectorId;
+            if (currentRun.evalId) {
+                // Eval-based run
+                const evalItem = getEval(currentRun.evalId);
+                if (!evalItem) {
+                    throw new Error(`Eval not found: ${currentRun.evalId}`);
+                }
+                if (!evalItem.connectorId) {
+                    throw new Error("Eval has no connector assigned");
+                }
+                connectorId = evalItem.connectorId;
+            }
+            else {
+                // Playground run - connector stored on run
+                if (!currentRun.connectorId) {
+                    throw new Error("Playground run has no connector assigned");
+                }
+                connectorId = currentRun.connectorId;
+            }
+            // Get project for LLM settings
+            const project = getProject(currentRun.projectId);
+            if (!project) {
+                throw new Error(`Project not found: ${currentRun.projectId}`);
+            }
+            // Resolve LLM configuration from project settings
+            const projectSettings = project.llmSettings;
+            const evaluationProviderId = projectSettings?.evaluation?.providerId;
+            const evaluationModel = projectSettings?.evaluation?.model;
+            if (!evaluationProviderId) {
+                throw new Error("LLM Provider for evaluation is required. Configure in project Settings > LLM Defaults.");
+            }
+            // Validate evaluation provider exists
+            const evaluationProvider = getLLMProvider(evaluationProviderId);
+            if (!evaluationProvider) {
+                throw new Error(`LLM Provider "${evaluationProviderId}" not found`);
+            }
+            // Persona: use project persona settings, fallback to evaluation settings
+            const personaProviderId = projectSettings?.persona?.providerId || evaluationProviderId;
+            const personaModel = projectSettings?.persona?.model || evaluationModel;
+            // Validate persona provider exists
+            const personaProvider = getLLMProvider(personaProviderId);
+            if (!personaProvider) {
+                throw new Error(`LLM Provider "${personaProviderId}" not found`);
+            }
+            const llmConfig = {
+                evaluationProviderId,
+                evaluationModel,
+                personaProviderId,
+                personaModel,
+            };
+            // Get scenario and persona from stored IDs
+            const scenario = getScenario(currentRun.scenarioId);
+            if (!scenario) {
+                throw new Error(`Scenario not found: ${currentRun.scenarioId}`);
+            }
+            const persona = currentRun.personaId
+                ? getPersona(currentRun.personaId)
+                : undefined;
+            // Get eval input messages if this is an eval-based run
+            const evalInput = currentRun.evalId ? getEval(currentRun.evalId)?.input : undefined;
+            // Build all messages including system prompt
+            const allMessages = this.buildAllMessages(scenario, persona, evalInput);
+            // Store all initial messages in the run (so they're visible in UI)
+            const runWithMessages = updateRun(currentRun.id, {
+                messages: allMessages,
+            });
+            if (runWithMessages) {
+                currentRun = runWithMessages;
+            }
+            // Determine max messages (default to 10 if not specified)
+            const maxMessages = scenario.maxMessages ?? 10;
+            // Check if we have criteria to evaluate against
+            if (!scenario.successCriteria && !scenario.failureCriteria) {
+                throw new Error("Scenario must have success or failure criteria defined");
+            }
+            // Run the evaluation loop
+            await this.executeEvaluationLoop(currentRun, connectorId, llmConfig, scenario, persona, maxMessages);
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error("Unknown error");
+            const updatedRun = updateRun(currentRun.id, {
+                status: "error",
+                error: err.message,
+                completedAt: new Date().toISOString(),
+            });
+            if (updatedRun) {
+                this.options.onStatusChange?.(currentRun.id, "error", updatedRun);
+                this.options.onRunError?.(updatedRun, err);
+            }
+        }
+    }
+    /**
+     * Gets the thread ID for LangGraph.
+     * Uses the stored threadId if available (set on retry), otherwise uses run.id.
+     */
+    getThreadId(run) {
+        return run.threadId || run.id;
+    }
+    /**
+     * Gets the role of the last non-system message.
+     * Used to determine if we need to generate a persona message before invoking the connector.
+     */
+    getLastMessageRole(messages) {
+        const nonSystemMessages = messages.filter((m) => m.role !== "system");
+        if (nonSystemMessages.length === 0)
+            return undefined;
+        return nonSystemMessages[nonSystemMessages.length - 1].role;
+    }
+    /**
+     * Executes the main evaluation loop.
+     * Both success and failure criteria are evaluated at every turn.
+     * The loop stops when:
+     * - Success criteria is met (run succeeds)
+     * - Failure criteria is met AND failureCriteriaMode is "every_turn" (run fails early)
+     * - Max messages reached without success (run fails; default mode "on_max_messages")
+     */
+    async executeEvaluationLoop(currentRun, connectorId, llmConfig, scenario, persona, maxMessages) {
+        let messages = [...currentRun.messages];
+        let totalLatencyMs = 0;
+        let connectorCallCount = 0;
+        let lastResult;
+        let finalEvaluation;
+        // Track messages already in the LangGraph thread (for continuation calls)
+        let threadMessageCount = 0;
+        // Count only user and assistant messages for the limit (exclude system)
+        const countConversationMessages = () => messages.filter((m) => m.role === "user" || m.role === "assistant").length;
+        // Generate an initial user message if there are no conversation messages
+        // or the last seed message is from the assistant
+        const lastRole = this.getLastMessageRole(messages);
+        if (!lastRole || lastRole === "assistant") {
+            const personaResponse = await generatePersonaMessage({
+                messages,
+                persona,
+                scenario,
+                llmProviderId: llmConfig.personaProviderId,
+                model: llmConfig.personaModel,
+            });
+            const userMessage = {
+                role: "user",
+                content: personaResponse.content,
+            };
+            messages = [...messages, userMessage];
+            // Update run with the generated persona message
+            updateRun(currentRun.id, { messages });
+        }
+        while (countConversationMessages() < maxMessages) {
+            // Build conversation messages (without system prompt) for connector
+            const conversationMessages = messages.filter((m) => m.role !== "system");
+            // Invoke the connector (send to tested agent, pass thread ID for LangGraph)
+            // threadMessageCount tells LangGraph strategy how many messages are already in the thread
+            const result = await invokeConnector(connectorId, {
+                messages: conversationMessages,
+                runId: this.getThreadId(currentRun),
+                threadMessageCount,
+            });
+            lastResult = result;
+            if (!result.success || !result.messages || result.messages.length === 0) {
+                throw new Error(result.error || "No response messages from connector");
+            }
+            // Add all agent response messages (may include tool calls, tool results, and final response)
+            messages = [...messages, ...result.messages];
+            // Update thread message count for next iteration (thread now has all conversation messages)
+            threadMessageCount = messages.filter((m) => m.role !== "system").length;
+            totalLatencyMs += result.latencyMs;
+            connectorCallCount++;
+            // Update run with current messages
+            updateRun(currentRun.id, { messages });
+            // Evaluate the conversation against criteria
+            const evaluation = await evaluateCriteria({
+                messages,
+                successCriteria: scenario.successCriteria,
+                failureCriteria: scenario.failureCriteria,
+                llmProviderId: llmConfig.evaluationProviderId,
+                model: llmConfig.evaluationModel,
+            });
+            finalEvaluation = evaluation;
+            // Determine failure criteria check mode (default: "every_turn")
+            const failureMode = scenario.failureCriteriaMode ?? "on_max_messages";
+            // Stop the loop on success, or on failure when failureCriteriaMode is "every_turn"
+            if (evaluation.successMet || (evaluation.failureMet && failureMode === "every_turn")) {
+                const runResult = {
+                    success: evaluation.successMet,
+                    score: evaluation.confidence,
+                    reason: evaluation.successMet
+                        ? evaluation.reasoning
+                        : `Failure criteria was triggered. ${evaluation.reasoning}`,
+                };
+                const updatedRun = updateRun(currentRun.id, {
+                    status: "completed",
+                    messages,
+                    result: runResult,
+                    output: {
+                        avgLatencyMs: connectorCallCount > 0 ? Math.round(totalLatencyMs / connectorCallCount) : 0,
+                        totalLatencyMs,
+                        messageCount: countConversationMessages(),
+                        evaluation: {
+                            successMet: evaluation.successMet,
+                            failureMet: evaluation.failureMet,
+                            confidence: evaluation.confidence,
+                            reasoning: evaluation.reasoning,
+                        },
+                    },
+                    completedAt: new Date().toISOString(),
+                });
+                if (updatedRun && lastResult) {
+                    this.options.onStatusChange?.(currentRun.id, "completed", updatedRun);
+                    this.options.onRunComplete?.(updatedRun, lastResult);
+                }
+                return;
+            }
+            // Check if we've reached max messages
+            if (countConversationMessages() >= maxMessages) {
+                break;
+            }
+            // Generate a new user message to continue the conversation
+            // If persona is provided, the message will be personalized; otherwise generic
+            const personaResponse = await generatePersonaMessage({
+                messages,
+                persona,
+                scenario,
+                llmProviderId: llmConfig.personaProviderId,
+                model: llmConfig.personaModel,
+            });
+            // Add persona message as user message
+            const userMessage = {
+                role: "user",
+                content: personaResponse.content,
+            };
+            messages = [...messages, userMessage];
+            // Update run with new user message
+            updateRun(currentRun.id, { messages });
+        }
+        // Max messages reached without meeting success criteria
+        const failureTriggered = finalEvaluation?.failureMet ?? false;
+        const runResult = {
+            success: false,
+            score: finalEvaluation?.confidence ?? 0,
+            reason: failureTriggered
+                ? `Failure criteria was triggered. ${finalEvaluation?.reasoning || ""}`
+                : `Max messages (${maxMessages}) reached without meeting success criteria. ${finalEvaluation?.reasoning || ""}`,
+        };
+        const updatedRun = updateRun(currentRun.id, {
+            status: "completed",
+            messages,
+            result: runResult,
+            output: {
+                avgLatencyMs: connectorCallCount > 0 ? Math.round(totalLatencyMs / connectorCallCount) : 0,
+                totalLatencyMs,
+                messageCount: countConversationMessages(),
+                maxMessagesReached: true,
+                evaluation: finalEvaluation
+                    ? {
+                        successMet: finalEvaluation.successMet,
+                        failureMet: finalEvaluation.failureMet,
+                        confidence: finalEvaluation.confidence,
+                        reasoning: finalEvaluation.reasoning,
+                    }
+                    : undefined,
+            },
+            completedAt: new Date().toISOString(),
+        });
+        if (updatedRun && lastResult) {
+            this.options.onStatusChange?.(currentRun.id, "completed", updatedRun);
+            this.options.onRunComplete?.(updatedRun, lastResult);
+        }
+    }
+    /**
+     * Builds all messages including system prompt for a run.
+     * These messages are stored in the run for visibility in the UI.
+     */
+    buildAllMessages(scenario, persona, evalInput) {
+        const messages = [];
+        // Add system prompt from persona/scenario
+        const systemPrompt = buildTestAgentSystemPrompt({
+            persona: persona
+                ? {
+                    name: persona.name,
+                    description: persona.description,
+                    systemPrompt: persona.systemPrompt,
+                }
+                : undefined,
+            scenario: {
+                name: scenario.name,
+                instructions: scenario.instructions,
+                messages: scenario.messages,
+            },
+        });
+        if (systemPrompt.trim()) {
+            messages.push({ role: "system", content: systemPrompt });
+        }
+        // Add scenario seed messages if present
+        if (scenario.messages) {
+            messages.push(...scenario.messages);
+        }
+        // Add eval input messages (only for eval-based runs)
+        if (Array.isArray(evalInput)) {
+            messages.push(...evalInput);
+        }
+        return messages;
+    }
+    /**
+     * Recovers runs that were interrupted by server crash.
+     * Only recovers runs for this processor's project filter.
+     */
+    recoverStuckRuns() {
+        const stuckRuns = listRuns({
+            status: "running",
+            projectId: this.options.projectId,
+        });
+        for (const run of stuckRuns) {
+            updateRun(run.id, { status: "queued" });
+        }
+    }
+}
+//# sourceMappingURL=run-processor.js.map

package/dist/run-processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-processor.js","sourceRoot":"","sources":["../src/run-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACL,eAAe,GAEhB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAgB,MAAM,cAAc,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAiB,MAAM,eAAe,CAAC;AAC3D,OAAO,EACL,MAAM,EACN,QAAQ,EACR,SAAS,GAIV,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAiC,MAAM,gBAAgB,CAAC;AACjF,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAyChE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,YAAY;IACf,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAA0B,IAAI,CAAC;IACzC,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC9C,OAAO,CAAkB;IAEjC,YAAY,UAA+B,EAAE;QAC3C,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI;YAC9C,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC;YACzC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,6DAA6D;QAC7D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAExB,qBAAqB;QACrB,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE9E,uBAAuB;QACvB,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK;QAChC,gEAAgE;QAChE,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEtE,4BAA4B;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QACzE,IAAI,cAAc,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,qDAAqD;QACrD,MAAM,UAAU,GAAG,QAAQ,CAAC;YAC1B,MAAM,EAAE,QAAQ;YAChB,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YACjC,KAAK,EAAE,cAAc;SACtB,CAAC,CAAC;QAEH,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAoB,EAAE,CAAC;QAErC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAE1C,sCAAsC;YACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC3B,SAAS,CAAC,uCAAuC;YACnD,CAAC;YAED,kBAAkB;YAClB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrC,OAAO,EAAE,CAAC;YAEV,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;YAED,qBAAqB;YACrB,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,kDAAkD;QAClD,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,QAAQ,CAAC,KAAa;QAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC,CAAC,mCAAmC;QACnD,CAAC;QAED,2BAA2B;QAC3B,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE;YAC/B,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,OAAO,OAAO,KAAK,SAAS,CAAC;IAC/B,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC/B,gDAAgD;QAChD,IAAI,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,eAAe;YACf,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,CAAC;YAEtC,yBAAyB;YACzB,IAAI,WAAmB,CAAC;YAExB,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACtB,iBAAiB;gBACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CAAC,mBAAmB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC1D,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACpD,CAAC;gBACD,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC9D,CAAC;gBACD,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;YACvC,CAAC;YAED,+BAA+B;YAC/B,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YACjD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,sBAAsB,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;YAChE,CAAC;YAED,kDAAkD;YAClD,MAAM,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;YAC5C,MAAM,oBAAoB,GAAG,eAAe,EAAE,UAAU,EAAE,UAAU,CAAC;YACrE,MAAM,eAAe,GAAG,eAAe,EAAE,UAAU,EAAE,KAAK,CAAC;YAE3D,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;YACJ,CAAC;YAED,sCAAsC;YACtC,MAAM,kBAAkB,GAAG,cAAc,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CAAC,iBAAiB,oBAAoB,aAAa,CAAC,CAAC;YACtE,CAAC;YAED,yEAAyE;YACzE,MAAM,iBAAiB,GACrB,eAAe,EAAE,OAAO,EAAE,UAAU,IAAI,oBAAoB,CAAC;YAC/D,MAAM,YAAY,GAChB,eAAe,EAAE,OAAO,EAAE,KAAK,IAAI,eAAe,CAAC;YAErD,mCAAmC;YACnC,MAAM,eAAe,GAAG,cAAc,CAAC,iBAAiB,CAAC,CAAC;YAC1D,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,iBAAiB,iBAAiB,aAAa,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,SAAS,GAAsB;gBACnC,oBAAoB;gBACpB,eAAe;gBACf,iBAAiB;gBACjB,YAAY;aACb,CAAC;YAEF,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS;gBAClC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC;gBAClC,CAAC,CAAC,SAAS,CAAC;YAEd,uDAAuD;YACvD,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAEpF,6CAA6C;YAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAExE,mEAAmE;YACnE,MAAM,eAAe,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE;gBAC/C,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;YACH,IAAI,eAAe,EAAE,CAAC;gBACpB,UAAU,GAAG,eAAe,CAAC;YAC/B,CAAC;YAED,0DAA0D;YAC1D,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;YAE/C,gDAAgD;YAChD,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC;gBAC3D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;YAC5E,CAAC;YAED,0BAA0B;YAC1B,MAAM,IAAI,CAAC,qBAAqB,CAC9B,UAAU,EACV,WAAW,EACX,SAAS,EACT,QAAQ,EACR,OAAO,EACP,WAAW,CACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YACxE,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE;gBAC1C,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,GAAG,CAAC,OAAO;gBAClB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACtC,CAAC,CAAC;YAEH,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;gBAClE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,GAAQ;QAC1B,OAAO,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,QAAmB;QAC5C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QACrD,OAAO,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,qBAAqB,CACjC,UAAe,EACf,WAAmB,EACnB,SAA4B,EAC5B,QAAkB,EAClB,OAA4B,EAC5B,WAAmB;QAEnB,IAAI,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,UAA6C,CAAC;QAClD,IAAI,eAAqD,CAAC;QAE1D,0EAA0E;QAC1E,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,wEAAwE;QACxE,MAAM,yBAAyB,GAAG,GAAG,EAAE,CACrC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAE7E,yEAAyE;QACzE,iDAAiD;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YAC1C,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC;gBACnD,QAAQ;gBACR,OAAO;gBACP,QAAQ;gBACR,aAAa,EAAE,SAAS,CAAC,iBAAiB;gBAC1C,KAAK,EAAE,SAAS,CAAC,YAAY;aAC9B,CAAC,CAAC;YAEH,MAAM,WAAW,GAAY;gBAC3B,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,eAAe,CAAC,OAAO;aACjC,CAAC;YACF,QAAQ,GAAG,CAAC,GAAG,QAAQ,EAAE,WAAW,CAAC,CAAC;YAEtC,gDAAgD;YAChD,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,OAAO,yBAAyB,EAAE,GAAG,WAAW,EAAE,CAAC;YACjD,oEAAoE;YACpE,MAAM,oBAAoB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;YAEzE,4EAA4E;YAC5E,0FAA0F;YAC1F,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE;gBAChD,QAAQ,EAAE,oBAAoB;gBAC9B,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC;gBACnC,kBAAkB;aACnB,CAAC,CAAC;YACH,UAAU,GAAG,MAAM,CAAC;YAEpB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,qCAAqC,CAAC,CAAC;YACzE,CAAC;YAED,6FAA6F;YAC7F,QAAQ,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAE7C,4FAA4F;YAC5F,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;YACxE,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC;YACnC,kBAAkB,EAAE,CAAC;YAErB,mCAAmC;YACnC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YAEvC,6CAA6C;YAC7C,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC;gBACxC,QAAQ;gBACR,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,aAAa,EAAE,SAAS,CAAC,oBAAoB;gBAC7C,KAAK,EAAE,SAAS,CAAC,eAAe;aACjC,CAAC,CAAC;YACH,eAAe,GAAG,UAAU,CAAC;YAE7B,gEAAgE;YAChE,MAAM,WAAW,GAAG,QAAQ,CAAC,mBAAmB,IAAI,iBAAiB,CAAC;YAEtE,mFAAmF;YACnF,IAAI,UAAU,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,WAAW,KAAK,YAAY,CAAC,EAAE,CAAC;gBACrF,MAAM,SAAS,GAAc;oBAC3B,OAAO,EAAE,UAAU,CAAC,UAAU;oBAC9B,KAAK,EAAE,UAAU,CAAC,UAAU;oBAC5B,MAAM,EAAE,UAAU,CAAC,UAAU;wBAC3B,CAAC,CAAC,UAAU,CAAC,SAAS;wBACtB,CAAC,CAAC,mCAAmC,UAAU,CAAC,SAAS,EAAE;iBAC9D,CAAC;gBAEF,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE;oBAC1C,MAAM,EAAE,WAAW;oBACnB,QAAQ;oBACR,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE;wBACN,YAAY,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC1F,cAAc;wBACd,YAAY,EAAE,yBAAyB,EAAE;wBACzC,UAAU,EAAE;4BACV,UAAU,EAAE,UAAU,CAAC,UAAU;4BACjC,UAAU,EAAE,UAAU,CAAC,UAAU;4BACjC,UAAU,EAAE,UAAU,CAAC,UAAU;4BACjC,SAAS,EAAE,UAAU,CAAC,SAAS;yBAChC;qBACF;oBACD,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC,CAAC,CAAC;gBAEH,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;oBACtE,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;gBACvD,CAAC;gBACD,OAAO;YACT,CAAC;YAED,sCAAsC;YACtC,IAAI,yBAAyB,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC/C,MAAM;YACR,CAAC;YAED,2DAA2D;YAC3D,8EAA8E;YAC9E,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC;gBACnD,QAAQ;gBACR,OAAO;gBACP,QAAQ;gBACR,aAAa,EAAE,SAAS,CAAC,iBAAiB;gBAC1C,KAAK,EAAE,SAAS,CAAC,YAAY;aAC9B,CAAC,CAAC;YAEH,sCAAsC;YACtC,MAAM,WAAW,GAAY;gBAC3B,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,eAAe,CAAC,OAAO;aACjC,CAAC;YACF,QAAQ,GAAG,CAAC,GAAG,QAAQ,EAAE,WAAW,CAAC,CAAC;YAEtC,mCAAmC;YACnC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,wDAAwD;QACxD,MAAM,gBAAgB,GAAG,eAAe,EAAE,UAAU,IAAI,KAAK,CAAC;QAC9D,MAAM,SAAS,GAAc;YAC3B,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,eAAe,EAAE,UAAU,IAAI,CAAC;YACvC,MAAM,EAAE,gBAAgB;gBACtB,CAAC,CAAC,mCAAmC,eAAe,EAAE,SAAS,IAAI,EAAE,EAAE;gBACvE,CAAC,CAAC,iBAAiB,WAAW,+CAA+C,eAAe,EAAE,SAAS,IAAI,EAAE,EAAE;SAClH,CAAC;QAEF,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE;YAC1C,MAAM,EAAE,WAAW;YACnB,QAAQ;YACR,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE;gBACN,YAAY,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1F,cAAc;gBACd,YAAY,EAAE,yBAAyB,EAAE;gBACzC,kBAAkB,EAAE,IAAI;gBACxB,UAAU,EAAE,eAAe;oBACzB,CAAC,CAAC;wBACE,UAAU,EAAE,eAAe,CAAC,UAAU;wBACtC,UAAU,EAAE,eAAe,CAAC,UAAU;wBACtC,UAAU,EAAE,eAAe,CAAC,UAAU;wBACtC,SAAS,EAAE,eAAe,CAAC,SAAS;qBACrC;oBACH,CAAC,CAAC,SAAS;aACd;YACD,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC,CAAC,CAAC;QAEH,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,gBAAgB,CACtB,QAAkB,EAClB,OAA4B,EAC5B,SAA+C;QAE/C,MAAM,QAAQ,GAAc,EAAE,CAAC;QAE/B,0CAA0C;QAC1C,MAAM,YAAY,GAAG,0BAA0B,CAAC;YAC9C,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC;oBACE,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,YAAY,EAAE,OAAO,CAAC,YAAY;iBACnC;gBACH,CAAC,CAAC,SAAS;YACb,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,YAAY,EAAE,QAAQ,CAAC,YAAY;gBACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B;SACF,CAAC,CAAC;QACH,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,wCAAwC;QACxC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,qDAAqD;QACrD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CAAC,GAAI,SAAuB,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,gBAAgB;QACtB,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;SAClC,CAAC,CAAC;QAEH,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;CACF"}

package/dist/run.d.ts

@@ -0,0 +1,101 @@
+import { type Message } from "./eval.js";
+/**
+ * Run status types:
+ * - queued: Waiting to be processed
+ * - pending: Reserved for future use
+ * - running: Currently executing
+ * - completed: Finished (check result.success for pass/fail)
+ * - error: System error occurred (retryable)
+ */
+export type RunStatus = "queued" | "pending" | "running" | "completed" | "error";
+export interface RunResult {
+    success: boolean;
+    score?: number;
+    reason?: string;
+}
+export interface RunMetadata {
+    latencyMs?: number;
+    tokenUsage?: {
+        input: number;
+        output: number;
+    };
+    [key: string]: unknown;
+}
+export interface Run {
+    id: string;
+    /** Eval ID (optional for playground runs) */
+    evalId?: string;
+    projectId: string;
+    personaId?: string;
+    scenarioId: string;
+    /** Connector ID (for playground runs without eval) */
+    connectorId?: string;
+    /** Execution ID - groups runs created together in a single execution */
+    executionId?: number;
+    status: RunStatus;
+    startedAt?: string;
+    completedAt?: string;
+    messages: Message[];
+    output?: Record<string, unknown>;
+    result?: RunResult;
+    error?: string;
+    metadata?: RunMetadata;
+    /** Thread ID for LangGraph (regenerated on retry to start fresh thread) */
+    threadId?: string;
+    createdAt: string;
+    updatedAt: string;
+}
+export interface CreateRunInput {
+    evalId: string;
+}
+export interface CreatePlaygroundRunInput {
+    scenarioId: string;
+    connectorId: string;
+    personaId?: string;
+}
+export interface UpdateRunInput {
+    status?: RunStatus;
+    startedAt?: string;
+    completedAt?: string;
+    messages?: Message[];
+    output?: Record<string, unknown>;
+    result?: RunResult;
+    error?: string;
+    metadata?: RunMetadata;
+    threadId?: string;
+}
+export declare function createRuns(input: CreateRunInput): Run[];
+export declare function createRun(input: CreateRunInput): Run;
+/**
+ * Creates a playground run for testing scenarios without an eval.
+ * Used by the Scenario Playground to create runs directly.
+ */
+export declare function createPlaygroundRun(input: CreatePlaygroundRunInput): Run;
+export declare function getRun(id: string): Run | undefined;
+export interface ListRunsOptions {
+    evalId?: string;
+    projectId?: string;
+    scenarioId?: string;
+    status?: RunStatus;
+    limit?: number;
+}
+export declare function listRuns(options?: ListRunsOptions): Run[];
+export declare function listRuns(evalId?: string, projectId?: string): Run[];
+export declare function listRunsByEval(evalId: string): Run[];
+export declare function listRunsByScenario(scenarioId: string): Run[];
+export declare function listRunsByPersona(personaId: string): Run[];
+export declare function updateRun(id: string, input: UpdateRunInput): Run | undefined;
+export declare function deleteRun(id: string): boolean;
+export declare function deleteRunsByEval(evalId: string): number;
+export declare function deleteRunsByProject(projectId: string): number;
+/**
+ * Retries a failed run by resetting it to queued status.
+ * Clears error, timing info, messages, and output.
+ * Increments retryCount to ensure a fresh LangGraph thread is used.
+ *
+ * @param id - The run ID to retry
+ * @returns The updated run, or undefined if not found
+ * @throws Error if the run status is not "failed"
+ */
+export declare function retryRun(id: string): Run | undefined;
+//# sourceMappingURL=run.d.ts.map

package/dist/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAW,KAAK,OAAO,EAAE,MAAM,WAAW,CAAC;AAMlD;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;AAEjF,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAoBD,wBAAgB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,GAAG,EAAE,CA2EvD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,GAAG,CAGpD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,GAAG,CA2CxE;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAGlD;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,QAAQ,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,GAAG,EAAE,CAAC;AAC3D,wBAAgB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC;AAwDrE,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAQpD;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,EAAE,CAQ5D;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE,CAQ1D;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,GAAG,GAAG,SAAS,CA6B5E;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAY7C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAUvD;AAED,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAU7D;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CA2BpD"}