@evalstudio/core 0.2.0

package/dist/run.js

@@ -0,0 +1,279 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getConnector } from "./connector.js";
+import { getEval } from "./eval.js";
+import { createExecution } from "./execution.js";
+import { getPersona } from "./persona.js";
+import { getScenario } from "./scenario.js";
+import { getStorageDir } from "./storage.js";
+function getStoragePath() {
+    return join(getStorageDir(), "runs.json");
+}
+function loadRuns() {
+    const path = getStoragePath();
+    if (!existsSync(path)) {
+        return [];
+    }
+    const data = readFileSync(path, "utf-8");
+    return JSON.parse(data);
+}
+function saveRuns(runs) {
+    const path = getStoragePath();
+    writeFileSync(path, JSON.stringify(runs, null, 2));
+}
+export function createRuns(input) {
+    const evalItem = getEval(input.evalId);
+    if (!evalItem) {
+        throw new Error(`Eval with id "${input.evalId}" not found`);
+    }
+    // Validate all scenarios exist
+    const scenarios = evalItem.scenarioIds.map((scenarioId) => {
+        const scenario = getScenario(scenarioId);
+        if (!scenario) {
+            throw new Error(`Scenario with id "${scenarioId}" not found`);
+        }
+        return scenario;
+    });
+    if (scenarios.length === 0) {
+        throw new Error("Eval has no scenarios configured");
+    }
+    // Collect all unique persona IDs and validate they exist
+    const allPersonaIds = new Set();
+    for (const scenario of scenarios) {
+        if (scenario.personaIds) {
+            for (const personaId of scenario.personaIds) {
+                allPersonaIds.add(personaId);
+            }
+        }
+    }
+    for (const personaId of allPersonaIds) {
+        const persona = getPersona(personaId);
+        if (!persona) {
+            throw new Error(`Persona with id "${personaId}" not found`);
+        }
+    }
+    const allRuns = loadRuns();
+    const now = new Date().toISOString();
+    const createdRuns = [];
+    // Create an execution to group all runs created in this batch
+    const execution = createExecution({
+        projectId: evalItem.projectId,
+        evalId: input.evalId,
+    });
+    // Create runs for each scenario/persona combination
+    for (const scenario of scenarios) {
+        // Determine personas for this scenario: use scenario.personaIds, or [undefined] if empty
+        const personaIds = scenario.personaIds && scenario.personaIds.length > 0
+            ? scenario.personaIds
+            : [undefined];
+        for (const personaId of personaIds) {
+            const run = {
+                id: randomUUID(),
+                evalId: input.evalId,
+                projectId: evalItem.projectId,
+                personaId,
+                scenarioId: scenario.id,
+                executionId: execution.id,
+                status: "queued",
+                messages: [],
+                createdAt: now,
+                updatedAt: now,
+            };
+            allRuns.push(run);
+            createdRuns.push(run);
+        }
+    }
+    saveRuns(allRuns);
+    return createdRuns;
+}
+export function createRun(input) {
+    const runs = createRuns(input);
+    return runs[0];
+}
+/**
+ * Creates a playground run for testing scenarios without an eval.
+ * Used by the Scenario Playground to create runs directly.
+ */
+export function createPlaygroundRun(input) {
+    const { scenarioId, connectorId, personaId } = input;
+    // Validate scenario exists
+    const scenario = getScenario(scenarioId);
+    if (!scenario) {
+        throw new Error(`Scenario with id "${scenarioId}" not found`);
+    }
+    // Validate connector exists
+    const connector = getConnector(connectorId);
+    if (!connector) {
+        throw new Error(`Connector with id "${connectorId}" not found`);
+    }
+    // Validate persona if provided
+    if (personaId) {
+        const persona = getPersona(personaId);
+        if (!persona) {
+            throw new Error(`Persona with id "${personaId}" not found`);
+        }
+    }
+    const allRuns = loadRuns();
+    const now = new Date().toISOString();
+    const run = {
+        id: randomUUID(),
+        // No evalId for playground runs
+        projectId: scenario.projectId,
+        scenarioId,
+        connectorId,
+        personaId,
+        status: "queued",
+        messages: [],
+        createdAt: now,
+        updatedAt: now,
+    };
+    allRuns.push(run);
+    saveRuns(allRuns);
+    return run;
+}
+export function getRun(id) {
+    const runs = loadRuns();
+    return runs.find((r) => r.id === id);
+}
+export function listRuns(evalIdOrOptions, projectId) {
+    const runs = loadRuns();
+    // Handle new object-based API
+    if (typeof evalIdOrOptions === "object" && evalIdOrOptions !== null) {
+        const { evalId, projectId, scenarioId, status, limit } = evalIdOrOptions;
+        let filtered = runs;
+        if (evalId) {
+            filtered = filtered.filter((r) => r.evalId === evalId);
+        }
+        if (projectId) {
+            filtered = filtered.filter((r) => r.projectId === projectId);
+        }
+        if (scenarioId) {
+            filtered = filtered.filter((r) => r.scenarioId === scenarioId);
+        }
+        if (status) {
+            filtered = filtered.filter((r) => r.status === status);
+        }
+        // Sort by createdAt (oldest first for queue processing)
+        filtered.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+        if (limit && limit > 0) {
+            filtered = filtered.slice(0, limit);
+        }
+        return filtered;
+    }
+    // Legacy API compatibility
+    const evalId = evalIdOrOptions;
+    if (evalId) {
+        return runs.filter((r) => r.evalId === evalId);
+    }
+    if (projectId) {
+        return runs.filter((r) => r.projectId === projectId);
+    }
+    return runs;
+}
+export function listRunsByEval(evalId) {
+    const runs = loadRuns();
+    return runs
+        .filter((r) => r.evalId === evalId)
+        .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+}
+export function listRunsByScenario(scenarioId) {
+    const runs = loadRuns();
+    return runs
+        .filter((r) => r.scenarioId === scenarioId)
+        .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+}
+export function listRunsByPersona(personaId) {
+    const runs = loadRuns();
+    return runs
+        .filter((r) => r.personaId === personaId)
+        .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+}
+export function updateRun(id, input) {
+    const runs = loadRuns();
+    const index = runs.findIndex((r) => r.id === id);
+    if (index === -1) {
+        return undefined;
+    }
+    const run = runs[index];
+    // Use "in" check to allow explicitly setting fields to undefined (to clear them)
+    const updated = {
+        ...run,
+        status: input.status ?? run.status,
+        startedAt: "startedAt" in input ? input.startedAt : run.startedAt,
+        completedAt: "completedAt" in input ? input.completedAt : run.completedAt,
+        messages: input.messages ?? run.messages,
+        output: "output" in input ? input.output : run.output,
+        result: "result" in input ? input.result : run.result,
+        error: "error" in input ? input.error : run.error,
+        metadata: "metadata" in input ? input.metadata : run.metadata,
+        threadId: "threadId" in input ? input.threadId : run.threadId,
+        updatedAt: new Date().toISOString(),
+    };
+    runs[index] = updated;
+    saveRuns(runs);
+    return updated;
+}
+export function deleteRun(id) {
+    const runs = loadRuns();
+    const index = runs.findIndex((r) => r.id === id);
+    if (index === -1) {
+        return false;
+    }
+    runs.splice(index, 1);
+    saveRuns(runs);
+    return true;
+}
+export function deleteRunsByEval(evalId) {
+    const runs = loadRuns();
+    const filtered = runs.filter((r) => r.evalId !== evalId);
+    const deletedCount = runs.length - filtered.length;
+    if (deletedCount > 0) {
+        saveRuns(filtered);
+    }
+    return deletedCount;
+}
+export function deleteRunsByProject(projectId) {
+    const runs = loadRuns();
+    const filtered = runs.filter((r) => r.projectId !== projectId);
+    const deletedCount = runs.length - filtered.length;
+    if (deletedCount > 0) {
+        saveRuns(filtered);
+    }
+    return deletedCount;
+}
+/**
+ * Retries a failed run by resetting it to queued status.
+ * Clears error, timing info, messages, and output.
+ * Increments retryCount to ensure a fresh LangGraph thread is used.
+ *
+ * @param id - The run ID to retry
+ * @returns The updated run, or undefined if not found
+ * @throws Error if the run status is not "failed"
+ */
+export function retryRun(id) {
+    const run = getRun(id);
+    if (!run) {
+        return undefined;
+    }
+    // Only allow retry on error runs (system failures)
+    // Completed runs with result.success=false are evaluation failures and should not be retried
+    if (run.status !== "error") {
+        throw new Error(`Cannot retry run with status "${run.status}". Only runs with system errors can be retried.`);
+    }
+    const updates = {
+        status: "queued",
+        error: undefined,
+        startedAt: undefined,
+        completedAt: undefined,
+        result: undefined,
+        metadata: undefined,
+        output: undefined,
+        // Always clear messages - they'll be rebuilt when the run starts
+        messages: [],
+        // Generate a new thread ID to start fresh in LangGraph
+        threadId: randomUUID(),
+    };
+    return updateRun(id, updates);
+}
+//# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAgB,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AA0E7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,WAAW,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,QAAQ;IACf,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AACnC,CAAC;AAED,SAAS,QAAQ,CAAC,IAAW;IAC3B,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAqB;IAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;IAC9D,CAAC;IAED,+BAA+B;IAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QACxD,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,aAAa,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAED,yDAAyD;IACzD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAC5C,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,oBAAoB,SAAS,aAAa,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,WAAW,GAAU,EAAE,CAAC;IAE9B,8DAA8D;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC;IAEH,oDAAoD;IACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,yFAAyF;QACzF,MAAM,UAAU,GACd,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YACnD,CAAC,CAAC,QAAQ,CAAC,UAAU;YACrB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAElB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAQ;gBACf,EAAE,EAAE,UAAU,EAAE;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,SAAS;gBACT,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,GAAG;gBACd,SAAS,EAAE,GAAG;aACf,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClB,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAqB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IAErD,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,aAAa,CAAC,CAAC;IAChE,CAAC;IAED,4BAA4B;IAC5B,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,WAAW,aAAa,CAAC,CAAC;IAClE,CAAC;IAED,+BAA+B;IAC/B,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,oBAAoB,SAAS,aAAa,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,MAAM,GAAG,GAAQ;QACf,EAAE,EAAE,UAAU,EAAE;QAChB,gCAAgC;QAChC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU;QACV,WAAW;QACX,SAAS;QACT,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,EAAE;QACZ,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClB,QAAQ,CAAC,OAAO,CAAC,CAAC;IAElB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,EAAU;IAC/B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACvC,CAAC;AAYD,MAAM,UAAU,QAAQ,CACtB,eAA0C,EAC1C,SAAkB;IAElB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IAExB,8BAA8B;IAC9B,IAAI,OAAO,eAAe,KAAK,QAAQ,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QACpE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC;QAEzE,IAAI,QAAQ,GAAG,IAAI,CAAC;QAEpB,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACzD,CAAC;QAED,wDAAwD;QACxD,QAAQ,CAAC,IAAI,CACX,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CACpE,CAAC;QAEF,IAAI,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAG,eAAe,CAAC;IAE/B,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;SAClC,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CACpE,CAAC;AACN,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACnD,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;SAC1C,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CACpE,CAAC;AACN,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;SACxC,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CACpE,CAAC;AACN,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAU,EAAE,KAAqB;IACzD,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEjD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAExB,iFAAiF;IACjF,MAAM,OAAO,GAAQ;QACnB,GAAG,GAAG;QACN,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM;QAClC,SAAS,EAAE,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS;QACjE,WAAW,EAAE,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW;QACzE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ;QACxC,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM;QACrD,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM;QACrD,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;QACjD,QAAQ,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ;QAC7D,QAAQ,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ;QAC7D,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEf,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAU;IAClC,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEjD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEf,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEnD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,SAAiB;IACnD,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEnD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACvB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mDAAmD;IACnD,6FAA6F;IAC7F,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,CAAC,MAAM,iDAAiD,CAAC,CAAC;IAChH,CAAC;IAED,MAAM,OAAO,GAAmB;QAC9B,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,SAAS;QAChB,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,SAAS;QACtB,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,SAAS;QACnB,MAAM,EAAE,SAAS;QACjB,iEAAiE;QACjE,QAAQ,EAAE,EAAE;QACZ,uDAAuD;QACvD,QAAQ,EAAE,UAAU,EAAE;KACvB,CAAC;IAEF,OAAO,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;AAChC,CAAC"}

package/dist/scenario.d.ts

@@ -0,0 +1,66 @@
+import type { Message } from "./types.js";
+export type { Message };
+/**
+ * Controls when failureCriteria is evaluated during a run:
+ * - "every_turn": Check at every turn and stop immediately on failure (like successCriteria)
+ * - "on_max_messages": Only check when maxMessages is reached without success
+ */
+export type FailureCriteriaMode = "every_turn" | "on_max_messages";
+export interface Scenario {
+    id: string;
+    projectId: string;
+    name: string;
+    instructions?: string;
+    messages?: Message[];
+    /** Maximum number of messages in a conversation before stopping */
+    maxMessages?: number;
+    /** Criteria for determining if the agent passed (used by evaluator) */
+    successCriteria?: string;
+    /** Criteria for determining if the agent failed (used by evaluator) */
+    failureCriteria?: string;
+    /** When to check failureCriteria: "on_max_messages" (default) or "every_turn" */
+    failureCriteriaMode?: FailureCriteriaMode;
+    /** IDs of personas associated with this scenario (optional, can be empty) */
+    personaIds?: string[];
+    createdAt: string;
+    updatedAt: string;
+}
+export interface CreateScenarioInput {
+    projectId: string;
+    name: string;
+    instructions?: string;
+    messages?: Message[];
+    /** Maximum number of messages in a conversation before stopping */
+    maxMessages?: number;
+    /** Criteria for determining if the agent passed */
+    successCriteria?: string;
+    /** Criteria for determining if the agent failed */
+    failureCriteria?: string;
+    /** When to check failureCriteria: "on_max_messages" (default) or "every_turn" */
+    failureCriteriaMode?: FailureCriteriaMode;
+    /** IDs of personas associated with this scenario */
+    personaIds?: string[];
+}
+export interface UpdateScenarioInput {
+    name?: string;
+    instructions?: string;
+    messages?: Message[];
+    /** Maximum number of messages in a conversation before stopping */
+    maxMessages?: number;
+    /** Criteria for determining if the agent passed */
+    successCriteria?: string;
+    /** Criteria for determining if the agent failed */
+    failureCriteria?: string;
+    /** When to check failureCriteria: "on_max_messages" (default) or "every_turn" */
+    failureCriteriaMode?: FailureCriteriaMode;
+    /** IDs of personas associated with this scenario */
+    personaIds?: string[];
+}
+export declare function createScenario(input: CreateScenarioInput): Scenario;
+export declare function getScenario(id: string): Scenario | undefined;
+export declare function getScenarioByName(projectId: string, name: string): Scenario | undefined;
+export declare function listScenarios(projectId?: string): Scenario[];
+export declare function updateScenario(id: string, input: UpdateScenarioInput): Scenario | undefined;
+export declare function deleteScenario(id: string): boolean;
+export declare function deleteScenariosByProject(projectId: string): number;
+//# sourceMappingURL=scenario.d.ts.map

package/dist/scenario.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.d.ts","sourceRoot":"","sources":["../src/scenario.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,YAAY,EAAE,OAAO,EAAE,CAAC;AAExB;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,iBAAiB,CAAC;AAEnE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAoBD,wBAAgB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,QAAQ,CAsCnE;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CAG5D;AAED,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,QAAQ,GAAG,SAAS,CAGtB;AAED,wBAAgB,aAAa,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAM5D;AAED,wBAAgB,cAAc,CAC5B,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,mBAAmB,GACzB,QAAQ,GAAG,SAAS,CAuCtB;AAED,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAYlD;AAED,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUlE"}

package/dist/scenario.js

@@ -0,0 +1,110 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getProject } from "./project.js";
+import { getStorageDir } from "./storage.js";
+function getStoragePath() {
+    return join(getStorageDir(), "scenarios.json");
+}
+function loadScenarios() {
+    const path = getStoragePath();
+    if (!existsSync(path)) {
+        return [];
+    }
+    const data = readFileSync(path, "utf-8");
+    return JSON.parse(data);
+}
+function saveScenarios(scenarios) {
+    const path = getStoragePath();
+    writeFileSync(path, JSON.stringify(scenarios, null, 2));
+}
+export function createScenario(input) {
+    const project = getProject(input.projectId);
+    if (!project) {
+        throw new Error(`Project with id "${input.projectId}" not found`);
+    }
+    const scenarios = loadScenarios();
+    if (scenarios.some((s) => s.projectId === input.projectId && s.name === input.name)) {
+        throw new Error(`Scenario with name "${input.name}" already exists in this project`);
+    }
+    const now = new Date().toISOString();
+    const scenario = {
+        id: randomUUID(),
+        projectId: input.projectId,
+        name: input.name,
+        instructions: input.instructions,
+        messages: input.messages,
+        maxMessages: input.maxMessages,
+        successCriteria: input.successCriteria,
+        failureCriteria: input.failureCriteria,
+        failureCriteriaMode: input.failureCriteriaMode,
+        personaIds: input.personaIds,
+        createdAt: now,
+        updatedAt: now,
+    };
+    scenarios.push(scenario);
+    saveScenarios(scenarios);
+    return scenario;
+}
+export function getScenario(id) {
+    const scenarios = loadScenarios();
+    return scenarios.find((s) => s.id === id);
+}
+export function getScenarioByName(projectId, name) {
+    const scenarios = loadScenarios();
+    return scenarios.find((s) => s.projectId === projectId && s.name === name);
+}
+export function listScenarios(projectId) {
+    const scenarios = loadScenarios();
+    if (projectId) {
+        return scenarios.filter((s) => s.projectId === projectId);
+    }
+    return scenarios;
+}
+export function updateScenario(id, input) {
+    const scenarios = loadScenarios();
+    const index = scenarios.findIndex((s) => s.id === id);
+    if (index === -1) {
+        return undefined;
+    }
+    const scenario = scenarios[index];
+    if (input.name &&
+        scenarios.some((s) => s.projectId === scenario.projectId && s.name === input.name && s.id !== id)) {
+        throw new Error(`Scenario with name "${input.name}" already exists in this project`);
+    }
+    const updated = {
+        ...scenario,
+        name: input.name ?? scenario.name,
+        instructions: input.instructions ?? scenario.instructions,
+        messages: input.messages ?? scenario.messages,
+        maxMessages: input.maxMessages ?? scenario.maxMessages,
+        successCriteria: input.successCriteria ?? scenario.successCriteria,
+        failureCriteria: input.failureCriteria ?? scenario.failureCriteria,
+        failureCriteriaMode: input.failureCriteriaMode ?? scenario.failureCriteriaMode,
+        personaIds: input.personaIds ?? scenario.personaIds,
+        updatedAt: new Date().toISOString(),
+    };
+    scenarios[index] = updated;
+    saveScenarios(scenarios);
+    return updated;
+}
+export function deleteScenario(id) {
+    const scenarios = loadScenarios();
+    const index = scenarios.findIndex((s) => s.id === id);
+    if (index === -1) {
+        return false;
+    }
+    scenarios.splice(index, 1);
+    saveScenarios(scenarios);
+    return true;
+}
+export function deleteScenariosByProject(projectId) {
+    const scenarios = loadScenarios();
+    const filtered = scenarios.filter((s) => s.projectId !== projectId);
+    const deletedCount = scenarios.length - filtered.length;
+    if (deletedCount > 0) {
+        saveScenarios(filtered);
+    }
+    return deletedCount;
+}
+//# sourceMappingURL=scenario.js.map

package/dist/scenario.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.js","sourceRoot":"","sources":["../src/scenario.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAiE7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,gBAAgB,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,SAAqB;IAC1C,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAA0B;IACvD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,SAAS,aAAa,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,IACE,SAAS,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAChE,EACD,CAAC;QACD,MAAM,IAAI,KAAK,CACb,uBAAuB,KAAK,CAAC,IAAI,kCAAkC,CACpE,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAa;QACzB,EAAE,EAAE,UAAU,EAAE;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,aAAa,CAAC,SAAS,CAAC,CAAC;IAEzB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,IAAY;IAEZ,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,SAAkB;IAC9C,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,EAAU,EACV,KAA0B;IAE1B,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEtD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAElC,IACE,KAAK,CAAC,IAAI;QACV,SAAS,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,CAC7E,EACD,CAAC;QACD,MAAM,IAAI,KAAK,CACb,uBAAuB,KAAK,CAAC,IAAI,kCAAkC,CACpE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAa;QACxB,GAAG,QAAQ;QACX,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACjC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY;QACzD,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC7C,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;QACtD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,QAAQ,CAAC,eAAe;QAClE,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,QAAQ,CAAC,eAAe;QAClE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,IAAI,QAAQ,CAAC,mBAAmB;QAC9E,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU;QACnD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,SAAS,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IAC3B,aAAa,CAAC,SAAS,CAAC,CAAC;IAEzB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAAU;IACvC,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEtD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3B,aAAa,CAAC,SAAS,CAAC,CAAC;IAEzB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,SAAiB;IACxD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACpE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAExD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/status.d.ts

@@ -0,0 +1,10 @@
+export interface Status {
+    name: string;
+    version: string;
+    status: "ok" | "error";
+    timestamp: string;
+    node: string;
+    storageDir: string;
+}
+export declare function getStatus(): Status;
+//# sourceMappingURL=status.d.ts.map

package/dist/status.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../src/status.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,MAAM;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,SAAS,IAAI,MAAM,CASlC"}

package/dist/status.js

@@ -0,0 +1,15 @@
+import { createRequire } from "module";
+import { getStorageDir } from "./storage.js";
+const require = createRequire(import.meta.url);
+const packageJson = require("../package.json");
+export function getStatus() {
+    return {
+        name: "evalstudio",
+        version: packageJson.version,
+        status: "ok",
+        timestamp: new Date().toISOString(),
+        node: process.version,
+        storageDir: getStorageDir(),
+    };
+}
+//# sourceMappingURL=status.js.map

package/dist/status.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"status.js","sourceRoot":"","sources":["../src/status.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAC;AAWtE,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,MAAM,EAAE,IAAI;QACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,IAAI,EAAE,OAAO,CAAC,OAAO;QACrB,UAAU,EAAE,aAAa,EAAE;KAC5B,CAAC;AACJ,CAAC"}

package/dist/storage.d.ts

@@ -0,0 +1,11 @@
+export declare const CONFIG_FILENAME = "evalstudio.config.json";
+export declare function getStorageDir(): string;
+export declare function setStorageDir(dir: string | null): void;
+export declare function resetStorageDir(): void;
+export interface InitLocalProjectResult {
+    projectDir: string;
+    configPath: string;
+    storagePath: string;
+}
+export declare function initLocalProject(parentDir: string, name: string): InitLocalProjectResult;
+//# sourceMappingURL=storage.d.ts.map

package/dist/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,eAAe,2BAA2B,CAAC;AAyBxD,wBAAgB,aAAa,IAAI,MAAM,CAQtC;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAEtD;AAED,wBAAgB,eAAe,IAAI,IAAI,CAGtC;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,sBAAsB,CAexB"}

package/dist/storage.js

@@ -0,0 +1,57 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+export const CONFIG_FILENAME = "evalstudio.config.json";
+const LOCAL_STORAGE_DIRNAME = ".evalstudio";
+let storageDir = null;
+let discoveredDir = undefined;
+function discoverLocalStorageDir(startDir) {
+    let current = startDir;
+    while (true) {
+        if (existsSync(join(current, CONFIG_FILENAME))) {
+            return join(current, LOCAL_STORAGE_DIRNAME);
+        }
+        const parent = dirname(current);
+        if (parent === current)
+            return null;
+        current = parent;
+    }
+}
+function ensureDir(dir) {
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    return dir;
+}
+export function getStorageDir() {
+    if (storageDir !== null)
+        return ensureDir(storageDir);
+    if (process.env.EVALSTUDIO_STORAGE_DIR)
+        return ensureDir(process.env.EVALSTUDIO_STORAGE_DIR);
+    if (discoveredDir === undefined) {
+        discoveredDir = discoverLocalStorageDir(process.cwd());
+    }
+    if (discoveredDir !== null)
+        return ensureDir(discoveredDir);
+    return ensureDir(join(homedir(), ".evalstudio"));
+}
+export function setStorageDir(dir) {
+    storageDir = dir;
+}
+export function resetStorageDir() {
+    storageDir = null;
+    discoveredDir = undefined;
+}
+export function initLocalProject(parentDir, name) {
+    const projectDir = join(parentDir, name);
+    const configPath = join(projectDir, CONFIG_FILENAME);
+    if (existsSync(configPath)) {
+        throw new Error(`Already initialized: ${configPath} already exists`);
+    }
+    mkdirSync(projectDir, { recursive: true });
+    writeFileSync(configPath, JSON.stringify({ version: 1 }, null, 2) + "\n");
+    const storagePath = join(projectDir, LOCAL_STORAGE_DIRNAME);
+    mkdirSync(storagePath, { recursive: true });
+    return { projectDir, configPath, storagePath };
+}
+//# sourceMappingURL=storage.js.map

package/dist/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,CAAC,MAAM,eAAe,GAAG,wBAAwB,CAAC;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAE5C,IAAI,UAAU,GAAkB,IAAI,CAAC;AACrC,IAAI,aAAa,GAA8B,SAAS,CAAC;AAEzD,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QACpC,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC,UAAU,CAAC,CAAC;IACtD,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB;QAAE,OAAO,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAC7F,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,aAAa,GAAG,uBAAuB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC,aAAa,CAAC,CAAC;IAC5D,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAkB;IAC9C,UAAU,GAAG,GAAG,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,UAAU,GAAG,IAAI,CAAC;IAClB,aAAa,GAAG,SAAS,CAAC;AAC5B,CAAC;AAQD,MAAM,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,IAAY;IAEZ,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IAErD,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,iBAAiB,CAAC,CAAC;IACvE,CAAC;IAED,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAE1E,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAC;IAC5D,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;AACjD,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Shared types used across multiple modules.
+ */
+/**
+ * A tool call made by an assistant message.
+ */
+export interface ToolCall {
+    id?: string;
+    name: string;
+    args: Record<string, unknown>;
+    type?: string;
+}
+/**
+ * A content block within a message (for multi-part content).
+ */
+export interface ContentBlock {
+    type: string;
+    text?: string;
+    [key: string]: unknown;
+}
+/**
+ * A message in OpenAI chat format with optional LangGraph extensions.
+ */
+export interface Message {
+    role: "user" | "assistant" | "system" | "tool";
+    content: string | ContentBlock[];
+    /** Tool calls made by the assistant */
+    tool_calls?: ToolCall[];
+    /** Tool call ID (for tool response messages) */
+    tool_call_id?: string;
+    /** Message name (e.g., tool name for tool messages) */
+    name?: string;
+    /** Additional metadata from the model/framework */
+    additional_kwargs?: Record<string, unknown>;
+    /** Response metadata from the model */
+    response_metadata?: Record<string, unknown>;
+    /** Message ID */
+    id?: string;
+}
+/**
+ * Normalizes message content to a string.
+ * If content is already a string, returns it as-is.
+ * If content is an array of ContentBlocks, extracts and joins text content.
+ */
+export declare function getMessageContentAsString(content: string | ContentBlock[]): string;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/C,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACjC,uCAAuC;IACvC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;IACxB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,uCAAuC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,GAC/B,MAAM,CAgBR"}