@evalstudio/core 0.2.0

package/dist/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEzD,YAAY,EAAE,OAAO,EAAE,CAAC;AAExB,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,2EAA2E;IAC3E,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,2EAA2E;IAC3E,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED,MAAM,WAAW,iBAAkB,SAAQ,IAAI;IAC7C,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE;QACV,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAoBD,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,CA+CvD;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAGpD;AAED,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,IAAI,GAAG,SAAS,CAKlB;AAED,wBAAgB,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,CAMpD;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,MAAM,GACT,iBAAiB,GAAG,SAAS,CA8C/B;AAED,wBAAgB,UAAU,CACxB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,eAAe,GACrB,IAAI,GAAG,SAAS,CAkDlB;AAED,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAe9C;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAU9D"}

package/dist/eval.js

@@ -0,0 +1,188 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getConnector } from "./connector.js";
+import { getProject } from "./project.js";
+import { deleteRunsByEval } from "./run.js";
+import { getScenario } from "./scenario.js";
+import { getStorageDir } from "./storage.js";
+function getStoragePath() {
+    return join(getStorageDir(), "evals.json");
+}
+function loadEvals() {
+    const path = getStoragePath();
+    if (!existsSync(path)) {
+        return [];
+    }
+    const data = readFileSync(path, "utf-8");
+    return JSON.parse(data);
+}
+function saveEvals(evals) {
+    const path = getStoragePath();
+    writeFileSync(path, JSON.stringify(evals, null, 2));
+}
+export function createEval(input) {
+    const project = getProject(input.projectId);
+    if (!project) {
+        throw new Error(`Project with id "${input.projectId}" not found`);
+    }
+    // Validate connector (required)
+    const connector = getConnector(input.connectorId);
+    if (!connector) {
+        throw new Error(`Connector with id "${input.connectorId}" not found`);
+    }
+    if (connector.projectId !== input.projectId) {
+        throw new Error("Connector does not belong to the specified project");
+    }
+    // Validate scenarios (required, at least one)
+    if (!input.scenarioIds || input.scenarioIds.length === 0) {
+        throw new Error("At least one scenario is required");
+    }
+    for (const scenarioId of input.scenarioIds) {
+        const scenario = getScenario(scenarioId);
+        if (!scenario) {
+            throw new Error(`Scenario with id "${scenarioId}" not found`);
+        }
+        if (scenario.projectId !== input.projectId) {
+            throw new Error("Scenario does not belong to the specified project");
+        }
+    }
+    const evals = loadEvals();
+    const now = new Date().toISOString();
+    const evalItem = {
+        id: randomUUID(),
+        projectId: input.projectId,
+        name: input.name,
+        input: input.input ?? [],
+        scenarioIds: input.scenarioIds,
+        connectorId: input.connectorId,
+        createdAt: now,
+        updatedAt: now,
+    };
+    evals.push(evalItem);
+    saveEvals(evals);
+    return evalItem;
+}
+export function getEval(id) {
+    const evals = loadEvals();
+    return evals.find((e) => e.id === id);
+}
+export function getEvalByScenario(projectId, scenarioId) {
+    const evals = loadEvals();
+    return evals.find((e) => e.projectId === projectId && e.scenarioIds.includes(scenarioId));
+}
+export function listEvals(projectId) {
+    const evals = loadEvals();
+    if (projectId) {
+        return evals.filter((e) => e.projectId === projectId);
+    }
+    return evals;
+}
+export function getEvalWithRelations(id) {
+    const evalItem = getEval(id);
+    if (!evalItem) {
+        return undefined;
+    }
+    // Fetch all scenarios (at least one is required)
+    const scenarios = [];
+    for (const scenarioId of evalItem.scenarioIds) {
+        const scenario = getScenario(scenarioId);
+        if (!scenario) {
+            continue; // Skip missing scenarios (shouldn't happen in normal operation)
+        }
+        scenarios.push({
+            id: scenario.id,
+            name: scenario.name,
+            instructions: scenario.instructions,
+            messages: scenario.messages,
+            maxMessages: scenario.maxMessages,
+            successCriteria: scenario.successCriteria,
+            failureCriteria: scenario.failureCriteria,
+            failureCriteriaMode: scenario.failureCriteriaMode,
+        });
+    }
+    if (scenarios.length === 0) {
+        return undefined; // At least one scenario must exist for a valid eval
+    }
+    const result = {
+        ...evalItem,
+        scenarios,
+    };
+    // Always populate connector (it's required on Eval)
+    const connector = getConnector(evalItem.connectorId);
+    if (connector) {
+        result.connector = {
+            id: connector.id,
+            name: connector.name,
+            type: connector.type,
+            baseUrl: connector.baseUrl,
+        };
+    }
+    return result;
+}
+export function updateEval(id, input) {
+    const evals = loadEvals();
+    const index = evals.findIndex((e) => e.id === id);
+    if (index === -1) {
+        return undefined;
+    }
+    const evalItem = evals[index];
+    // Validate connector if being updated
+    if (input.connectorId) {
+        const connector = getConnector(input.connectorId);
+        if (!connector) {
+            throw new Error(`Connector with id "${input.connectorId}" not found`);
+        }
+        if (connector.projectId !== evalItem.projectId) {
+            throw new Error("Connector does not belong to the specified project");
+        }
+    }
+    // Validate scenarios if being updated
+    if (input.scenarioIds) {
+        if (input.scenarioIds.length === 0) {
+            throw new Error("At least one scenario is required");
+        }
+        for (const scenarioId of input.scenarioIds) {
+            const scenario = getScenario(scenarioId);
+            if (!scenario) {
+                throw new Error(`Scenario with id "${scenarioId}" not found`);
+            }
+            if (scenario.projectId !== evalItem.projectId) {
+                throw new Error("Scenario does not belong to the specified project");
+            }
+        }
+    }
+    const updated = {
+        ...evalItem,
+        name: input.name ?? evalItem.name,
+        input: input.input ?? evalItem.input,
+        scenarioIds: input.scenarioIds ?? evalItem.scenarioIds,
+        connectorId: input.connectorId ?? evalItem.connectorId,
+        updatedAt: new Date().toISOString(),
+    };
+    evals[index] = updated;
+    saveEvals(evals);
+    return updated;
+}
+export function deleteEval(id) {
+    const evals = loadEvals();
+    const index = evals.findIndex((e) => e.id === id);
+    if (index === -1) {
+        return false;
+    }
+    // Delete associated runs first
+    deleteRunsByEval(id);
+    evals.splice(index, 1);
+    saveEvals(evals);
+    return true;
+}
+export function deleteEvalsByProject(projectId) {
+    const evals = loadEvals();
+    const filtered = evals.filter((e) => e.projectId !== projectId);
+    const deletedCount = evals.length - filtered.length;
+    if (deletedCount > 0) {
+        saveEvals(filtered);
+    }
+    return deletedCount;
+}
+//# sourceMappingURL=eval.js.map

package/dist/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAiE7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,YAAY,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,SAAS;IAChB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAW,CAAC;AACpC,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAsB;IAC/C,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,SAAS,aAAa,CAAC,CAAC;IACpE,CAAC;IAED,gCAAgC;IAChC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,WAAW,aAAa,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,SAAS,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,aAAa,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAS;QACrB,EAAE,EAAE,UAAU,EAAE;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;QACxB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,SAAS,CAAC,KAAK,CAAC,CAAC;IAEjB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,EAAU;IAChC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,UAAkB;IAElB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,OAAO,KAAK,CAAC,IAAI,CACf,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,CACvE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,SAAkB;IAC1C,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAU;IAEV,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,iDAAiD;IACjD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,SAAS,CAAC,gEAAgE;QAC5E,CAAC;QACD,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;SAClD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,SAAS,CAAC,CAAC,oDAAoD;IACxE,CAAC;IAED,MAAM,MAAM,GAAsB;QAChC,GAAG,QAAQ;QACX,SAAS;KACV,CAAC;IAEF,oDAAoD;IACpD,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACrD,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,CAAC,SAAS,GAAG;YACjB,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO,EAAE,SAAS,CAAC,OAAO;SAC3B,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,EAAU,EACV,KAAsB;IAEtB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAElD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAE9B,sCAAsC;IACtC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,WAAW,aAAa,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,SAAS,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;YACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,aAAa,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAS;QACpB,GAAG,QAAQ;QACX,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACjC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK;QACpC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;QACtD,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;QACtD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IACvB,SAAS,CAAC,KAAK,CAAC,CAAC;IAEjB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAU;IACnC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAElD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,+BAA+B;IAC/B,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAErB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACvB,SAAS,CAAC,KAAK,CAAC,CAAC;IAEjB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IACpD,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEpD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/evaluator.d.ts

@@ -0,0 +1,37 @@
+import type { Message } from "./types.js";
+/**
+ * Result of evaluating conversation against success/failure criteria
+ */
+export interface CriteriaEvaluationResult {
+    /** Whether the success criteria has been met */
+    successMet: boolean;
+    /** Whether the failure criteria has been met */
+    failureMet: boolean;
+    /** Confidence score (0-1) for the evaluation decision */
+    confidence: number;
+    /** Reasoning for the evaluation decision */
+    reasoning: string;
+    /** Raw response from the LLM evaluator */
+    rawResponse?: string;
+}
+/**
+ * Input for criteria evaluation
+ */
+export interface EvaluateCriteriaInput {
+    /** The conversation history to evaluate */
+    messages: Message[];
+    /** Success criteria to check against */
+    successCriteria?: string;
+    /** Failure criteria to check against */
+    failureCriteria?: string;
+    /** LLM provider ID to use for evaluation */
+    llmProviderId: string;
+    /** Model to use (optional, defaults based on provider) */
+    model?: string;
+}
+/**
+ * Evaluates conversation against success/failure criteria using an LLM judge.
+ * Returns the evaluation result indicating whether criteria have been met.
+ */
+export declare function evaluateCriteria(input: EvaluateCriteriaInput): Promise<CriteriaEvaluationResult>;
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,gDAAgD;IAChD,UAAU,EAAE,OAAO,CAAC;IACpB,gDAAgD;IAChD,UAAU,EAAE,OAAO,CAAC;IACpB,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,2CAA2C;IAC3C,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,wCAAwC;IACxC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAmED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,qBAAqB,GAC3B,OAAO,CAAC,wBAAwB,CAAC,CA6DnC"}

package/dist/evaluator.js

@@ -0,0 +1,121 @@
+import { getLLMProvider } from "./llm-provider.js";
+import { chatCompletion } from "./llm-client.js";
+/**
+ * Validates and extracts the evaluation result from parsed JSON
+ */
+function validateEvaluationResult(parsed) {
+    if (typeof parsed !== "object" || parsed === null ||
+        typeof parsed.successMet !== "boolean" ||
+        typeof parsed.failureMet !== "boolean" ||
+        typeof parsed.confidence !== "number" ||
+        parsed.confidence < 0 ||
+        parsed.confidence > 1 ||
+        typeof parsed.reasoning !== "string") {
+        throw new Error("Invalid evaluation result shape");
+    }
+    const p = parsed;
+    return {
+        successMet: p.successMet,
+        failureMet: p.failureMet,
+        confidence: p.confidence,
+        reasoning: p.reasoning,
+    };
+}
+/**
+ * Formats conversation messages for evaluation
+ */
+function formatConversation(messages) {
+    return messages
+        .filter((m) => m.role !== "system")
+        .map((m) => {
+        const role = m.role === "assistant" ? "Agent" : "User";
+        const content = typeof m.content === "string" ? m.content : JSON.stringify(m.content);
+        return `${role}: ${content}`;
+    })
+        .join("\n\n");
+}
+/**
+ * Parses the LLM response into a CriteriaEvaluationResult
+ */
+function parseEvaluationResponse(response) {
+    try {
+        // Try to extract JSON from the response
+        const jsonMatch = response.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            throw new Error("No JSON found in response");
+        }
+        const parsed = JSON.parse(jsonMatch[0]);
+        const validated = validateEvaluationResult(parsed);
+        return validated;
+    }
+    catch {
+        // Default to inconclusive if parsing fails
+        return {
+            successMet: false,
+            failureMet: false,
+            confidence: 0,
+            reasoning: `Failed to parse evaluation response: ${response.slice(0, 200)}`,
+        };
+    }
+}
+/**
+ * Evaluates conversation against success/failure criteria using an LLM judge.
+ * Returns the evaluation result indicating whether criteria have been met.
+ */
+export async function evaluateCriteria(input) {
+    const { messages, successCriteria, failureCriteria, llmProviderId, model } = input;
+    // If no criteria defined, return inconclusive
+    if (!successCriteria && !failureCriteria) {
+        return {
+            successMet: false,
+            failureMet: false,
+            confidence: 1,
+            reasoning: "No evaluation criteria defined",
+        };
+    }
+    // Get the LLM provider
+    const llmProvider = getLLMProvider(llmProviderId);
+    if (!llmProvider) {
+        throw new Error(`LLM Provider with id "${llmProviderId}" not found`);
+    }
+    // Format conversation and build messages
+    const conversation = formatConversation(messages);
+    const chatMessages = [
+        {
+            role: "system",
+            content: `You are an evaluation judge. Analyze conversations between a User and an Agent to determine if specific criteria have been met.
+
+You must respond with a JSON object containing:
+- successMet: boolean - whether the success criteria has been met
+- failureMet: boolean - whether the failure criteria has been met
+- confidence: number - your confidence level from 0.0 to 1.0
+- reasoning: string - brief explanation of your evaluation decision
+
+Respond ONLY with the JSON object, no additional text.`,
+        },
+        {
+            role: "user",
+            content: `## Conversation
+${conversation}
+
+## Evaluation Criteria
+
+### Success Criteria
+${successCriteria || "No success criteria defined."}
+
+### Failure Criteria
+${failureCriteria || "No failure criteria defined."}
+
+Analyze the conversation and determine if the criteria have been met.`,
+        },
+    ];
+    // Invoke the LLM
+    const response = await chatCompletion(llmProvider, chatMessages, { model });
+    // Parse and return result
+    const result = parseEvaluationResponse(response.content);
+    return {
+        ...result,
+        rawResponse: response.content,
+    };
+}
+//# sourceMappingURL=evaluator.js.map

package/dist/evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,cAAc,EAA8B,MAAM,iBAAiB,CAAC;AAkC7E;;GAEG;AACH,SAAS,wBAAwB,CAC/B,MAAe;IAEf,IACE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAC7C,OAAQ,MAAkC,CAAC,UAAU,KAAK,SAAS;QACnE,OAAQ,MAAkC,CAAC,UAAU,KAAK,SAAS;QACnE,OAAQ,MAAkC,CAAC,UAAU,KAAK,QAAQ;QAChE,MAAkC,CAAC,UAAqB,GAAG,CAAC;QAC5D,MAAkC,CAAC,UAAqB,GAAG,CAAC;QAC9D,OAAQ,MAAkC,CAAC,SAAS,KAAK,QAAQ,EACjE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,CAAC,GAAG,MAAiC,CAAC;IAC5C,OAAO;QACL,UAAU,EAAE,CAAC,CAAC,UAAqB;QACnC,UAAU,EAAE,CAAC,CAAC,UAAqB;QACnC,UAAU,EAAE,CAAC,CAAC,UAAoB;QAClC,SAAS,EAAE,CAAC,CAAC,SAAmB;KACjC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,QAAmB;IAC7C,OAAO,QAAQ;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QACvD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtF,OAAO,GAAG,IAAI,KAAK,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;QACnD,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,2CAA2C;QAC3C,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,wCAAwC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAA4B;IAE5B,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAEnF,8CAA8C;IAC9C,IAAI,CAAC,eAAe,IAAI,CAAC,eAAe,EAAE,CAAC;QACzC,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,gCAAgC;SAC5C,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,MAAM,WAAW,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;IAClD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,yBAAyB,aAAa,aAAa,CAAC,CAAC;IACvE,CAAC;IAED,yCAAyC;IACzC,MAAM,YAAY,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAElD,MAAM,YAAY,GAA4B;QAC5C;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE;;;;;;;;uDAQwC;SAClD;QACD;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;EACb,YAAY;;;;;EAKZ,eAAe,IAAI,8BAA8B;;;EAGjD,eAAe,IAAI,8BAA8B;;sEAEmB;SACjE;KACF,CAAC;IAEF,iBAAiB;IACjB,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAE5E,0BAA0B;IAC1B,MAAM,MAAM,GAAG,uBAAuB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzD,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,QAAQ,CAAC,OAAO;KAC9B,CAAC;AACJ,CAAC"}

package/dist/execution.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Execution represents a batch of runs created together from a single eval run.
+ * Each execution groups all runs (scenario/persona combinations) created at the same time.
+ */
+export interface Execution {
+    /** Auto-increment integer ID (1, 2, 3, ...) */
+    id: number;
+    /** The project this execution belongs to */
+    projectId: string;
+    /** The eval that triggered this execution */
+    evalId: string;
+    /** When this execution was created */
+    createdAt: string;
+}
+export interface CreateExecutionInput {
+    projectId: string;
+    evalId: string;
+}
+/**
+ * Creates a new execution for a batch of runs.
+ */
+export declare function createExecution(input: CreateExecutionInput): Execution;
+export declare function getExecution(id: number): Execution | undefined;
+export declare function listExecutions(evalId?: string): Execution[];
+export declare function listExecutionsByProject(projectId: string): Execution[];
+export declare function deleteExecution(id: number): boolean;
+export declare function deleteExecutionsByEval(evalId: string): number;
+export declare function deleteExecutionsByProject(projectId: string): number;
+//# sourceMappingURL=execution.d.ts.map

package/dist/execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution.d.ts","sourceRoot":"","sources":["../src/execution.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAiCD;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,CAetE;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAG9D;AAED,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,CAQ3D;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,CAKtE;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAYnD;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAU7D;AAED,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUnE"}

package/dist/execution.js

@@ -0,0 +1,94 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getStorageDir } from "./storage.js";
+function getStoragePath() {
+    return join(getStorageDir(), "executions.json");
+}
+function loadExecutions() {
+    const path = getStoragePath();
+    if (!existsSync(path)) {
+        return [];
+    }
+    const data = readFileSync(path, "utf-8");
+    return JSON.parse(data);
+}
+function saveExecutions(executions) {
+    const path = getStoragePath();
+    writeFileSync(path, JSON.stringify(executions, null, 2));
+}
+/**
+ * Get the next auto-increment ID for executions within a project.
+ * Finds the maximum existing ID for the project and returns max + 1.
+ */
+function getNextId(executions, projectId) {
+    const projectExecutions = executions.filter((e) => e.projectId === projectId);
+    if (projectExecutions.length === 0) {
+        return 1;
+    }
+    const maxId = Math.max(...projectExecutions.map((e) => e.id));
+    return maxId + 1;
+}
+/**
+ * Creates a new execution for a batch of runs.
+ */
+export function createExecution(input) {
+    const executions = loadExecutions();
+    const now = new Date().toISOString();
+    const execution = {
+        id: getNextId(executions, input.projectId),
+        projectId: input.projectId,
+        evalId: input.evalId,
+        createdAt: now,
+    };
+    executions.push(execution);
+    saveExecutions(executions);
+    return execution;
+}
+export function getExecution(id) {
+    const executions = loadExecutions();
+    return executions.find((e) => e.id === id);
+}
+export function listExecutions(evalId) {
+    const executions = loadExecutions();
+    if (evalId) {
+        return executions
+            .filter((e) => e.evalId === evalId)
+            .sort((a, b) => b.id - a.id);
+    }
+    return executions.sort((a, b) => b.id - a.id);
+}
+export function listExecutionsByProject(projectId) {
+    const executions = loadExecutions();
+    return executions
+        .filter((e) => e.projectId === projectId)
+        .sort((a, b) => b.id - a.id);
+}
+export function deleteExecution(id) {
+    const executions = loadExecutions();
+    const index = executions.findIndex((e) => e.id === id);
+    if (index === -1) {
+        return false;
+    }
+    executions.splice(index, 1);
+    saveExecutions(executions);
+    return true;
+}
+export function deleteExecutionsByEval(evalId) {
+    const executions = loadExecutions();
+    const filtered = executions.filter((e) => e.evalId !== evalId);
+    const deletedCount = executions.length - filtered.length;
+    if (deletedCount > 0) {
+        saveExecutions(filtered);
+    }
+    return deletedCount;
+}
+export function deleteExecutionsByProject(projectId) {
+    const executions = loadExecutions();
+    const filtered = executions.filter((e) => e.projectId !== projectId);
+    const deletedCount = executions.length - filtered.length;
+    if (deletedCount > 0) {
+        saveExecutions(filtered);
+    }
+    return deletedCount;
+}
+//# sourceMappingURL=execution.js.map

package/dist/execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution.js","sourceRoot":"","sources":["../src/execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAsB7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,iBAAiB,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAgB,CAAC;AACzC,CAAC;AAED,SAAS,cAAc,CAAC,UAAuB;IAC7C,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,UAAuB,EAAE,SAAiB;IAC3D,MAAM,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IAC9E,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9D,OAAO,KAAK,GAAG,CAAC,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAA2B;IACzD,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,MAAM,SAAS,GAAc;QAC3B,EAAE,EAAE,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC;QAC1C,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3B,cAAc,CAAC,UAAU,CAAC,CAAC;IAE3B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAAU;IACrC,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAe;IAC5C,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,UAAU;aACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,OAAO,UAAU;SACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;SACxC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEvD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5B,cAAc,CAAC,UAAU,CAAC,CAAC;IAE3B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEzD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,SAAiB;IACzD,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACrE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEzD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,17 @@
+export { getStatus, type Status } from "./status.js";
+export type { Message } from "./types.js";
+export { getStorageDir, setStorageDir, resetStorageDir, initLocalProject, CONFIG_FILENAME, type InitLocalProjectResult, } from "./storage.js";
+export { createProject, deleteProject, getProject, getProjectByName, listProjects, updateProject, type CreateProjectInput, type LLMUseCaseSettings, type Project, type ProjectLLMSettings, type UpdateProjectInput, } from "./project.js";
+export { createPersona, deletePersona, deletePersonasByProject, getPersona, getPersonaByName, listPersonas, updatePersona, type CreatePersonaInput, type Persona, type UpdatePersonaInput, } from "./persona.js";
+export { createScenario, deleteScenario, deleteScenariosByProject, getScenario, getScenarioByName, listScenarios, updateScenario, type CreateScenarioInput, type FailureCriteriaMode, type Scenario, type UpdateScenarioInput, } from "./scenario.js";
+export { createEval, deleteEval, deleteEvalsByProject, getEval, getEvalByScenario, getEvalWithRelations, listEvals, updateEval, type CreateEvalInput, type Eval, type EvalWithRelations, type UpdateEvalInput, } from "./eval.js";
+export { createLLMProvider, deleteLLMProvider, deleteLLMProvidersByProject, fetchProviderModels, getDefaultModels, getLLMProvider, getLLMProviderByName, listLLMProviders, updateLLMProvider, type CreateLLMProviderInput, type LLMProvider, type LLMProviderConfig, type ProviderType, type UpdateLLMProviderInput, } from "./llm-provider.js";
+export { createConnector, deleteConnector, deleteConnectorsByProject, getConnector, getConnectorByName, getConnectorTypes, invokeConnector, listConnectors, testConnector, updateConnector, type AuthType, type Connector, type ConnectorConfig, type ConnectorInvokeInput, type ConnectorInvokeResult, type ConnectorTestResult, type ConnectorType, type CreateConnectorInput, type HttpConnectorConfig, type LangGraphConnectorConfig, type UpdateConnectorInput, } from "./connector.js";
+export { createPlaygroundRun, createRun, createRuns, deleteRun, deleteRunsByEval, deleteRunsByProject, getRun, listRuns, listRunsByEval, listRunsByScenario, listRunsByPersona, retryRun, updateRun, type CreatePlaygroundRunInput, type CreateRunInput, type ListRunsOptions, type Run, type RunMetadata, type RunResult, type RunStatus, type UpdateRunInput, } from "./run.js";
+export { createExecution, deleteExecution, deleteExecutionsByEval, deleteExecutionsByProject, getExecution, listExecutions, listExecutionsByProject, type CreateExecutionInput, type Execution, } from "./execution.js";
+export { buildTestAgentSystemPrompt, buildTestAgentMessages, type BuildTestAgentPromptInput, } from "./prompt.js";
+export { RunProcessor, type RunProcessorOptions } from "./run-processor.js";
+export { evaluateCriteria, type CriteriaEvaluationResult, type EvaluateCriteriaInput, } from "./evaluator.js";
+export { generatePersonaMessage, type GeneratePersonaMessageInput, type GeneratePersonaMessageResult, } from "./persona-generator.js";
+export { chatCompletion, getDefaultModelForProvider, type ChatCompletionMessage, type ChatCompletionOptions, type ChatCompletionResult, } from "./llm-client.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AACrD,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EACL,aAAa,EACb,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,KAAK,sBAAsB,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,OAAO,EACZ,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,GACxB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,KAAK,kBAAkB,EACvB,KAAK,OAAO,EACZ,KAAK,kBAAkB,GACxB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,cAAc,EACd,cAAc,EACd,wBAAwB,EACxB,WAAW,EACX,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,mBAAmB,GACzB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,iBAAiB,EACjB,oBAAoB,EACpB,SAAS,EACT,UAAU,EACV,KAAK,eAAe,EACpB,KAAK,IAAI,EACT,KAAK,iBAAiB,EACtB,KAAK,eAAe,GACrB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,2BAA2B,EAC3B,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,KAAK,sBAAsB,EAC3B,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,KAAK,sBAAsB,GAC5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,eAAe,EACf,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,GAC1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,SAAS,EACT,gBAAgB,EAChB,mBAAmB,EACnB,MAAM,EACN,QAAQ,EACR,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,QAAQ,EACR,SAAS,EACT,KAAK,wBAAwB,EAC7B,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,GAAG,EACR,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,cAAc,GACpB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,eAAe,EACf,eAAe,EACf,sBAAsB,EACtB,yBAAyB,EACzB,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,KAAK,oBAAoB,EACzB,KAAK,SAAS,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,EACtB,KAAK,yBAAyB,GAC/B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EACL,gBAAgB,EAChB,KAAK,wBAAwB,EAC7B,KAAK,qBAAqB,GAC3B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,sBAAsB,EACtB,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,GAClC,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,cAAc,EACd,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,GAC1B,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -0,0 +1,16 @@
+export { getStatus } from "./status.js";
+export { getStorageDir, setStorageDir, resetStorageDir, initLocalProject, CONFIG_FILENAME, } from "./storage.js";
+export { createProject, deleteProject, getProject, getProjectByName, listProjects, updateProject, } from "./project.js";
+export { createPersona, deletePersona, deletePersonasByProject, getPersona, getPersonaByName, listPersonas, updatePersona, } from "./persona.js";
+export { createScenario, deleteScenario, deleteScenariosByProject, getScenario, getScenarioByName, listScenarios, updateScenario, } from "./scenario.js";
+export { createEval, deleteEval, deleteEvalsByProject, getEval, getEvalByScenario, getEvalWithRelations, listEvals, updateEval, } from "./eval.js";
+export { createLLMProvider, deleteLLMProvider, deleteLLMProvidersByProject, fetchProviderModels, getDefaultModels, getLLMProvider, getLLMProviderByName, listLLMProviders, updateLLMProvider, } from "./llm-provider.js";
+export { createConnector, deleteConnector, deleteConnectorsByProject, getConnector, getConnectorByName, getConnectorTypes, invokeConnector, listConnectors, testConnector, updateConnector, } from "./connector.js";
+export { createPlaygroundRun, createRun, createRuns, deleteRun, deleteRunsByEval, deleteRunsByProject, getRun, listRuns, listRunsByEval, listRunsByScenario, listRunsByPersona, retryRun, updateRun, } from "./run.js";
+export { createExecution, deleteExecution, deleteExecutionsByEval, deleteExecutionsByProject, getExecution, listExecutions, listExecutionsByProject, } from "./execution.js";
+export { buildTestAgentSystemPrompt, buildTestAgentMessages, } from "./prompt.js";
+export { RunProcessor } from "./run-processor.js";
+export { evaluateCriteria, } from "./evaluator.js";
+export { generatePersonaMessage, } from "./persona-generator.js";
+export { chatCompletion, getDefaultModelForProvider, } from "./llm-client.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,aAAa,CAAC;AAErD,OAAO,EACL,aAAa,EACb,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,eAAe,GAEhB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,aAAa,GAMd,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,aAAa,GAId,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,cAAc,EACd,cAAc,EACd,wBAAwB,EACxB,WAAW,EACX,iBAAiB,EACjB,aAAa,EACb,cAAc,GAKf,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,iBAAiB,EACjB,oBAAoB,EACpB,SAAS,EACT,UAAU,GAKX,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,2BAA2B,EAC3B,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAMlB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,eAAe,EACf,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,GAYhB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,SAAS,EACT,gBAAgB,EAChB,mBAAmB,EACnB,MAAM,EACN,QAAQ,EACR,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,QAAQ,EACR,SAAS,GASV,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,eAAe,EACf,eAAe,EACf,sBAAsB,EACtB,yBAAyB,EACzB,YAAY,EACZ,cAAc,EACd,uBAAuB,GAGxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,GAEvB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAA4B,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EACL,gBAAgB,GAGjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,cAAc,EACd,0BAA0B,GAI3B,MAAM,iBAAiB,CAAC"}

package/dist/llm-client.d.ts

@@ -0,0 +1,31 @@
+import type { LLMProvider, ProviderType } from "./llm-provider.js";
+/**
+ * A message in the standard chat completion format.
+ */
+export interface ChatCompletionMessage {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+/**
+ * The result of a chat completion call.
+ */
+export interface ChatCompletionResult {
+    content: string;
+}
+/**
+ * Options for the chat completion call.
+ */
+export interface ChatCompletionOptions {
+    model?: string;
+    temperature?: number;
+}
+/**
+ * Returns the default model for a given provider type.
+ */
+export declare function getDefaultModelForProvider(providerType: ProviderType): string;
+/**
+ * Sends a chat completion request to the appropriate provider API.
+ * Supports OpenAI and Anthropic via native fetch().
+ */
+export declare function chatCompletion(provider: LLMProvider, messages: ChatCompletionMessage[], options?: ChatCompletionOptions): Promise<ChatCompletionResult>;
+//# sourceMappingURL=llm-client.d.ts.map

package/dist/llm-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-client.d.ts","sourceRoot":"","sources":["../src/llm-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,YAAY,EAAE,YAAY,GAAG,MAAM,CAS7E;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,WAAW,EACrB,QAAQ,EAAE,qBAAqB,EAAE,EACjC,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAW/B"}