@evalstudio/core 0.2.0

package/dist/persona.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"persona.js","sourceRoot":"","sources":["../src/persona.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAyB7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,eAAe,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAc,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,QAAmB;IACvC,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAyB;IACrD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,SAAS,aAAa,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAEhC,IACE,QAAQ,CAAC,IAAI,CACX,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAChE,EACD,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,CAAC,IAAI,kCAAkC,CACnE,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,OAAO,GAAY;QACvB,EAAE,EAAE,UAAU,EAAE;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAU;IACnC,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,IAAY;IAEZ,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAkB;IAC7C,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,EAAU,EACV,KAAyB;IAEzB,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAErD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEhC,IACE,KAAK,CAAC,IAAI;QACV,QAAQ,CAAC,IAAI,CACX,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,CAC5E,EACD,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,CAAC,IAAI,kCAAkC,CACnE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,GAAG,OAAO;QACV,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI;QAChC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW;QACrD,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY;QACxD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,QAAQ,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IAC1B,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAErD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1B,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACnE,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEvD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/project.d.ts

@@ -0,0 +1,43 @@
+/**
+ * LLM settings for a specific use-case (evaluation or persona generation)
+ */
+export interface LLMUseCaseSettings {
+    providerId: string;
+    model?: string;
+}
+/**
+ * Project-level LLM configuration for different use-cases
+ */
+export interface ProjectLLMSettings {
+    /** LLM settings for evaluation/judging conversations */
+    evaluation?: LLMUseCaseSettings;
+    /** LLM settings for persona response generation */
+    persona?: LLMUseCaseSettings;
+}
+export interface Project {
+    id: string;
+    name: string;
+    description?: string;
+    /** Default LLM settings for the project */
+    llmSettings?: ProjectLLMSettings;
+    createdAt: string;
+    updatedAt: string;
+}
+export interface CreateProjectInput {
+    name: string;
+    description?: string;
+    llmSettings?: ProjectLLMSettings;
+}
+export interface UpdateProjectInput {
+    name?: string;
+    description?: string;
+    /** Set to null to clear LLM settings */
+    llmSettings?: ProjectLLMSettings | null;
+}
+export declare function createProject(input: CreateProjectInput): Project;
+export declare function getProject(id: string): Project | undefined;
+export declare function getProjectByName(name: string): Project | undefined;
+export declare function listProjects(): Project[];
+export declare function updateProject(id: string, input: UpdateProjectInput): Project | undefined;
+export declare function deleteProject(id: string): boolean;
+//# sourceMappingURL=project.d.ts.map

package/dist/project.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAMA;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,mDAAmD;IACnD,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,WAAW,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;CACzC;AAoBD,wBAAgB,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAqBhE;AAED,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAG1D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAGlE;AAED,wBAAgB,YAAY,IAAI,OAAO,EAAE,CAExC;AAED,wBAAgB,aAAa,CAC3B,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,kBAAkB,GACxB,OAAO,GAAG,SAAS,CA+DrB;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAYjD"}

package/dist/project.js

@@ -0,0 +1,114 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getLLMProvider } from "./llm-provider.js";
+import { getStorageDir } from "./storage.js";
+function getStoragePath() {
+    return join(getStorageDir(), "projects.json");
+}
+function loadProjects() {
+    const path = getStoragePath();
+    if (!existsSync(path)) {
+        return [];
+    }
+    const data = readFileSync(path, "utf-8");
+    return JSON.parse(data);
+}
+function saveProjects(projects) {
+    const path = getStoragePath();
+    writeFileSync(path, JSON.stringify(projects, null, 2));
+}
+export function createProject(input) {
+    const projects = loadProjects();
+    if (projects.some((p) => p.name === input.name)) {
+        throw new Error(`Project with name "${input.name}" already exists`);
+    }
+    const now = new Date().toISOString();
+    const project = {
+        id: randomUUID(),
+        name: input.name,
+        description: input.description,
+        llmSettings: input.llmSettings,
+        createdAt: now,
+        updatedAt: now,
+    };
+    projects.push(project);
+    saveProjects(projects);
+    return project;
+}
+export function getProject(id) {
+    const projects = loadProjects();
+    return projects.find((p) => p.id === id);
+}
+export function getProjectByName(name) {
+    const projects = loadProjects();
+    return projects.find((p) => p.name === name);
+}
+export function listProjects() {
+    return loadProjects();
+}
+export function updateProject(id, input) {
+    const projects = loadProjects();
+    const index = projects.findIndex((p) => p.id === id);
+    if (index === -1) {
+        return undefined;
+    }
+    if (input.name && projects.some((p) => p.name === input.name && p.id !== id)) {
+        throw new Error(`Project with name "${input.name}" already exists`);
+    }
+    // Validate LLM settings if provided
+    if (input.llmSettings) {
+        const { evaluation, persona } = input.llmSettings;
+        if (evaluation?.providerId) {
+            const provider = getLLMProvider(evaluation.providerId);
+            if (!provider) {
+                throw new Error(`LLM Provider with id "${evaluation.providerId}" not found`);
+            }
+            if (provider.projectId !== id) {
+                throw new Error("Evaluation LLM Provider does not belong to this project");
+            }
+        }
+        if (persona?.providerId) {
+            const provider = getLLMProvider(persona.providerId);
+            if (!provider) {
+                throw new Error(`LLM Provider with id "${persona.providerId}" not found`);
+            }
+            if (provider.projectId !== id) {
+                throw new Error("Persona LLM Provider does not belong to this project");
+            }
+        }
+    }
+    const project = projects[index];
+    // Handle llmSettings: null clears, undefined keeps existing, object updates
+    let newLLMSettings;
+    if (input.llmSettings === null) {
+        newLLMSettings = undefined;
+    }
+    else if (input.llmSettings !== undefined) {
+        newLLMSettings = input.llmSettings;
+    }
+    else {
+        newLLMSettings = project.llmSettings;
+    }
+    const updated = {
+        ...project,
+        name: input.name ?? project.name,
+        description: input.description ?? project.description,
+        llmSettings: newLLMSettings,
+        updatedAt: new Date().toISOString(),
+    };
+    projects[index] = updated;
+    saveProjects(projects);
+    return updated;
+}
+export function deleteProject(id) {
+    const projects = loadProjects();
+    const index = projects.findIndex((p) => p.id === id);
+    if (index === -1) {
+        return false;
+    }
+    projects.splice(index, 1);
+    saveProjects(projects);
+    return true;
+}
+//# sourceMappingURL=project.js.map

package/dist/project.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AA2C7C,SAAS,cAAc;IACrB,OAAO,IAAI,CAAC,aAAa,EAAE,EAAE,eAAe,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAc,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,QAAmB;IACvC,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAyB;IACrD,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAEhC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,IAAI,kBAAkB,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,OAAO,GAAY;QACvB,EAAE,EAAE,UAAU,EAAE;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAU;IACnC,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,EAAU,EACV,KAAyB;IAEzB,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAErD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,IAAI,kBAAkB,CAAC,CAAC;IACtE,CAAC;IAED,oCAAoC;IACpC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC;QAClD,IAAI,UAAU,EAAE,UAAU,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YACvD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CACb,yBAAyB,UAAU,CAAC,UAAU,aAAa,CAC5D,CAAC;YACJ,CAAC;YACD,IAAI,QAAQ,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QACD,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CACb,yBAAyB,OAAO,CAAC,UAAU,aAAa,CACzD,CAAC;YACJ,CAAC;YACD,IAAI,QAAQ,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEhC,4EAA4E;IAC5E,IAAI,cAA8C,CAAC;IACnD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;QAC/B,cAAc,GAAG,SAAS,CAAC;IAC7B,CAAC;SAAM,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QAC3C,cAAc,GAAG,KAAK,CAAC,WAAW,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IACvC,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,GAAG,OAAO;QACV,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI;QAChC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW;QACrD,WAAW,EAAE,cAAc;QAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,QAAQ,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IAC1B,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAErD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1B,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/prompt.d.ts

@@ -0,0 +1,31 @@
+import type { Persona } from "./persona.js";
+import type { Scenario } from "./scenario.js";
+/**
+ * Input for building the test agent system prompt.
+ * Accepts partial persona/scenario objects so it can work with
+ * both full entities and the embedded relations from EvalWithRelations.
+ */
+export interface BuildTestAgentPromptInput {
+    persona?: Pick<Persona, "name" | "description" | "systemPrompt"> | null;
+    scenario?: Pick<Scenario, "name" | "instructions" | "messages"> | null;
+}
+/**
+ * Builds the system prompt for the test agent that will impersonate
+ * a user persona and simulate a scenario when interacting with the
+ * chatbot being tested.
+ *
+ * The test agent's role is to act as a realistic user, following the
+ * persona's characteristics and the scenario's context to evaluate
+ * how well the chatbot handles the interaction.
+ */
+export declare function buildTestAgentSystemPrompt(input: BuildTestAgentPromptInput): string;
+/**
+ * Builds a messages array in OpenAI format for the test agent,
+ * starting with the system prompt and including any initial seed
+ * messages from the scenario.
+ */
+export declare function buildTestAgentMessages(input: BuildTestAgentPromptInput): Array<{
+    role: "system" | "user" | "assistant";
+    content: string;
+}>;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC;IACxE,QAAQ,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC;CACxE;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,CA4CnF;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,yBAAyB,GAC/B,KAAK,CAAC;IAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAiBnE"}

package/dist/prompt.js

@@ -0,0 +1,73 @@
+import { getMessageContentAsString } from "./types.js";
+/**
+ * Builds the system prompt for the test agent that will impersonate
+ * a user persona and simulate a scenario when interacting with the
+ * chatbot being tested.
+ *
+ * The test agent's role is to act as a realistic user, following the
+ * persona's characteristics and the scenario's context to evaluate
+ * how well the chatbot handles the interaction.
+ */
+export function buildTestAgentSystemPrompt(input) {
+    const { persona, scenario } = input;
+    let personaContent = "";
+    if (persona) {
+        personaContent = `
+## User Persona
+
+Name:
+${persona.name || "N/A"}
+
+Character Instructions:
+${persona.systemPrompt || "N/A"}`;
+    }
+    let scenarioContent = "";
+    if (scenario?.instructions) {
+        scenarioContent = `
+## Scenario
+
+${scenario.instructions}
+`;
+    }
+    const systemPrompt = `
+You are a test agent simulating a user interaction with a chatbot.
+Your role is to impersonate a specific user persona and simulate a realistic conversation
+based on the given scenario. Behave naturally as this user would, staying in character
+throughout the conversation.
+
+${personaContent}
+
+${scenarioContent}
+
+## Guidelines
+
+- Stay in character as the user persona throughout the conversation
+- Follow the scenario context to guide your messages and goals
+- Respond naturally as a real user would, with realistic questions, concerns, or requests
+- Do not break character or reveal that you are a test agent
+- If the chatbot asks clarifying questions, answer them based on the persona and scenario
+- Express appropriate emotions based on the scenario (frustration, curiosity, urgency, etc.)`;
+    return systemPrompt;
+}
+/**
+ * Builds a messages array in OpenAI format for the test agent,
+ * starting with the system prompt and including any initial seed
+ * messages from the scenario.
+ */
+export function buildTestAgentMessages(input) {
+    const systemPrompt = buildTestAgentSystemPrompt(input);
+    const messages = [
+        { role: "system", content: systemPrompt },
+    ];
+    // Include scenario's initial seed messages if present
+    if (input.scenario?.messages && input.scenario.messages.length > 0) {
+        for (const msg of input.scenario.messages) {
+            messages.push({
+                role: msg.role,
+                content: getMessageContentAsString(msg.content),
+            });
+        }
+    }
+    return messages;
+}
+//# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAYvD;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,KAAgC;IACzE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAEpC,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,OAAO,EAAE,CAAC;QACZ,cAAc,GAAG;;;;EAInB,OAAO,CAAC,IAAI,IAAI,KAAK;;;EAGrB,OAAO,CAAC,YAAY,IAAI,KAAK,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,QAAQ,EAAE,YAAY,EAAE,CAAC;QAC3B,eAAe,GAAG;;;EAGpB,QAAQ,CAAC,YAAY;CACtB,CAAC;IACA,CAAC;IAED,MAAM,YAAY,GAAW;;;;;;EAM7B,cAAc;;EAEd,eAAe;;;;;;;;;6FAS4E,CAAC;IAE5F,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAgC;IAEhC,MAAM,YAAY,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAsE;QAClF,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;KAC1C,CAAC;IAEF,sDAAsD;IACtD,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnE,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAuC;gBACjD,OAAO,EAAE,yBAAyB,CAAC,GAAG,CAAC,OAAO,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/run-processor.d.ts

@@ -0,0 +1,127 @@
+import { type ConnectorInvokeResult } from "./connector.js";
+import { type Run, type RunStatus } from "./run.js";
+export type { RunStatus };
+export interface RunProcessorOptions {
+    /** Polling interval in milliseconds (default: 5000) */
+    pollIntervalMs?: number;
+    /** Maximum concurrent run executions (default: 3) */
+    maxConcurrent?: number;
+    /** Filter runs by project ID (optional) */
+    projectId?: string;
+    /** Callback for status changes */
+    onStatusChange?: (runId: string, status: RunStatus, run: Run) => void;
+    /** Callback when a run starts */
+    onRunStart?: (run: Run) => void;
+    /** Callback when a run completes */
+    onRunComplete?: (run: Run, result: ConnectorInvokeResult) => void;
+    /** Callback when a run fails */
+    onRunError?: (run: Run, error: Error) => void;
+}
+/**
+ * Background processor for executing queued evaluation runs.
+ *
+ * The RunProcessor polls for runs with status "queued" and executes them
+ * via the configured connector. It supports concurrent execution and provides
+ * callbacks for monitoring status changes.
+ *
+ * Works from both CLI and API contexts - the same processor logic can be used
+ * with different status update mechanisms (terminal output vs WebSocket).
+ *
+ * @example
+ * ```typescript
+ * const processor = new RunProcessor({
+ *   pollIntervalMs: 5000,
+ *   maxConcurrent: 3,
+ *   onStatusChange: (runId, status, run) => {
+ *     console.log(`Run ${runId} is now ${status}`);
+ *   },
+ * });
+ *
+ * processor.start();
+ *
+ * // Later: graceful shutdown
+ * await processor.stop();
+ * ```
+ */
+export declare class RunProcessor {
+    private running;
+    private intervalId;
+    private activeRuns;
+    private options;
+    constructor(options?: RunProcessorOptions);
+    /**
+     * Starts the processor loop.
+     * Call this on server/CLI startup.
+     */
+    start(): void;
+    /**
+     * Stops the processor gracefully.
+     * Waits for active runs to complete.
+     */
+    stop(): Promise<void>;
+    /**
+     * Process a single tick (useful for testing or one-shot processing).
+     * Returns the number of runs started and waits for them to complete.
+     */
+    processOnce(): Promise<number>;
+    /**
+     * Returns true if the processor is currently running.
+     */
+    isRunning(): boolean;
+    /**
+     * Returns the number of currently active runs.
+     */
+    getActiveRunCount(): number;
+    /**
+     * Main processing tick - picks up queued runs and executes them.
+     * Returns the number of runs started.
+     * @param oneShot If true, waits for runs to complete before returning
+     */
+    private tick;
+    /**
+     * Atomically claims a run for processing.
+     * Returns true if successful, false if already claimed.
+     */
+    private claimRun;
+    /**
+     * Executes a single run with the evaluation loop.
+     *
+     * The loop:
+     * 1. Sends conversation to tested agent
+     * 2. Evaluates agent response against success/failure criteria
+     * 3. If success criteria met or failure criteria met, finish the run
+     * 4. If max messages reached, finish the run
+     * 5. Otherwise, generate a new persona message and continue
+     */
+    private executeRun;
+    /**
+     * Gets the thread ID for LangGraph.
+     * Uses the stored threadId if available (set on retry), otherwise uses run.id.
+     */
+    private getThreadId;
+    /**
+     * Gets the role of the last non-system message.
+     * Used to determine if we need to generate a persona message before invoking the connector.
+     */
+    private getLastMessageRole;
+    /**
+     * Executes the main evaluation loop.
+     * Both success and failure criteria are evaluated at every turn.
+     * The loop stops when:
+     * - Success criteria is met (run succeeds)
+     * - Failure criteria is met AND failureCriteriaMode is "every_turn" (run fails early)
+     * - Max messages reached without success (run fails; default mode "on_max_messages")
+     */
+    private executeEvaluationLoop;
+    /**
+     * Builds all messages including system prompt for a run.
+     * These messages are stored in the run for visibility in the UI.
+     */
+    private buildAllMessages;
+    /**
+     * Recovers runs that were interrupted by server crash.
+     * Only recovers runs for this processor's project filter.
+     */
+    private recoverStuckRuns;
+}
+//# sourceMappingURL=run-processor.d.ts.map

package/dist/run-processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-processor.d.ts","sourceRoot":"","sources":["../src/run-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,gBAAgB,CAAC;AAKxB,OAAO,EAIL,KAAK,GAAG,EACR,KAAK,SAAS,EAEf,MAAM,UAAU,CAAC;AAgBlB,YAAY,EAAE,SAAS,EAAE,CAAC;AAE1B,MAAM,WAAW,mBAAmB;IAClC,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC;IACtE,iCAAiC;IACjC,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC;IAChC,oCAAoC;IACpC,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAClE,gCAAgC;IAChC,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAC/C;AAYD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,OAAO,CAAkB;gBAErB,OAAO,GAAE,mBAAwB;IAY7C;;;OAGG;IACH,KAAK,IAAI,IAAI;IAeb;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAY3B;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAIpC;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;;OAIG;YACW,IAAI;IA+ClB;;;OAGG;IACH,OAAO,CAAC,QAAQ;IAehB;;;;;;;;;OASG;YACW,UAAU;IAmIxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAInB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;;;;;;OAOG;YACW,qBAAqB;IAsLnC;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAuCxB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;CAUzB"}