@axiom-lattice/agent-eval 2.1.9

package/dist/index.mjs

@@ -0,0 +1,1055 @@
+// src/LatticeEval.ts
+import { getAgentClient } from "@axiom-lattice/core";
+import { HumanMessage } from "@langchain/core/messages";
+import { v4 } from "uuid";
+var LatticeEval = class {
+  /**
+   * Create a new LatticeEval instance
+   * @param config Optional server configuration (defaults to localhost:3203)
+   */
+  constructor(config) {
+    this.inMemoryLogs = [];
+    this.lastDurationMs = 0;
+    this.config = config;
+    this.baseUrl = this.config.base_url;
+    this.verbose = this.config.verbose ?? true;
+  }
+  getLastRunMeta() {
+    return {
+      duration_ms: this.lastDurationMs,
+      thread_id: this.lastThreadId,
+      judge_thread_id: this.lastJudgeThreadId,
+      test_prompt: this.lastTestPrompt,
+      final_output: this.lastFinalOutput
+    };
+  }
+  getInMemoryLogs() {
+    return [...this.inMemoryLogs];
+  }
+  record(level, message, data) {
+    const event = {
+      ts: (/* @__PURE__ */ new Date()).toISOString(),
+      level,
+      message,
+      data
+    };
+    this.inMemoryLogs.push(event);
+    if (!this.verbose) return;
+    if (level === "error") {
+      const keyInfo = this.getKeyInfo(data);
+      console.log(`  \u2717 ${message}${keyInfo ? ` ${keyInfo}` : ""}`);
+    } else if (message.includes("Starting case") || message.includes("Case evaluation completed")) {
+      const keyInfo = this.getKeyInfo(data);
+      console.log(`  ${message}${keyInfo ? ` ${keyInfo}` : ""}`);
+    }
+  }
+  log(message, data) {
+    this.record("info", message, data);
+  }
+  getKeyInfo(data) {
+    if (!data) return "";
+    const parts = [];
+    if (data.case_id) parts.push(`case=${data.case_id}`);
+    if (data.pass !== void 0) parts.push(`pass=${data.pass}`);
+    if (data.final_score !== void 0) parts.push(`score=${data.final_score}`);
+    if (data.error) parts.push(`error=${data.error}`);
+    return parts.length > 0 ? `(${parts.join(" ")})` : "";
+  }
+  /**
+   * Execute a single agent step and return the thread ID and response data
+   */
+  async executeAgentStep(step, threadId, inputMessage, files) {
+    this.log("Executing agent step", {
+      agent_id: step.agent_id,
+      thread_id: threadId,
+      has_override_input_message: Boolean(step.override_input_message),
+      input_message_length: step.override_input_message ? step.override_input_message.length : inputMessage.length,
+      files_count: Object.keys(files || {}).length
+    });
+    const response = await fetch(`${this.baseUrl}/api/runs`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        assistant_id: step.agent_id,
+        thread_id: threadId,
+        files: Object.keys(files).reduce((acc, key) => {
+          acc[key] = { content: files[key].split("\n"), created_at: (/* @__PURE__ */ new Date()).toISOString(), modified_at: (/* @__PURE__ */ new Date()).toISOString() };
+          return acc;
+        }, {}),
+        message: step.override_input_message || inputMessage
+      })
+    });
+    const responseData = await response.json();
+    if (responseData.error) {
+      this.log("Agent step failed", {
+        agent_id: step.agent_id,
+        thread_id: threadId,
+        error: responseData.error
+      });
+      throw new Error(
+        `Failed to run agent ${step.agent_id}: ${responseData.error}`
+      );
+    }
+    this.log("Agent step completed", {
+      agent_id: step.agent_id,
+      thread_id: threadId,
+      response_keys: responseData ? Object.keys(responseData) : []
+    });
+    return { threadId, responseData };
+  }
+  /**
+   * Extract output content based on OutputType
+   */
+  async extractOutput(outputType, agentId, threadId, runResponseData) {
+    if (outputType.type === "file_content") {
+      this.log("Extracting file output", {
+        agent_id: agentId,
+        thread_id: threadId,
+        file_path: outputType.file_path
+      });
+      const responseState = await fetch(
+        `${this.baseUrl}/api/assistants/${agentId}/${threadId}/state`,
+        {
+          method: "GET"
+        }
+      );
+      if (!responseState.ok) {
+        this.log("Failed to fetch assistant state", {
+          agent_id: agentId,
+          thread_id: threadId,
+          status: responseState.status,
+          statusText: responseState.statusText
+        });
+        throw new Error(`Failed to get state: ${responseState.statusText}`);
+      }
+      const state = await responseState.json();
+      const stateValues = state.values;
+      const fileContent = stateValues.files[outputType.file_path]?.content;
+      if (!fileContent) {
+        this.log("File output not found in state", {
+          agent_id: agentId,
+          thread_id: threadId,
+          file_path: outputType.file_path
+        });
+        throw new Error(
+          `File not found in output: ${outputType.file_path}`
+        );
+      }
+      const content = Array.isArray(fileContent) ? fileContent.join("\n") : fileContent;
+      this.log("File output extracted", {
+        agent_id: agentId,
+        thread_id: threadId,
+        file_path: outputType.file_path,
+        output_length: typeof content === "string" ? content.length : void 0
+      });
+      return content;
+    } else {
+      if (runResponseData?.messages && runResponseData.messages.length > 0) {
+        const content = runResponseData.messages[runResponseData.messages.length - 1]?.content || "";
+        this.log("Message output extracted", {
+          agent_id: agentId,
+          thread_id: threadId,
+          output_length: typeof content === "string" ? content.length : void 0
+        });
+        return content;
+      }
+      this.log("No message content found in run response", {
+        agent_id: agentId,
+        thread_id: threadId
+      });
+      throw new Error("No message content found in run response");
+    }
+  }
+  /**
+   * Evaluate a single Lattice evaluation case
+   * @param evalCase The evaluation case to run
+   * @returns Evaluation result with pass/fail status and scores
+   */
+  async evaluateCase(evalCase) {
+    const startedAt = Date.now();
+    const threadId = `${evalCase.caseId}||${v4()}`;
+    this.inMemoryLogs = [];
+    this.lastThreadId = threadId;
+    this.lastJudgeThreadId = void 0;
+    this.lastTestPrompt = void 0;
+    this.lastFinalOutput = void 0;
+    this.lastDurationMs = 0;
+    this.log("Starting case evaluation", {
+      case_id: evalCase.caseId,
+      thread_id: threadId,
+      steps_count: evalCase.steps?.length,
+      output_type: evalCase.output?.type
+    });
+    let currentThreadId = threadId;
+    let lastResponseData = null;
+    for (const step of evalCase.steps) {
+      const result = await this.executeAgentStep(
+        step,
+        currentThreadId,
+        evalCase.input.message,
+        evalCase.input.files || {}
+      );
+      currentThreadId = result.threadId;
+      lastResponseData = result.responseData;
+    }
+    const finalAgentId = evalCase.steps[evalCase.steps.length - 1]?.agent_id || "";
+    this.log("All agent steps completed", {
+      case_id: evalCase.caseId,
+      final_agent_id: finalAgentId,
+      final_thread_id: currentThreadId
+    });
+    const finalOutput = await this.extractOutput(
+      evalCase.output,
+      finalAgentId,
+      currentThreadId,
+      lastResponseData
+    );
+    this.lastFinalOutput = finalOutput;
+    this.log("Final output extracted", {
+      case_id: evalCase.caseId,
+      output_type: evalCase.output.type,
+      output_length: finalOutput.length
+    });
+    const testCaseFilesContent = evalCase.input.files ? Object.keys(evalCase.input.files).map(
+      (key) => `File name: ${key}
+File content: ${evalCase.input.files[key]}`
+    ).join("\n\n") : "";
+    const outputTypeDescription = evalCase.output.type === "file_content" ? `\u865A\u62DF\u4EA7\u7269\uFF08\u6587\u4EF6\uFF1A${evalCase.output.file_path}\uFF09` : "\u6D88\u606F\u5185\u5BB9";
+    const defaultRubrics = [
+      {
+        dimension: "correctness",
+        weight: 100,
+        description: "\u6574\u4F53\u6B63\u786E\u6027\uFF0C\u662F\u5426\u7B26\u5408\u9884\u671F\u8F93\u51FA\u63CF\u8FF0\u3002"
+      }
+    ];
+    const evalRubrics = evalCase.eval.eval_rubrics && evalCase.eval.eval_rubrics.length > 0 ? evalCase.eval.eval_rubrics : defaultRubrics;
+    this.log("Prepared evaluation rubrics", {
+      case_id: evalCase.caseId,
+      rubrics_count: evalRubrics.length,
+      rubric_dimensions: evalRubrics.map((r) => r.dimension)
+    });
+    const rubricsSection = `
+## \u8BC4\u4F30\u6307\u6807\uFF08Evaluation Rubrics\uFF09
+${evalRubrics.map(
+      (r) => `- **${r.dimension}**\uFF08\u6743\u91CD\uFF1A${r.weight}\uFF09\uFF1A${r.description}`
+    ).join("\n")}`;
+    const testPrompt = `# \u89D2\u8272
+\u4F60\u662F\u4E00\u540D\u8D44\u6DF1\u7684 AI Agent \u8BC4\u4F30\u4E13\u5BB6\uFF0C\u8D1F\u8D23\u6839\u636E\u9884\u8BBE\u7684\u6307\u6807\uFF08Rubrics\uFF09\u5BF9 Agent \u7684\u6267\u884C\u7ED3\u679C\u8FDB\u884C"\u9ED1\u76D2\u6D4B\u8BD5"\u5224\u5B9A\u3002
+
+# \u8F93\u5165\u4FE1\u606F
+\u6D4B\u8BD5\u6846\u67B6\u5C06\u4E3A\u4F60\u63D0\u4F9B\u4EE5\u4E0B\u56DB\u4E2A\u6838\u5FC3\u4E0A\u4E0B\u6587\uFF1A
+
+1. **\u7528\u6237\u610F\u56FE\uFF08User Intent\uFF09**\uFF1A${evalCase.input.message}
+
+2. **\u8F93\u5165\u6587\u4EF6\uFF08Input Files\uFF09**\uFF1A${testCaseFilesContent || "\u65E0"}
+
+3. **\u5B9E\u9645\u8F93\u51FA\uFF08Actual Output\uFF0C${outputTypeDescription}\uFF09**\uFF1A 
+${finalOutput}
+
+4. **\u671F\u671B\u8F93\u51FA\u63CF\u8FF0\uFF08Expected Output Description\uFF09**\uFF1A${evalCase.eval.content_assertion}
+${rubricsSection}
+
+# \u4EFB\u52A1
+\u4F60\u5FC5\u987B\u4E25\u683C\u5BF9\u7167"\u8BC4\u4F30\u6307\u6807\uFF08Evaluation Rubrics\uFF09"\u4E2D\u7684\u6BCF\u4E00\u9879\u6307\u6807\uFF0C\u5206\u6790"\u5B9E\u9645\u8F93\u51FA\uFF08Actual Output\uFF09"\u662F\u5426\u8FBE\u6807\u3002
+
+# \u89C4\u5219
+1. **\u5BA2\u89C2\u6027**\uFF1A\u4EC5\u6839\u636E\u63D0\u4F9B\u7684\u4E0A\u4E0B\u6587\u5224\u5B9A\u3002\u5982\u679C\u6807\u51C6\u8981\u6C42"\u5305\u542B\u6570\u5B57"\uFF0C\u4F46\u8F93\u51FA\u53EA\u6709\u6587\u5B57\uFF0C\u5373\u4F7F\u8BED\u6C14\u518D\u597D\u4E5F\u5FC5\u987B\u6263\u5206\u3002
+2. **\u7ED3\u679C\u6821\u9A8C**\uFF1A\u5982\u679C"\u5B9E\u9645\u8F93\u51FA"\u4E2D\u7F3A\u5931\u9884\u671F\u7684\u5185\u5BB9\uFF0C\u6216\u5185\u5BB9\u4E0D\u7B26\u5408"\u8BC4\u4F30\u6307\u6807"\u4E2D\u7684\u6807\u51C6\uFF0C\u5BF9\u5E94\u7684\u6307\u6807\u5E94\u5224\u5B9A\u4E3A\u5931\u8D25\u3002
+3. **\u8BC1\u636E\u5BFC\u5411**\uFF1A\u5728\u7ED9\u51FA\u539F\u56E0\uFF08reason\uFF09\u65F6\uFF0C\u5FC5\u987B\u5F15\u7528\u8F93\u51FA\u4E2D\u7684\u539F\u6587\u6216\u865A\u62DF\u4EA7\u7269\u4E2D\u7684\u5177\u4F53\u6570\u636E\u7247\u6BB5\u3002
+4. **\u52A0\u6743\u8BA1\u7B97**\uFF1A\u6700\u7EC8\u5206\u6570\u4E3A\u5404\u9879\u6307\u6807\u5F97\u5206\u4E0E\u5176\u6743\u91CD\u7684\u4E58\u79EF\u4E4B\u548C\uFF080-100\u5206\u5236\uFF09\u3002
+
+# \u8F93\u51FA\u683C\u5F0F\uFF08\u4EC5JSON\uFF09
+\u4F60\u5FC5\u987B\u4EC5\u4EE5 JSON \u683C\u5F0F\u56DE\u590D\uFF0C\u7ED3\u6784\u5982\u4E0B\uFF1A
+{
+  "pass": true | false,
+  "final_score": number,
+  "dimension_results": [
+    {
+      "name": "\u6307\u6807\u540D\u79F0",
+      "score": number,
+      "reason": "\u5177\u4F53\u7684\u6263\u5206\u6216\u7ED9\u5206\u7406\u7531\uFF0C\u9700\u5F15\u7528\u8BC1\u636E"
+    }
+  ],
+  "summary": "\u5BF9 Agent \u8868\u73B0\u7684\u6574\u4F53\u8BC4\u4EF7"
+}
+
+\u6CE8\u610F\uFF1A\u5982\u679C final_score >= 80 \u4E14\u6CA1\u6709\u81F4\u547D\u6027\u9519\u8BEF\uFF0Cpass \u5E94\u4E3A true\uFF1B\u5426\u5219\u4E3A false\u3002`;
+    this.lastTestPrompt = testPrompt;
+    const judgeThreadId = v4();
+    this.lastJudgeThreadId = judgeThreadId;
+    this.log("Invoking judge agent", { agent_key: "LatticeTest", case_id: evalCase.caseId });
+    const testResponse = await getAgentClient("LatticeTest").invoke(
+      {
+        messages: [new HumanMessage(testPrompt)]
+      },
+      {
+        configurable: {
+          thread_id: judgeThreadId
+        }
+      }
+    );
+    this.log("Judge agent responded", {
+      case_id: evalCase.caseId,
+      messages_count: testResponse?.messages?.length
+    });
+    const testResultContent = testResponse.messages[testResponse.messages.length - 1]?.content || "";
+    this.log("Judge raw output received", {
+      case_id: evalCase.caseId,
+      output_length: typeof testResultContent === "string" ? testResultContent.length : void 0
+    });
+    let parsedResult = {};
+    try {
+      const jsonMatch = testResultContent.match(/```(?:json)?\s*(\{[\s\S]*\})\s*```/) || testResultContent.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        parsedResult = JSON.parse(jsonMatch[1] || jsonMatch[0]);
+        this.log("Parsed judge JSON successfully", {
+          case_id: evalCase.caseId,
+          parsed_keys: Object.keys(parsedResult || {})
+        });
+      } else {
+        this.log("No JSON detected in judge output; will fallback", {
+          case_id: evalCase.caseId
+        });
+      }
+    } catch (error) {
+      console.warn("Failed to parse JSON from judge agent response, falling back to keyword-based parsing:", error);
+      this.log("Failed to parse judge JSON; falling back", {
+        case_id: evalCase.caseId,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+    let pass;
+    if (parsedResult.pass !== void 0) {
+      pass = parsedResult.pass;
+      this.log("Pass determined from parsedResult.pass", { case_id: evalCase.caseId, pass });
+    } else if (parsedResult.final_score !== void 0) {
+      pass = parsedResult.final_score >= 80;
+      this.log("Pass determined from parsedResult.final_score", {
+        case_id: evalCase.caseId,
+        final_score: parsedResult.final_score,
+        pass
+      });
+    } else {
+      pass = testResultContent.toLowerCase().includes("pass") || testResultContent.toLowerCase().includes("success") || testResultContent.toLowerCase().includes("\u901A\u8FC7") || testResultContent.toLowerCase().includes("\u7B26\u5408");
+      this.log("Pass determined from keyword fallback", { case_id: evalCase.caseId, pass });
+    }
+    let dimensionResults = [];
+    if (parsedResult.dimension_results && parsedResult.dimension_results.length > 0) {
+      dimensionResults = parsedResult.dimension_results;
+      this.log("Using parsed dimension_results from judge", {
+        case_id: evalCase.caseId,
+        dimensions_count: dimensionResults.length
+      });
+    } else if (evalRubrics.length > 0) {
+      dimensionResults = evalRubrics.map((rubric) => ({
+        name: rubric.dimension,
+        score: 0,
+        reason: ""
+      }));
+      this.log("No dimension_results parsed; using rubric skeleton", {
+        case_id: evalCase.caseId,
+        dimensions_count: dimensionResults.length
+      });
+    }
+    let finalScore;
+    if (parsedResult.final_score !== void 0) {
+      finalScore = parsedResult.final_score;
+      this.log("Final score taken from parsedResult.final_score", {
+        case_id: evalCase.caseId,
+        final_score: finalScore
+      });
+    } else if (dimensionResults.length > 0 && evalRubrics.length > 0) {
+      const rubricMap = new Map(evalRubrics.map((r) => [r.dimension, r.weight]));
+      const totalWeight = Array.from(rubricMap.values()).reduce((sum, w) => sum + w, 0);
+      if (totalWeight > 0) {
+        finalScore = dimensionResults.reduce((sum, result) => {
+          const weight = rubricMap.get(result.name) || 1;
+          return sum + result.score * weight;
+        }, 0) / totalWeight;
+      } else {
+        finalScore = dimensionResults.reduce((sum, result) => sum + result.score, 0) / dimensionResults.length;
+      }
+      this.log("Final score computed from dimension_results", {
+        case_id: evalCase.caseId,
+        final_score: finalScore,
+        total_weight: totalWeight
+      });
+    } else {
+      finalScore = pass ? 100 : 0;
+      this.log("Final score fallback (pass-based)", {
+        case_id: evalCase.caseId,
+        final_score: finalScore,
+        pass
+      });
+    }
+    this.log("Case evaluation completed", {
+      case_id: evalCase.caseId,
+      pass,
+      final_score: finalScore
+    });
+    const finishedAt = Date.now();
+    this.lastDurationMs = finishedAt - startedAt;
+    this.record("info", "Case duration recorded", {
+      case_id: evalCase.caseId,
+      duration_ms: this.lastDurationMs
+    });
+    return {
+      pass,
+      final_score: finalScore,
+      dimension_results: dimensionResults,
+      summary: parsedResult.summary || testResultContent
+    };
+  }
+};
+async function evaluateLatticeCase(evalCase, config) {
+  const defaultConfig = {
+    base_url: "http://localhost:3203"
+  };
+  const evaluator = new LatticeEval(config || defaultConfig);
+  return evaluator.evaluateCase(evalCase);
+}
+async function evaluateLatticeCaseWithLogs(evalCase, config) {
+  const defaultConfig = {
+    base_url: "http://localhost:3203"
+  };
+  const evaluator = new LatticeEval(config || defaultConfig);
+  try {
+    const result = await evaluator.evaluateCase(evalCase);
+    const meta = evaluator.getLastRunMeta();
+    return {
+      caseId: evalCase.caseId,
+      result,
+      duration_ms: meta.duration_ms,
+      thread_id: meta.thread_id,
+      judge_thread_id: meta.judge_thread_id,
+      test_prompt: meta.test_prompt,
+      final_output: meta.final_output,
+      logs: evaluator.getInMemoryLogs()
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    const errorStack = error instanceof Error ? error.stack : void 0;
+    evaluator.record("error", "Case evaluation failed", {
+      case_id: evalCase.caseId,
+      error: errorMessage
+    });
+    const meta = evaluator.getLastRunMeta();
+    return {
+      caseId: evalCase.caseId,
+      error: errorMessage,
+      error_stack: errorStack,
+      duration_ms: meta.duration_ms,
+      thread_id: meta.thread_id,
+      judge_thread_id: meta.judge_thread_id,
+      test_prompt: meta.test_prompt,
+      final_output: meta.final_output,
+      logs: evaluator.getInMemoryLogs()
+    };
+  }
+}
+
+// src/LatticeEvalSuite.ts
+async function limitConcurrency(tasks, concurrency) {
+  const results = [];
+  const executing = [];
+  let index = 0;
+  const executeTask = async (task, taskIndex) => {
+    try {
+      const result = await task();
+      results[taskIndex] = { success: true, result };
+    } catch (error) {
+      results[taskIndex] = {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  };
+  while (index < tasks.length || executing.length > 0) {
+    while (executing.length < concurrency && index < tasks.length) {
+      const task = tasks[index];
+      const currentIndex = index++;
+      const promise = executeTask(task, currentIndex).catch((err) => {
+        console.error(`Unexpected error in task execution:`, err);
+      }).finally(() => {
+        const idx = executing.indexOf(promise);
+        if (idx > -1) {
+          executing.splice(idx, 1);
+        }
+      });
+      executing.push(promise);
+    }
+    if (executing.length > 0) {
+      await Promise.race(executing);
+    }
+  }
+  await Promise.allSettled(executing);
+  return results;
+}
+function resolveTemplateCase(templateCase, templates) {
+  const template = templates.get(templateCase.templateId);
+  if (!template) {
+    throw new Error(`Template not found: ${templateCase.templateId}`);
+  }
+  const resolvedCase = {
+    caseId: templateCase.caseId,
+    input: {
+      message: templateCase.input.message ?? template.default_case.input.message,
+      files: {
+        ...template.default_case.input.files,
+        ...templateCase.input.files
+      }
+    },
+    steps: template.default_case.steps,
+    output: templateCase.output || template.default_case.output,
+    eval: {
+      content_assertion: templateCase.eval.content_assertion,
+      eval_rubrics: templateCase.eval.eval_rubrics || template.default_case.eval?.eval_rubrics
+    }
+  };
+  return resolvedCase;
+}
+function isTemplateCase(case_) {
+  return "templateId" in case_;
+}
+var LatticeEvalSuite = class {
+  constructor(suite, projectConfig, templates = /* @__PURE__ */ new Map()) {
+    this.suite = suite;
+    this.projectConfig = projectConfig;
+    this.templates = templates;
+  }
+  /**
+   * Get resolved configuration from project
+   */
+  getResolvedConfig() {
+    return this.projectConfig;
+  }
+  /**
+   * Get suite name
+   */
+  getSuiteName() {
+    return this.suite.suiteName;
+  }
+  /**
+   * Get suite version
+   */
+  getVersion() {
+    return this.suite.version;
+  }
+  /**
+   * Get all cases in this suite (resolved from templates if needed)
+   */
+  getCases() {
+    return this.suite.cases.map((case_) => {
+      if (isTemplateCase(case_)) {
+        return resolveTemplateCase(case_, this.templates);
+      }
+      return case_;
+    });
+  }
+  /**
+   * Get a specific case by ID (resolved from template if needed)
+   */
+  getCase(caseId) {
+    const case_ = this.suite.cases.find((c) => c.caseId === caseId);
+    if (!case_) {
+      return void 0;
+    }
+    if (isTemplateCase(case_)) {
+      return resolveTemplateCase(case_, this.templates);
+    }
+    return case_;
+  }
+  /**
+   * Run a single case in this suite with error handling
+   * @param caseId The case ID to run
+   * @returns Case run result with error handling
+   */
+  async runCase(caseId) {
+    try {
+      const evalCase = this.getCase(caseId);
+      if (!evalCase) {
+        return {
+          caseId,
+          error: `Case not found: ${caseId}`,
+          logs: []
+        };
+      }
+      const config = this.getResolvedConfig();
+      const evalConfig = {
+        base_url: config.lattice_server_config.base_url,
+        api_key: config.lattice_server_config.api_key
+      };
+      const run = await evaluateLatticeCaseWithLogs(evalCase, evalConfig);
+      return {
+        caseId,
+        result: run.result,
+        error: run.error,
+        error_stack: run.error_stack,
+        duration_ms: run.duration_ms,
+        thread_id: run.thread_id,
+        judge_thread_id: run.judge_thread_id,
+        test_prompt: run.test_prompt,
+        final_output: run.final_output,
+        logs: run.logs
+      };
+    } catch (error) {
+      return {
+        caseId,
+        error: error instanceof Error ? error.message : String(error),
+        logs: []
+      };
+    }
+  }
+  /**
+   * Run all cases in this suite with concurrency control and error isolation
+   * @param concurrency Optional concurrency limit (overrides project config)
+   * @returns Array of case run results with error handling
+   */
+  async runAllCases(concurrency) {
+    const config = this.getResolvedConfig();
+    const maxConcurrency = concurrency ?? config.concurrency;
+    const tasks = this.suite.cases.map((case_) => async () => {
+      try {
+        const evalCase = isTemplateCase(case_) ? resolveTemplateCase(case_, this.templates) : case_;
+        const evalConfig = {
+          base_url: config.lattice_server_config.base_url,
+          api_key: config.lattice_server_config.api_key
+        };
+        const run = await evaluateLatticeCaseWithLogs(evalCase, evalConfig);
+        return {
+          caseId: evalCase.caseId,
+          result: run.result,
+          error: run.error,
+          error_stack: run.error_stack,
+          duration_ms: run.duration_ms,
+          thread_id: run.thread_id,
+          judge_thread_id: run.judge_thread_id,
+          test_prompt: run.test_prompt,
+          final_output: run.final_output,
+          logs: run.logs
+        };
+      } catch (error) {
+        return {
+          caseId: case_.caseId,
+          error: error instanceof Error ? error.message : String(error),
+          logs: []
+        };
+      }
+    });
+    const taskResults = await limitConcurrency(tasks, maxConcurrency);
+    return taskResults.map((taskResult, index) => {
+      if (taskResult.success && taskResult.result) {
+        return taskResult.result;
+      }
+      return {
+        caseId: this.suite.cases[index].caseId,
+        error: taskResult.error || "Unknown error",
+        logs: []
+      };
+    });
+  }
+};
+
+// src/LatticeEvalProject.ts
+import {
+  registerModelLattice,
+  registerAgentLattice,
+  AgentType
+} from "@axiom-lattice/core";
+import { mkdir, writeFile } from "fs/promises";
+import path from "path";
+var LatticeEvalProject = class {
+  constructor(project) {
+    this.suites = /* @__PURE__ */ new Map();
+    this.project = project;
+    this.reportConfig = project.report_config;
+    const judgeModelKey = `${this.project.projectName}_judge_model`;
+    registerModelLattice(judgeModelKey, this.project.judge_agent_config.model);
+    const judgeAgentKey = "LatticeTest";
+    const judgeAgentConfig = {
+      key: judgeAgentKey,
+      name: "Lattice Test Judge Agent",
+      description: "Judge agent for evaluating Lattice test cases",
+      type: AgentType.REACT,
+      prompt: "",
+      // No prompt as requested
+      modelKey: judgeModelKey
+    };
+    registerAgentLattice(judgeAgentConfig);
+    const projectConfig = {
+      lattice_server_config: {
+        base_url: this.project.lattice_server_config.base_url,
+        api_key: this.project.lattice_server_config.api_key
+      },
+      judge_agent_config: this.project.judge_agent_config,
+      concurrency: this.project.concurrency ?? 1
+    };
+    const templatesMap = /* @__PURE__ */ new Map();
+    if (this.project.templates) {
+      for (const template of this.project.templates) {
+        templatesMap.set(template.templateId, template);
+      }
+    }
+    for (const suite of this.project.suites) {
+      this.suites.set(
+        suite.suiteName,
+        new LatticeEvalSuite(suite, projectConfig, templatesMap)
+      );
+    }
+  }
+  /**
+   * Get project name
+   */
+  getProjectName() {
+    return this.project.projectName;
+  }
+  /**
+   * Get project version
+   */
+  getVersion() {
+    return this.project.version;
+  }
+  /**
+   * Get project description
+   */
+  getDescription() {
+    return this.project.description;
+  }
+  /**
+   * Get all suite names
+   */
+  getSuiteNames() {
+    return Array.from(this.suites.keys());
+  }
+  /**
+   * Get a specific suite by name
+   */
+  getSuite(suiteName) {
+    return this.suites.get(suiteName);
+  }
+  /**
+   * Run a specific case in a specific suite
+   * @param suiteName The suite name
+   * @param caseId The case ID to run
+   * @returns Case run result with error handling
+   */
+  async runCase(suiteName, caseId) {
+    const suite = this.getSuite(suiteName);
+    if (!suite) {
+      return {
+        caseId,
+        error: `Suite not found: ${suiteName}`,
+        logs: []
+      };
+    }
+    return suite.runCase(caseId);
+  }
+  /**
+   * Run all cases in a specific suite with concurrency control and error isolation
+   * @param suiteName The suite name
+   * @param concurrency Optional concurrency limit (overrides project config)
+   * @returns Array of case run results with error handling
+   */
+  async runSuite(suiteName, concurrency) {
+    const suite = this.getSuite(suiteName);
+    if (!suite) {
+      throw new Error(`Suite not found: ${suiteName}`);
+    }
+    return suite.runAllCases(concurrency);
+  }
+  /**
+   * Run all cases in all suites with concurrency control and error isolation
+   * @param concurrency Optional concurrency limit (overrides project config)
+   * @returns Map of suite names to their case run results
+   */
+  async runAllSuites(concurrency) {
+    const results = /* @__PURE__ */ new Map();
+    for (const suiteName of this.getSuiteNames()) {
+      try {
+        const suiteResults = await this.runSuite(suiteName, concurrency);
+        results.set(suiteName, suiteResults);
+      } catch (error) {
+        const suite = this.getSuite(suiteName);
+        if (suite) {
+          const errorResults = suite.getCases().map((c) => ({
+            caseId: c.caseId,
+            error: error instanceof Error ? error.message : String(error),
+            logs: []
+          }));
+          results.set(suiteName, errorResults);
+        }
+      }
+    }
+    return results;
+  }
+  /**
+   * Run all suites as a "batch", build a report, and optionally write it to disk.
+   */
+  async runAllSuitesBatch(concurrency) {
+    const started_at = (/* @__PURE__ */ new Date()).toISOString();
+    const batch_id = this.reportConfig?.batch_id || `${Date.now()}`;
+    console.log(`
+Running batch: ${this.project.projectName} (${this.getSuiteNames().length} suites)`);
+    const results = await this.runAllSuites(concurrency);
+    let total_cases = 0;
+    let passed_cases = 0;
+    let failed_cases = 0;
+    const suites = [];
+    const durations = [];
+    for (const [suiteName, caseResults] of results.entries()) {
+      const suiteTotal = caseResults.length;
+      const suitePassed = caseResults.filter((r) => r.result?.pass).length;
+      const suiteFailed = suiteTotal - suitePassed;
+      total_cases += suiteTotal;
+      passed_cases += suitePassed;
+      failed_cases += suiteFailed;
+      suites.push({
+        suiteName,
+        total_cases: suiteTotal,
+        passed_cases: suitePassed,
+        failed_cases: suiteFailed,
+        cases: caseResults.map((r) => ({
+          caseId: r.caseId,
+          pass: r.result?.pass,
+          final_score: r.result?.final_score,
+          error: r.error
+        }))
+      });
+      for (const r of caseResults) {
+        if (typeof r.duration_ms === "number") durations.push(r.duration_ms);
+      }
+    }
+    const finished_at = (/* @__PURE__ */ new Date()).toISOString();
+    const report = {
+      batch_id,
+      started_at,
+      finished_at,
+      project: {
+        projectName: this.project.projectName,
+        version: this.project.version,
+        description: this.project.description
+      },
+      summary: {
+        total_cases,
+        passed_cases,
+        failed_cases,
+        pass_rate: total_cases > 0 ? passed_cases / total_cases : 0
+      },
+      suites
+    };
+    const batch_dir = await this.maybeWriteBatchArtifacts(
+      batch_id,
+      report,
+      results
+    );
+    console.log(`
+=== Summary ===`);
+    console.log(`Total: ${report.summary.total_cases} | Passed: ${report.summary.passed_cases} | Failed: ${report.summary.failed_cases} | Pass Rate: ${(report.summary.pass_rate * 100).toFixed(2)}%`);
+    if (batch_dir) {
+      console.log(`
+Results saved to: ${batch_dir}`);
+    }
+    return { batch_id, batch_dir, results, report };
+  }
+  generateCaseMarkdown(index, suiteName, caseResult, payload) {
+    const lines = [];
+    const status = caseResult.result?.pass ? "\u2705 PASS" : "\u274C FAIL";
+    lines.push(`# Test ${index}: ${status}`);
+    lines.push(``);
+    lines.push(`- **Suite**: ${suiteName}`);
+    lines.push(`- **Case ID**: ${caseResult.caseId}`);
+    lines.push(`- **Status**: ${caseResult.result?.pass ? "PASS" : "FAIL"}`);
+    if (typeof payload.duration === "number") {
+      lines.push(`- **Duration**: ${(payload.duration / 1e3).toFixed(2)}s`);
+    }
+    if (payload.threadId) {
+      lines.push(`- **Thread ID**: ${payload.threadId}`);
+    }
+    if (payload.judgeThreadId) {
+      lines.push(`- **Judge Thread ID**: ${payload.judgeThreadId}`);
+    }
+    lines.push(``);
+    if (caseResult.result) {
+      lines.push(`## Result`);
+      lines.push(``);
+      lines.push(`- **Final Score**: ${caseResult.result.final_score}`);
+      lines.push(`- **Summary**: ${caseResult.result.summary || "N/A"}`);
+      lines.push(``);
+      if (caseResult.result.dimension_results && caseResult.result.dimension_results.length > 0) {
+        lines.push(`## Dimension Results`);
+        lines.push(``);
+        for (const dim of caseResult.result.dimension_results) {
+          lines.push(`### ${dim.name}`);
+          lines.push(`- **Score**: ${dim.score}`);
+          lines.push(`- **Reason**: ${dim.reason}`);
+          lines.push(``);
+        }
+      }
+    }
+    if (caseResult.error) {
+      lines.push(`## Error`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      lines.push(caseResult.error);
+      if (caseResult.error_stack) {
+        lines.push(``);
+        lines.push(caseResult.error_stack);
+      }
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+    if (payload.finalOutput) {
+      lines.push(`## Final Output`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      const output = payload.finalOutput.length > 5e3 ? payload.finalOutput.substring(0, 5e3) + "\n\n... (truncated, see JSON for full output)" : payload.finalOutput;
+      lines.push(output);
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+    if (payload.testPrompt) {
+      lines.push(`## Test Prompt`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      const prompt = payload.testPrompt.length > 5e3 ? payload.testPrompt.substring(0, 5e3) + "\n\n... (truncated, see JSON for full prompt)" : payload.testPrompt;
+      lines.push(prompt);
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+    return lines.join("\n");
+  }
+  generateMarkdownSummary(batch_id, report, results) {
+    const lines = [];
+    lines.push(`# Lattice Eval Batch Summary`);
+    lines.push(``);
+    lines.push(`- **Project**: ${report.project.projectName}`);
+    if (report.project.version) lines.push(`- **Version**: ${report.project.version}`);
+    if (report.project.description) lines.push(`- **Description**: ${report.project.description}`);
+    lines.push(`- **Batch ID**: ${batch_id}`);
+    lines.push(`- **Started**: ${report.started_at}`);
+    lines.push(`- **Finished**: ${report.finished_at}`);
+    lines.push(``);
+    lines.push(`## Overview`);
+    lines.push(``);
+    lines.push(`| Metric | Value |`);
+    lines.push(`|---|---:|`);
+    lines.push(`| Total cases | ${report.summary.total_cases} |`);
+    lines.push(`| Passed | ${report.summary.passed_cases} |`);
+    lines.push(`| Failed | ${report.summary.failed_cases} |`);
+    lines.push(`| Pass rate | ${(report.summary.pass_rate * 100).toFixed(2)}% |`);
+    lines.push(``);
+    lines.push(`## Suites`);
+    lines.push(``);
+    for (const suite of report.suites) {
+      lines.push(`### ${suite.suiteName}`);
+      lines.push(``);
+      lines.push(`| Case | Status | Score | Duration (ms) | Thread |`);
+      lines.push(`|---|---|---:|---:|---|`);
+      const suiteResults = results.get(suite.suiteName) || [];
+      for (const r of suiteResults) {
+        const status = r.result?.pass ? "PASS" : "FAIL";
+        const score = r.result?.final_score ?? "";
+        const dur = typeof r.duration_ms === "number" ? r.duration_ms : "";
+        const thread = r.thread_id ?? "";
+        lines.push(`| ${r.caseId} | ${status} | ${score} | ${dur} | ${thread} |`);
+      }
+      lines.push(``);
+    }
+    return lines.join("\n");
+  }
+  async maybeWriteBatchArtifacts(batch_id, report, results) {
+    const config = this.reportConfig;
+    if (!config?.output_dir) return void 0;
+    const batchDir = path.join(config.output_dir, batch_id);
+    await mkdir(batchDir, { recursive: true });
+    const writeReportJson = config.write_report_json ?? true;
+    const writeCaseLogs = config.write_case_logs ?? true;
+    if (writeReportJson) {
+      await writeFile(
+        path.join(batchDir, "report.json"),
+        JSON.stringify(report, null, 2),
+        "utf-8"
+      );
+    }
+    const resultsJsonPath = path.join(batchDir, "results.json");
+    const resultsJson = {
+      executionTimestamp: batch_id,
+      summary: report.summary,
+      report,
+      results: Array.from(results.entries()).map(([suiteName, caseResults]) => ({
+        suiteName,
+        cases: caseResults.map((r) => ({
+          caseId: r.caseId,
+          passed: r.result?.pass === true,
+          message: r.result?.summary || r.error || "",
+          error: r.error ? {
+            message: r.error,
+            stack: r.error_stack
+          } : void 0,
+          duration: r.duration_ms,
+          testPrompt: r.test_prompt,
+          finalOutput: r.final_output,
+          threadId: r.thread_id,
+          judgeThreadId: r.judge_thread_id
+        }))
+      }))
+    };
+    await writeFile(resultsJsonPath, JSON.stringify(resultsJson, null, 2), "utf-8");
+    const summaryMdPath = path.join(batchDir, "summary.md");
+    const summaryMd = this.generateMarkdownSummary(batch_id, report, results);
+    await writeFile(summaryMdPath, summaryMd, "utf-8");
+    const individualDir = path.join(batchDir, "individual");
+    await mkdir(individualDir, { recursive: true });
+    let index = 1;
+    for (const [suiteName, caseResults] of results.entries()) {
+      for (const r of caseResults) {
+        const status = r.result?.pass ? "PASS" : "FAIL";
+        const baseFilename = `test-${index}-${suiteName}-${r.caseId}-${status}`.replace(/[\/\\]/g, "_");
+        const jsonPath = path.join(individualDir, `${baseFilename}.json`);
+        const payload = {
+          index,
+          suiteName,
+          caseId: r.caseId,
+          passed: r.result?.pass === true,
+          result: r.result,
+          message: r.result?.summary || r.error || "",
+          error: r.error ? { message: r.error, stack: r.error_stack } : void 0,
+          duration: r.duration_ms,
+          threadId: r.thread_id,
+          judgeThreadId: r.judge_thread_id,
+          finalOutput: r.final_output,
+          testPrompt: r.test_prompt
+        };
+        await writeFile(jsonPath, JSON.stringify(payload, null, 2), "utf-8");
+        const mdPath = path.join(individualDir, `${baseFilename}.md`);
+        const mdContent = this.generateCaseMarkdown(index, suiteName, r, payload);
+        await writeFile(mdPath, mdContent, "utf-8");
+        index += 1;
+      }
+    }
+    if (writeCaseLogs) {
+      for (const [suiteName, caseResults] of results.entries()) {
+        const suiteDir = path.join(batchDir, "cases", suiteName);
+        await mkdir(suiteDir, { recursive: true });
+        for (const r of caseResults) {
+          await writeFile(
+            path.join(suiteDir, `${r.caseId}.logs.json`),
+            JSON.stringify(r.logs || [], null, 2),
+            "utf-8"
+          );
+        }
+      }
+    }
+    return batchDir;
+  }
+};
+export {
+  LatticeEval,
+  LatticeEvalProject,
+  LatticeEvalSuite,
+  evaluateLatticeCase,
+  evaluateLatticeCaseWithLogs
+};
+//# sourceMappingURL=index.mjs.map