@axiom-lattice/agent-eval 2.1.9

package/src/LatticeEval.ts

@@ -0,0 +1,615 @@
+import { getAgentClient } from "@axiom-lattice/core";
+import { HumanMessage } from "@langchain/core/messages";
+import { v4 } from "uuid";
+import type {
+  LatticeEvalCase,
+  LatticeEvalLogEvent,
+  LatticeEvalLogLevel,
+  LatticeEvalRubric,
+  LatticeEvalResult,
+  OutputType,
+  LatticeAgentStepConfig,
+} from "./types";
+
+/**
+ * Configuration for Lattice evaluation server
+ */
+export interface LatticeEvalConfig {
+  base_url: string;
+  api_key?: string;
+  /**
+   * When true, prints detailed execution logs for each action.
+   * Defaults to true.
+   */
+  verbose?: boolean;
+}
+
+export interface LatticeEvalCaseRunResult {
+  caseId: string;
+  result?: LatticeEvalResult;
+  error?: string;
+  error_stack?: string;
+  duration_ms: number;
+  thread_id?: string;
+  judge_thread_id?: string;
+  test_prompt?: string;
+  final_output?: string;
+  logs: LatticeEvalLogEvent[];
+}
+
+
+/**
+ * LatticeEval class for evaluating Lattice evaluation cases
+ */
+export class LatticeEval {
+  private config: LatticeEvalConfig;
+  private baseUrl: string;
+  private verbose: boolean;
+  private inMemoryLogs: LatticeEvalLogEvent[] = [];
+  private lastThreadId?: string;
+  private lastJudgeThreadId?: string;
+  private lastTestPrompt?: string;
+  private lastFinalOutput?: string;
+  private lastDurationMs: number = 0;
+
+  public getLastRunMeta() {
+    return {
+      duration_ms: this.lastDurationMs,
+      thread_id: this.lastThreadId,
+      judge_thread_id: this.lastJudgeThreadId,
+      test_prompt: this.lastTestPrompt,
+      final_output: this.lastFinalOutput,
+    };
+  }
+
+  /**
+   * Create a new LatticeEval instance
+   * @param config Optional server configuration (defaults to localhost:3203)
+   */
+  constructor(config: LatticeEvalConfig) {
+    this.config = config;
+    this.baseUrl = this.config.base_url;
+    this.verbose = this.config.verbose ?? true;
+  }
+
+  public getInMemoryLogs(): LatticeEvalLogEvent[] {
+    return [...this.inMemoryLogs];
+  }
+
+  public record(
+    level: LatticeEvalLogLevel,
+    message: string,
+    data?: Record<string, unknown>
+  ) {
+    const event: LatticeEvalLogEvent = {
+      ts: new Date().toISOString(),
+      level,
+      message,
+      data,
+    };
+    this.inMemoryLogs.push(event);
+
+    if (!this.verbose) return;
+    // Simple console output - only show key info, no verbose details
+    if (level === "error") {
+      const keyInfo = this.getKeyInfo(data);
+      console.log(`  ✗ ${message}${keyInfo ? ` ${keyInfo}` : ""}`);
+    } else if (message.includes("Starting case") || message.includes("Case evaluation completed")) {
+      // Only show start/end of case evaluation
+      const keyInfo = this.getKeyInfo(data);
+      console.log(`  ${message}${keyInfo ? ` ${keyInfo}` : ""}`);
+    }
+    // Skip other verbose logs in console (they're still in memory for file output)
+  }
+
+  private log(message: string, data?: Record<string, unknown>) {
+    this.record("info", message, data);
+  }
+
+  private getKeyInfo(data?: Record<string, unknown>): string {
+    if (!data) return "";
+    const parts: string[] = [];
+    // Only show case_id, pass, final_score, error - key info only
+    if (data.case_id) parts.push(`case=${data.case_id}`);
+    if (data.pass !== undefined) parts.push(`pass=${data.pass}`);
+    if (data.final_score !== undefined) parts.push(`score=${data.final_score}`);
+    if (data.error) parts.push(`error=${data.error}`);
+    return parts.length > 0 ? `(${parts.join(" ")})` : "";
+  }
+
+  /**
+   * Execute a single agent step and return the thread ID and response data
+   */
+  private async executeAgentStep(
+    step: LatticeAgentStepConfig,
+    threadId: string,
+    inputMessage: string,
+    files: Record<string, string>
+  ): Promise<{ threadId: string; responseData: any }> {
+    this.log("Executing agent step", {
+      agent_id: step.agent_id,
+      thread_id: threadId,
+      has_override_input_message: Boolean(step.override_input_message),
+      input_message_length: step.override_input_message
+        ? step.override_input_message.length
+        : inputMessage.length,
+      files_count: Object.keys(files || {}).length,
+    });
+
+    const response = await fetch(`${this.baseUrl}/api/runs`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        assistant_id: step.agent_id,
+        thread_id: threadId,
+        files: Object.keys(files).reduce((acc, key) => {
+          acc[key] = { content: files[key].split("\n"), created_at: new Date().toISOString(), modified_at: new Date().toISOString() };
+          return acc;
+        }, {} as Record<string, any>),
+        message: step.override_input_message || inputMessage,
+      }),
+    });
+
+
+    const responseData: any = await response.json();
+    if (responseData.error) {
+      this.log("Agent step failed", {
+        agent_id: step.agent_id,
+        thread_id: threadId,
+        error: responseData.error,
+      });
+      throw new Error(
+        `Failed to run agent ${step.agent_id}: ${responseData.error}`
+      );
+    }
+    this.log("Agent step completed", {
+      agent_id: step.agent_id,
+      thread_id: threadId,
+      response_keys: responseData ? Object.keys(responseData) : [],
+    });
+    return { threadId, responseData };
+  }
+
+  /**
+   * Extract output content based on OutputType
+   */
+  private async extractOutput(
+    outputType: OutputType,
+    agentId: string,
+    threadId: string,
+    runResponseData?: any
+  ): Promise<string> {
+    if (outputType.type === "file_content") {
+      this.log("Extracting file output", {
+        agent_id: agentId,
+        thread_id: threadId,
+        file_path: outputType.file_path,
+      });
+      const responseState = await fetch(
+        `${this.baseUrl}/api/assistants/${agentId}/${threadId}/state`,
+        {
+          method: "GET",
+        }
+      );
+
+      if (!responseState.ok) {
+        this.log("Failed to fetch assistant state", {
+          agent_id: agentId,
+          thread_id: threadId,
+          status: responseState.status,
+          statusText: responseState.statusText,
+        });
+        throw new Error(`Failed to get state: ${responseState.statusText}`);
+      }
+
+      const state: any = await responseState.json();
+      const stateValues: any = state.values;
+      const fileContent = stateValues.files[outputType.file_path]?.content;
+
+      if (!fileContent) {
+        this.log("File output not found in state", {
+          agent_id: agentId,
+          thread_id: threadId,
+          file_path: outputType.file_path,
+        });
+        throw new Error(
+          `File not found in output: ${outputType.file_path}`
+        );
+      }
+      const content = Array.isArray(fileContent)
+        ? fileContent.join("\n")
+        : fileContent;
+      this.log("File output extracted", {
+        agent_id: agentId,
+        thread_id: threadId,
+        file_path: outputType.file_path,
+        output_length: typeof content === "string" ? content.length : undefined,
+      });
+      return content;
+    } else {
+      // For message_content type, get the last message from the run response
+      if (runResponseData?.messages && runResponseData.messages.length > 0) {
+        const content =
+          runResponseData.messages[runResponseData.messages.length - 1]?.content ||
+          "";
+        this.log("Message output extracted", {
+          agent_id: agentId,
+          thread_id: threadId,
+          output_length: typeof content === "string" ? content.length : undefined,
+        });
+        return content;
+      }
+      this.log("No message content found in run response", {
+        agent_id: agentId,
+        thread_id: threadId,
+      });
+      throw new Error("No message content found in run response");
+    }
+  }
+
+  /**
+   * Evaluate a single Lattice evaluation case
+   * @param evalCase The evaluation case to run
+   * @returns Evaluation result with pass/fail status and scores
+   */
+  async evaluateCase(evalCase: LatticeEvalCase): Promise<LatticeEvalResult> {
+    const startedAt = Date.now();
+    const threadId = `${evalCase.caseId}||${v4()}`;
+    this.inMemoryLogs = [];
+    this.lastThreadId = threadId;
+    this.lastJudgeThreadId = undefined;
+    this.lastTestPrompt = undefined;
+    this.lastFinalOutput = undefined;
+    this.lastDurationMs = 0;
+    this.log("Starting case evaluation", {
+      case_id: evalCase.caseId,
+      thread_id: threadId,
+      steps_count: evalCase.steps?.length,
+      output_type: evalCase.output?.type,
+    });
+
+    // Execute all agent steps sequentially
+    let currentThreadId = threadId;
+    let lastResponseData: any = null;
+    for (const step of evalCase.steps) {
+      const result = await this.executeAgentStep(
+        step,
+        currentThreadId,
+        evalCase.input.message,
+        evalCase.input.files || {}
+      );
+      currentThreadId = result.threadId;
+      lastResponseData = result.responseData;
+    }
+
+    // Get the final agent ID from the last step
+    const finalAgentId =
+      evalCase.steps[evalCase.steps.length - 1]?.agent_id || "";
+    this.log("All agent steps completed", {
+      case_id: evalCase.caseId,
+      final_agent_id: finalAgentId,
+      final_thread_id: currentThreadId,
+    });
+
+    // Extract output based on output type
+    const finalOutput = await this.extractOutput(
+      evalCase.output,
+      finalAgentId,
+      currentThreadId,
+      lastResponseData
+    );
+    this.lastFinalOutput = finalOutput;
+    this.log("Final output extracted", {
+      case_id: evalCase.caseId,
+      output_type: evalCase.output.type,
+      output_length: finalOutput.length,
+    });
+
+    // Build test prompt
+    const testCaseFilesContent = evalCase.input.files
+      ? Object.keys(evalCase.input.files)
+        .map(
+          (key: string) =>
+            `File name: ${key}\nFile content: ${evalCase.input.files![key]}`
+        )
+        .join("\n\n")
+      : "";
+
+    // Determine output type description
+    const outputTypeDescription = evalCase.output.type === "file_content"
+      ? `虚拟产物（文件：${evalCase.output.file_path}）`
+      : "消息内容";
+
+    const defaultRubrics: LatticeEvalRubric[] = [
+      {
+        dimension: "correctness",
+        weight: 100,
+        description:
+          "整体正确性，是否符合预期输出描述。",
+      },
+    ];
+
+    const evalRubrics =
+      evalCase.eval.eval_rubrics && evalCase.eval.eval_rubrics.length > 0
+        ? evalCase.eval.eval_rubrics
+        : defaultRubrics;
+    this.log("Prepared evaluation rubrics", {
+      case_id: evalCase.caseId,
+      rubrics_count: evalRubrics.length,
+      rubric_dimensions: evalRubrics.map((r) => r.dimension),
+    });
+
+    const rubricsSection = `\n## 评估指标（Evaluation Rubrics）\n${evalRubrics
+      .map(
+        (r) =>
+          `- **${r.dimension}**（权重：${r.weight}）：${r.description}`
+      )
+      .join("\n")}`;
+
+    const testPrompt = `# 角色
+你是一名资深的 AI Agent 评估专家，负责根据预设的指标（Rubrics）对 Agent 的执行结果进行"黑盒测试"判定。
+
+# 输入信息
+测试框架将为你提供以下四个核心上下文：
+
+1. **用户意图（User Intent）**：${evalCase.input.message}
+
+2. **输入文件（Input Files）**：${testCaseFilesContent || "无"}
+
+3. **实际输出（Actual Output，${outputTypeDescription}）**： 
+${finalOutput}
+
+4. **期望输出描述（Expected Output Description）**：${evalCase.eval.content_assertion}
+${rubricsSection}
+
+# 任务
+你必须严格对照"评估指标（Evaluation Rubrics）"中的每一项指标，分析"实际输出（Actual Output）"是否达标。
+
+# 规则
+1. **客观性**：仅根据提供的上下文判定。如果标准要求"包含数字"，但输出只有文字，即使语气再好也必须扣分。
+2. **结果校验**：如果"实际输出"中缺失预期的内容，或内容不符合"评估指标"中的标准，对应的指标应判定为失败。
+3. **证据导向**：在给出原因（reason）时，必须引用输出中的原文或虚拟产物中的具体数据片段。
+4. **加权计算**：最终分数为各项指标得分与其权重的乘积之和（0-100分制）。
+
+# 输出格式（仅JSON）
+你必须仅以 JSON 格式回复，结构如下：
+{
+  "pass": true | false,
+  "final_score": number,
+  "dimension_results": [
+    {
+      "name": "指标名称",
+      "score": number,
+      "reason": "具体的扣分或给分理由，需引用证据"
+    }
+  ],
+  "summary": "对 Agent 表现的整体评价"
+}
+
+注意：如果 final_score >= 80 且没有致命性错误，pass 应为 true；否则为 false。`;
+    this.lastTestPrompt = testPrompt;
+
+    // Invoke judge agent
+    const judgeThreadId = v4();
+    this.lastJudgeThreadId = judgeThreadId;
+    this.log("Invoking judge agent", { agent_key: "LatticeTest", case_id: evalCase.caseId });
+    const testResponse = await getAgentClient("LatticeTest").invoke(
+      {
+        messages: [new HumanMessage(testPrompt)],
+      },
+      {
+        configurable: {
+          thread_id: judgeThreadId,
+        },
+      }
+    );
+    this.log("Judge agent responded", {
+      case_id: evalCase.caseId,
+      messages_count: testResponse?.messages?.length,
+    });
+
+    const testResultContent: any =
+      testResponse.messages[testResponse.messages.length - 1]?.content || "";
+    this.log("Judge raw output received", {
+      case_id: evalCase.caseId,
+      output_length:
+        typeof testResultContent === "string" ? testResultContent.length : undefined,
+    });
+
+    // Parse JSON response from judge agent
+    let parsedResult: {
+      pass?: boolean;
+      final_score?: number;
+      dimension_results?: Array<{ name: string; score: number; reason: string }>;
+      summary?: string;
+    } = {};
+
+    try {
+      // Try to extract JSON from the response (handle code blocks or plain JSON)
+      const jsonMatch = testResultContent.match(/```(?:json)?\s*(\{[\s\S]*\})\s*```/) ||
+        testResultContent.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        parsedResult = JSON.parse(jsonMatch[1] || jsonMatch[0]);
+        this.log("Parsed judge JSON successfully", {
+          case_id: evalCase.caseId,
+          parsed_keys: Object.keys(parsedResult || {}),
+        });
+      } else {
+        this.log("No JSON detected in judge output; will fallback", {
+          case_id: evalCase.caseId,
+        });
+      }
+    } catch (error) {
+      // If JSON parsing fails, fall back to keyword-based parsing
+      console.warn("Failed to parse JSON from judge agent response, falling back to keyword-based parsing:", error);
+      this.log("Failed to parse judge JSON; falling back", {
+        case_id: evalCase.caseId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+
+    // Determine pass status
+    let pass: boolean;
+    if (parsedResult.pass !== undefined) {
+      pass = parsedResult.pass;
+      this.log("Pass determined from parsedResult.pass", { case_id: evalCase.caseId, pass });
+    } else if (parsedResult.final_score !== undefined) {
+      pass = parsedResult.final_score >= 80;
+      this.log("Pass determined from parsedResult.final_score", {
+        case_id: evalCase.caseId,
+        final_score: parsedResult.final_score,
+        pass,
+      });
+    } else {
+      // Fallback to keyword-based parsing
+      pass = testResultContent.toLowerCase().includes("pass") ||
+        testResultContent.toLowerCase().includes("success") ||
+        testResultContent.toLowerCase().includes("通过") ||
+        testResultContent.toLowerCase().includes("符合");
+      this.log("Pass determined from keyword fallback", { case_id: evalCase.caseId, pass });
+    }
+
+    // Extract dimension results
+    let dimensionResults: Array<{ name: string; score: number; reason: string }> = [];
+
+    if (parsedResult.dimension_results && parsedResult.dimension_results.length > 0) {
+      // Use parsed dimension results from JSON
+      dimensionResults = parsedResult.dimension_results;
+      this.log("Using parsed dimension_results from judge", {
+        case_id: evalCase.caseId,
+        dimensions_count: dimensionResults.length,
+      });
+    } else if (evalRubrics.length > 0) {
+      // Fallback: create dimension results structure if rubrics are provided but not parsed
+      dimensionResults = evalRubrics.map((rubric) => ({
+        name: rubric.dimension,
+        score: 0,
+        reason: "",
+      }));
+      this.log("No dimension_results parsed; using rubric skeleton", {
+        case_id: evalCase.caseId,
+        dimensions_count: dimensionResults.length,
+      });
+    }
+
+    // Calculate final score
+    let finalScore: number;
+    if (parsedResult.final_score !== undefined) {
+      finalScore = parsedResult.final_score;
+      this.log("Final score taken from parsedResult.final_score", {
+        case_id: evalCase.caseId,
+        final_score: finalScore,
+      });
+    } else if (dimensionResults.length > 0 && evalRubrics.length > 0) {
+      // Calculate weighted average if rubrics are provided
+      const rubricMap = new Map(evalRubrics.map((r) => [r.dimension, r.weight]));
+      const totalWeight = Array.from(rubricMap.values()).reduce((sum, w) => sum + w, 0);
+
+      if (totalWeight > 0) {
+        finalScore = dimensionResults.reduce((sum, result) => {
+          const weight = rubricMap.get(result.name) || 1;
+          return sum + (result.score * weight);
+        }, 0) / totalWeight;
+      } else {
+        // Fallback to simple average if no weights
+        finalScore = dimensionResults.reduce((sum, result) => sum + result.score, 0) / dimensionResults.length;
+      }
+      this.log("Final score computed from dimension_results", {
+        case_id: evalCase.caseId,
+        final_score: finalScore,
+        total_weight: totalWeight,
+      });
+    } else {
+      finalScore = pass ? 100 : 0;
+      this.log("Final score fallback (pass-based)", {
+        case_id: evalCase.caseId,
+        final_score: finalScore,
+        pass,
+      });
+    }
+
+    this.log("Case evaluation completed", {
+      case_id: evalCase.caseId,
+      pass,
+      final_score: finalScore,
+    });
+    const finishedAt = Date.now();
+    this.lastDurationMs = finishedAt - startedAt;
+    this.record("info", "Case duration recorded", {
+      case_id: evalCase.caseId,
+      duration_ms: this.lastDurationMs,
+    });
+    return {
+      pass,
+      final_score: finalScore,
+      dimension_results: dimensionResults,
+      summary: parsedResult.summary || testResultContent,
+    };
+  }
+}
+
+/**
+ * Evaluate a single Lattice evaluation case (backward compatibility function)
+ * @param evalCase The evaluation case to run
+ * @param config Optional server configuration (defaults to localhost:3203)
+ * @returns Evaluation result with pass/fail status and scores
+ * @deprecated Use LatticeEval class instead
+ */
+export async function evaluateLatticeCase(
+  evalCase: LatticeEvalCase,
+  config?: LatticeEvalConfig
+): Promise<LatticeEvalResult> {
+  const defaultConfig: LatticeEvalConfig = {
+    base_url: "http://localhost:3203",
+  };
+  const evaluator = new LatticeEval(config || defaultConfig);
+  return evaluator.evaluateCase(evalCase);
+}
+
+/**
+ * Evaluate a single Lattice evaluation case and always return logs (never throws).
+ */
+export async function evaluateLatticeCaseWithLogs(
+  evalCase: LatticeEvalCase,
+  config?: LatticeEvalConfig
+): Promise<LatticeEvalCaseRunResult> {
+  const defaultConfig: LatticeEvalConfig = {
+    base_url: "http://localhost:3203",
+  };
+  const evaluator = new LatticeEval(config || defaultConfig);
+  try {
+    const result = await evaluator.evaluateCase(evalCase);
+    const meta = evaluator.getLastRunMeta();
+    return {
+      caseId: evalCase.caseId,
+      result,
+      duration_ms: meta.duration_ms,
+      thread_id: meta.thread_id,
+      judge_thread_id: meta.judge_thread_id,
+      test_prompt: meta.test_prompt,
+      final_output: meta.final_output,
+      logs: evaluator.getInMemoryLogs(),
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    const errorStack = error instanceof Error ? error.stack : undefined;
+    evaluator.record("error", "Case evaluation failed", {
+      case_id: evalCase.caseId,
+      error: errorMessage,
+    });
+    const meta = evaluator.getLastRunMeta();
+    return {
+      caseId: evalCase.caseId,
+      error: errorMessage,
+      error_stack: errorStack,
+      duration_ms: meta.duration_ms,
+      thread_id: meta.thread_id,
+      judge_thread_id: meta.judge_thread_id,
+      test_prompt: meta.test_prompt,
+      final_output: meta.final_output,
+      logs: evaluator.getInMemoryLogs(),
+    };
+  }
+}