@axiom-lattice/agent-eval 2.1.9

package/src/LatticeEvalProject.ts

@@ -0,0 +1,496 @@
+import type {
+  LatticeEvalBatchReport,
+  LatticeEvalProjectType,
+  LatticeEvalReportConfig,
+  LatticeEvalTemplate,
+} from "./types";
+import { LatticeEvalSuite, ResolvedConfig, CaseRunResult } from "./LatticeEvalSuite";
+import {
+  registerModelLattice,
+  registerAgentLattice,
+  AgentType,
+  AgentConfig,
+} from "@axiom-lattice/core";
+import { mkdir, writeFile } from "fs/promises";
+import path from "path";
+
+/**
+ * LatticeEvalProject class manages a project with multiple evaluation suites
+ * with project-level configuration
+ */
+export class LatticeEvalProject {
+  private project: LatticeEvalProjectType;
+  private suites: Map<string, LatticeEvalSuite> = new Map();
+  private reportConfig?: LatticeEvalReportConfig;
+
+  constructor(project: LatticeEvalProjectType) {
+    this.project = project;
+    this.reportConfig = project.report_config;
+
+    // Register judge model
+    const judgeModelKey = `${this.project.projectName}_judge_model`;
+    registerModelLattice(judgeModelKey, this.project.judge_agent_config.model);
+
+    // Register judge agent
+    const judgeAgentKey = "LatticeTest";
+    const judgeAgentConfig: AgentConfig = {
+      key: judgeAgentKey,
+      name: "Lattice Test Judge Agent",
+      description: "Judge agent for evaluating Lattice test cases",
+      type: AgentType.REACT,
+      prompt: "", // No prompt as requested
+      modelKey: judgeModelKey,
+    };
+    registerAgentLattice(judgeAgentConfig);
+
+    // Build resolved config for project
+    const projectConfig: ResolvedConfig = {
+      lattice_server_config: {
+        base_url: this.project.lattice_server_config.base_url,
+        api_key: this.project.lattice_server_config.api_key,
+      },
+      judge_agent_config: this.project.judge_agent_config,
+      concurrency: this.project.concurrency ?? 1,
+    };
+
+    // Build templates map
+    const templatesMap = new Map<string, LatticeEvalTemplate>();
+    if (this.project.templates) {
+      for (const template of this.project.templates) {
+        templatesMap.set(template.templateId, template);
+      }
+    }
+
+    // Initialize all suites with project config and templates
+    for (const suite of this.project.suites) {
+      this.suites.set(
+        suite.suiteName,
+        new LatticeEvalSuite(suite, projectConfig, templatesMap)
+      );
+    }
+  }
+
+  /**
+   * Get project name
+   */
+  getProjectName(): string {
+    return this.project.projectName;
+  }
+
+  /**
+   * Get project version
+   */
+  getVersion(): string | undefined {
+    return this.project.version;
+  }
+
+  /**
+   * Get project description
+   */
+  getDescription(): string | undefined {
+    return this.project.description;
+  }
+
+  /**
+   * Get all suite names
+   */
+  getSuiteNames(): string[] {
+    return Array.from(this.suites.keys());
+  }
+
+  /**
+   * Get a specific suite by name
+   */
+  getSuite(suiteName: string): LatticeEvalSuite | undefined {
+    return this.suites.get(suiteName);
+  }
+
+  /**
+   * Run a specific case in a specific suite
+   * @param suiteName The suite name
+   * @param caseId The case ID to run
+   * @returns Case run result with error handling
+   */
+  async runCase(suiteName: string, caseId: string): Promise<CaseRunResult> {
+    const suite = this.getSuite(suiteName);
+    if (!suite) {
+      return {
+        caseId,
+        error: `Suite not found: ${suiteName}`,
+        logs: [],
+      };
+    }
+    return suite.runCase(caseId);
+  }
+
+  /**
+   * Run all cases in a specific suite with concurrency control and error isolation
+   * @param suiteName The suite name
+   * @param concurrency Optional concurrency limit (overrides project config)
+   * @returns Array of case run results with error handling
+   */
+  async runSuite(suiteName: string, concurrency?: number): Promise<CaseRunResult[]> {
+    const suite = this.getSuite(suiteName);
+    if (!suite) {
+      throw new Error(`Suite not found: ${suiteName}`);
+    }
+    return suite.runAllCases(concurrency);
+  }
+
+  /**
+   * Run all cases in all suites with concurrency control and error isolation
+   * @param concurrency Optional concurrency limit (overrides project config)
+   * @returns Map of suite names to their case run results
+   */
+  async runAllSuites(concurrency?: number): Promise<Map<string, CaseRunResult[]>> {
+    const results = new Map<string, CaseRunResult[]>();
+
+    for (const suiteName of this.getSuiteNames()) {
+      try {
+        const suiteResults = await this.runSuite(suiteName, concurrency);
+        results.set(suiteName, suiteResults);
+      } catch (error) {
+        // If suite execution fails, create error results for all cases
+        const suite = this.getSuite(suiteName);
+        if (suite) {
+          const errorResults: CaseRunResult[] = suite.getCases().map((c) => ({
+            caseId: c.caseId,
+            error: error instanceof Error ? error.message : String(error),
+            logs: [],
+          }));
+          results.set(suiteName, errorResults);
+        }
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Run all suites as a "batch", build a report, and optionally write it to disk.
+   */
+  async runAllSuitesBatch(concurrency?: number): Promise<{
+    batch_id: string;
+    batch_dir?: string;
+    results: Map<string, CaseRunResult[]>;
+    report: LatticeEvalBatchReport;
+  }> {
+    const started_at = new Date().toISOString();
+    const batch_id =
+      this.reportConfig?.batch_id ||
+      `${Date.now()}`;
+
+    console.log(`\nRunning batch: ${this.project.projectName} (${this.getSuiteNames().length} suites)`);
+
+    const results = await this.runAllSuites(concurrency);
+
+    let total_cases = 0;
+    let passed_cases = 0;
+    let failed_cases = 0;
+
+    const suites: LatticeEvalBatchReport["suites"] = [];
+    const durations: number[] = [];
+    for (const [suiteName, caseResults] of results.entries()) {
+      const suiteTotal = caseResults.length;
+      const suitePassed = caseResults.filter((r) => r.result?.pass).length;
+      const suiteFailed = suiteTotal - suitePassed;
+
+      total_cases += suiteTotal;
+      passed_cases += suitePassed;
+      failed_cases += suiteFailed;
+
+      suites.push({
+        suiteName,
+        total_cases: suiteTotal,
+        passed_cases: suitePassed,
+        failed_cases: suiteFailed,
+        cases: caseResults.map((r) => ({
+          caseId: r.caseId,
+          pass: r.result?.pass,
+          final_score: r.result?.final_score,
+          error: r.error,
+        })),
+      });
+
+      for (const r of caseResults) {
+        if (typeof r.duration_ms === "number") durations.push(r.duration_ms);
+      }
+    }
+
+    const finished_at = new Date().toISOString();
+    const report: LatticeEvalBatchReport = {
+      batch_id,
+      started_at,
+      finished_at,
+      project: {
+        projectName: this.project.projectName,
+        version: this.project.version,
+        description: this.project.description,
+      },
+      summary: {
+        total_cases,
+        passed_cases,
+        failed_cases,
+        pass_rate: total_cases > 0 ? passed_cases / total_cases : 0,
+      },
+      suites,
+    };
+
+    const batch_dir = await this.maybeWriteBatchArtifacts(
+      batch_id,
+      report,
+      results
+    );
+
+    console.log(`\n=== Summary ===`);
+    console.log(`Total: ${report.summary.total_cases} | Passed: ${report.summary.passed_cases} | Failed: ${report.summary.failed_cases} | Pass Rate: ${(report.summary.pass_rate * 100).toFixed(2)}%`);
+    if (batch_dir) {
+      console.log(`\nResults saved to: ${batch_dir}`);
+    }
+
+    return { batch_id, batch_dir, results, report };
+  }
+
+  private generateCaseMarkdown(
+    index: number,
+    suiteName: string,
+    caseResult: CaseRunResult,
+    payload: any
+  ): string {
+    const lines: string[] = [];
+    const status = caseResult.result?.pass ? "✅ PASS" : "❌ FAIL";
+
+    lines.push(`# Test ${index}: ${status}`);
+    lines.push(``);
+    lines.push(`- **Suite**: ${suiteName}`);
+    lines.push(`- **Case ID**: ${caseResult.caseId}`);
+    lines.push(`- **Status**: ${caseResult.result?.pass ? "PASS" : "FAIL"}`);
+    if (typeof payload.duration === "number") {
+      lines.push(`- **Duration**: ${(payload.duration / 1000).toFixed(2)}s`);
+    }
+    if (payload.threadId) {
+      lines.push(`- **Thread ID**: ${payload.threadId}`);
+    }
+    if (payload.judgeThreadId) {
+      lines.push(`- **Judge Thread ID**: ${payload.judgeThreadId}`);
+    }
+    lines.push(``);
+
+    if (caseResult.result) {
+      lines.push(`## Result`);
+      lines.push(``);
+      lines.push(`- **Final Score**: ${caseResult.result.final_score}`);
+      lines.push(`- **Summary**: ${caseResult.result.summary || "N/A"}`);
+      lines.push(``);
+
+      if (caseResult.result.dimension_results && caseResult.result.dimension_results.length > 0) {
+        lines.push(`## Dimension Results`);
+        lines.push(``);
+        for (const dim of caseResult.result.dimension_results) {
+          lines.push(`### ${dim.name}`);
+          lines.push(`- **Score**: ${dim.score}`);
+          lines.push(`- **Reason**: ${dim.reason}`);
+          lines.push(``);
+        }
+      }
+    }
+
+    if (caseResult.error) {
+      lines.push(`## Error`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      lines.push(caseResult.error);
+      if (caseResult.error_stack) {
+        lines.push(``);
+        lines.push(caseResult.error_stack);
+      }
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+
+    if (payload.finalOutput) {
+      lines.push(`## Final Output`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      // Truncate very long outputs in markdown for readability
+      const output = payload.finalOutput.length > 5000
+        ? payload.finalOutput.substring(0, 5000) + "\n\n... (truncated, see JSON for full output)"
+        : payload.finalOutput;
+      lines.push(output);
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+
+    if (payload.testPrompt) {
+      lines.push(`## Test Prompt`);
+      lines.push(``);
+      lines.push(`\`\`\``);
+      // Truncate very long prompts in markdown for readability
+      const prompt = payload.testPrompt.length > 5000
+        ? payload.testPrompt.substring(0, 5000) + "\n\n... (truncated, see JSON for full prompt)"
+        : payload.testPrompt;
+      lines.push(prompt);
+      lines.push(`\`\`\``);
+      lines.push(``);
+    }
+
+    return lines.join("\n");
+  }
+
+  private generateMarkdownSummary(
+    batch_id: string,
+    report: LatticeEvalBatchReport,
+    results: Map<string, CaseRunResult[]>
+  ): string {
+    const lines: string[] = [];
+    lines.push(`# Lattice Eval Batch Summary`);
+    lines.push(``);
+    lines.push(`- **Project**: ${report.project.projectName}`);
+    if (report.project.version) lines.push(`- **Version**: ${report.project.version}`);
+    if (report.project.description) lines.push(`- **Description**: ${report.project.description}`);
+    lines.push(`- **Batch ID**: ${batch_id}`);
+    lines.push(`- **Started**: ${report.started_at}`);
+    lines.push(`- **Finished**: ${report.finished_at}`);
+    lines.push(``);
+
+    lines.push(`## Overview`);
+    lines.push(``);
+    lines.push(`| Metric | Value |`);
+    lines.push(`|---|---:|`);
+    lines.push(`| Total cases | ${report.summary.total_cases} |`);
+    lines.push(`| Passed | ${report.summary.passed_cases} |`);
+    lines.push(`| Failed | ${report.summary.failed_cases} |`);
+    lines.push(`| Pass rate | ${(report.summary.pass_rate * 100).toFixed(2)}% |`);
+    lines.push(``);
+
+    lines.push(`## Suites`);
+    lines.push(``);
+    for (const suite of report.suites) {
+      lines.push(`### ${suite.suiteName}`);
+      lines.push(``);
+      lines.push(`| Case | Status | Score | Duration (ms) | Thread |`);
+      lines.push(`|---|---|---:|---:|---|`);
+      const suiteResults = results.get(suite.suiteName) || [];
+      for (const r of suiteResults) {
+        const status = r.result?.pass ? "PASS" : "FAIL";
+        const score = r.result?.final_score ?? "";
+        const dur = typeof r.duration_ms === "number" ? r.duration_ms : "";
+        const thread = r.thread_id ?? "";
+        lines.push(`| ${r.caseId} | ${status} | ${score} | ${dur} | ${thread} |`);
+      }
+      lines.push(``);
+    }
+
+    return lines.join("\n");
+  }
+
+  private async maybeWriteBatchArtifacts(
+    batch_id: string,
+    report: LatticeEvalBatchReport,
+    results: Map<string, CaseRunResult[]>
+  ): Promise<string | undefined> {
+    const config = this.reportConfig;
+    if (!config?.output_dir) return undefined;
+
+    const batchDir = path.join(config.output_dir, batch_id);
+    await mkdir(batchDir, { recursive: true });
+
+    const writeReportJson = config.write_report_json ?? true;
+    const writeCaseLogs = config.write_case_logs ?? true;
+
+    if (writeReportJson) {
+      await writeFile(
+        path.join(batchDir, "report.json"),
+        JSON.stringify(report, null, 2),
+        "utf-8"
+      );
+    }
+
+    // Write richer results.json (similar to TestRunner)
+    const resultsJsonPath = path.join(batchDir, "results.json");
+    const resultsJson = {
+      executionTimestamp: batch_id,
+      summary: report.summary,
+      report,
+      results: Array.from(results.entries()).map(([suiteName, caseResults]) => ({
+        suiteName,
+        cases: caseResults.map((r) => ({
+          caseId: r.caseId,
+          passed: r.result?.pass === true,
+          message: r.result?.summary || r.error || "",
+          error: r.error
+            ? {
+              message: r.error,
+              stack: r.error_stack,
+            }
+            : undefined,
+          duration: r.duration_ms,
+          testPrompt: r.test_prompt,
+          finalOutput: r.final_output,
+          threadId: r.thread_id,
+          judgeThreadId: r.judge_thread_id,
+        })),
+      })),
+    };
+    await writeFile(resultsJsonPath, JSON.stringify(resultsJson, null, 2), "utf-8");
+
+    // Write summary.md
+    const summaryMdPath = path.join(batchDir, "summary.md");
+    const summaryMd = this.generateMarkdownSummary(batch_id, report, results);
+    await writeFile(summaryMdPath, summaryMd, "utf-8");
+
+    // Write per-case detailed json and markdown
+    const individualDir = path.join(batchDir, "individual");
+    await mkdir(individualDir, { recursive: true });
+    let index = 1;
+    for (const [suiteName, caseResults] of results.entries()) {
+      for (const r of caseResults) {
+        const status = r.result?.pass ? "PASS" : "FAIL";
+        const baseFilename = `test-${index}-${suiteName}-${r.caseId}-${status}`.replace(/[\/\\]/g, "_");
+
+        // Write JSON
+        const jsonPath = path.join(individualDir, `${baseFilename}.json`);
+        const payload = {
+          index,
+          suiteName,
+          caseId: r.caseId,
+          passed: r.result?.pass === true,
+          result: r.result,
+          message: r.result?.summary || r.error || "",
+          error: r.error
+            ? { message: r.error, stack: r.error_stack }
+            : undefined,
+          duration: r.duration_ms,
+          threadId: r.thread_id,
+          judgeThreadId: r.judge_thread_id,
+          finalOutput: r.final_output,
+          testPrompt: r.test_prompt,
+        };
+        await writeFile(jsonPath, JSON.stringify(payload, null, 2), "utf-8");
+
+        // Write Markdown
+        const mdPath = path.join(individualDir, `${baseFilename}.md`);
+        const mdContent = this.generateCaseMarkdown(index, suiteName, r, payload);
+        await writeFile(mdPath, mdContent, "utf-8");
+
+        index += 1;
+      }
+    }
+
+    if (writeCaseLogs) {
+      for (const [suiteName, caseResults] of results.entries()) {
+        const suiteDir = path.join(batchDir, "cases", suiteName);
+        await mkdir(suiteDir, { recursive: true });
+        for (const r of caseResults) {
+          await writeFile(
+            path.join(suiteDir, `${r.caseId}.logs.json`),
+            JSON.stringify(r.logs || [], null, 2),
+            "utf-8"
+          );
+        }
+      }
+    }
+
+    return batchDir;
+  }
+}