@axiom-lattice/agent-eval 2.1.9

package/dist/index.d.ts

@@ -0,0 +1,366 @@
+import { LLMConfig } from '@axiom-lattice/protocols';
+
+interface LatticeAgentStepConfig {
+    agent_id: string;
+    override_input_message?: string;
+}
+type OutputFileContent = {
+    type: "file_content";
+    file_path: string;
+};
+type OutputMessageContent = {
+    type: "message_content";
+    message: string;
+};
+type OutputType = OutputFileContent | OutputMessageContent;
+interface LatticeEvalProjectType {
+    projectName: string;
+    version?: string;
+    description?: string;
+    suites: LatticeEvalSuiteType[];
+    templates?: LatticeEvalTemplate[];
+    report_config?: LatticeEvalReportConfig;
+    judge_agent_config: {
+        model: LLMConfig;
+    };
+    lattice_server_config: {
+        base_url: string;
+        api_key: string;
+    };
+    concurrency?: number;
+}
+type LatticeEvalLogLevel = "debug" | "info" | "warn" | "error";
+interface LatticeEvalLogEvent {
+    ts: string;
+    level: LatticeEvalLogLevel;
+    message: string;
+    data?: Record<string, unknown>;
+}
+interface LatticeEvalReportConfig {
+    /**
+     * Output directory for each batch run.
+     * A subfolder will be created per batch.
+     */
+    output_dir: string;
+    /**
+     * Optional batch id. If not set, a timestamp-based id will be generated per run.
+     */
+    batch_id?: string;
+    /**
+     * When true, writes `report.json` into the batch folder.
+     * Defaults to true.
+     */
+    write_report_json?: boolean;
+    /**
+     * When true, writes per-case log files into the batch folder.
+     * Defaults to true.
+     */
+    write_case_logs?: boolean;
+}
+interface LatticeEvalBatchReport {
+    batch_id: string;
+    started_at: string;
+    finished_at: string;
+    project: {
+        projectName: string;
+        version?: string;
+        description?: string;
+    };
+    summary: {
+        total_cases: number;
+        passed_cases: number;
+        failed_cases: number;
+        pass_rate: number;
+    };
+    suites: Array<{
+        suiteName: string;
+        total_cases: number;
+        passed_cases: number;
+        failed_cases: number;
+        cases: Array<{
+            caseId: string;
+            pass?: boolean;
+            final_score?: number;
+            error?: string;
+        }>;
+    }>;
+}
+type LatticeEvalCaseType = LatticeEvalCase | LatticeEvalCaseWithTemplate;
+interface LatticeEvalSuiteType {
+    suiteName: string;
+    version?: string;
+    cases: LatticeEvalCaseType[];
+}
+interface LatticeEvalRubric {
+    dimension: string;
+    weight: number;
+    description: string;
+}
+interface LatticeEvalCase {
+    caseId: string;
+    input: {
+        message: string;
+        files?: Record<string, string>;
+    };
+    steps: LatticeAgentStepConfig[];
+    output: OutputType;
+    eval: {
+        content_assertion: string;
+        eval_rubrics?: LatticeEvalRubric[];
+    };
+}
+interface LatticeEvalCaseWithTemplate {
+    caseId: string;
+    templateId: string;
+    input: {
+        message?: string;
+        files?: Record<string, string>;
+        variables?: Record<string, string>;
+    };
+    output?: OutputType;
+    eval: {
+        content_assertion: string;
+        eval_rubrics?: LatticeEvalRubric[];
+    };
+}
+interface LatticeEvalTemplate {
+    templateId: string;
+    description?: string;
+    input_schema: {
+        required_files?: string[];
+        variables?: string[];
+    };
+    default_case: Omit<LatticeEvalCase, "caseId" | "eval"> & {
+        eval?: {
+            eval_rubrics?: LatticeEvalRubric[];
+        };
+    };
+}
+interface LatticeEvalResult {
+    pass: boolean;
+    final_score: number;
+    dimension_results: {
+        name: string;
+        score: number;
+        reason: string;
+    }[];
+    summary: string;
+    error?: string;
+}
+
+/**
+ * Configuration for Lattice evaluation server
+ */
+interface LatticeEvalConfig {
+    base_url: string;
+    api_key?: string;
+    /**
+     * When true, prints detailed execution logs for each action.
+     * Defaults to true.
+     */
+    verbose?: boolean;
+}
+interface LatticeEvalCaseRunResult {
+    caseId: string;
+    result?: LatticeEvalResult;
+    error?: string;
+    error_stack?: string;
+    duration_ms: number;
+    thread_id?: string;
+    judge_thread_id?: string;
+    test_prompt?: string;
+    final_output?: string;
+    logs: LatticeEvalLogEvent[];
+}
+/**
+ * LatticeEval class for evaluating Lattice evaluation cases
+ */
+declare class LatticeEval {
+    private config;
+    private baseUrl;
+    private verbose;
+    private inMemoryLogs;
+    private lastThreadId?;
+    private lastJudgeThreadId?;
+    private lastTestPrompt?;
+    private lastFinalOutput?;
+    private lastDurationMs;
+    getLastRunMeta(): {
+        duration_ms: number;
+        thread_id: string | undefined;
+        judge_thread_id: string | undefined;
+        test_prompt: string | undefined;
+        final_output: string | undefined;
+    };
+    /**
+     * Create a new LatticeEval instance
+     * @param config Optional server configuration (defaults to localhost:3203)
+     */
+    constructor(config: LatticeEvalConfig);
+    getInMemoryLogs(): LatticeEvalLogEvent[];
+    record(level: LatticeEvalLogLevel, message: string, data?: Record<string, unknown>): void;
+    private log;
+    private getKeyInfo;
+    /**
+     * Execute a single agent step and return the thread ID and response data
+     */
+    private executeAgentStep;
+    /**
+     * Extract output content based on OutputType
+     */
+    private extractOutput;
+    /**
+     * Evaluate a single Lattice evaluation case
+     * @param evalCase The evaluation case to run
+     * @returns Evaluation result with pass/fail status and scores
+     */
+    evaluateCase(evalCase: LatticeEvalCase): Promise<LatticeEvalResult>;
+}
+/**
+ * Evaluate a single Lattice evaluation case (backward compatibility function)
+ * @param evalCase The evaluation case to run
+ * @param config Optional server configuration (defaults to localhost:3203)
+ * @returns Evaluation result with pass/fail status and scores
+ * @deprecated Use LatticeEval class instead
+ */
+declare function evaluateLatticeCase(evalCase: LatticeEvalCase, config?: LatticeEvalConfig): Promise<LatticeEvalResult>;
+/**
+ * Evaluate a single Lattice evaluation case and always return logs (never throws).
+ */
+declare function evaluateLatticeCaseWithLogs(evalCase: LatticeEvalCase, config?: LatticeEvalConfig): Promise<LatticeEvalCaseRunResult>;
+
+/**
+ * Configuration resolved from project/suite hierarchy
+ */
+interface ResolvedConfig {
+    lattice_server_config: {
+        base_url: string;
+        api_key?: string;
+    };
+    judge_agent_config?: {
+        model: LLMConfig;
+    };
+    concurrency: number;
+}
+/**
+ * Result with error handling
+ */
+interface CaseRunResult {
+    caseId: string;
+    result?: LatticeEvalResult;
+    error?: string;
+    logs: LatticeEvalLogEvent[];
+    duration_ms?: number;
+    thread_id?: string;
+    judge_thread_id?: string;
+    test_prompt?: string;
+    final_output?: string;
+    error_stack?: string;
+}
+/**
+ * LatticeEvalSuite class manages a suite of evaluation cases
+ * with suite-level configuration
+ */
+declare class LatticeEvalSuite {
+    private suite;
+    private projectConfig;
+    private templates;
+    constructor(suite: LatticeEvalSuiteType, projectConfig: ResolvedConfig, templates?: Map<string, LatticeEvalTemplate>);
+    /**
+     * Get resolved configuration from project
+     */
+    private getResolvedConfig;
+    /**
+     * Get suite name
+     */
+    getSuiteName(): string;
+    /**
+     * Get suite version
+     */
+    getVersion(): string | undefined;
+    /**
+     * Get all cases in this suite (resolved from templates if needed)
+     */
+    getCases(): LatticeEvalCase[];
+    /**
+     * Get a specific case by ID (resolved from template if needed)
+     */
+    getCase(caseId: string): LatticeEvalCase | undefined;
+    /**
+     * Run a single case in this suite with error handling
+     * @param caseId The case ID to run
+     * @returns Case run result with error handling
+     */
+    runCase(caseId: string): Promise<CaseRunResult>;
+    /**
+     * Run all cases in this suite with concurrency control and error isolation
+     * @param concurrency Optional concurrency limit (overrides project config)
+     * @returns Array of case run results with error handling
+     */
+    runAllCases(concurrency?: number): Promise<CaseRunResult[]>;
+}
+
+/**
+ * LatticeEvalProject class manages a project with multiple evaluation suites
+ * with project-level configuration
+ */
+declare class LatticeEvalProject {
+    private project;
+    private suites;
+    private reportConfig?;
+    constructor(project: LatticeEvalProjectType);
+    /**
+     * Get project name
+     */
+    getProjectName(): string;
+    /**
+     * Get project version
+     */
+    getVersion(): string | undefined;
+    /**
+     * Get project description
+     */
+    getDescription(): string | undefined;
+    /**
+     * Get all suite names
+     */
+    getSuiteNames(): string[];
+    /**
+     * Get a specific suite by name
+     */
+    getSuite(suiteName: string): LatticeEvalSuite | undefined;
+    /**
+     * Run a specific case in a specific suite
+     * @param suiteName The suite name
+     * @param caseId The case ID to run
+     * @returns Case run result with error handling
+     */
+    runCase(suiteName: string, caseId: string): Promise<CaseRunResult>;
+    /**
+     * Run all cases in a specific suite with concurrency control and error isolation
+     * @param suiteName The suite name
+     * @param concurrency Optional concurrency limit (overrides project config)
+     * @returns Array of case run results with error handling
+     */
+    runSuite(suiteName: string, concurrency?: number): Promise<CaseRunResult[]>;
+    /**
+     * Run all cases in all suites with concurrency control and error isolation
+     * @param concurrency Optional concurrency limit (overrides project config)
+     * @returns Map of suite names to their case run results
+     */
+    runAllSuites(concurrency?: number): Promise<Map<string, CaseRunResult[]>>;
+    /**
+     * Run all suites as a "batch", build a report, and optionally write it to disk.
+     */
+    runAllSuitesBatch(concurrency?: number): Promise<{
+        batch_id: string;
+        batch_dir?: string;
+        results: Map<string, CaseRunResult[]>;
+        report: LatticeEvalBatchReport;
+    }>;
+    private generateCaseMarkdown;
+    private generateMarkdownSummary;
+    private maybeWriteBatchArtifacts;
+}
+
+export { type CaseRunResult, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseRunResult, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalReportConfig, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, type OutputFileContent, type OutputMessageContent, type OutputType, type ResolvedConfig, evaluateLatticeCase, evaluateLatticeCaseWithLogs };