@hemia-ai/agents-evals 0.0.1

package/dist/index.d.ts

@@ -0,0 +1,340 @@
+import { AgentKey, JsonObject, AgentExecutionResult } from '@hemia-ai/agents-core';
+import { ModelGateway } from '@hemia-ai/agents-models';
+
+interface EvalCase<TInput = unknown, TExpected = unknown> {
+    id: string;
+    name: string;
+    agentKey: AgentKey;
+    input: TInput;
+    expected?: TExpected;
+    tags?: string[];
+    metadata?: JsonObject;
+}
+
+interface EvalRunContext {
+    runId: string;
+    datasetId: string;
+    datasetVersion: string;
+    agentKey?: AgentKey;
+    modelId?: string;
+    promptVersion?: string;
+    metadata?: JsonObject;
+}
+
+interface EvalEvaluatorInput {
+    testCase: EvalCase;
+    result: AgentExecutionResult;
+    context: EvalRunContext;
+}
+interface EvalScore {
+    name: string;
+    score: number;
+    passed: boolean;
+    reason?: string;
+    metadata?: JsonObject;
+}
+interface EvalEvaluator {
+    name: string;
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+declare function createEvalScore(input: EvalScore): EvalScore;
+
+interface EvalMetric {
+    name: string;
+    value: number;
+    unit?: string;
+    metadata?: JsonObject;
+}
+
+interface EvalCaseResult {
+    caseId: string;
+    testCase: EvalCase;
+    result: AgentExecutionResult;
+    scores: EvalScore[];
+    passed: boolean;
+    metadata?: JsonObject;
+}
+interface EvalRunSummary {
+    totalCases: number;
+    passedCases: number;
+    failedCases: number;
+    passRate: number;
+    averageScore: number;
+}
+interface EvalRunResult {
+    runId: string;
+    datasetId: string;
+    datasetVersion: string;
+    results: EvalCaseResult[];
+    summary: EvalRunSummary;
+    metrics: EvalMetric[];
+    passed: boolean;
+    metadata?: JsonObject;
+}
+
+interface EvalBaseline {
+    id: string;
+    name: string;
+    run: EvalRunResult;
+    metadata?: JsonObject;
+}
+
+interface EvalRegressionResult {
+    metricName: string;
+    baselineValue: number;
+    currentValue: number;
+    delta: number;
+    regressed: boolean;
+    threshold: number;
+    metadata?: JsonObject;
+}
+
+interface BaselineComparatorOptions {
+    thresholds?: Record<string, number>;
+}
+declare class BaselineComparator {
+    private readonly thresholds;
+    constructor(options?: BaselineComparatorOptions);
+    compare(baseline: EvalRunResult, current: EvalRunResult): EvalRegressionResult[];
+}
+
+interface EvalCaseBuilderInput<TInput = unknown, TExpected = unknown> {
+    id: string;
+    name: string;
+    agentKey: AgentKey;
+    input: TInput;
+    expected?: TExpected;
+    tags?: string[];
+    metadata?: JsonObject;
+}
+declare function createEvalCase<TInput = unknown, TExpected = unknown>(input: EvalCaseBuilderInput<TInput, TExpected>): EvalCase<TInput, TExpected>;
+
+type EvalDatasetSplit = 'dev' | 'test' | 'holdout';
+
+interface EvalDataset {
+    id: string;
+    name: string;
+    version: string;
+    split: EvalDatasetSplit;
+    cases: EvalCase[];
+    metadata?: JsonObject;
+}
+
+interface EvalDatasetStore {
+    list(): Promise<EvalDataset[]>;
+    get(id: string, version?: string): Promise<EvalDataset | undefined>;
+    put(dataset: EvalDataset): Promise<void>;
+}
+
+declare class InMemoryEvalDatasetStore implements EvalDatasetStore {
+    private readonly datasets;
+    constructor(datasets?: EvalDataset[]);
+    list(): Promise<EvalDataset[]>;
+    get(id: string, version?: string): Promise<EvalDataset | undefined>;
+    put(dataset: EvalDataset): Promise<void>;
+    private key;
+}
+
+type EvalErrorCode = 'EVAL_DATASET_NOT_FOUND' | 'EVAL_CASE_EXECUTION_FAILED' | 'EVAL_EVALUATOR_FAILED' | 'EVAL_REPORT_FAILED' | 'EVAL_GATE_FAILED' | 'EVAL_UNKNOWN_ERROR';
+
+declare class EvalError extends Error {
+    readonly code: EvalErrorCode;
+    readonly metadata?: JsonObject;
+    constructor(code: EvalErrorCode, message: string, metadata?: JsonObject);
+}
+
+interface WeightedEvalEvaluator {
+    evaluator: EvalEvaluator;
+    weight?: number;
+}
+interface CompositeEvaluatorOptions {
+    name?: string;
+    evaluators: WeightedEvalEvaluator[];
+    threshold?: number;
+}
+declare class CompositeEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly evaluators;
+    private readonly threshold;
+    constructor(options: CompositeEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+interface ExactMatchEvaluatorOptions {
+    name?: string;
+    ignoreCase?: boolean;
+    trim?: boolean;
+}
+declare class ExactMatchEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly ignoreCase;
+    private readonly trim;
+    constructor(options?: ExactMatchEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+    private normalize;
+}
+
+interface JsonSchemaEvaluatorOptions {
+    name?: string;
+    schema: JsonObject;
+}
+declare class JsonSchemaEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly schema;
+    constructor(options: JsonSchemaEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+interface LlmJudgeEvaluatorOptions {
+    name?: string;
+    modelGateway: Pick<ModelGateway, 'generate'>;
+    model?: string;
+    threshold?: number;
+    rubric?: string;
+    metadata?: JsonObject;
+}
+declare class LlmJudgeEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly modelGateway;
+    private readonly model?;
+    private readonly threshold;
+    private readonly rubric?;
+    private readonly metadata?;
+    constructor(options: LlmJudgeEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+interface RegexEvaluatorOptions {
+    name?: string;
+    pattern: RegExp;
+}
+declare class RegexEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly pattern;
+    constructor(options: RegexEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+interface SemanticSimilarityEvaluatorOptions {
+    name?: string;
+    threshold?: number;
+    similarity: (actual: string, expected: string) => Promise<number> | number;
+}
+declare class SemanticSimilarityEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly threshold;
+    private readonly similarity;
+    constructor(options: SemanticSimilarityEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+interface ExpectedToolCall {
+    name: string;
+    input?: unknown;
+}
+interface ToolCallEvaluatorOptions {
+    name?: string;
+    expectedToolCalls?: ExpectedToolCall[];
+    requireOrder?: boolean;
+}
+declare class ToolCallEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly expectedToolCalls?;
+    private readonly requireOrder;
+    constructor(options?: ToolCallEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+declare function createPassRateMetric(passed: number, total: number): EvalMetric;
+declare function createAverageScoreMetric(scores: number[]): EvalMetric;
+declare function createCostMetric(results: AgentExecutionResult[]): EvalMetric;
+
+interface EvalReport {
+    run: EvalRunResult;
+    regressions?: EvalRegressionResult[];
+    metadata?: JsonObject;
+}
+interface EvalReportRenderer {
+    render(report: EvalReport): string;
+}
+
+declare class JsonReportRenderer implements EvalReportRenderer {
+    render(report: EvalReport): string;
+}
+
+declare class MarkdownReportRenderer implements EvalReportRenderer {
+    render(report: EvalReport): string;
+}
+
+interface EvalRubricCriterion {
+    key: string;
+    description: string;
+    weight?: number;
+    threshold?: number;
+    metadata?: JsonObject;
+}
+interface EvalRubric {
+    id: string;
+    name: string;
+    version: string;
+    criteria: EvalRubricCriterion[];
+    threshold?: number;
+    metadata?: JsonObject;
+}
+
+interface RubricCriterionScore {
+    key: string;
+    score: number;
+    passed: boolean;
+    reason?: string;
+    metadata?: JsonObject;
+}
+interface RubricScore {
+    rubricId: string;
+    rubricVersion: string;
+    score: number;
+    passed: boolean;
+    criteria: RubricCriterionScore[];
+    reason?: string;
+    metadata?: JsonObject;
+}
+
+interface WeightedRubricEvaluatorOptions {
+    rubric: EvalRubric;
+    scoreCriterion: (criterionKey: string, input: EvalEvaluatorInput) => Promise<RubricCriterionScore> | RubricCriterionScore;
+}
+declare class WeightedRubricEvaluator implements EvalEvaluator {
+    readonly name: string;
+    private readonly rubric;
+    private readonly scoreCriterion;
+    constructor(options: WeightedRubricEvaluatorOptions);
+    evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;
+}
+
+type EvalCaseExecutor = (testCase: EvalCase, context: EvalRunContext) => Promise<AgentExecutionResult>;
+
+interface EvalRunConfig {
+    runId?: string;
+    datasetId: string;
+    datasetVersion?: string;
+    agentKey?: AgentKey;
+    modelId?: string;
+    promptVersion?: string;
+    threshold?: number;
+    metadata?: JsonObject;
+}
+
+interface EvalRunnerOptions {
+    datasetStore: EvalDatasetStore;
+    evaluators: EvalEvaluator[];
+    executor: EvalCaseExecutor;
+}
+declare class EvalRunner {
+    private readonly datasetStore;
+    private readonly evaluators;
+    private readonly executor;
+    constructor(options: EvalRunnerOptions);
+    run(config: EvalRunConfig): Promise<EvalRunResult>;
+}
+
+export { BaselineComparator, type BaselineComparatorOptions, CompositeEvaluator, type CompositeEvaluatorOptions, type EvalBaseline, type EvalCase, type EvalCaseBuilderInput, type EvalCaseExecutor, type EvalCaseResult, type EvalDataset, type EvalDatasetSplit, type EvalDatasetStore, EvalError, type EvalErrorCode, type EvalEvaluator, type EvalEvaluatorInput, type EvalMetric, type EvalRegressionResult, type EvalReport, type EvalReportRenderer, type EvalRubric, type EvalRubricCriterion, type EvalRunConfig, type EvalRunContext, type EvalRunResult, type EvalRunSummary, EvalRunner, type EvalRunnerOptions, type EvalScore, ExactMatchEvaluator, type ExactMatchEvaluatorOptions, type ExpectedToolCall, InMemoryEvalDatasetStore, JsonReportRenderer, JsonSchemaEvaluator, type JsonSchemaEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorOptions, MarkdownReportRenderer, RegexEvaluator, type RegexEvaluatorOptions, type RubricCriterionScore, type RubricScore, SemanticSimilarityEvaluator, type SemanticSimilarityEvaluatorOptions, ToolCallEvaluator, type ToolCallEvaluatorOptions, type WeightedEvalEvaluator, WeightedRubricEvaluator, type WeightedRubricEvaluatorOptions, createAverageScoreMetric, createCostMetric, createEvalCase, createEvalScore, createPassRateMetric };

package/dist/index.js

@@ -0,0 +1,578 @@
+// src/baselines/baseline-comparator.ts
+var BaselineComparator = class {
+  thresholds;
+  constructor(options = {}) {
+    this.thresholds = options.thresholds ?? {};
+  }
+  compare(baseline, current) {
+    const baselineMetrics = new Map(baseline.metrics.map((metric) => [metric.name, metric.value]));
+    return current.metrics.filter((metric) => baselineMetrics.has(metric.name)).map((metric) => {
+      const baselineValue = baselineMetrics.get(metric.name) ?? 0;
+      const threshold = this.thresholds[metric.name] ?? 0;
+      const delta = metric.value - baselineValue;
+      const lowerIsBetter = metric.name.includes("cost") || metric.name.includes("latency");
+      const regressed = lowerIsBetter ? delta > threshold : delta < -threshold;
+      return {
+        metricName: metric.name,
+        baselineValue,
+        currentValue: metric.value,
+        delta,
+        regressed,
+        threshold
+      };
+    });
+  }
+};
+
+// src/cases/eval-case-builder.ts
+function createEvalCase(input) {
+  const testCase = {
+    id: input.id,
+    name: input.name,
+    agentKey: input.agentKey,
+    input: input.input
+  };
+  if (input.expected !== void 0) testCase.expected = input.expected;
+  if (input.tags !== void 0) testCase.tags = input.tags;
+  if (input.metadata !== void 0) testCase.metadata = input.metadata;
+  return testCase;
+}
+
+// src/datasets/in-memory-dataset-store.ts
+var InMemoryEvalDatasetStore = class {
+  datasets = /* @__PURE__ */ new Map();
+  constructor(datasets = []) {
+    for (const dataset of datasets) {
+      this.datasets.set(this.key(dataset.id, dataset.version), dataset);
+    }
+  }
+  async list() {
+    return [...this.datasets.values()];
+  }
+  async get(id, version) {
+    if (version !== void 0) return this.datasets.get(this.key(id, version));
+    return [...this.datasets.values()].filter((dataset) => dataset.id === id).sort((a, b) => b.version.localeCompare(a.version))[0];
+  }
+  async put(dataset) {
+    this.datasets.set(this.key(dataset.id, dataset.version), dataset);
+  }
+  key(id, version) {
+    return `${id}@${version}`;
+  }
+};
+
+// src/errors/eval-error.ts
+var EvalError = class extends Error {
+  code;
+  metadata;
+  constructor(code, message, metadata) {
+    super(message);
+    this.name = "EvalError";
+    this.code = code;
+    if (metadata !== void 0) this.metadata = metadata;
+  }
+};
+
+// src/evaluators/composite-evaluator.ts
+var CompositeEvaluator = class {
+  name;
+  evaluators;
+  threshold;
+  constructor(options) {
+    this.name = options.name ?? "composite";
+    this.evaluators = options.evaluators;
+    this.threshold = options.threshold ?? 1;
+  }
+  async evaluate(input) {
+    const scores = await Promise.all(this.evaluators.map(({ evaluator }) => evaluator.evaluate(input)));
+    const totalWeight = this.evaluators.reduce((sum, item) => sum + (item.weight ?? 1), 0) || 1;
+    const score = scores.reduce((sum, item, index) => sum + item.score * (this.evaluators[index]?.weight ?? 1), 0) / totalWeight;
+    const passed = score >= this.threshold && scores.every((item) => item.passed);
+    return {
+      name: this.name,
+      score,
+      passed,
+      reason: passed ? "Composite score passed." : "Composite score failed.",
+      metadata: {
+        threshold: this.threshold,
+        scores: scores.map((item) => ({
+          name: item.name,
+          score: item.score,
+          passed: item.passed,
+          reason: item.reason ?? null
+        }))
+      }
+    };
+  }
+};
+
+// src/evaluators/evaluator.interface.ts
+function createEvalScore(input) {
+  return input;
+}
+
+// src/evaluators/exact-match-evaluator.ts
+var ExactMatchEvaluator = class {
+  name;
+  ignoreCase;
+  trim;
+  constructor(options = {}) {
+    this.name = options.name ?? "exact_match";
+    this.ignoreCase = options.ignoreCase ?? false;
+    this.trim = options.trim ?? true;
+  }
+  async evaluate(input) {
+    const actual = input.result.status === "completed" ? input.result.output : void 0;
+    const expected = input.testCase.expected;
+    const passed = this.normalize(actual) === this.normalize(expected);
+    return {
+      name: this.name,
+      score: passed ? 1 : 0,
+      passed,
+      reason: passed ? "Output matches expected value." : "Output does not match expected value."
+    };
+  }
+  normalize(value) {
+    const raw = typeof value === "string" ? value : JSON.stringify(value) ?? "";
+    const trimmed = this.trim ? raw.trim() : raw;
+    return this.ignoreCase ? trimmed.toLowerCase() : trimmed;
+  }
+};
+
+// src/evaluators/json-schema-evaluator.ts
+var JsonSchemaEvaluator = class {
+  name;
+  schema;
+  constructor(options) {
+    this.name = options.name ?? "json_schema";
+    this.schema = options.schema;
+  }
+  async evaluate(input) {
+    const actual = input.result.status === "completed" ? input.result.output : void 0;
+    const errors = validateAgainstSchema(actual, this.schema);
+    const passed = errors.length === 0;
+    return {
+      name: this.name,
+      score: passed ? 1 : 0,
+      passed,
+      reason: passed ? "Output matches schema." : errors.join("; ")
+    };
+  }
+};
+function validateAgainstSchema(value, schema, path = "$") {
+  const errors = [];
+  const type = schema.type;
+  if (typeof type === "string" && !matchesType(value, type)) {
+    errors.push(`${path} expected ${type}`);
+    return errors;
+  }
+  if (type === "object" && isRecord(value)) {
+    const required = Array.isArray(schema.required) ? schema.required.filter((item) => typeof item === "string") : [];
+    for (const key of required) {
+      if (!(key in value)) errors.push(`${path}.${key} is required`);
+    }
+    const properties = isRecord(schema.properties) ? schema.properties : {};
+    for (const [key, childSchema] of Object.entries(properties)) {
+      if (key in value && isRecord(childSchema)) {
+        errors.push(...validateAgainstSchema(value[key], childSchema, `${path}.${key}`));
+      }
+    }
+  }
+  return errors;
+}
+function matchesType(value, type) {
+  if (type === "array") return Array.isArray(value);
+  if (type === "null") return value === null;
+  if (type === "integer") return Number.isInteger(value);
+  return typeof value === type;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/evaluators/llm-judge-evaluator.ts
+var LlmJudgeEvaluator = class {
+  name;
+  modelGateway;
+  model;
+  threshold;
+  rubric;
+  metadata;
+  constructor(options) {
+    this.name = options.name ?? "llm_judge";
+    this.modelGateway = options.modelGateway;
+    this.threshold = options.threshold ?? 0.8;
+    if (options.model !== void 0) this.model = options.model;
+    if (options.rubric !== void 0) this.rubric = options.rubric;
+    if (options.metadata !== void 0) this.metadata = options.metadata;
+  }
+  async evaluate(input) {
+    const request = {
+      taskType: "eval_judge",
+      agentKey: input.testCase.agentKey,
+      messages: [
+        {
+          role: "system",
+          content: 'You are an evaluator. Return JSON only with shape {"score": number, "passed": boolean, "reason": string}.'
+        },
+        {
+          role: "user",
+          content: JSON.stringify({
+            rubric: this.rubric,
+            input: input.testCase.input,
+            expected: input.testCase.expected,
+            actual: input.result.status === "completed" ? input.result.output : input.result
+          })
+        }
+      ]
+    };
+    const response = await this.modelGateway.generate({
+      ...request,
+      ...this.model !== void 0 ? { model: this.model } : {},
+      ...this.metadata !== void 0 ? { metadata: this.metadata } : {}
+    });
+    const parsed = parseJudgeResponse(response.content);
+    const score = parsed.score ?? 0;
+    const passed = parsed.passed ?? score >= this.threshold;
+    return {
+      name: this.name,
+      score,
+      passed,
+      reason: parsed.reason ?? response.content,
+      metadata: {
+        threshold: this.threshold,
+        judgeModel: this.model ?? null,
+        usage: response.usage === void 0 ? null : JSON.parse(JSON.stringify(response.usage))
+      }
+    };
+  }
+};
+function parseJudgeResponse(content) {
+  try {
+    const parsed = JSON.parse(content);
+    if (!isRecord2(parsed)) return {};
+    const result = {};
+    if (typeof parsed.score === "number") result.score = parsed.score;
+    if (typeof parsed.passed === "boolean") result.passed = parsed.passed;
+    if (typeof parsed.reason === "string") result.reason = parsed.reason;
+    return result;
+  } catch {
+    return {};
+  }
+}
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/evaluators/regex-evaluator.ts
+var RegexEvaluator = class {
+  name;
+  pattern;
+  constructor(options) {
+    this.name = options.name ?? "regex_match";
+    this.pattern = options.pattern;
+  }
+  async evaluate(input) {
+    const actual = input.result.status === "completed" ? input.result.output : void 0;
+    const content = typeof actual === "string" ? actual : JSON.stringify(actual);
+    const passed = this.pattern.test(content);
+    return {
+      name: this.name,
+      score: passed ? 1 : 0,
+      passed,
+      reason: passed ? "Output matches pattern." : "Output does not match pattern.",
+      metadata: { pattern: this.pattern.source }
+    };
+  }
+};
+
+// src/evaluators/semantic-similarity-evaluator.ts
+var SemanticSimilarityEvaluator = class {
+  name;
+  threshold;
+  similarity;
+  constructor(options) {
+    this.name = options.name ?? "semantic_similarity";
+    this.threshold = options.threshold ?? 0.8;
+    this.similarity = options.similarity;
+  }
+  async evaluate(input) {
+    const actual = stringify(input.result.status === "completed" ? input.result.output : void 0);
+    const expected = stringify(input.testCase.expected);
+    const score = await this.similarity(actual, expected);
+    return {
+      name: this.name,
+      score,
+      passed: score >= this.threshold,
+      reason: score >= this.threshold ? "Semantic similarity passed." : "Semantic similarity below threshold.",
+      metadata: { threshold: this.threshold }
+    };
+  }
+};
+function stringify(value) {
+  return typeof value === "string" ? value : JSON.stringify(value);
+}
+
+// src/evaluators/tool-call-evaluator.ts
+var ToolCallEvaluator = class {
+  name;
+  expectedToolCalls;
+  requireOrder;
+  constructor(options = {}) {
+    this.name = options.name ?? "tool_calls";
+    if (options.expectedToolCalls !== void 0) this.expectedToolCalls = options.expectedToolCalls;
+    this.requireOrder = options.requireOrder ?? true;
+  }
+  async evaluate(input) {
+    const actualToolCalls = input.result.status === "completed" ? extractToolCalls(input.result.output) : [];
+    const expectedToolCalls = this.expectedToolCalls ?? extractExpectedToolCalls(input.testCase.expected);
+    const passed = this.requireOrder ? matchesInOrder(actualToolCalls, expectedToolCalls) : matchesAnyOrder(actualToolCalls, expectedToolCalls);
+    return {
+      name: this.name,
+      score: passed ? 1 : 0,
+      passed,
+      reason: passed ? "Tool calls match expected calls." : "Tool calls do not match expected calls.",
+      metadata: {
+        expected: expectedToolCalls.map((call) => call.name),
+        actual: actualToolCalls.map((call) => call.name)
+      }
+    };
+  }
+};
+function extractToolCalls(value) {
+  if (isRecord3(value) && Array.isArray(value.toolCalls)) {
+    return value.toolCalls.filter(isToolCall);
+  }
+  return [];
+}
+function extractExpectedToolCalls(value) {
+  if (isRecord3(value) && Array.isArray(value.toolCalls)) {
+    return value.toolCalls.filter(isExpectedToolCall);
+  }
+  return [];
+}
+function matchesInOrder(actual, expected) {
+  return expected.every((expectedCall, index) => matchesCall(actual[index], expectedCall));
+}
+function matchesAnyOrder(actual, expected) {
+  return expected.every((expectedCall) => actual.some((actualCall) => matchesCall(actualCall, expectedCall)));
+}
+function matchesCall(actual, expected) {
+  if (actual === void 0 || actual.name !== expected.name) return false;
+  if (expected.input === void 0) return true;
+  return JSON.stringify(actual.input) === JSON.stringify(expected.input);
+}
+function isToolCall(value) {
+  return isRecord3(value) && typeof value.name === "string";
+}
+function isExpectedToolCall(value) {
+  return isRecord3(value) && typeof value.name === "string";
+}
+function isRecord3(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/metrics/metric-helpers.ts
+function createPassRateMetric(passed, total) {
+  return {
+    name: "pass_rate",
+    value: total === 0 ? 0 : passed / total,
+    unit: "ratio",
+    metadata: { passed, total }
+  };
+}
+function createAverageScoreMetric(scores) {
+  return {
+    name: "average_score",
+    value: scores.length === 0 ? 0 : scores.reduce((sum, score) => sum + score, 0) / scores.length,
+    unit: "ratio"
+  };
+}
+function createCostMetric(results) {
+  const cost = results.reduce((sum, result) => {
+    const value = result.metadata?.estimatedCostUsd;
+    return sum + (typeof value === "number" ? value : 0);
+  }, 0);
+  return {
+    name: "estimated_cost_usd",
+    value: cost,
+    unit: "usd"
+  };
+}
+
+// src/reports/json-report-renderer.ts
+var JsonReportRenderer = class {
+  render(report) {
+    return JSON.stringify(report, null, 2);
+  }
+};
+
+// src/reports/markdown-report-renderer.ts
+var MarkdownReportRenderer = class {
+  render(report) {
+    const failed = report.run.results.filter((result) => !result.passed);
+    const regressions = report.regressions?.filter((result) => result.regressed) ?? [];
+    return [
+      `# Eval Report ${report.run.runId}`,
+      "",
+      `Dataset: ${report.run.datasetId}@${report.run.datasetVersion}`,
+      `Passed: ${report.run.passed ? "yes" : "no"}`,
+      `Pass rate: ${formatRatio(report.run.summary.passRate)}`,
+      `Average score: ${formatRatio(report.run.summary.averageScore)}`,
+      "",
+      "## Metrics",
+      ...report.run.metrics.map((metric) => `- ${metric.name}: ${metric.value}${metric.unit ? ` ${metric.unit}` : ""}`),
+      "",
+      "## Failed Cases",
+      ...failed.length === 0 ? ["- none"] : failed.map((result) => {
+        const reasons = result.scores.filter((score) => !score.passed).map((score) => `${score.name}: ${score.reason ?? "failed"}`);
+        return `- ${result.caseId}: ${reasons.join(", ")}`;
+      }),
+      "",
+      "## Regressions",
+      ...regressions.length === 0 ? ["- none"] : regressions.map((result) => `- ${result.metricName}: ${result.baselineValue} -> ${result.currentValue}`)
+    ].join("\n");
+  }
+};
+function formatRatio(value) {
+  return `${Math.round(value * 1e4) / 100}%`;
+}
+
+// src/rubrics/weighted-rubric-evaluator.ts
+var WeightedRubricEvaluator = class {
+  name;
+  rubric;
+  scoreCriterion;
+  constructor(options) {
+    this.name = `rubric:${options.rubric.id}`;
+    this.rubric = options.rubric;
+    this.scoreCriterion = options.scoreCriterion;
+  }
+  async evaluate(input) {
+    const criteria = await Promise.all(
+      this.rubric.criteria.map((criterion) => this.scoreCriterion(criterion.key, input))
+    );
+    const totalWeight = this.rubric.criteria.reduce((sum, item) => sum + (item.weight ?? 1), 0) || 1;
+    const score = criteria.reduce((sum, item) => {
+      const weight = this.rubric.criteria.find((criterion) => criterion.key === item.key)?.weight ?? 1;
+      return sum + item.score * weight;
+    }, 0) / totalWeight;
+    const threshold = this.rubric.threshold ?? 1;
+    const passed = score >= threshold && criteria.every((criterion) => criterion.passed);
+    return {
+      name: this.name,
+      score,
+      passed,
+      reason: passed ? "Rubric passed." : "Rubric failed.",
+      metadata: {
+        rubricId: this.rubric.id,
+        rubricVersion: this.rubric.version,
+        threshold,
+        criteria: criteria.map((criterion) => ({
+          key: criterion.key,
+          score: criterion.score,
+          passed: criterion.passed,
+          reason: criterion.reason ?? null
+        }))
+      }
+    };
+  }
+};
+
+// src/runners/eval-runner.ts
+var EvalRunner = class {
+  datasetStore;
+  evaluators;
+  executor;
+  constructor(options) {
+    this.datasetStore = options.datasetStore;
+    this.evaluators = options.evaluators;
+    this.executor = options.executor;
+  }
+  async run(config) {
+    const dataset = await this.datasetStore.get(config.datasetId, config.datasetVersion);
+    if (dataset === void 0) {
+      throw new EvalError("EVAL_DATASET_NOT_FOUND", `Eval dataset not found: ${config.datasetId}`);
+    }
+    const context = {
+      runId: config.runId ?? createRunId(),
+      datasetId: dataset.id,
+      datasetVersion: dataset.version
+    };
+    if (config.agentKey !== void 0) context.agentKey = config.agentKey;
+    if (config.modelId !== void 0) context.modelId = config.modelId;
+    if (config.promptVersion !== void 0) context.promptVersion = config.promptVersion;
+    if (config.metadata !== void 0) context.metadata = config.metadata;
+    const results = [];
+    for (const testCase of dataset.cases) {
+      const result = await this.executor(testCase, context);
+      const scores2 = await Promise.all(
+        this.evaluators.map(
+          (evaluator) => evaluator.evaluate({
+            testCase,
+            result,
+            context
+          })
+        )
+      );
+      const averageScore = scores2.length === 0 ? 0 : scores2.reduce((sum, score) => sum + score.score, 0) / scores2.length;
+      const passed = result.status === "completed" && scores2.every((score) => score.passed) && averageScore >= (config.threshold ?? 1);
+      results.push({
+        caseId: testCase.id,
+        testCase,
+        result,
+        scores: scores2,
+        passed
+      });
+    }
+    const passedCases = results.filter((result) => result.passed).length;
+    const scores = results.flatMap((result) => result.scores.map((score) => score.score));
+    const summary = {
+      totalCases: results.length,
+      passedCases,
+      failedCases: results.length - passedCases,
+      passRate: results.length === 0 ? 0 : passedCases / results.length,
+      averageScore: scores.length === 0 ? 0 : scores.reduce((sum, score) => sum + score, 0) / scores.length
+    };
+    const runResult = {
+      runId: context.runId,
+      datasetId: dataset.id,
+      datasetVersion: dataset.version,
+      results,
+      summary,
+      metrics: [
+        createPassRateMetric(summary.passedCases, summary.totalCases),
+        createAverageScoreMetric(scores),
+        createCostMetric(results.map((result) => result.result))
+      ],
+      passed: summary.failedCases === 0
+    };
+    if (config.metadata !== void 0) runResult.metadata = config.metadata;
+    return runResult;
+  }
+};
+function createRunId() {
+  return `eval_${Date.now().toString(36)}`;
+}
+export {
+  BaselineComparator,
+  CompositeEvaluator,
+  EvalError,
+  EvalRunner,
+  ExactMatchEvaluator,
+  InMemoryEvalDatasetStore,
+  JsonReportRenderer,
+  JsonSchemaEvaluator,
+  LlmJudgeEvaluator,
+  MarkdownReportRenderer,
+  RegexEvaluator,
+  SemanticSimilarityEvaluator,
+  ToolCallEvaluator,
+  WeightedRubricEvaluator,
+  createAverageScoreMetric,
+  createCostMetric,
+  createEvalCase,
+  createEvalScore,
+  createPassRateMetric
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/baselines/baseline-comparator.ts","../src/cases/eval-case-builder.ts","../src/datasets/in-memory-dataset-store.ts","../src/errors/eval-error.ts","../src/evaluators/composite-evaluator.ts","../src/evaluators/evaluator.interface.ts","../src/evaluators/exact-match-evaluator.ts","../src/evaluators/json-schema-evaluator.ts","../src/evaluators/llm-judge-evaluator.ts","../src/evaluators/regex-evaluator.ts","../src/evaluators/semantic-similarity-evaluator.ts","../src/evaluators/tool-call-evaluator.ts","../src/metrics/metric-helpers.ts","../src/reports/json-report-renderer.ts","../src/reports/markdown-report-renderer.ts","../src/rubrics/weighted-rubric-evaluator.ts","../src/runners/eval-runner.ts"],"sourcesContent":["import type { EvalRunResult } from '../runners/eval-run-result';\nimport type { EvalRegressionResult } from './regression-result';\n\nexport interface BaselineComparatorOptions {\n  thresholds?: Record<string, number>;\n}\n\nexport class BaselineComparator {\n  private readonly thresholds: Record<string, number>;\n\n  constructor(options: BaselineComparatorOptions = {}) {\n    this.thresholds = options.thresholds ?? {};\n  }\n\n  compare(baseline: EvalRunResult, current: EvalRunResult): EvalRegressionResult[] {\n    const baselineMetrics = new Map(baseline.metrics.map((metric) => [metric.name, metric.value]));\n\n    return current.metrics\n      .filter((metric) => baselineMetrics.has(metric.name))\n      .map((metric) => {\n        const baselineValue = baselineMetrics.get(metric.name) ?? 0;\n        const threshold = this.thresholds[metric.name] ?? 0;\n        const delta = metric.value - baselineValue;\n        const lowerIsBetter = metric.name.includes('cost') || metric.name.includes('latency');\n        const regressed = lowerIsBetter ? delta > threshold : delta < -threshold;\n\n        return {\n          metricName: metric.name,\n          baselineValue,\n          currentValue: metric.value,\n          delta,\n          regressed,\n          threshold,\n        };\n      });\n  }\n}\n","import type { AgentKey, JsonObject } from '@hemia-ai/agents-core';\nimport type { EvalCase } from './eval-case';\n\nexport interface EvalCaseBuilderInput<TInput = unknown, TExpected = unknown> {\n  id: string;\n  name: string;\n  agentKey: AgentKey;\n  input: TInput;\n  expected?: TExpected;\n  tags?: string[];\n  metadata?: JsonObject;\n}\n\nexport function createEvalCase<TInput = unknown, TExpected = unknown>(\n  input: EvalCaseBuilderInput<TInput, TExpected>,\n): EvalCase<TInput, TExpected> {\n  const testCase: EvalCase<TInput, TExpected> = {\n    id: input.id,\n    name: input.name,\n    agentKey: input.agentKey,\n    input: input.input,\n  };\n\n  if (input.expected !== undefined) testCase.expected = input.expected;\n  if (input.tags !== undefined) testCase.tags = input.tags;\n  if (input.metadata !== undefined) testCase.metadata = input.metadata;\n\n  return testCase;\n}\n","import type { EvalDatasetStore } from './dataset-registry';\nimport type { EvalDataset } from './eval-dataset';\n\nexport class InMemoryEvalDatasetStore implements EvalDatasetStore {\n  private readonly datasets = new Map<string, EvalDataset>();\n\n  constructor(datasets: EvalDataset[] = []) {\n    for (const dataset of datasets) {\n      this.datasets.set(this.key(dataset.id, dataset.version), dataset);\n    }\n  }\n\n  async list(): Promise<EvalDataset[]> {\n    return [...this.datasets.values()];\n  }\n\n  async get(id: string, version?: string): Promise<EvalDataset | undefined> {\n    if (version !== undefined) return this.datasets.get(this.key(id, version));\n\n    return [...this.datasets.values()]\n      .filter((dataset) => dataset.id === id)\n      .sort((a, b) => b.version.localeCompare(a.version))[0];\n  }\n\n  async put(dataset: EvalDataset): Promise<void> {\n    this.datasets.set(this.key(dataset.id, dataset.version), dataset);\n  }\n\n  private key(id: string, version: string): string {\n    return `${id}@${version}`;\n  }\n}\n","import type { JsonObject } from '@hemia-ai/agents-core';\nimport type { EvalErrorCode } from './eval-error-code';\n\nexport class EvalError extends Error {\n  readonly code: EvalErrorCode;\n  readonly metadata?: JsonObject;\n\n  constructor(code: EvalErrorCode, message: string, metadata?: JsonObject) {\n    super(message);\n    this.name = 'EvalError';\n    this.code = code;\n    if (metadata !== undefined) this.metadata = metadata;\n  }\n}\n","import type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface WeightedEvalEvaluator {\n  evaluator: EvalEvaluator;\n  weight?: number;\n}\n\nexport interface CompositeEvaluatorOptions {\n  name?: string;\n  evaluators: WeightedEvalEvaluator[];\n  threshold?: number;\n}\n\nexport class CompositeEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly evaluators: WeightedEvalEvaluator[];\n  private readonly threshold: number;\n\n  constructor(options: CompositeEvaluatorOptions) {\n    this.name = options.name ?? 'composite';\n    this.evaluators = options.evaluators;\n    this.threshold = options.threshold ?? 1;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const scores = await Promise.all(this.evaluators.map(({ evaluator }) => evaluator.evaluate(input)));\n    const totalWeight = this.evaluators.reduce((sum, item) => sum + (item.weight ?? 1), 0) || 1;\n    const score = scores.reduce((sum, item, index) => sum + item.score * (this.evaluators[index]?.weight ?? 1), 0) / totalWeight;\n    const passed = score >= this.threshold && scores.every((item) => item.passed);\n\n    return {\n      name: this.name,\n      score,\n      passed,\n      reason: passed ? 'Composite score passed.' : 'Composite score failed.',\n      metadata: {\n        threshold: this.threshold,\n        scores: scores.map((item) => ({\n          name: item.name,\n          score: item.score,\n          passed: item.passed,\n          reason: item.reason ?? null,\n        })),\n      },\n    };\n  }\n}\n","import type { AgentExecutionResult, JsonObject } from '@hemia-ai/agents-core';\nimport type { EvalCase } from '../cases/eval-case';\nimport type { EvalRunContext } from '../runners/eval-run-context';\n\nexport interface EvalEvaluatorInput {\n  testCase: EvalCase;\n  result: AgentExecutionResult;\n  context: EvalRunContext;\n}\n\nexport interface EvalScore {\n  name: string;\n  score: number;\n  passed: boolean;\n  reason?: string;\n  metadata?: JsonObject;\n}\n\nexport interface EvalEvaluator {\n  name: string;\n  evaluate(input: EvalEvaluatorInput): Promise<EvalScore>;\n}\n\nexport function createEvalScore(input: EvalScore): EvalScore {\n  return input;\n}\n","import type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface ExactMatchEvaluatorOptions {\n  name?: string;\n  ignoreCase?: boolean;\n  trim?: boolean;\n}\n\nexport class ExactMatchEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly ignoreCase: boolean;\n  private readonly trim: boolean;\n\n  constructor(options: ExactMatchEvaluatorOptions = {}) {\n    this.name = options.name ?? 'exact_match';\n    this.ignoreCase = options.ignoreCase ?? false;\n    this.trim = options.trim ?? true;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const actual = input.result.status === 'completed' ? input.result.output : undefined;\n    const expected = input.testCase.expected;\n    const passed = this.normalize(actual) === this.normalize(expected);\n\n    return {\n      name: this.name,\n      score: passed ? 1 : 0,\n      passed,\n      reason: passed ? 'Output matches expected value.' : 'Output does not match expected value.',\n    };\n  }\n\n  private normalize(value: unknown): string {\n    const raw = typeof value === 'string' ? value : JSON.stringify(value) ?? '';\n    const trimmed = this.trim ? raw.trim() : raw;\n    return this.ignoreCase ? trimmed.toLowerCase() : trimmed;\n  }\n}\n","import type { JsonObject } from '@hemia-ai/agents-core';\nimport type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface JsonSchemaEvaluatorOptions {\n  name?: string;\n  schema: JsonObject;\n}\n\nexport class JsonSchemaEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly schema: JsonObject;\n\n  constructor(options: JsonSchemaEvaluatorOptions) {\n    this.name = options.name ?? 'json_schema';\n    this.schema = options.schema;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const actual = input.result.status === 'completed' ? input.result.output : undefined;\n    const errors = validateAgainstSchema(actual, this.schema);\n    const passed = errors.length === 0;\n\n    return {\n      name: this.name,\n      score: passed ? 1 : 0,\n      passed,\n      reason: passed ? 'Output matches schema.' : errors.join('; '),\n    };\n  }\n}\n\nfunction validateAgainstSchema(value: unknown, schema: JsonObject, path = '$'): string[] {\n  const errors: string[] = [];\n  const type = schema.type;\n\n  if (typeof type === 'string' && !matchesType(value, type)) {\n    errors.push(`${path} expected ${type}`);\n    return errors;\n  }\n\n  if (type === 'object' && isRecord(value)) {\n    const required = Array.isArray(schema.required) ? schema.required.filter((item): item is string => typeof item === 'string') : [];\n    for (const key of required) {\n      if (!(key in value)) errors.push(`${path}.${key} is required`);\n    }\n\n    const properties = isRecord(schema.properties) ? schema.properties : {};\n    for (const [key, childSchema] of Object.entries(properties)) {\n      if (key in value && isRecord(childSchema)) {\n        errors.push(...validateAgainstSchema(value[key], childSchema, `${path}.${key}`));\n      }\n    }\n  }\n\n  return errors;\n}\n\nfunction matchesType(value: unknown, type: string): boolean {\n  if (type === 'array') return Array.isArray(value);\n  if (type === 'null') return value === null;\n  if (type === 'integer') return Number.isInteger(value);\n  return typeof value === type;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n","import type { JsonObject } from '@hemia-ai/agents-core';\nimport type { AiGenerateInput } from '@hemia-ai/agents-models';\nimport type { ModelGateway } from '@hemia-ai/agents-models';\nimport type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface LlmJudgeEvaluatorOptions {\n  name?: string;\n  modelGateway: Pick<ModelGateway, 'generate'>;\n  model?: string;\n  threshold?: number;\n  rubric?: string;\n  metadata?: JsonObject;\n}\n\nexport class LlmJudgeEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly modelGateway: Pick<ModelGateway, 'generate'>;\n  private readonly model?: string;\n  private readonly threshold: number;\n  private readonly rubric?: string;\n  private readonly metadata?: JsonObject;\n\n  constructor(options: LlmJudgeEvaluatorOptions) {\n    this.name = options.name ?? 'llm_judge';\n    this.modelGateway = options.modelGateway;\n    this.threshold = options.threshold ?? 0.8;\n    if (options.model !== undefined) this.model = options.model;\n    if (options.rubric !== undefined) this.rubric = options.rubric;\n    if (options.metadata !== undefined) this.metadata = options.metadata;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const request: AiGenerateInput = {\n      taskType: 'eval_judge',\n      agentKey: input.testCase.agentKey,\n      messages: [\n        {\n          role: 'system',\n          content:\n            'You are an evaluator. Return JSON only with shape {\"score\": number, \"passed\": boolean, \"reason\": string}.',\n        },\n        {\n          role: 'user',\n          content: JSON.stringify({\n            rubric: this.rubric,\n            input: input.testCase.input,\n            expected: input.testCase.expected,\n            actual: input.result.status === 'completed' ? input.result.output : input.result,\n          }),\n        },\n      ],\n    };\n    const response = await this.modelGateway.generate({\n      ...request,\n      ...(this.model !== undefined ? { model: this.model } : {}),\n      ...(this.metadata !== undefined ? { metadata: this.metadata } : {}),\n    });\n\n    const parsed = parseJudgeResponse(response.content);\n    const score = parsed.score ?? 0;\n    const passed = parsed.passed ?? score >= this.threshold;\n\n    return {\n      name: this.name,\n      score,\n      passed,\n      reason: parsed.reason ?? response.content,\n      metadata: {\n        threshold: this.threshold,\n        judgeModel: this.model ?? null,\n        usage: response.usage === undefined ? null : JSON.parse(JSON.stringify(response.usage)),\n      },\n    };\n  }\n}\n\nfunction parseJudgeResponse(content: string): { score?: number; passed?: boolean; reason?: string } {\n  try {\n    const parsed = JSON.parse(content) as unknown;\n    if (!isRecord(parsed)) return {};\n\n    const result: { score?: number; passed?: boolean; reason?: string } = {};\n    if (typeof parsed.score === 'number') result.score = parsed.score;\n    if (typeof parsed.passed === 'boolean') result.passed = parsed.passed;\n    if (typeof parsed.reason === 'string') result.reason = parsed.reason;\n    return result;\n  } catch {\n    return {};\n  }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n","import type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface RegexEvaluatorOptions {\n  name?: string;\n  pattern: RegExp;\n}\n\nexport class RegexEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly pattern: RegExp;\n\n  constructor(options: RegexEvaluatorOptions) {\n    this.name = options.name ?? 'regex_match';\n    this.pattern = options.pattern;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const actual = input.result.status === 'completed' ? input.result.output : undefined;\n    const content = typeof actual === 'string' ? actual : JSON.stringify(actual);\n    const passed = this.pattern.test(content);\n\n    return {\n      name: this.name,\n      score: passed ? 1 : 0,\n      passed,\n      reason: passed ? 'Output matches pattern.' : 'Output does not match pattern.',\n      metadata: { pattern: this.pattern.source },\n    };\n  }\n}\n","import type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface SemanticSimilarityEvaluatorOptions {\n  name?: string;\n  threshold?: number;\n  similarity: (actual: string, expected: string) => Promise<number> | number;\n}\n\nexport class SemanticSimilarityEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly threshold: number;\n  private readonly similarity: (actual: string, expected: string) => Promise<number> | number;\n\n  constructor(options: SemanticSimilarityEvaluatorOptions) {\n    this.name = options.name ?? 'semantic_similarity';\n    this.threshold = options.threshold ?? 0.8;\n    this.similarity = options.similarity;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const actual = stringify(input.result.status === 'completed' ? input.result.output : undefined);\n    const expected = stringify(input.testCase.expected);\n    const score = await this.similarity(actual, expected);\n\n    return {\n      name: this.name,\n      score,\n      passed: score >= this.threshold,\n      reason: score >= this.threshold ? 'Semantic similarity passed.' : 'Semantic similarity below threshold.',\n      metadata: { threshold: this.threshold },\n    };\n  }\n}\n\nfunction stringify(value: unknown): string {\n  return typeof value === 'string' ? value : JSON.stringify(value);\n}\n","import type { ToolCall } from '@hemia-ai/agents-core';\nimport type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from './evaluator.interface';\n\nexport interface ExpectedToolCall {\n  name: string;\n  input?: unknown;\n}\n\nexport interface ToolCallEvaluatorOptions {\n  name?: string;\n  expectedToolCalls?: ExpectedToolCall[];\n  requireOrder?: boolean;\n}\n\nexport class ToolCallEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly expectedToolCalls?: ExpectedToolCall[];\n  private readonly requireOrder: boolean;\n\n  constructor(options: ToolCallEvaluatorOptions = {}) {\n    this.name = options.name ?? 'tool_calls';\n    if (options.expectedToolCalls !== undefined) this.expectedToolCalls = options.expectedToolCalls;\n    this.requireOrder = options.requireOrder ?? true;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const actualToolCalls = input.result.status === 'completed' ? extractToolCalls(input.result.output) : [];\n    const expectedToolCalls = this.expectedToolCalls ?? extractExpectedToolCalls(input.testCase.expected);\n    const passed = this.requireOrder\n      ? matchesInOrder(actualToolCalls, expectedToolCalls)\n      : matchesAnyOrder(actualToolCalls, expectedToolCalls);\n\n    return {\n      name: this.name,\n      score: passed ? 1 : 0,\n      passed,\n      reason: passed ? 'Tool calls match expected calls.' : 'Tool calls do not match expected calls.',\n      metadata: {\n        expected: expectedToolCalls.map((call) => call.name),\n        actual: actualToolCalls.map((call) => call.name),\n      },\n    };\n  }\n}\n\nfunction extractToolCalls(value: unknown): ToolCall[] {\n  if (isRecord(value) && Array.isArray(value.toolCalls)) {\n    return value.toolCalls.filter(isToolCall);\n  }\n\n  return [];\n}\n\nfunction extractExpectedToolCalls(value: unknown): ExpectedToolCall[] {\n  if (isRecord(value) && Array.isArray(value.toolCalls)) {\n    return value.toolCalls.filter(isExpectedToolCall);\n  }\n\n  return [];\n}\n\nfunction matchesInOrder(actual: ToolCall[], expected: ExpectedToolCall[]): boolean {\n  return expected.every((expectedCall, index) => matchesCall(actual[index], expectedCall));\n}\n\nfunction matchesAnyOrder(actual: ToolCall[], expected: ExpectedToolCall[]): boolean {\n  return expected.every((expectedCall) => actual.some((actualCall) => matchesCall(actualCall, expectedCall)));\n}\n\nfunction matchesCall(actual: ToolCall | undefined, expected: ExpectedToolCall): boolean {\n  if (actual === undefined || actual.name !== expected.name) return false;\n  if (expected.input === undefined) return true;\n  return JSON.stringify(actual.input) === JSON.stringify(expected.input);\n}\n\nfunction isToolCall(value: unknown): value is ToolCall {\n  return isRecord(value) && typeof value.name === 'string';\n}\n\nfunction isExpectedToolCall(value: unknown): value is ExpectedToolCall {\n  return isRecord(value) && typeof value.name === 'string';\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n","import type { AgentExecutionResult } from '@hemia-ai/agents-core';\nimport type { EvalMetric } from './eval-metric';\n\nexport function createPassRateMetric(passed: number, total: number): EvalMetric {\n  return {\n    name: 'pass_rate',\n    value: total === 0 ? 0 : passed / total,\n    unit: 'ratio',\n    metadata: { passed, total },\n  };\n}\n\nexport function createAverageScoreMetric(scores: number[]): EvalMetric {\n  return {\n    name: 'average_score',\n    value: scores.length === 0 ? 0 : scores.reduce((sum, score) => sum + score, 0) / scores.length,\n    unit: 'ratio',\n  };\n}\n\nexport function createCostMetric(results: AgentExecutionResult[]): EvalMetric {\n  const cost = results.reduce((sum, result) => {\n    const value = result.metadata?.estimatedCostUsd;\n    return sum + (typeof value === 'number' ? value : 0);\n  }, 0);\n\n  return {\n    name: 'estimated_cost_usd',\n    value: cost,\n    unit: 'usd',\n  };\n}\n","import type { EvalReport, EvalReportRenderer } from './eval-report';\n\nexport class JsonReportRenderer implements EvalReportRenderer {\n  render(report: EvalReport): string {\n    return JSON.stringify(report, null, 2);\n  }\n}\n","import type { EvalReport, EvalReportRenderer } from './eval-report';\n\nexport class MarkdownReportRenderer implements EvalReportRenderer {\n  render(report: EvalReport): string {\n    const failed = report.run.results.filter((result) => !result.passed);\n    const regressions = report.regressions?.filter((result) => result.regressed) ?? [];\n\n    return [\n      `# Eval Report ${report.run.runId}`,\n      '',\n      `Dataset: ${report.run.datasetId}@${report.run.datasetVersion}`,\n      `Passed: ${report.run.passed ? 'yes' : 'no'}`,\n      `Pass rate: ${formatRatio(report.run.summary.passRate)}`,\n      `Average score: ${formatRatio(report.run.summary.averageScore)}`,\n      '',\n      '## Metrics',\n      ...report.run.metrics.map((metric) => `- ${metric.name}: ${metric.value}${metric.unit ? ` ${metric.unit}` : ''}`),\n      '',\n      '## Failed Cases',\n      ...(failed.length === 0\n        ? ['- none']\n        : failed.map((result) => {\n            const reasons = result.scores.filter((score) => !score.passed).map((score) => `${score.name}: ${score.reason ?? 'failed'}`);\n            return `- ${result.caseId}: ${reasons.join(', ')}`;\n          })),\n      '',\n      '## Regressions',\n      ...(regressions.length === 0\n        ? ['- none']\n        : regressions.map((result) => `- ${result.metricName}: ${result.baselineValue} -> ${result.currentValue}`)),\n    ].join('\\n');\n  }\n}\n\nfunction formatRatio(value: number): string {\n  return `${Math.round(value * 10000) / 100}%`;\n}\n","import type { EvalEvaluator, EvalEvaluatorInput, EvalScore } from '../evaluators/evaluator.interface';\nimport type { EvalRubric } from './eval-rubric';\nimport type { RubricCriterionScore } from './rubric-score';\n\nexport interface WeightedRubricEvaluatorOptions {\n  rubric: EvalRubric;\n  scoreCriterion: (criterionKey: string, input: EvalEvaluatorInput) => Promise<RubricCriterionScore> | RubricCriterionScore;\n}\n\nexport class WeightedRubricEvaluator implements EvalEvaluator {\n  readonly name: string;\n  private readonly rubric: EvalRubric;\n  private readonly scoreCriterion: (criterionKey: string, input: EvalEvaluatorInput) => Promise<RubricCriterionScore> | RubricCriterionScore;\n\n  constructor(options: WeightedRubricEvaluatorOptions) {\n    this.name = `rubric:${options.rubric.id}`;\n    this.rubric = options.rubric;\n    this.scoreCriterion = options.scoreCriterion;\n  }\n\n  async evaluate(input: EvalEvaluatorInput): Promise<EvalScore> {\n    const criteria = await Promise.all(\n      this.rubric.criteria.map((criterion) => this.scoreCriterion(criterion.key, input)),\n    );\n    const totalWeight = this.rubric.criteria.reduce((sum, item) => sum + (item.weight ?? 1), 0) || 1;\n    const score = criteria.reduce((sum, item) => {\n      const weight = this.rubric.criteria.find((criterion) => criterion.key === item.key)?.weight ?? 1;\n      return sum + item.score * weight;\n    }, 0) / totalWeight;\n    const threshold = this.rubric.threshold ?? 1;\n    const passed = score >= threshold && criteria.every((criterion) => criterion.passed);\n\n    return {\n      name: this.name,\n      score,\n      passed,\n      reason: passed ? 'Rubric passed.' : 'Rubric failed.',\n      metadata: {\n        rubricId: this.rubric.id,\n        rubricVersion: this.rubric.version,\n        threshold,\n        criteria: criteria.map((criterion) => ({\n          key: criterion.key,\n          score: criterion.score,\n          passed: criterion.passed,\n          reason: criterion.reason ?? null,\n        })),\n      },\n    };\n  }\n}\n","import type { EvalDatasetStore } from '../datasets/dataset-registry';\nimport { EvalError } from '../errors/eval-error';\nimport type { EvalEvaluator } from '../evaluators/evaluator.interface';\nimport { createAverageScoreMetric, createCostMetric, createPassRateMetric } from '../metrics/metric-helpers';\nimport type { EvalCaseExecutor } from './eval-case-executor';\nimport type { EvalRunConfig } from './eval-run-config';\nimport type { EvalRunContext } from './eval-run-context';\nimport type { EvalCaseResult, EvalRunResult } from './eval-run-result';\n\nexport interface EvalRunnerOptions {\n  datasetStore: EvalDatasetStore;\n  evaluators: EvalEvaluator[];\n  executor: EvalCaseExecutor;\n}\n\nexport class EvalRunner {\n  private readonly datasetStore: EvalDatasetStore;\n  private readonly evaluators: EvalEvaluator[];\n  private readonly executor: EvalCaseExecutor;\n\n  constructor(options: EvalRunnerOptions) {\n    this.datasetStore = options.datasetStore;\n    this.evaluators = options.evaluators;\n    this.executor = options.executor;\n  }\n\n  async run(config: EvalRunConfig): Promise<EvalRunResult> {\n    const dataset = await this.datasetStore.get(config.datasetId, config.datasetVersion);\n    if (dataset === undefined) {\n      throw new EvalError('EVAL_DATASET_NOT_FOUND', `Eval dataset not found: ${config.datasetId}`);\n    }\n\n    const context: EvalRunContext = {\n      runId: config.runId ?? createRunId(),\n      datasetId: dataset.id,\n      datasetVersion: dataset.version,\n    };\n    if (config.agentKey !== undefined) context.agentKey = config.agentKey;\n    if (config.modelId !== undefined) context.modelId = config.modelId;\n    if (config.promptVersion !== undefined) context.promptVersion = config.promptVersion;\n    if (config.metadata !== undefined) context.metadata = config.metadata;\n\n    const results: EvalCaseResult[] = [];\n\n    for (const testCase of dataset.cases) {\n      const result = await this.executor(testCase, context);\n      const scores = await Promise.all(\n        this.evaluators.map((evaluator) =>\n          evaluator.evaluate({\n            testCase,\n            result,\n            context,\n          }),\n        ),\n      );\n      const averageScore = scores.length === 0 ? 0 : scores.reduce((sum, score) => sum + score.score, 0) / scores.length;\n      const passed = result.status === 'completed' && scores.every((score) => score.passed) && averageScore >= (config.threshold ?? 1);\n\n      results.push({\n        caseId: testCase.id,\n        testCase,\n        result,\n        scores,\n        passed,\n      });\n    }\n\n    const passedCases = results.filter((result) => result.passed).length;\n    const scores = results.flatMap((result) => result.scores.map((score) => score.score));\n    const summary = {\n      totalCases: results.length,\n      passedCases,\n      failedCases: results.length - passedCases,\n      passRate: results.length === 0 ? 0 : passedCases / results.length,\n      averageScore: scores.length === 0 ? 0 : scores.reduce((sum, score) => sum + score, 0) / scores.length,\n    };\n\n    const runResult: EvalRunResult = {\n      runId: context.runId,\n      datasetId: dataset.id,\n      datasetVersion: dataset.version,\n      results,\n      summary,\n      metrics: [\n        createPassRateMetric(summary.passedCases, summary.totalCases),\n        createAverageScoreMetric(scores),\n        createCostMetric(results.map((result) => result.result)),\n      ],\n      passed: summary.failedCases === 0,\n    };\n    if (config.metadata !== undefined) runResult.metadata = config.metadata;\n\n    return runResult;\n  }\n}\n\nfunction createRunId(): string {\n  return `eval_${Date.now().toString(36)}`;\n}\n"],"mappings":";AAOO,IAAM,qBAAN,MAAyB;AAAA,EACb;AAAA,EAEjB,YAAY,UAAqC,CAAC,GAAG;AACnD,SAAK,aAAa,QAAQ,cAAc,CAAC;AAAA,EAC3C;AAAA,EAEA,QAAQ,UAAyB,SAAgD;AAC/E,UAAM,kBAAkB,IAAI,IAAI,SAAS,QAAQ,IAAI,CAAC,WAAW,CAAC,OAAO,MAAM,OAAO,KAAK,CAAC,CAAC;AAE7F,WAAO,QAAQ,QACZ,OAAO,CAAC,WAAW,gBAAgB,IAAI,OAAO,IAAI,CAAC,EACnD,IAAI,CAAC,WAAW;AACf,YAAM,gBAAgB,gBAAgB,IAAI,OAAO,IAAI,KAAK;AAC1D,YAAM,YAAY,KAAK,WAAW,OAAO,IAAI,KAAK;AAClD,YAAM,QAAQ,OAAO,QAAQ;AAC7B,YAAM,gBAAgB,OAAO,KAAK,SAAS,MAAM,KAAK,OAAO,KAAK,SAAS,SAAS;AACpF,YAAM,YAAY,gBAAgB,QAAQ,YAAY,QAAQ,CAAC;AAE/D,aAAO;AAAA,QACL,YAAY,OAAO;AAAA,QACnB;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACL;AACF;;;ACvBO,SAAS,eACd,OAC6B;AAC7B,QAAM,WAAwC;AAAA,IAC5C,IAAI,MAAM;AAAA,IACV,MAAM,MAAM;AAAA,IACZ,UAAU,MAAM;AAAA,IAChB,OAAO,MAAM;AAAA,EACf;AAEA,MAAI,MAAM,aAAa,OAAW,UAAS,WAAW,MAAM;AAC5D,MAAI,MAAM,SAAS,OAAW,UAAS,OAAO,MAAM;AACpD,MAAI,MAAM,aAAa,OAAW,UAAS,WAAW,MAAM;AAE5D,SAAO;AACT;;;ACzBO,IAAM,2BAAN,MAA2D;AAAA,EAC/C,WAAW,oBAAI,IAAyB;AAAA,EAEzD,YAAY,WAA0B,CAAC,GAAG;AACxC,eAAW,WAAW,UAAU;AAC9B,WAAK,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,QAAQ,OAAO,GAAG,OAAO;AAAA,IAClE;AAAA,EACF;AAAA,EAEA,MAAM,OAA+B;AACnC,WAAO,CAAC,GAAG,KAAK,SAAS,OAAO,CAAC;AAAA,EACnC;AAAA,EAEA,MAAM,IAAI,IAAY,SAAoD;AACxE,QAAI,YAAY,OAAW,QAAO,KAAK,SAAS,IAAI,KAAK,IAAI,IAAI,OAAO,CAAC;AAEzE,WAAO,CAAC,GAAG,KAAK,SAAS,OAAO,CAAC,EAC9B,OAAO,CAAC,YAAY,QAAQ,OAAO,EAAE,EACrC,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,cAAc,EAAE,OAAO,CAAC,EAAE,CAAC;AAAA,EACzD;AAAA,EAEA,MAAM,IAAI,SAAqC;AAC7C,SAAK,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,QAAQ,OAAO,GAAG,OAAO;AAAA,EAClE;AAAA,EAEQ,IAAI,IAAY,SAAyB;AAC/C,WAAO,GAAG,EAAE,IAAI,OAAO;AAAA,EACzB;AACF;;;AC5BO,IAAM,YAAN,cAAwB,MAAM;AAAA,EAC1B;AAAA,EACA;AAAA,EAET,YAAY,MAAqB,SAAiB,UAAuB;AACvE,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,QAAI,aAAa,OAAW,MAAK,WAAW;AAAA,EAC9C;AACF;;;ACAO,IAAM,qBAAN,MAAkD;AAAA,EAC9C;AAAA,EACQ;AAAA,EACA;AAAA,EAEjB,YAAY,SAAoC;AAC9C,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,aAAa,QAAQ;AAC1B,SAAK,YAAY,QAAQ,aAAa;AAAA,EACxC;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,SAAS,MAAM,QAAQ,IAAI,KAAK,WAAW,IAAI,CAAC,EAAE,UAAU,MAAM,UAAU,SAAS,KAAK,CAAC,CAAC;AAClG,UAAM,cAAc,KAAK,WAAW,OAAO,CAAC,KAAK,SAAS,OAAO,KAAK,UAAU,IAAI,CAAC,KAAK;AAC1F,UAAM,QAAQ,OAAO,OAAO,CAAC,KAAK,MAAM,UAAU,MAAM,KAAK,SAAS,KAAK,WAAW,KAAK,GAAG,UAAU,IAAI,CAAC,IAAI;AACjH,UAAM,SAAS,SAAS,KAAK,aAAa,OAAO,MAAM,CAAC,SAAS,KAAK,MAAM;AAE5E,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,SAAS,4BAA4B;AAAA,MAC7C,UAAU;AAAA,QACR,WAAW,KAAK;AAAA,QAChB,QAAQ,OAAO,IAAI,CAAC,UAAU;AAAA,UAC5B,MAAM,KAAK;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,QAAQ,KAAK;AAAA,UACb,QAAQ,KAAK,UAAU;AAAA,QACzB,EAAE;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AACF;;;ACvBO,SAAS,gBAAgB,OAA6B;AAC3D,SAAO;AACT;;;ACjBO,IAAM,sBAAN,MAAmD;AAAA,EAC/C;AAAA,EACQ;AAAA,EACA;AAAA,EAEjB,YAAY,UAAsC,CAAC,GAAG;AACpD,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,aAAa,QAAQ,cAAc;AACxC,SAAK,OAAO,QAAQ,QAAQ;AAAA,EAC9B;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,SAAS,MAAM,OAAO,WAAW,cAAc,MAAM,OAAO,SAAS;AAC3E,UAAM,WAAW,MAAM,SAAS;AAChC,UAAM,SAAS,KAAK,UAAU,MAAM,MAAM,KAAK,UAAU,QAAQ;AAEjE,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,OAAO,SAAS,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,mCAAmC;AAAA,IACtD;AAAA,EACF;AAAA,EAEQ,UAAU,OAAwB;AACxC,UAAM,MAAM,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK,KAAK;AACzE,UAAM,UAAU,KAAK,OAAO,IAAI,KAAK,IAAI;AACzC,WAAO,KAAK,aAAa,QAAQ,YAAY,IAAI;AAAA,EACnD;AACF;;;AC7BO,IAAM,sBAAN,MAAmD;AAAA,EAC/C;AAAA,EACQ;AAAA,EAEjB,YAAY,SAAqC;AAC/C,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,SAAS,MAAM,OAAO,WAAW,cAAc,MAAM,OAAO,SAAS;AAC3E,UAAM,SAAS,sBAAsB,QAAQ,KAAK,MAAM;AACxD,UAAM,SAAS,OAAO,WAAW;AAEjC,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,OAAO,SAAS,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,2BAA2B,OAAO,KAAK,IAAI;AAAA,IAC9D;AAAA,EACF;AACF;AAEA,SAAS,sBAAsB,OAAgB,QAAoB,OAAO,KAAe;AACvF,QAAM,SAAmB,CAAC;AAC1B,QAAM,OAAO,OAAO;AAEpB,MAAI,OAAO,SAAS,YAAY,CAAC,YAAY,OAAO,IAAI,GAAG;AACzD,WAAO,KAAK,GAAG,IAAI,aAAa,IAAI,EAAE;AACtC,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,YAAY,SAAS,KAAK,GAAG;AACxC,UAAM,WAAW,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,CAAC,SAAyB,OAAO,SAAS,QAAQ,IAAI,CAAC;AAChI,eAAW,OAAO,UAAU;AAC1B,UAAI,EAAE,OAAO,OAAQ,QAAO,KAAK,GAAG,IAAI,IAAI,GAAG,cAAc;AAAA,IAC/D;AAEA,UAAM,aAAa,SAAS,OAAO,UAAU,IAAI,OAAO,aAAa,CAAC;AACtE,eAAW,CAAC,KAAK,WAAW,KAAK,OAAO,QAAQ,UAAU,GAAG;AAC3D,UAAI,OAAO,SAAS,SAAS,WAAW,GAAG;AACzC,eAAO,KAAK,GAAG,sBAAsB,MAAM,GAAG,GAAG,aAAa,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;AAAA,MACjF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,YAAY,OAAgB,MAAuB;AAC1D,MAAI,SAAS,QAAS,QAAO,MAAM,QAAQ,KAAK;AAChD,MAAI,SAAS,OAAQ,QAAO,UAAU;AACtC,MAAI,SAAS,UAAW,QAAO,OAAO,UAAU,KAAK;AACrD,SAAO,OAAO,UAAU;AAC1B;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;ACpDO,IAAM,oBAAN,MAAiD;AAAA,EAC7C;AAAA,EACQ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,SAAmC;AAC7C,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,eAAe,QAAQ;AAC5B,SAAK,YAAY,QAAQ,aAAa;AACtC,QAAI,QAAQ,UAAU,OAAW,MAAK,QAAQ,QAAQ;AACtD,QAAI,QAAQ,WAAW,OAAW,MAAK,SAAS,QAAQ;AACxD,QAAI,QAAQ,aAAa,OAAW,MAAK,WAAW,QAAQ;AAAA,EAC9D;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,UAA2B;AAAA,MAC/B,UAAU;AAAA,MACV,UAAU,MAAM,SAAS;AAAA,MACzB,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,KAAK,UAAU;AAAA,YACtB,QAAQ,KAAK;AAAA,YACb,OAAO,MAAM,SAAS;AAAA,YACtB,UAAU,MAAM,SAAS;AAAA,YACzB,QAAQ,MAAM,OAAO,WAAW,cAAc,MAAM,OAAO,SAAS,MAAM;AAAA,UAC5E,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AACA,UAAM,WAAW,MAAM,KAAK,aAAa,SAAS;AAAA,MAChD,GAAG;AAAA,MACH,GAAI,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,MACxD,GAAI,KAAK,aAAa,SAAY,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC;AAAA,IACnE,CAAC;AAED,UAAM,SAAS,mBAAmB,SAAS,OAAO;AAClD,UAAM,QAAQ,OAAO,SAAS;AAC9B,UAAM,SAAS,OAAO,UAAU,SAAS,KAAK;AAE9C,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,OAAO,UAAU,SAAS;AAAA,MAClC,UAAU;AAAA,QACR,WAAW,KAAK;AAAA,QAChB,YAAY,KAAK,SAAS;AAAA,QAC1B,OAAO,SAAS,UAAU,SAAY,OAAO,KAAK,MAAM,KAAK,UAAU,SAAS,KAAK,CAAC;AAAA,MACxF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,SAAwE;AAClG,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,CAACA,UAAS,MAAM,EAAG,QAAO,CAAC;AAE/B,UAAM,SAAgE,CAAC;AACvE,QAAI,OAAO,OAAO,UAAU,SAAU,QAAO,QAAQ,OAAO;AAC5D,QAAI,OAAO,OAAO,WAAW,UAAW,QAAO,SAAS,OAAO;AAC/D,QAAI,OAAO,OAAO,WAAW,SAAU,QAAO,SAAS,OAAO;AAC9D,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,SAASA,UAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;ACtFO,IAAM,iBAAN,MAA8C;AAAA,EAC1C;AAAA,EACQ;AAAA,EAEjB,YAAY,SAAgC;AAC1C,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,SAAS,MAAM,OAAO,WAAW,cAAc,MAAM,OAAO,SAAS;AAC3E,UAAM,UAAU,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAC3E,UAAM,SAAS,KAAK,QAAQ,KAAK,OAAO;AAExC,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,OAAO,SAAS,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,4BAA4B;AAAA,MAC7C,UAAU,EAAE,SAAS,KAAK,QAAQ,OAAO;AAAA,IAC3C;AAAA,EACF;AACF;;;ACrBO,IAAM,8BAAN,MAA2D;AAAA,EACvD;AAAA,EACQ;AAAA,EACA;AAAA,EAEjB,YAAY,SAA6C;AACvD,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,aAAa,QAAQ;AAAA,EAC5B;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,SAAS,UAAU,MAAM,OAAO,WAAW,cAAc,MAAM,OAAO,SAAS,MAAS;AAC9F,UAAM,WAAW,UAAU,MAAM,SAAS,QAAQ;AAClD,UAAM,QAAQ,MAAM,KAAK,WAAW,QAAQ,QAAQ;AAEpD,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX;AAAA,MACA,QAAQ,SAAS,KAAK;AAAA,MACtB,QAAQ,SAAS,KAAK,YAAY,gCAAgC;AAAA,MAClE,UAAU,EAAE,WAAW,KAAK,UAAU;AAAA,IACxC;AAAA,EACF;AACF;AAEA,SAAS,UAAU,OAAwB;AACzC,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;;;ACtBO,IAAM,oBAAN,MAAiD;AAAA,EAC7C;AAAA,EACQ;AAAA,EACA;AAAA,EAEjB,YAAY,UAAoC,CAAC,GAAG;AAClD,SAAK,OAAO,QAAQ,QAAQ;AAC5B,QAAI,QAAQ,sBAAsB,OAAW,MAAK,oBAAoB,QAAQ;AAC9E,SAAK,eAAe,QAAQ,gBAAgB;AAAA,EAC9C;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,kBAAkB,MAAM,OAAO,WAAW,cAAc,iBAAiB,MAAM,OAAO,MAAM,IAAI,CAAC;AACvG,UAAM,oBAAoB,KAAK,qBAAqB,yBAAyB,MAAM,SAAS,QAAQ;AACpG,UAAM,SAAS,KAAK,eAChB,eAAe,iBAAiB,iBAAiB,IACjD,gBAAgB,iBAAiB,iBAAiB;AAEtD,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,OAAO,SAAS,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,qCAAqC;AAAA,MACtD,UAAU;AAAA,QACR,UAAU,kBAAkB,IAAI,CAAC,SAAS,KAAK,IAAI;AAAA,QACnD,QAAQ,gBAAgB,IAAI,CAAC,SAAS,KAAK,IAAI;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,OAA4B;AACpD,MAAIC,UAAS,KAAK,KAAK,MAAM,QAAQ,MAAM,SAAS,GAAG;AACrD,WAAO,MAAM,UAAU,OAAO,UAAU;AAAA,EAC1C;AAEA,SAAO,CAAC;AACV;AAEA,SAAS,yBAAyB,OAAoC;AACpE,MAAIA,UAAS,KAAK,KAAK,MAAM,QAAQ,MAAM,SAAS,GAAG;AACrD,WAAO,MAAM,UAAU,OAAO,kBAAkB;AAAA,EAClD;AAEA,SAAO,CAAC;AACV;AAEA,SAAS,eAAe,QAAoB,UAAuC;AACjF,SAAO,SAAS,MAAM,CAAC,cAAc,UAAU,YAAY,OAAO,KAAK,GAAG,YAAY,CAAC;AACzF;AAEA,SAAS,gBAAgB,QAAoB,UAAuC;AAClF,SAAO,SAAS,MAAM,CAAC,iBAAiB,OAAO,KAAK,CAAC,eAAe,YAAY,YAAY,YAAY,CAAC,CAAC;AAC5G;AAEA,SAAS,YAAY,QAA8B,UAAqC;AACtF,MAAI,WAAW,UAAa,OAAO,SAAS,SAAS,KAAM,QAAO;AAClE,MAAI,SAAS,UAAU,OAAW,QAAO;AACzC,SAAO,KAAK,UAAU,OAAO,KAAK,MAAM,KAAK,UAAU,SAAS,KAAK;AACvE;AAEA,SAAS,WAAW,OAAmC;AACrD,SAAOA,UAAS,KAAK,KAAK,OAAO,MAAM,SAAS;AAClD;AAEA,SAAS,mBAAmB,OAA2C;AACrE,SAAOA,UAAS,KAAK,KAAK,OAAO,MAAM,SAAS;AAClD;AAEA,SAASA,UAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AClFO,SAAS,qBAAqB,QAAgB,OAA2B;AAC9E,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,UAAU,IAAI,IAAI,SAAS;AAAA,IAClC,MAAM;AAAA,IACN,UAAU,EAAE,QAAQ,MAAM;AAAA,EAC5B;AACF;AAEO,SAAS,yBAAyB,QAA8B;AACrE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,WAAW,IAAI,IAAI,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAAA,IACxF,MAAM;AAAA,EACR;AACF;AAEO,SAAS,iBAAiB,SAA6C;AAC5E,QAAM,OAAO,QAAQ,OAAO,CAAC,KAAK,WAAW;AAC3C,UAAM,QAAQ,OAAO,UAAU;AAC/B,WAAO,OAAO,OAAO,UAAU,WAAW,QAAQ;AAAA,EACpD,GAAG,CAAC;AAEJ,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO;AAAA,IACP,MAAM;AAAA,EACR;AACF;;;AC7BO,IAAM,qBAAN,MAAuD;AAAA,EAC5D,OAAO,QAA4B;AACjC,WAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;AAAA,EACvC;AACF;;;ACJO,IAAM,yBAAN,MAA2D;AAAA,EAChE,OAAO,QAA4B;AACjC,UAAM,SAAS,OAAO,IAAI,QAAQ,OAAO,CAAC,WAAW,CAAC,OAAO,MAAM;AACnE,UAAM,cAAc,OAAO,aAAa,OAAO,CAAC,WAAW,OAAO,SAAS,KAAK,CAAC;AAEjF,WAAO;AAAA,MACL,iBAAiB,OAAO,IAAI,KAAK;AAAA,MACjC;AAAA,MACA,YAAY,OAAO,IAAI,SAAS,IAAI,OAAO,IAAI,cAAc;AAAA,MAC7D,WAAW,OAAO,IAAI,SAAS,QAAQ,IAAI;AAAA,MAC3C,cAAc,YAAY,OAAO,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACtD,kBAAkB,YAAY,OAAO,IAAI,QAAQ,YAAY,CAAC;AAAA,MAC9D;AAAA,MACA;AAAA,MACA,GAAG,OAAO,IAAI,QAAQ,IAAI,CAAC,WAAW,KAAK,OAAO,IAAI,KAAK,OAAO,KAAK,GAAG,OAAO,OAAO,IAAI,OAAO,IAAI,KAAK,EAAE,EAAE;AAAA,MAChH;AAAA,MACA;AAAA,MACA,GAAI,OAAO,WAAW,IAClB,CAAC,QAAQ,IACT,OAAO,IAAI,CAAC,WAAW;AACrB,cAAM,UAAU,OAAO,OAAO,OAAO,CAAC,UAAU,CAAC,MAAM,MAAM,EAAE,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,KAAK,MAAM,UAAU,QAAQ,EAAE;AAC1H,eAAO,KAAK,OAAO,MAAM,KAAK,QAAQ,KAAK,IAAI,CAAC;AAAA,MAClD,CAAC;AAAA,MACL;AAAA,MACA;AAAA,MACA,GAAI,YAAY,WAAW,IACvB,CAAC,QAAQ,IACT,YAAY,IAAI,CAAC,WAAW,KAAK,OAAO,UAAU,KAAK,OAAO,aAAa,OAAO,OAAO,YAAY,EAAE;AAAA,IAC7G,EAAE,KAAK,IAAI;AAAA,EACb;AACF;AAEA,SAAS,YAAY,OAAuB;AAC1C,SAAO,GAAG,KAAK,MAAM,QAAQ,GAAK,IAAI,GAAG;AAC3C;;;AC3BO,IAAM,0BAAN,MAAuD;AAAA,EACnD;AAAA,EACQ;AAAA,EACA;AAAA,EAEjB,YAAY,SAAyC;AACnD,SAAK,OAAO,UAAU,QAAQ,OAAO,EAAE;AACvC,SAAK,SAAS,QAAQ;AACtB,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA,EAEA,MAAM,SAAS,OAA+C;AAC5D,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,KAAK,OAAO,SAAS,IAAI,CAAC,cAAc,KAAK,eAAe,UAAU,KAAK,KAAK,CAAC;AAAA,IACnF;AACA,UAAM,cAAc,KAAK,OAAO,SAAS,OAAO,CAAC,KAAK,SAAS,OAAO,KAAK,UAAU,IAAI,CAAC,KAAK;AAC/F,UAAM,QAAQ,SAAS,OAAO,CAAC,KAAK,SAAS;AAC3C,YAAM,SAAS,KAAK,OAAO,SAAS,KAAK,CAAC,cAAc,UAAU,QAAQ,KAAK,GAAG,GAAG,UAAU;AAC/F,aAAO,MAAM,KAAK,QAAQ;AAAA,IAC5B,GAAG,CAAC,IAAI;AACR,UAAM,YAAY,KAAK,OAAO,aAAa;AAC3C,UAAM,SAAS,SAAS,aAAa,SAAS,MAAM,CAAC,cAAc,UAAU,MAAM;AAEnF,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,SAAS,mBAAmB;AAAA,MACpC,UAAU;AAAA,QACR,UAAU,KAAK,OAAO;AAAA,QACtB,eAAe,KAAK,OAAO;AAAA,QAC3B;AAAA,QACA,UAAU,SAAS,IAAI,CAAC,eAAe;AAAA,UACrC,KAAK,UAAU;AAAA,UACf,OAAO,UAAU;AAAA,UACjB,QAAQ,UAAU;AAAA,UAClB,QAAQ,UAAU,UAAU;AAAA,QAC9B,EAAE;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AACF;;;ACnCO,IAAM,aAAN,MAAiB;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,SAA4B;AACtC,SAAK,eAAe,QAAQ;AAC5B,SAAK,aAAa,QAAQ;AAC1B,SAAK,WAAW,QAAQ;AAAA,EAC1B;AAAA,EAEA,MAAM,IAAI,QAA+C;AACvD,UAAM,UAAU,MAAM,KAAK,aAAa,IAAI,OAAO,WAAW,OAAO,cAAc;AACnF,QAAI,YAAY,QAAW;AACzB,YAAM,IAAI,UAAU,0BAA0B,2BAA2B,OAAO,SAAS,EAAE;AAAA,IAC7F;AAEA,UAAM,UAA0B;AAAA,MAC9B,OAAO,OAAO,SAAS,YAAY;AAAA,MACnC,WAAW,QAAQ;AAAA,MACnB,gBAAgB,QAAQ;AAAA,IAC1B;AACA,QAAI,OAAO,aAAa,OAAW,SAAQ,WAAW,OAAO;AAC7D,QAAI,OAAO,YAAY,OAAW,SAAQ,UAAU,OAAO;AAC3D,QAAI,OAAO,kBAAkB,OAAW,SAAQ,gBAAgB,OAAO;AACvE,QAAI,OAAO,aAAa,OAAW,SAAQ,WAAW,OAAO;AAE7D,UAAM,UAA4B,CAAC;AAEnC,eAAW,YAAY,QAAQ,OAAO;AACpC,YAAM,SAAS,MAAM,KAAK,SAAS,UAAU,OAAO;AACpD,YAAMC,UAAS,MAAM,QAAQ;AAAA,QAC3B,KAAK,WAAW;AAAA,UAAI,CAAC,cACnB,UAAU,SAAS;AAAA,YACjB;AAAA,YACA;AAAA,YACA;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AACA,YAAM,eAAeA,QAAO,WAAW,IAAI,IAAIA,QAAO,OAAO,CAAC,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAIA,QAAO;AAC5G,YAAM,SAAS,OAAO,WAAW,eAAeA,QAAO,MAAM,CAAC,UAAU,MAAM,MAAM,KAAK,iBAAiB,OAAO,aAAa;AAE9H,cAAQ,KAAK;AAAA,QACX,QAAQ,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA,QAAAA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,cAAc,QAAQ,OAAO,CAAC,WAAW,OAAO,MAAM,EAAE;AAC9D,UAAM,SAAS,QAAQ,QAAQ,CAAC,WAAW,OAAO,OAAO,IAAI,CAAC,UAAU,MAAM,KAAK,CAAC;AACpF,UAAM,UAAU;AAAA,MACd,YAAY,QAAQ;AAAA,MACpB;AAAA,MACA,aAAa,QAAQ,SAAS;AAAA,MAC9B,UAAU,QAAQ,WAAW,IAAI,IAAI,cAAc,QAAQ;AAAA,MAC3D,cAAc,OAAO,WAAW,IAAI,IAAI,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAAA,IACjG;AAEA,UAAM,YAA2B;AAAA,MAC/B,OAAO,QAAQ;AAAA,MACf,WAAW,QAAQ;AAAA,MACnB,gBAAgB,QAAQ;AAAA,MACxB;AAAA,MACA;AAAA,MACA,SAAS;AAAA,QACP,qBAAqB,QAAQ,aAAa,QAAQ,UAAU;AAAA,QAC5D,yBAAyB,MAAM;AAAA,QAC/B,iBAAiB,QAAQ,IAAI,CAAC,WAAW,OAAO,MAAM,CAAC;AAAA,MACzD;AAAA,MACA,QAAQ,QAAQ,gBAAgB;AAAA,IAClC;AACA,QAAI,OAAO,aAAa,OAAW,WAAU,WAAW,OAAO;AAE/D,WAAO;AAAA,EACT;AACF;AAEA,SAAS,cAAsB;AAC7B,SAAO,QAAQ,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC;AACxC;","names":["isRecord","isRecord","scores"]}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@hemia-ai/agents-evals",
+  "version": "0.0.1",
+  "description": "Evaluation contracts for Hemia AI agent quality, golden datasets, and regression checks.",
+  "type": "module",
+  "sideEffects": false,
+  "files": ["dist"],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --sourcemap --clean",
+    "clean": "rm -rf dist *.tsbuildinfo",
+    "dev": "tsup src/index.ts --format esm --dts --watch",
+    "lint": "tsc --noEmit",
+    "test": "vitest run --passWithNoTests",
+    "typecheck": "tsc --noEmit",
+    "prepack": "npm run build"
+  },
+  "dependencies": {
+    "@hemia-ai/agents-core": "^0.0.1",
+    "@hemia-ai/agents-models": "^0.0.1"
+  }
+}