@assemble-dev/evals 0.1.0

package/dist/index.js

@@ -0,0 +1,1138 @@
+// src/eval-case.ts
+import { z } from "zod";
+var ExpectedToolCallSchema = z.object({
+  toolName: z.string().min(1),
+  count: z.number().int().min(1).optional()
+});
+var ExpectedApprovalDecisionSchema = z.object({
+  toolName: z.string().min(1).optional(),
+  status: z.enum(["approved", "rejected"])
+});
+
+// src/eval-suite.ts
+import { AppErrorCode } from "@assemble-dev/shared-types/errors";
+import { AppError } from "@assemble-dev/shared-utils/errors";
+
+// src/schema-validation-formatting.ts
+function formatSchemaValidationError(error) {
+  if (error.issues === void 0 || error.issues.length === 0) {
+    return error.message;
+  }
+  return error.issues.map(formatSchemaValidationIssue).join("; ");
+}
+function formatSchemaValidationIssue(issue) {
+  if (issue.path === void 0 || issue.path.length === 0) {
+    return issue.message;
+  }
+  return `${issue.path.map(String).join(".")}: ${issue.message}`;
+}
+
+// src/eval-suite.ts
+function evalSuite(config) {
+  ensureSuiteNameIsPresent(config.name);
+  const rawCases = config.cases;
+  ensureSuiteHasCases(config.name, rawCases);
+  ensureCaseNamesAreUnique(config.name, rawCases);
+  const validatedCases = rawCases.map(
+    (evalCase) => buildValidatedEvalCase(config.workflow, evalCase)
+  );
+  return Object.freeze({
+    name: config.name,
+    workflow: config.workflow,
+    cases: Object.freeze(validatedCases)
+  });
+}
+function ensureSuiteNameIsPresent(suiteName) {
+  if (suiteName.trim() !== "") {
+    return;
+  }
+  throw new AppError({
+    code: AppErrorCode.ValidationFailed,
+    message: "eval suite name must not be empty",
+    statusCode: 422
+  });
+}
+function ensureSuiteHasCases(suiteName, cases) {
+  if (cases.length > 0) {
+    return;
+  }
+  throw new AppError({
+    code: AppErrorCode.ValidationFailed,
+    message: `eval suite "${suiteName}" must define at least one case`,
+    statusCode: 422
+  });
+}
+function ensureCaseNamesAreUnique(suiteName, cases) {
+  const seenCaseNames = /* @__PURE__ */ new Set();
+  for (const evalCase of cases) {
+    if (evalCase.name.trim() === "") {
+      throw new AppError({
+        code: AppErrorCode.ValidationFailed,
+        message: `eval suite "${suiteName}" contains a case with an empty name`,
+        statusCode: 422
+      });
+    }
+    if (seenCaseNames.has(evalCase.name)) {
+      throw new AppError({
+        code: AppErrorCode.ValidationFailed,
+        message: `eval suite "${suiteName}" contains duplicate case name "${evalCase.name}"`,
+        statusCode: 422
+      });
+    }
+    seenCaseNames.add(evalCase.name);
+  }
+}
+function buildValidatedEvalCase(workflowDefinition, evalCase) {
+  const validatedInput = parseEvalCaseInput(workflowDefinition, evalCase);
+  const validatedExpectation = buildValidatedExpectation(
+    workflowDefinition,
+    evalCase
+  );
+  if (validatedExpectation === void 0) {
+    return Object.freeze({ name: evalCase.name, input: validatedInput });
+  }
+  return Object.freeze({
+    name: evalCase.name,
+    input: validatedInput,
+    expected: validatedExpectation
+  });
+}
+function parseEvalCaseInput(workflowDefinition, evalCase) {
+  const parsedInput = workflowDefinition.inputSchema.safeParse(evalCase.input);
+  if (!parsedInput.success) {
+    throw new AppError({
+      code: AppErrorCode.ValidationFailed,
+      message: `eval case "${evalCase.name}" input failed workflow "${workflowDefinition.name}" input schema validation: ${formatSchemaValidationError(parsedInput.error)}`,
+      statusCode: 422
+    });
+  }
+  return parsedInput.data;
+}
+function buildValidatedExpectation(workflowDefinition, evalCase) {
+  const expectation = evalCase.expected;
+  if (expectation === void 0) {
+    return void 0;
+  }
+  const validatedExpectation = {};
+  if (expectation.output !== void 0) {
+    validatedExpectation.output = parseExpectedOutput(
+      workflowDefinition,
+      evalCase.name,
+      expectation.output
+    );
+  }
+  if (expectation.outputFields !== void 0) {
+    validatedExpectation.outputFields = { ...expectation.outputFields };
+  }
+  if (expectation.outputPredicate !== void 0) {
+    validatedExpectation.outputPredicate = expectation.outputPredicate;
+  }
+  if (expectation.toolCalls !== void 0) {
+    validatedExpectation.toolCalls = expectation.toolCalls.map((toolCall) => ({
+      ...toolCall
+    }));
+  }
+  if (expectation.approvalDecisions !== void 0) {
+    validatedExpectation.approvalDecisions = expectation.approvalDecisions.map(
+      (approvalDecision) => ({ ...approvalDecision })
+    );
+  }
+  return Object.freeze(validatedExpectation);
+}
+function parseExpectedOutput(workflowDefinition, caseName, expectedOutput) {
+  if (workflowDefinition.outputSchema === null) {
+    return expectedOutput;
+  }
+  const parsedOutput = workflowDefinition.outputSchema.safeParse(expectedOutput);
+  if (!parsedOutput.success) {
+    throw new AppError({
+      code: AppErrorCode.ValidationFailed,
+      message: `eval case "${caseName}" expected output failed workflow "${workflowDefinition.name}" output schema validation: ${formatSchemaValidationError(parsedOutput.error)}`,
+      statusCode: 422
+    });
+  }
+  return parsedOutput.data;
+}
+
+// src/dataset-loading.ts
+import { existsSync, readFileSync } from "fs";
+import { AppErrorCode as AppErrorCode2 } from "@assemble-dev/shared-types/errors";
+import { JsonValueSchema } from "@assemble-dev/shared-types/json";
+import { AppError as AppError2 } from "@assemble-dev/shared-utils/errors";
+import { z as z2 } from "zod";
+var SerializedEvalCaseExpectationSchema = z2.strictObject({
+  output: JsonValueSchema.optional(),
+  outputFields: z2.record(z2.string(), JsonValueSchema).optional(),
+  toolCalls: z2.array(ExpectedToolCallSchema).optional(),
+  approvalDecisions: z2.array(ExpectedApprovalDecisionSchema).optional()
+});
+var SerializedEvalCaseSchema = z2.strictObject({
+  name: z2.string().min(1),
+  input: JsonValueSchema,
+  expected: SerializedEvalCaseExpectationSchema.optional()
+});
+function loadEvalCasesFromJsonlFile(input) {
+  const fileContents = readEvalDatasetFile(input.filePath);
+  const lines = fileContents.split(/\r?\n/);
+  const cases = [];
+  for (const [lineIndex, line] of lines.entries()) {
+    if (line.trim() === "") {
+      continue;
+    }
+    const lineNumber = lineIndex + 1;
+    const lineValue = parseJsonlLine(line, lineNumber, input.filePath);
+    cases.push(parseSerializedEvalCase(lineValue, lineNumber, input.filePath));
+  }
+  return cases;
+}
+function loadEvalCases(source) {
+  if ("jsonlFilePath" in source) {
+    return loadEvalCasesFromJsonlFile({ filePath: source.jsonlFilePath });
+  }
+  return source;
+}
+function readEvalDatasetFile(filePath) {
+  if (!existsSync(filePath)) {
+    throw new AppError2({
+      code: AppErrorCode2.NotFound,
+      message: `eval dataset file not found: ${filePath}`,
+      statusCode: 404
+    });
+  }
+  return readFileSync(filePath, "utf8");
+}
+function parseJsonlLine(line, lineNumber, filePath) {
+  try {
+    return JsonValueSchema.parse(JSON.parse(line));
+  } catch (error) {
+    throw new AppError2({
+      code: AppErrorCode2.ValidationFailed,
+      message: `invalid JSON on line ${lineNumber} of ${filePath}`,
+      statusCode: 422,
+      cause: error instanceof Error ? error : void 0
+    });
+  }
+}
+function parseSerializedEvalCase(lineValue, lineNumber, filePath) {
+  const parsedCase = SerializedEvalCaseSchema.safeParse(lineValue);
+  if (!parsedCase.success) {
+    throw new AppError2({
+      code: AppErrorCode2.ValidationFailed,
+      message: `invalid eval case on line ${lineNumber} of ${filePath}: ${formatSchemaValidationError(parsedCase.error)}`,
+      statusCode: 422
+    });
+  }
+  return parsedCase.data;
+}
+
+// src/judge/judge-metric.ts
+import { z as z3 } from "zod";
+
+// src/metrics/metric-result.ts
+function createPassedMetricResult(name, reason) {
+  return { name, passed: true, score: 1, reason, skipped: false };
+}
+function createFailedMetricResult(name, score, reason) {
+  return { name, passed: false, score, reason, skipped: false };
+}
+function createSkippedMetricResult(name, reason) {
+  return { name, passed: false, score: 0, reason, skipped: true };
+}
+function createScoredMetricResult(name, outcomes) {
+  const satisfiedCount = outcomes.filter(
+    (outcome) => outcome.satisfied
+  ).length;
+  const reason = outcomes.map((outcome) => outcome.reason).join("; ");
+  if (satisfiedCount === outcomes.length) {
+    return createPassedMetricResult(name, reason);
+  }
+  return createFailedMetricResult(
+    name,
+    satisfiedCount / outcomes.length,
+    reason
+  );
+}
+
+// src/judge/judge-metric.ts
+var defaultJudgeMetricName = "llm-judge";
+var defaultJudgePassThreshold = 0.7;
+var JudgeVerdictSchema = z3.object({
+  score: z3.number().min(0).max(1),
+  reasoning: z3.string().min(1)
+});
+function defineJudgeMetric(config) {
+  return {
+    name: config.name ?? defaultJudgeMetricName,
+    model: config.model,
+    rubric: config.rubric,
+    passThreshold: config.passThreshold ?? defaultJudgePassThreshold
+  };
+}
+async function evaluateJudgeMetric(input) {
+  if (input.runState.status !== "completed") {
+    return createSkippedMetricResult(
+      input.definition.name,
+      `judge skipped because run finished with status "${input.runState.status}" instead of "completed"`
+    );
+  }
+  try {
+    const structuredOutput = await input.definition.model.generateStructuredOutput({
+      messages: buildJudgeChatMessages(input),
+      schema: JudgeVerdictSchema
+    });
+    return mapJudgeVerdictToMetricResult(
+      input.definition,
+      structuredOutput.value
+    );
+  } catch (caughtError) {
+    const failureReason = caughtError instanceof Error ? caughtError.message : String(caughtError);
+    return createFailedMetricResult(
+      input.definition.name,
+      0,
+      `judge model call failed: ${failureReason}`
+    );
+  }
+}
+function mapJudgeVerdictToMetricResult(definition, verdict) {
+  return {
+    name: definition.name,
+    passed: verdict.score >= definition.passThreshold,
+    score: verdict.score,
+    reason: verdict.reasoning,
+    skipped: false
+  };
+}
+function buildJudgeChatMessages(input) {
+  return [
+    { role: "system", content: buildJudgeSystemPrompt(input.definition.rubric) },
+    {
+      role: "user",
+      content: JSON.stringify(buildJudgeCaseSummary(input), null, 2)
+    }
+  ];
+}
+function buildJudgeSystemPrompt(rubric) {
+  return [
+    "You are an impartial judge that evaluates one eval case from an AI workflow run.",
+    "Score how well the final output satisfies the rubric, from 0 (fails completely) to 1 (fully satisfies).",
+    "Rubric:",
+    rubric,
+    'Respond as pure JSON matching {"score": number between 0 and 1, "reasoning": string} with no surrounding text.'
+  ].join("\n");
+}
+function buildJudgeCaseSummary(input) {
+  return {
+    caseName: input.evalCase.name,
+    input: input.evalCase.input,
+    expectedSummary: buildExpectedSummary(input.evalCase.expected),
+    finalOutput: input.runState.finalOutput
+  };
+}
+function buildExpectedSummary(expectation) {
+  if (expectation === void 0) {
+    return null;
+  }
+  const expectedSummary = {};
+  if (expectation.output !== void 0) {
+    expectedSummary.output = expectation.output;
+  }
+  if (expectation.outputFields !== void 0) {
+    expectedSummary.outputFields = expectation.outputFields;
+  }
+  if (expectation.outputPredicate !== void 0) {
+    expectedSummary.outputPredicateDefined = true;
+  }
+  if (expectation.toolCalls !== void 0) {
+    expectedSummary.toolCalls = expectation.toolCalls.map((expectedToolCall) => ({
+      toolName: expectedToolCall.toolName,
+      count: expectedToolCall.count ?? null
+    }));
+  }
+  if (expectation.approvalDecisions !== void 0) {
+    expectedSummary.approvalDecisions = expectation.approvalDecisions.map(
+      (expectedApprovalDecision) => ({
+        toolName: expectedApprovalDecision.toolName ?? null,
+        status: expectedApprovalDecision.status
+      })
+    );
+  }
+  return expectedSummary;
+}
+
+// src/metrics/approval-correctness-metric.ts
+var approvalCorrectnessMetricName = "approval-correctness";
+function evaluateApprovalCorrectnessMetric(input) {
+  const expectedApprovalDecisions = input.expectedApprovalDecisions;
+  if (expectedApprovalDecisions === void 0 || expectedApprovalDecisions.length === 0) {
+    return createSkippedMetricResult(
+      approvalCorrectnessMetricName,
+      "no approval decision expectations defined for this case"
+    );
+  }
+  const outcomes = expectedApprovalDecisions.map(
+    (expectedApprovalDecision) => evaluateExpectedApprovalDecision(expectedApprovalDecision, input.approvals)
+  );
+  return createScoredMetricResult(approvalCorrectnessMetricName, outcomes);
+}
+function evaluateExpectedApprovalDecision(expectedApprovalDecision, approvals) {
+  const candidateApprovals = filterApprovalsByToolName(
+    approvals,
+    expectedApprovalDecision.toolName
+  );
+  const approvalScope = describeApprovalScope(
+    expectedApprovalDecision.toolName
+  );
+  if (candidateApprovals.length === 0) {
+    return {
+      satisfied: false,
+      reason: `no approval found ${approvalScope}`
+    };
+  }
+  const hasMatchingStatus = candidateApprovals.some(
+    (approval) => approval.status === expectedApprovalDecision.status
+  );
+  if (hasMatchingStatus) {
+    return {
+      satisfied: true,
+      reason: `approval ${approvalScope} has status "${expectedApprovalDecision.status}"`
+    };
+  }
+  const observedStatuses = candidateApprovals.map((approval) => `"${approval.status}"`).join(", ");
+  return {
+    satisfied: false,
+    reason: `expected approval ${approvalScope} with status "${expectedApprovalDecision.status}" but found ${observedStatuses}`
+  };
+}
+function filterApprovalsByToolName(approvals, toolName) {
+  if (toolName === void 0) {
+    return approvals;
+  }
+  return approvals.filter((approval) => approval.toolName === toolName);
+}
+function describeApprovalScope(toolName) {
+  if (toolName === void 0) {
+    return "for the run";
+  }
+  return `for tool "${toolName}"`;
+}
+
+// src/metrics/schema-validity-metric.ts
+var schemaValidityMetricName = "schema-validity";
+function evaluateSchemaValidityMetric(input) {
+  if (input.outputSchema === null) {
+    return createSkippedMetricResult(
+      schemaValidityMetricName,
+      "workflow does not define an output schema"
+    );
+  }
+  const parsedOutput = input.outputSchema.safeParse(input.finalOutput);
+  if (parsedOutput.success) {
+    return createPassedMetricResult(
+      schemaValidityMetricName,
+      "final output matches the workflow output schema"
+    );
+  }
+  return createFailedMetricResult(
+    schemaValidityMetricName,
+    0,
+    `final output failed workflow output schema validation: ${formatSchemaValidationError(parsedOutput.error)}`
+  );
+}
+
+// src/metrics/json-value-equality.ts
+function isJsonObjectValue(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function areJsonValuesDeeplyEqual(left, right) {
+  if (left === right) {
+    return true;
+  }
+  if (Array.isArray(left) && Array.isArray(right)) {
+    return areJsonArraysDeeplyEqual(left, right);
+  }
+  if (isJsonObjectValue(left) && isJsonObjectValue(right)) {
+    return areJsonObjectsDeeplyEqual(left, right);
+  }
+  return false;
+}
+function areJsonArraysDeeplyEqual(left, right) {
+  if (left.length !== right.length) {
+    return false;
+  }
+  return left.every(
+    (item, index) => areJsonValuesDeeplyEqual(item, right[index] ?? null)
+  );
+}
+function areJsonObjectsDeeplyEqual(left, right) {
+  const leftKeys = Object.keys(left);
+  const rightKeys = Object.keys(right);
+  if (leftKeys.length !== rightKeys.length) {
+    return false;
+  }
+  return leftKeys.every(
+    (key) => key in right && areJsonValuesDeeplyEqual(left[key] ?? null, right[key] ?? null)
+  );
+}
+
+// src/metrics/task-success-metric.ts
+var taskSuccessMetricName = "task-success";
+function evaluateTaskSuccessMetric(input) {
+  const expectation = input.expectation;
+  if (expectation === void 0 || hasNoOutputExpectations(expectation)) {
+    return createSkippedMetricResult(
+      taskSuccessMetricName,
+      "no output expectations defined for this case"
+    );
+  }
+  const failureReasons = [
+    ...collectExactOutputFailureReasons(expectation, input.finalOutput),
+    ...collectOutputFieldFailureReasons(expectation, input.finalOutput),
+    ...collectOutputPredicateFailureReasons(expectation, input.finalOutput)
+  ];
+  if (failureReasons.length > 0) {
+    return createFailedMetricResult(
+      taskSuccessMetricName,
+      0,
+      failureReasons.join("; ")
+    );
+  }
+  return createPassedMetricResult(
+    taskSuccessMetricName,
+    "all output expectations matched the final output"
+  );
+}
+function hasNoOutputExpectations(expectation) {
+  return expectation.output === void 0 && expectation.outputFields === void 0 && expectation.outputPredicate === void 0;
+}
+function collectExactOutputFailureReasons(expectation, finalOutput) {
+  if (expectation.output === void 0) {
+    return [];
+  }
+  if (areJsonValuesDeeplyEqual(expectation.output, finalOutput)) {
+    return [];
+  }
+  return [
+    `final output ${JSON.stringify(finalOutput)} does not deeply equal expected output ${JSON.stringify(expectation.output)}`
+  ];
+}
+function collectOutputFieldFailureReasons(expectation, finalOutput) {
+  const outputFields = expectation.outputFields;
+  if (outputFields === void 0) {
+    return [];
+  }
+  if (!isJsonObjectValue(finalOutput)) {
+    return [
+      "final output is not an object, so expected output fields cannot match"
+    ];
+  }
+  const failureReasons = [];
+  for (const [fieldName, expectedValue] of Object.entries(outputFields)) {
+    if (!(fieldName in finalOutput)) {
+      failureReasons.push(
+        `expected output field "${fieldName}" is missing from the final output`
+      );
+      continue;
+    }
+    if (!areJsonValuesDeeplyEqual(finalOutput[fieldName] ?? null, expectedValue)) {
+      failureReasons.push(
+        `output field "${fieldName}" is ${JSON.stringify(finalOutput[fieldName])} but expected ${JSON.stringify(expectedValue)}`
+      );
+    }
+  }
+  return failureReasons;
+}
+function collectOutputPredicateFailureReasons(expectation, finalOutput) {
+  const outputPredicate = expectation.outputPredicate;
+  if (outputPredicate === void 0) {
+    return [];
+  }
+  try {
+    const predicateResult = outputPredicate(finalOutput);
+    if (predicateResult === true) {
+      return [];
+    }
+    if (predicateResult === false) {
+      return ["output predicate returned false"];
+    }
+    return [`output predicate failed: ${predicateResult}`];
+  } catch (caughtError) {
+    const errorMessage = caughtError instanceof Error ? caughtError.message : String(caughtError);
+    return [`output predicate threw an error: ${errorMessage}`];
+  }
+}
+
+// src/metrics/tool-accuracy-metric.ts
+var toolAccuracyMetricName = "tool-accuracy";
+function evaluateToolAccuracyMetric(input) {
+  const expectedToolCalls = input.expectedToolCalls;
+  if (expectedToolCalls === void 0 || expectedToolCalls.length === 0) {
+    return createSkippedMetricResult(
+      toolAccuracyMetricName,
+      "no tool call expectations defined for this case"
+    );
+  }
+  const completedCallCountsByToolName = countCompletedToolCallsByName(
+    input.toolCalls
+  );
+  const outcomes = expectedToolCalls.map(
+    (expectedToolCall) => evaluateExpectedToolCall(expectedToolCall, completedCallCountsByToolName)
+  );
+  return createScoredMetricResult(toolAccuracyMetricName, outcomes);
+}
+function countCompletedToolCallsByName(toolCalls) {
+  const completedCallCounts = /* @__PURE__ */ new Map();
+  for (const toolCall of toolCalls) {
+    if (toolCall.status !== "completed") {
+      continue;
+    }
+    const currentCount = completedCallCounts.get(toolCall.toolName) ?? 0;
+    completedCallCounts.set(toolCall.toolName, currentCount + 1);
+  }
+  return completedCallCounts;
+}
+function evaluateExpectedToolCall(expectedToolCall, completedCallCountsByToolName) {
+  const completedCount = completedCallCountsByToolName.get(expectedToolCall.toolName) ?? 0;
+  if (expectedToolCall.count !== void 0) {
+    return {
+      satisfied: completedCount === expectedToolCall.count,
+      reason: `tool "${expectedToolCall.toolName}" completed ${completedCount} time(s), expected exactly ${expectedToolCall.count}`
+    };
+  }
+  return {
+    satisfied: completedCount >= 1,
+    reason: `tool "${expectedToolCall.toolName}" completed ${completedCount} time(s), expected at least 1`
+  };
+}
+
+// src/metrics/case-metrics.ts
+var runStatusMetricName = "run-status";
+function collectRunPerformance(runState) {
+  const tokenUsage = runState.metrics.tokenUsage;
+  return {
+    totalLatencyMs: runState.metrics.totalLatencyMs,
+    tokenUsage: tokenUsage === null ? null : { ...tokenUsage }
+  };
+}
+function evaluateCaseMetrics(input) {
+  const performance = collectRunPerformance(input.runState);
+  if (input.runState.status !== "completed") {
+    return {
+      caseName: input.evalCase.name,
+      metrics: [buildRunStatusFailureMetric(input.runState)],
+      passed: false,
+      performance
+    };
+  }
+  const metrics = evaluateCompletedRunMetrics(input);
+  return {
+    caseName: input.evalCase.name,
+    metrics,
+    passed: metrics.every((metric) => metric.skipped || metric.passed),
+    performance
+  };
+}
+function buildRunStatusFailureMetric(runState) {
+  return createFailedMetricResult(
+    runStatusMetricName,
+    0,
+    `run finished with status "${runState.status}" instead of "completed"`
+  );
+}
+function evaluateCompletedRunMetrics(input) {
+  return [
+    evaluateTaskSuccessMetric({
+      expectation: input.evalCase.expected,
+      finalOutput: input.runState.finalOutput
+    }),
+    evaluateSchemaValidityMetric({
+      outputSchema: input.workflow.outputSchema,
+      finalOutput: input.runState.finalOutput
+    }),
+    evaluateToolAccuracyMetric({
+      expectedToolCalls: input.evalCase.expected?.toolCalls,
+      toolCalls: input.runState.toolCalls
+    }),
+    evaluateApprovalCorrectnessMetric({
+      expectedApprovalDecisions: input.evalCase.expected?.approvalDecisions,
+      approvals: input.runState.approvals
+    })
+  ];
+}
+
+// src/metrics/suite-metrics.ts
+function determineEvalSuiteStatus(summary) {
+  return summary.failedCases > 0 ? "failed" : "passed";
+}
+function aggregateSuiteMetrics(caseResults) {
+  const totalCases = caseResults.length;
+  const passedCases = caseResults.filter(
+    (caseResult) => caseResult.passed
+  ).length;
+  return {
+    totalCases,
+    passedCases,
+    failedCases: totalCases - passedCases,
+    passRate: totalCases === 0 ? 0 : passedCases / totalCases,
+    metricBreakdown: buildMetricBreakdown(caseResults),
+    performance: summarizeSuitePerformance(caseResults)
+  };
+}
+function buildMetricBreakdown(caseResults) {
+  const metricBreakdown = {};
+  for (const caseResult of caseResults) {
+    for (const metric of caseResult.metrics) {
+      const entry = metricBreakdown[metric.name] ?? {
+        evaluated: 0,
+        passed: 0,
+        failed: 0,
+        skipped: 0
+      };
+      if (metric.skipped) {
+        entry.skipped += 1;
+      } else {
+        entry.evaluated += 1;
+        if (metric.passed) {
+          entry.passed += 1;
+        } else {
+          entry.failed += 1;
+        }
+      }
+      metricBreakdown[metric.name] = entry;
+    }
+  }
+  return metricBreakdown;
+}
+function summarizeSuitePerformance(caseResults) {
+  const latencies = caseResults.map((caseResult) => caseResult.performance.totalLatencyMs).filter((latencyMs) => latencyMs !== null);
+  const tokenUsages = caseResults.map((caseResult) => caseResult.performance.tokenUsage).filter((tokenUsage) => tokenUsage !== null);
+  const totalLatencyMs = latencies.length === 0 ? null : latencies.reduce((sum, latencyMs) => sum + latencyMs, 0);
+  return {
+    totalLatencyMs,
+    averageLatencyMs: totalLatencyMs === null ? null : totalLatencyMs / latencies.length,
+    tokenUsage: tokenUsages.length === 0 ? null : sumTokenUsages(tokenUsages)
+  };
+}
+function sumTokenUsages(tokenUsages) {
+  return tokenUsages.reduce(
+    (summedUsage, tokenUsage) => ({
+      inputTokens: summedUsage.inputTokens + tokenUsage.inputTokens,
+      outputTokens: summedUsage.outputTokens + tokenUsage.outputTokens,
+      totalTokens: summedUsage.totalTokens + tokenUsage.totalTokens
+    }),
+    { inputTokens: 0, outputTokens: 0, totalTokens: 0 }
+  );
+}
+
+// src/eval-webhook.ts
+import { z as z5 } from "zod";
+
+// src/runner/eval-run-file.ts
+import { mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
+import { join } from "path";
+import { TokenUsageSchema } from "@assemble-dev/shared-types/runs";
+import { TraceEventSchema } from "@assemble-dev/shared-types/trace-events";
+import { z as z4 } from "zod";
+var defaultEvalResultsDirectory = join(".assemble", "evals");
+var EvalMetricResultSchema = z4.object({
+  name: z4.string().min(1),
+  passed: z4.boolean(),
+  score: z4.number(),
+  reason: z4.string(),
+  skipped: z4.boolean()
+});
+var EvalCasePerformanceSchema = z4.object({
+  totalLatencyMs: z4.number().nonnegative().nullable(),
+  tokenUsage: TokenUsageSchema.nullable()
+});
+var EvalCaseResultSchema = z4.object({
+  caseName: z4.string().min(1),
+  runId: z4.string().min(1),
+  passed: z4.boolean(),
+  metrics: z4.array(EvalMetricResultSchema),
+  performance: EvalCasePerformanceSchema
+});
+var EvalMetricBreakdownSchema = z4.object({
+  evaluated: z4.number().int().nonnegative(),
+  passed: z4.number().int().nonnegative(),
+  failed: z4.number().int().nonnegative(),
+  skipped: z4.number().int().nonnegative()
+});
+var EvalSuitePerformanceSummarySchema = z4.object({
+  totalLatencyMs: z4.number().nonnegative().nullable(),
+  averageLatencyMs: z4.number().nonnegative().nullable(),
+  tokenUsage: TokenUsageSchema.nullable()
+});
+var EvalSuiteSummarySchema = z4.object({
+  totalCases: z4.number().int().nonnegative(),
+  passedCases: z4.number().int().nonnegative(),
+  failedCases: z4.number().int().nonnegative(),
+  passRate: z4.number().min(0).max(1),
+  metricBreakdown: z4.record(z4.string(), EvalMetricBreakdownSchema),
+  performance: EvalSuitePerformanceSummarySchema
+});
+var EvalRunFileSchema = z4.object({
+  evalRunId: z4.string().min(1),
+  suiteName: z4.string().min(1),
+  workflowName: z4.string().min(1),
+  startedAt: z4.iso.datetime(),
+  completedAt: z4.iso.datetime(),
+  summary: EvalSuiteSummarySchema,
+  caseResults: z4.array(EvalCaseResultSchema),
+  events: z4.array(TraceEventSchema)
+});
+function resolveEvalRunFilePath(evalRunId, directory = defaultEvalResultsDirectory) {
+  return join(directory, `${evalRunId}.json`);
+}
+function saveEvalRunFile(input) {
+  const directory = input.directory ?? defaultEvalResultsDirectory;
+  const filePath = resolveEvalRunFilePath(
+    input.evalRunFile.evalRunId,
+    directory
+  );
+  const validatedEvalRunFile = EvalRunFileSchema.parse(input.evalRunFile);
+  mkdirSync(directory, { recursive: true });
+  writeFileSync(
+    filePath,
+    JSON.stringify(validatedEvalRunFile, null, 2),
+    "utf8"
+  );
+  return filePath;
+}
+function loadEvalRunFile(input) {
+  const filePath = resolveEvalRunFilePath(input.evalRunId, input.directory);
+  const rawContents = readFileSync2(filePath, "utf8");
+  return EvalRunFileSchema.parse(JSON.parse(rawContents));
+}
+
+// src/eval-webhook.ts
+var EvalWebhookCaseSchema = z5.object({
+  caseName: z5.string().min(1),
+  passed: z5.boolean(),
+  metrics: z5.array(EvalMetricResultSchema)
+});
+var EvalWebhookPayloadSchema = z5.object({
+  suiteName: z5.string().min(1),
+  status: z5.enum(["passed", "failed"]),
+  totalCases: z5.number().int().nonnegative(),
+  passedCases: z5.number().int().nonnegative(),
+  failedCases: z5.number().int().nonnegative(),
+  cases: z5.array(EvalWebhookCaseSchema)
+});
+function buildEvalWebhookPayload(runResult) {
+  return {
+    suiteName: runResult.suiteName,
+    status: determineEvalSuiteStatus(runResult.summary),
+    totalCases: runResult.summary.totalCases,
+    passedCases: runResult.summary.passedCases,
+    failedCases: runResult.summary.failedCases,
+    cases: runResult.caseResults.map((caseResult) => ({
+      caseName: caseResult.caseName,
+      passed: caseResult.passed,
+      metrics: caseResult.metrics
+    }))
+  };
+}
+async function postEvalResultsToWebhook(input) {
+  const validatedPayload = EvalWebhookPayloadSchema.parse(input.payload);
+  const webhookFetch = input.fetchImplementation ?? fetch;
+  try {
+    const response = await webhookFetch(input.webhookUrl, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(validatedPayload)
+    });
+    if (response.ok) {
+      return { outcome: "delivered" };
+    }
+    return {
+      outcome: "failed",
+      message: `Webhook responded with status ${response.status}`
+    };
+  } catch (caughtError) {
+    const reason = caughtError instanceof Error ? caughtError.message : String(caughtError);
+    return {
+      outcome: "failed",
+      message: `Webhook request failed: ${reason}`
+    };
+  }
+}
+
+// src/runner/run-eval-suite.ts
+import {
+  buildMissingKeyMessage,
+  createTraceBus,
+  resolvePlatformConfig,
+  runWorkflow
+} from "@assemble-dev/sdk";
+import { AppErrorCode as AppErrorCode3 } from "@assemble-dev/shared-types/errors";
+import { AppError as AppError3 } from "@assemble-dev/shared-utils/errors";
+import {
+  generateEvalRunId,
+  generateRunId,
+  generateTraceEventId
+} from "@assemble-dev/shared-utils/ids";
+async function runEvalSuite(input) {
+  const context = buildEvalRunContext(input.suite, input.options);
+  emitEvalStartedEvent(context);
+  const caseResults = await executeEvalSuiteCases(context);
+  const summary = aggregateSuiteMetrics(caseResults);
+  emitEvalCompletedEvent(context, summary);
+  await context.evalTraceBus.flushSinks();
+  const evalRunFile = buildEvalRunFile(context, summary, caseResults);
+  const resultsFilePath = saveEvalRunFile({
+    evalRunFile,
+    directory: context.resultsDirectory
+  });
+  const uploadStatus = await uploadEvalRunFile(context, evalRunFile);
+  return {
+    evalRunId: context.evalRunId,
+    suiteName: context.suite.name,
+    summary,
+    caseResults,
+    resultsFilePath,
+    uploadStatus
+  };
+}
+function buildEvalRunContext(suite, options) {
+  const resolvedOptions = options ?? {};
+  const now = resolvedOptions.now ?? (() => /* @__PURE__ */ new Date());
+  const startedAtDate = now();
+  const caseRunIds = suite.cases.map(() => generateRunId());
+  const evalEventsRunId = caseRunIds[0] ?? generateRunId();
+  return {
+    suite,
+    effectiveWorkflow: buildWorkflowWithMockedTools(
+      suite.workflow,
+      resolvedOptions.toolMocks
+    ),
+    options: resolvedOptions,
+    evalRunId: resolvedOptions.evalRunId ?? generateEvalRunId(),
+    caseRunIds,
+    evalEventsRunId,
+    now,
+    startedAt: startedAtDate.toISOString(),
+    startedAtMs: startedAtDate.getTime(),
+    evalTraceBus: createTraceBus({
+      runId: evalEventsRunId,
+      workflowName: suite.workflow.name,
+      input: null,
+      sinks: resolvedOptions.sinks ?? [],
+      now
+    }),
+    resultsDirectory: resolvedOptions.resultsDirectory ?? defaultEvalResultsDirectory
+  };
+}
+function buildWorkflowWithMockedTools(workflowDefinition, toolMocks) {
+  if (toolMocks === void 0 || Object.keys(toolMocks).length === 0) {
+    return workflowDefinition;
+  }
+  ensureToolMocksMatchRegisteredTools(workflowDefinition, toolMocks);
+  const toolsWithMocks = workflowDefinition.tools.map((workflowTool) => {
+    const toolMock = toolMocks[workflowTool.name];
+    if (toolMock === void 0) {
+      return workflowTool;
+    }
+    return {
+      ...workflowTool,
+      execute: async (toolInput) => await Promise.resolve(toolMock(toolInput))
+    };
+  });
+  return { ...workflowDefinition, tools: toolsWithMocks };
+}
+function ensureToolMocksMatchRegisteredTools(workflowDefinition, toolMocks) {
+  const registeredToolNames = new Set(
+    workflowDefinition.tools.map((workflowTool) => workflowTool.name)
+  );
+  const unknownToolNames = Object.keys(toolMocks).filter(
+    (mockedToolName) => !registeredToolNames.has(mockedToolName)
+  );
+  if (unknownToolNames.length === 0) {
+    return;
+  }
+  throw new AppError3({
+    code: AppErrorCode3.ValidationFailed,
+    message: `toolMocks reference tools not registered on workflow "${workflowDefinition.name}": ${unknownToolNames.join(", ")}`,
+    statusCode: 422
+  });
+}
+async function executeEvalSuiteCases(context) {
+  const caseResults = [];
+  for (const [caseIndex, evalCase] of context.suite.cases.entries()) {
+    const runId = context.caseRunIds[caseIndex] ?? generateRunId();
+    const runResult = await runWorkflow({
+      definition: context.effectiveWorkflow,
+      input: evalCase.input,
+      options: {
+        runId,
+        now: context.options.now,
+        sinks: context.options.sinks,
+        persistRunState: false
+      }
+    });
+    const caseMetrics = evaluateCaseMetrics({
+      runState: runResult.state,
+      evalCase,
+      workflow: context.suite.workflow
+    });
+    const metrics = await appendJudgeMetricResults(
+      context.options.judges,
+      evalCase,
+      runResult.state,
+      caseMetrics.metrics
+    );
+    caseResults.push({
+      caseName: caseMetrics.caseName,
+      runId,
+      passed: metrics.every((metric) => metric.skipped || metric.passed),
+      metrics,
+      performance: caseMetrics.performance
+    });
+  }
+  return caseResults;
+}
+async function appendJudgeMetricResults(judges, evalCase, runState, metrics) {
+  if (judges === void 0 || judges.length === 0) {
+    return metrics;
+  }
+  const judgeResults = [];
+  for (const definition of judges) {
+    judgeResults.push(
+      await evaluateJudgeMetric({ definition, evalCase, runState })
+    );
+  }
+  return [...metrics, ...judgeResults];
+}
+function emitEvalStartedEvent(context) {
+  context.evalTraceBus.emitTraceEvent({
+    id: generateTraceEventId(),
+    runId: context.evalEventsRunId,
+    workflowName: context.suite.workflow.name,
+    timestamp: context.now().toISOString(),
+    parentEventId: null,
+    redacted: false,
+    type: "eval.started",
+    evalRunId: context.evalRunId,
+    suiteName: context.suite.name,
+    caseCount: context.suite.cases.length
+  });
+}
+function emitEvalCompletedEvent(context, summary) {
+  context.evalTraceBus.emitTraceEvent({
+    id: generateTraceEventId(),
+    runId: context.evalEventsRunId,
+    workflowName: context.suite.workflow.name,
+    timestamp: context.now().toISOString(),
+    parentEventId: null,
+    redacted: false,
+    type: "eval.completed",
+    evalRunId: context.evalRunId,
+    passedCaseCount: summary.passedCases,
+    failedCaseCount: summary.failedCases,
+    latencyMs: Math.max(0, context.now().getTime() - context.startedAtMs)
+  });
+}
+function buildEvalRunFile(context, summary, caseResults) {
+  return {
+    evalRunId: context.evalRunId,
+    suiteName: context.suite.name,
+    workflowName: context.suite.workflow.name,
+    startedAt: context.startedAt,
+    completedAt: context.now().toISOString(),
+    summary,
+    caseResults,
+    events: [...context.evalTraceBus.listTraceEvents()]
+  };
+}
+function resolveEvalRunsUploadUrl(baseUrl) {
+  return `${baseUrl}/v1/eval-runs`;
+}
+function writeEvalLineToStdout(line) {
+  process.stdout.write(`${line}
+`);
+}
+async function uploadEvalRunFile(context, evalRunFile) {
+  const platformOptions = context.options.platform ?? {};
+  const platformConfig = resolvePlatformConfig({
+    apiKey: platformOptions.apiKey,
+    baseUrl: platformOptions.baseUrl,
+    environmentVariables: platformOptions.environmentVariables,
+    homeDirectory: platformOptions.homeDirectory
+  });
+  if (platformConfig.apiKey === null) {
+    const writeLine = context.options.writeLine ?? writeEvalLineToStdout;
+    writeLine(buildMissingKeyMessage());
+    return "skipped-no-key";
+  }
+  return await postEvalRunFileToPlatform(
+    platformConfig.apiKey,
+    platformConfig.baseUrl,
+    evalRunFile,
+    platformOptions.fetchImplementation
+  );
+}
+async function postEvalRunFileToPlatform(apiKey, baseUrl, evalRunFile, fetchImplementation) {
+  const uploadFetch = fetchImplementation ?? fetch;
+  try {
+    const response = await uploadFetch(resolveEvalRunsUploadUrl(baseUrl), {
+      method: "POST",
+      headers: {
+        authorization: `Bearer ${apiKey}`,
+        "content-type": "application/json"
+      },
+      body: JSON.stringify(evalRunFile)
+    });
+    return response.ok ? "uploaded" : "failed";
+  } catch {
+    return "failed";
+  }
+}
+export {
+  EvalCasePerformanceSchema,
+  EvalCaseResultSchema,
+  EvalMetricBreakdownSchema,
+  EvalMetricResultSchema,
+  EvalRunFileSchema,
+  EvalSuitePerformanceSummarySchema,
+  EvalSuiteSummarySchema,
+  EvalWebhookCaseSchema,
+  EvalWebhookPayloadSchema,
+  ExpectedApprovalDecisionSchema,
+  ExpectedToolCallSchema,
+  JudgeVerdictSchema,
+  SerializedEvalCaseExpectationSchema,
+  SerializedEvalCaseSchema,
+  aggregateSuiteMetrics,
+  approvalCorrectnessMetricName,
+  areJsonValuesDeeplyEqual,
+  buildEvalWebhookPayload,
+  collectRunPerformance,
+  createFailedMetricResult,
+  createPassedMetricResult,
+  createScoredMetricResult,
+  createSkippedMetricResult,
+  defaultEvalResultsDirectory,
+  defaultJudgeMetricName,
+  defaultJudgePassThreshold,
+  defineJudgeMetric,
+  determineEvalSuiteStatus,
+  evalSuite,
+  evaluateApprovalCorrectnessMetric,
+  evaluateCaseMetrics,
+  evaluateJudgeMetric,
+  evaluateSchemaValidityMetric,
+  evaluateTaskSuccessMetric,
+  evaluateToolAccuracyMetric,
+  isJsonObjectValue,
+  loadEvalCases,
+  loadEvalCasesFromJsonlFile,
+  loadEvalRunFile,
+  postEvalResultsToWebhook,
+  resolveEvalRunFilePath,
+  resolveEvalRunsUploadUrl,
+  runEvalSuite,
+  runStatusMetricName,
+  saveEvalRunFile,
+  schemaValidityMetricName,
+  taskSuccessMetricName,
+  toolAccuracyMetricName
+};