evalsense 0.2.0

package/dist/index.js

@@ -0,0 +1,1043 @@
+import { getCurrentSuite, setCurrentSuite, addSuite, addTestToCurrentSuite, DatasetError, IntegrityError, buildConfusionMatrix, getTruePositives, getFalsePositives, getFalseNegatives, getSupport, AssertionError, recordAssertion, recordFieldMetrics } from './chunk-5P7LNNO6.js';
+export { AssertionError, ConfigurationError, ConsoleReporter, DatasetError, EvalSenseError, ExitCodes, IntegrityError, JsonReporter, TestExecutionError, buildConfusionMatrix, discoverEvalFiles, executeEvalFiles, formatConfusionMatrix, getExitCode, parseReport } from './chunk-5P7LNNO6.js';
+import { readFileSync } from 'fs';
+import { resolve, extname } from 'path';
+
+// src/core/describe.ts
+function describe(name, fn) {
+  const parentSuite = getCurrentSuite();
+  const suite = {
+    name,
+    tests: [],
+    beforeAll: [],
+    afterAll: [],
+    beforeEach: [],
+    afterEach: []
+  };
+  setCurrentSuite(suite);
+  try {
+    fn();
+  } finally {
+    setCurrentSuite(parentSuite);
+  }
+  addSuite(suite);
+}
+function beforeAll(fn) {
+  const suite = getCurrentSuite();
+  if (!suite) {
+    throw new Error("beforeAll() must be called inside a describe() block");
+  }
+  suite.beforeAll?.push(fn);
+}
+function afterAll(fn) {
+  const suite = getCurrentSuite();
+  if (!suite) {
+    throw new Error("afterAll() must be called inside a describe() block");
+  }
+  suite.afterAll?.push(fn);
+}
+function beforeEach(fn) {
+  const suite = getCurrentSuite();
+  if (!suite) {
+    throw new Error("beforeEach() must be called inside a describe() block");
+  }
+  suite.beforeEach?.push(fn);
+}
+function afterEach(fn) {
+  const suite = getCurrentSuite();
+  if (!suite) {
+    throw new Error("afterEach() must be called inside a describe() block");
+  }
+  suite.afterEach?.push(fn);
+}
+
+// src/core/eval-test.ts
+function evalTest(name, fn) {
+  const currentSuite = getCurrentSuite();
+  if (!currentSuite) {
+    throw new Error("evalTest() must be called inside a describe() block");
+  }
+  const test2 = {
+    name,
+    fn
+  };
+  addTestToCurrentSuite(test2);
+}
+var test = evalTest;
+var it = evalTest;
+function evalTestSkip(name, _fn) {
+  const currentSuite = getCurrentSuite();
+  if (!currentSuite) {
+    throw new Error("evalTest.skip() must be called inside a describe() block");
+  }
+  const test2 = {
+    name: `[SKIPPED] ${name}`,
+    fn: async () => {
+    }
+  };
+  addTestToCurrentSuite(test2);
+}
+function evalTestOnly(name, fn) {
+  const currentSuite = getCurrentSuite();
+  if (!currentSuite) {
+    throw new Error("evalTest.only() must be called inside a describe() block");
+  }
+  const test2 = {
+    name: `[ONLY] ${name}`,
+    fn
+  };
+  addTestToCurrentSuite(test2);
+}
+evalTest.skip = evalTestSkip;
+evalTest.only = evalTestOnly;
+function loadDataset(path) {
+  const absolutePath = resolve(process.cwd(), path);
+  const ext = extname(absolutePath).toLowerCase();
+  let records;
+  try {
+    const content = readFileSync(absolutePath, "utf-8");
+    if (ext === ".ndjson" || ext === ".jsonl") {
+      records = parseNDJSON(content);
+    } else if (ext === ".json") {
+      records = parseJSON(content);
+    } else {
+      throw new DatasetError(
+        `Unsupported file format: ${ext}. Use .json, .ndjson, or .jsonl`,
+        path
+      );
+    }
+  } catch (error) {
+    if (error instanceof DatasetError) {
+      throw error;
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    throw new DatasetError(`Failed to load dataset from ${path}: ${message}`, path);
+  }
+  return {
+    records,
+    metadata: {
+      source: path,
+      count: records.length,
+      loadedAt: /* @__PURE__ */ new Date()
+    }
+  };
+}
+function parseJSON(content) {
+  const parsed = JSON.parse(content);
+  if (!Array.isArray(parsed)) {
+    throw new DatasetError("JSON dataset must be an array of records");
+  }
+  return parsed;
+}
+function parseNDJSON(content) {
+  const lines = content.split("\n").filter((line) => line.trim() !== "");
+  const records = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === void 0) continue;
+    try {
+      records.push(JSON.parse(line));
+    } catch {
+      throw new DatasetError(`Invalid JSON at line ${i + 1} in NDJSON file`);
+    }
+  }
+  return records;
+}
+function createDataset(records, source = "inline") {
+  return {
+    records,
+    metadata: {
+      source,
+      count: records.length,
+      loadedAt: /* @__PURE__ */ new Date()
+    }
+  };
+}
+
+// src/dataset/run-model.ts
+async function runModel(dataset, modelFn) {
+  const startTime = Date.now();
+  const predictions = [];
+  const aligned = [];
+  for (const record of dataset.records) {
+    const id = getRecordId(record);
+    const prediction = await modelFn(record);
+    if (prediction.id !== id) {
+      throw new DatasetError(
+        `Prediction ID mismatch: expected "${id}", got "${prediction.id}". Model function must return the same ID as the input record.`
+      );
+    }
+    predictions.push(prediction);
+    aligned.push({
+      id,
+      actual: { ...prediction },
+      expected: { ...record }
+    });
+  }
+  return {
+    predictions,
+    aligned,
+    duration: Date.now() - startTime
+  };
+}
+function getRecordId(record) {
+  const id = record.id ?? record._id;
+  if (id === void 0 || id === null) {
+    throw new DatasetError(
+      'Dataset records must have an "id" or "_id" field for alignment'
+    );
+  }
+  return String(id);
+}
+async function runModelParallel(dataset, modelFn, concurrency = 10) {
+  const startTime = Date.now();
+  const results = [];
+  for (let i = 0; i < dataset.records.length; i += concurrency) {
+    const batch = dataset.records.slice(i, i + concurrency);
+    const batchResults = await Promise.all(
+      batch.map(async (record) => {
+        const prediction = await modelFn(record);
+        return { prediction, record };
+      })
+    );
+    results.push(...batchResults);
+  }
+  const predictions = [];
+  const aligned = [];
+  for (const { prediction, record } of results) {
+    const id = getRecordId(record);
+    if (prediction.id !== id) {
+      throw new DatasetError(
+        `Prediction ID mismatch: expected "${id}", got "${prediction.id}".`
+      );
+    }
+    predictions.push(prediction);
+    aligned.push({
+      id,
+      actual: { ...prediction },
+      expected: { ...record }
+    });
+  }
+  return {
+    predictions,
+    aligned,
+    duration: Date.now() - startTime
+  };
+}
+
+// src/dataset/alignment.ts
+function alignByKey(predictions, expected, options = {}) {
+  const { strict = false, idField = "id" } = options;
+  const expectedMap = /* @__PURE__ */ new Map();
+  for (const record of expected) {
+    const id = String(record[idField] ?? record._id);
+    expectedMap.set(id, record);
+  }
+  const aligned = [];
+  const missingIds = [];
+  for (const prediction of predictions) {
+    const id = prediction.id;
+    const expectedRecord = expectedMap.get(id);
+    if (!expectedRecord) {
+      missingIds.push(id);
+      if (strict) {
+        continue;
+      }
+      aligned.push({
+        id,
+        actual: { ...prediction },
+        expected: {}
+      });
+    } else {
+      aligned.push({
+        id,
+        actual: { ...prediction },
+        expected: { ...expectedRecord }
+      });
+    }
+  }
+  if (strict && missingIds.length > 0) {
+    throw new IntegrityError(
+      `${missingIds.length} prediction(s) have no matching expected record`,
+      missingIds
+    );
+  }
+  return aligned;
+}
+function extractFieldValues(aligned, field) {
+  const actual = [];
+  const expected = [];
+  const ids = [];
+  for (const record of aligned) {
+    actual.push(record.actual[field]);
+    expected.push(record.expected[field]);
+    ids.push(record.id);
+  }
+  return { actual, expected, ids };
+}
+function filterComplete(aligned, field) {
+  return aligned.filter((record) => {
+    const actualValue = record.actual[field];
+    const expectedValue = record.expected[field];
+    return actualValue !== void 0 && expectedValue !== void 0;
+  });
+}
+
+// src/dataset/integrity.ts
+function checkIntegrity(dataset, options = {}) {
+  const { requiredFields = [], throwOnFailure = false } = options;
+  const seenIds = /* @__PURE__ */ new Map();
+  const missingIds = [];
+  const duplicateIds = [];
+  const missingFields = [];
+  for (let i = 0; i < dataset.records.length; i++) {
+    const record = dataset.records[i];
+    if (!record) continue;
+    const id = record.id ?? record._id;
+    if (id === void 0 || id === null) {
+      missingIds.push(`record[${i}]`);
+    } else {
+      const idStr = String(id);
+      const previousIndex = seenIds.get(idStr);
+      if (previousIndex !== void 0) {
+        duplicateIds.push(idStr);
+      } else {
+        seenIds.set(idStr, i);
+      }
+    }
+    if (requiredFields.length > 0) {
+      const missing = requiredFields.filter(
+        (field) => record[field] === void 0
+      );
+      if (missing.length > 0) {
+        missingFields.push({
+          id: String(id ?? `record[${i}]`),
+          fields: missing
+        });
+      }
+    }
+  }
+  const valid = missingIds.length === 0 && duplicateIds.length === 0 && missingFields.length === 0;
+  const result = {
+    valid,
+    totalRecords: dataset.records.length,
+    missingIds,
+    duplicateIds,
+    missingFields
+  };
+  if (throwOnFailure && !valid) {
+    const issues = [];
+    if (missingIds.length > 0) {
+      issues.push(`${missingIds.length} record(s) missing ID`);
+    }
+    if (duplicateIds.length > 0) {
+      issues.push(`${duplicateIds.length} duplicate ID(s): ${duplicateIds.slice(0, 3).join(", ")}${duplicateIds.length > 3 ? "..." : ""}`);
+    }
+    if (missingFields.length > 0) {
+      issues.push(`${missingFields.length} record(s) missing required fields`);
+    }
+    throw new IntegrityError(`Dataset integrity check failed: ${issues.join("; ")}`);
+  }
+  return result;
+}
+function validatePredictions(predictions, expectedIds) {
+  const predictionIds = new Set(predictions.map((p) => p.id));
+  const expectedIdSet = new Set(expectedIds);
+  const missing = expectedIds.filter((id) => !predictionIds.has(id));
+  const extra = predictions.map((p) => p.id).filter((id) => !expectedIdSet.has(id));
+  return {
+    valid: missing.length === 0 && extra.length === 0,
+    missing,
+    extra
+  };
+}
+
+// src/statistics/classification.ts
+function computeClassificationMetrics(actual, expected) {
+  const confusionMatrix = buildConfusionMatrix(actual, expected);
+  return computeMetricsFromMatrix(confusionMatrix);
+}
+function computeMetricsFromMatrix(cm) {
+  const perClass = {};
+  let totalSupport = 0;
+  let correctPredictions = 0;
+  for (const label of cm.labels) {
+    const tp = getTruePositives(cm, label);
+    const fp = getFalsePositives(cm, label);
+    const fn = getFalseNegatives(cm, label);
+    const support = getSupport(cm, label);
+    const precision = tp + fp > 0 ? tp / (tp + fp) : 0;
+    const recall = tp + fn > 0 ? tp / (tp + fn) : 0;
+    const f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0;
+    perClass[label] = { precision, recall, f1, support };
+    totalSupport += support;
+    correctPredictions += tp;
+  }
+  const accuracy = totalSupport > 0 ? correctPredictions / totalSupport : 0;
+  const classCount = cm.labels.length;
+  const macroAvg = {
+    precision: classCount > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.precision, 0) / classCount : 0,
+    recall: classCount > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.recall, 0) / classCount : 0,
+    f1: classCount > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.f1, 0) / classCount : 0
+  };
+  const weightedAvg = {
+    precision: totalSupport > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.precision * m.support, 0) / totalSupport : 0,
+    recall: totalSupport > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.recall * m.support, 0) / totalSupport : 0,
+    f1: totalSupport > 0 ? Object.values(perClass).reduce((sum, m) => sum + m.f1 * m.support, 0) / totalSupport : 0
+  };
+  return {
+    accuracy,
+    perClass,
+    macroAvg,
+    weightedAvg,
+    confusionMatrix: cm
+  };
+}
+function computePrecision(actual, expected, targetClass) {
+  const cm = buildConfusionMatrix(actual, expected);
+  const tp = getTruePositives(cm, targetClass);
+  const fp = getFalsePositives(cm, targetClass);
+  return tp + fp > 0 ? tp / (tp + fp) : 0;
+}
+function computeRecall(actual, expected, targetClass) {
+  const cm = buildConfusionMatrix(actual, expected);
+  const tp = getTruePositives(cm, targetClass);
+  const fn = getFalseNegatives(cm, targetClass);
+  return tp + fn > 0 ? tp / (tp + fn) : 0;
+}
+function computeF1(actual, expected, targetClass) {
+  const precision = computePrecision(actual, expected, targetClass);
+  const recall = computeRecall(actual, expected, targetClass);
+  return precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0;
+}
+function computeAccuracy(actual, expected) {
+  if (actual.length !== expected.length || actual.length === 0) {
+    return 0;
+  }
+  let correct = 0;
+  let total = 0;
+  for (let i = 0; i < actual.length; i++) {
+    const a = actual[i];
+    const e = expected[i];
+    if (a !== void 0 && a !== null && e !== void 0 && e !== null) {
+      total++;
+      if (String(a) === String(e)) {
+        correct++;
+      }
+    }
+  }
+  return total > 0 ? correct / total : 0;
+}
+
+// src/statistics/distribution.ts
+function filterNumericValues(values) {
+  return values.filter(
+    (v) => typeof v === "number" && !Number.isNaN(v) && v !== null && v !== void 0
+  );
+}
+function calculatePercentageBelow(values, threshold) {
+  if (values.length === 0) {
+    return 0;
+  }
+  const countBelow = values.filter((v) => v <= threshold).length;
+  return countBelow / values.length;
+}
+function calculatePercentageAbove(values, threshold) {
+  if (values.length === 0) {
+    return 0;
+  }
+  const countAbove = values.filter((v) => v > threshold).length;
+  return countAbove / values.length;
+}
+
+// src/assertions/binarize.ts
+var BinarizeSelector = class {
+  fieldName;
+  threshold;
+  binaryActual;
+  binaryExpected;
+  assertions = [];
+  constructor(aligned, fieldName, threshold) {
+    this.fieldName = fieldName;
+    this.threshold = threshold;
+    this.binaryActual = [];
+    this.binaryExpected = [];
+    for (const record of aligned) {
+      const actualVal = record.actual[fieldName];
+      const expectedVal = record.expected[fieldName];
+      if (typeof actualVal === "number") {
+        this.binaryActual.push(actualVal >= threshold ? "true" : "false");
+      } else if (typeof actualVal === "boolean") {
+        this.binaryActual.push(String(actualVal));
+      } else {
+        this.binaryActual.push(String(actualVal));
+      }
+      if (typeof expectedVal === "number") {
+        this.binaryExpected.push(expectedVal >= threshold ? "true" : "false");
+      } else if (typeof expectedVal === "boolean") {
+        this.binaryExpected.push(String(expectedVal));
+      } else {
+        this.binaryExpected.push(String(expectedVal));
+      }
+    }
+  }
+  /**
+   * Asserts that accuracy is above a threshold
+   */
+  toHaveAccuracyAbove(threshold) {
+    const metrics = computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+    const passed = metrics.accuracy >= threshold;
+    const result = {
+      type: "accuracy",
+      passed,
+      message: passed ? `Accuracy ${(metrics.accuracy * 100).toFixed(1)}% is above ${(threshold * 100).toFixed(1)}% (binarized at ${this.threshold})` : `Accuracy ${(metrics.accuracy * 100).toFixed(1)}% is below threshold ${(threshold * 100).toFixed(1)}% (binarized at ${this.threshold})`,
+      expected: threshold,
+      actual: metrics.accuracy,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, threshold, metrics.accuracy, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that precision is above a threshold
+   * @param classOrThreshold - Either the class (true/false) or threshold
+   * @param threshold - Threshold when class is specified
+   */
+  toHavePrecisionAbove(classOrThreshold, threshold) {
+    const metrics = computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+    let actualPrecision;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualPrecision = metrics.macroAvg.precision;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = String(classOrThreshold);
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in binarized predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualPrecision = classMetrics.precision;
+    }
+    const passed = actualPrecision >= actualThreshold;
+    const result = {
+      type: "precision",
+      passed,
+      message: passed ? `Precision${targetClass ? ` for ${targetClass}` : ""} ${(actualPrecision * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `Precision${targetClass ? ` for ${targetClass}` : ""} ${(actualPrecision * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualPrecision,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualPrecision, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that recall is above a threshold
+   * @param classOrThreshold - Either the class (true/false) or threshold
+   * @param threshold - Threshold when class is specified
+   */
+  toHaveRecallAbove(classOrThreshold, threshold) {
+    const metrics = computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+    let actualRecall;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualRecall = metrics.macroAvg.recall;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = String(classOrThreshold);
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in binarized predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualRecall = classMetrics.recall;
+    }
+    const passed = actualRecall >= actualThreshold;
+    const result = {
+      type: "recall",
+      passed,
+      message: passed ? `Recall${targetClass ? ` for ${targetClass}` : ""} ${(actualRecall * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `Recall${targetClass ? ` for ${targetClass}` : ""} ${(actualRecall * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualRecall,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualRecall, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that F1 score is above a threshold
+   */
+  toHaveF1Above(classOrThreshold, threshold) {
+    const metrics = computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+    let actualF1;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualF1 = metrics.macroAvg.f1;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = String(classOrThreshold);
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in binarized predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualF1 = classMetrics.f1;
+    }
+    const passed = actualF1 >= actualThreshold;
+    const result = {
+      type: "f1",
+      passed,
+      message: passed ? `F1${targetClass ? ` for ${targetClass}` : ""} ${(actualF1 * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `F1${targetClass ? ` for ${targetClass}` : ""} ${(actualF1 * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualF1,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualF1, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Includes the confusion matrix in the report
+   */
+  toHaveConfusionMatrix() {
+    const metrics = computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+    const fieldResult = {
+      field: this.fieldName,
+      metrics,
+      binarized: true,
+      binarizeThreshold: this.threshold
+    };
+    recordFieldMetrics(fieldResult);
+    const result = {
+      type: "confusionMatrix",
+      passed: true,
+      message: `Confusion matrix recorded for binarized field "${this.fieldName}" (threshold: ${this.threshold})`,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    return this;
+  }
+  /**
+   * Gets computed metrics
+   */
+  getMetrics() {
+    return computeClassificationMetrics(this.binaryActual, this.binaryExpected);
+  }
+  /**
+   * Gets all assertions made
+   */
+  getAssertions() {
+    return this.assertions;
+  }
+};
+
+// src/assertions/field-selector.ts
+var FieldSelector = class {
+  aligned;
+  fieldName;
+  actualValues;
+  expectedValues;
+  assertions = [];
+  constructor(aligned, fieldName) {
+    this.aligned = aligned;
+    this.fieldName = fieldName;
+    const extracted = extractFieldValues(aligned, fieldName);
+    this.actualValues = extracted.actual;
+    this.expectedValues = extracted.expected;
+  }
+  /**
+   * Transforms continuous scores to binary classification using a threshold
+   */
+  binarize(threshold) {
+    return new BinarizeSelector(this.aligned, this.fieldName, threshold);
+  }
+  /**
+   * Validates that ground truth exists for classification metrics.
+   * Throws a clear error if expected values are missing.
+   */
+  validateGroundTruth() {
+    const hasExpected = this.expectedValues.some(
+      (v) => v !== void 0 && v !== null
+    );
+    if (!hasExpected) {
+      throw new AssertionError(
+        `Classification metric requires ground truth, but field "${this.fieldName}" has no expected values. Use expectStats(predictions, groundTruth) to provide expected values.`,
+        void 0,
+        void 0,
+        this.fieldName
+      );
+    }
+  }
+  /**
+   * Asserts that accuracy is above a threshold
+   */
+  toHaveAccuracyAbove(threshold) {
+    this.validateGroundTruth();
+    const metrics = computeClassificationMetrics(this.actualValues, this.expectedValues);
+    const passed = metrics.accuracy >= threshold;
+    const result = {
+      type: "accuracy",
+      passed,
+      message: passed ? `Accuracy ${(metrics.accuracy * 100).toFixed(1)}% is above ${(threshold * 100).toFixed(1)}%` : `Accuracy ${(metrics.accuracy * 100).toFixed(1)}% is below threshold ${(threshold * 100).toFixed(1)}%`,
+      expected: threshold,
+      actual: metrics.accuracy,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, threshold, metrics.accuracy, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that precision is above a threshold
+   * @param classOrThreshold - Either the class name or threshold (if class is omitted, uses macro average)
+   * @param threshold - Threshold when class is specified
+   */
+  toHavePrecisionAbove(classOrThreshold, threshold) {
+    this.validateGroundTruth();
+    const metrics = computeClassificationMetrics(this.actualValues, this.expectedValues);
+    let actualPrecision;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualPrecision = metrics.macroAvg.precision;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = classOrThreshold;
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualPrecision = classMetrics.precision;
+    }
+    const passed = actualPrecision >= actualThreshold;
+    const result = {
+      type: "precision",
+      passed,
+      message: passed ? `Precision${targetClass ? ` for "${targetClass}"` : ""} ${(actualPrecision * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `Precision${targetClass ? ` for "${targetClass}"` : ""} ${(actualPrecision * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualPrecision,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualPrecision, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that recall is above a threshold
+   * @param classOrThreshold - Either the class name or threshold (if class is omitted, uses macro average)
+   * @param threshold - Threshold when class is specified
+   */
+  toHaveRecallAbove(classOrThreshold, threshold) {
+    this.validateGroundTruth();
+    const metrics = computeClassificationMetrics(this.actualValues, this.expectedValues);
+    let actualRecall;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualRecall = metrics.macroAvg.recall;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = classOrThreshold;
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualRecall = classMetrics.recall;
+    }
+    const passed = actualRecall >= actualThreshold;
+    const result = {
+      type: "recall",
+      passed,
+      message: passed ? `Recall${targetClass ? ` for "${targetClass}"` : ""} ${(actualRecall * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `Recall${targetClass ? ` for "${targetClass}"` : ""} ${(actualRecall * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualRecall,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualRecall, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that F1 score is above a threshold
+   * @param classOrThreshold - Either the class name or threshold (if class is omitted, uses macro average)
+   * @param threshold - Threshold when class is specified
+   */
+  toHaveF1Above(classOrThreshold, threshold) {
+    this.validateGroundTruth();
+    const metrics = computeClassificationMetrics(this.actualValues, this.expectedValues);
+    let actualF1;
+    let targetClass;
+    let actualThreshold;
+    if (typeof classOrThreshold === "number") {
+      actualF1 = metrics.macroAvg.f1;
+      actualThreshold = classOrThreshold;
+    } else {
+      targetClass = classOrThreshold;
+      actualThreshold = threshold;
+      const classMetrics = metrics.perClass[targetClass];
+      if (!classMetrics) {
+        throw new AssertionError(
+          `Class "${targetClass}" not found in predictions`,
+          targetClass,
+          Object.keys(metrics.perClass),
+          this.fieldName
+        );
+      }
+      actualF1 = classMetrics.f1;
+    }
+    const passed = actualF1 >= actualThreshold;
+    const result = {
+      type: "f1",
+      passed,
+      message: passed ? `F1${targetClass ? ` for "${targetClass}"` : ""} ${(actualF1 * 100).toFixed(1)}% is above ${(actualThreshold * 100).toFixed(1)}%` : `F1${targetClass ? ` for "${targetClass}"` : ""} ${(actualF1 * 100).toFixed(1)}% is below threshold ${(actualThreshold * 100).toFixed(1)}%`,
+      expected: actualThreshold,
+      actual: actualF1,
+      field: this.fieldName,
+      class: targetClass
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, actualThreshold, actualF1, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Includes the confusion matrix in the report
+   */
+  toHaveConfusionMatrix() {
+    const metrics = computeClassificationMetrics(this.actualValues, this.expectedValues);
+    const fieldResult = {
+      field: this.fieldName,
+      metrics,
+      binarized: false
+    };
+    recordFieldMetrics(fieldResult);
+    const result = {
+      type: "confusionMatrix",
+      passed: true,
+      message: `Confusion matrix recorded for field "${this.fieldName}"`,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    return this;
+  }
+  /**
+   * Asserts that a percentage of values are below or equal to a threshold.
+   * This is a distributional assertion that only looks at actual values (no ground truth required).
+   *
+   * @param valueThreshold - The value threshold to compare against
+   * @param percentageThreshold - The minimum percentage (0-1) of values that should be <= valueThreshold
+   * @returns this for method chaining
+   *
+   * @example
+   * // Assert that 90% of confidence scores are below 0.5
+   * expectStats(predictions)
+   *   .field("confidence")
+   *   .toHavePercentageBelow(0.5, 0.9)
+   */
+  toHavePercentageBelow(valueThreshold, percentageThreshold) {
+    const numericActual = filterNumericValues(this.actualValues);
+    if (numericActual.length === 0) {
+      throw new AssertionError(
+        `Field '${this.fieldName}' contains no numeric values (found 0 numeric out of ${this.actualValues.length} total values)`,
+        percentageThreshold,
+        void 0,
+        this.fieldName
+      );
+    }
+    const actualPercentage = calculatePercentageBelow(numericActual, valueThreshold);
+    const passed = actualPercentage >= percentageThreshold;
+    const result = {
+      type: "percentageBelow",
+      passed,
+      message: passed ? `${(actualPercentage * 100).toFixed(1)}% of '${this.fieldName}' values are below or equal to ${valueThreshold} (expected >= ${(percentageThreshold * 100).toFixed(1)}%)` : `Only ${(actualPercentage * 100).toFixed(1)}% of '${this.fieldName}' values are below or equal to ${valueThreshold} (expected >= ${(percentageThreshold * 100).toFixed(1)}%)`,
+      expected: percentageThreshold,
+      actual: actualPercentage,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, percentageThreshold, actualPercentage, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Asserts that a percentage of values are above a threshold.
+   * This is a distributional assertion that only looks at actual values (no ground truth required).
+   *
+   * @param valueThreshold - The value threshold to compare against
+   * @param percentageThreshold - The minimum percentage (0-1) of values that should be > valueThreshold
+   * @returns this for method chaining
+   *
+   * @example
+   * // Assert that 80% of quality scores are above 0.7
+   * expectStats(predictions)
+   *   .field("quality")
+   *   .toHavePercentageAbove(0.7, 0.8)
+   */
+  toHavePercentageAbove(valueThreshold, percentageThreshold) {
+    const numericActual = filterNumericValues(this.actualValues);
+    if (numericActual.length === 0) {
+      throw new AssertionError(
+        `Field '${this.fieldName}' contains no numeric values (found 0 numeric out of ${this.actualValues.length} total values)`,
+        percentageThreshold,
+        void 0,
+        this.fieldName
+      );
+    }
+    const actualPercentage = calculatePercentageAbove(numericActual, valueThreshold);
+    const passed = actualPercentage >= percentageThreshold;
+    const result = {
+      type: "percentageAbove",
+      passed,
+      message: passed ? `${(actualPercentage * 100).toFixed(1)}% of '${this.fieldName}' values are above ${valueThreshold} (expected >= ${(percentageThreshold * 100).toFixed(1)}%)` : `Only ${(actualPercentage * 100).toFixed(1)}% of '${this.fieldName}' values are above ${valueThreshold} (expected >= ${(percentageThreshold * 100).toFixed(1)}%)`,
+      expected: percentageThreshold,
+      actual: actualPercentage,
+      field: this.fieldName
+    };
+    this.assertions.push(result);
+    recordAssertion(result);
+    if (!passed) {
+      throw new AssertionError(result.message, percentageThreshold, actualPercentage, this.fieldName);
+    }
+    return this;
+  }
+  /**
+   * Gets the computed metrics for this field
+   */
+  getMetrics() {
+    return computeClassificationMetrics(this.actualValues, this.expectedValues);
+  }
+  /**
+   * Gets all assertions made on this field
+   */
+  getAssertions() {
+    return this.assertions;
+  }
+};
+
+// src/assertions/expect-stats.ts
+function normalizeInput(input) {
+  if ("aligned" in input && Array.isArray(input.aligned)) {
+    return input.aligned;
+  }
+  if (Array.isArray(input)) {
+    if (input.length === 0) {
+      return [];
+    }
+    const first = input[0];
+    if (first && "actual" in first && "expected" in first) {
+      return input;
+    }
+    return input.map((p) => ({
+      id: p.id,
+      actual: { ...p },
+      expected: {}
+    }));
+  }
+  throw new Error("Invalid input to expectStats(): expected ModelRunResult, Prediction[], or AlignedRecord[]");
+}
+function expectStats(inputOrActual, expected) {
+  if (expected !== void 0) {
+    if (!Array.isArray(inputOrActual)) {
+      throw new Error(
+        "When using two-argument expectStats(), first argument must be Prediction[]"
+      );
+    }
+    const aligned2 = alignByKey(inputOrActual, expected);
+    return new ExpectStats(aligned2);
+  }
+  const aligned = normalizeInput(inputOrActual);
+  return new ExpectStats(aligned);
+}
+var ExpectStats = class {
+  aligned;
+  constructor(aligned) {
+    this.aligned = aligned;
+  }
+  /**
+   * Selects a field to evaluate
+   */
+  field(fieldName) {
+    return new FieldSelector(this.aligned, fieldName);
+  }
+  /**
+   * Gets the raw aligned records (for advanced use)
+   */
+  getAligned() {
+    return this.aligned;
+  }
+  /**
+   * Gets the count of records
+   */
+  count() {
+    return this.aligned.length;
+  }
+};
+
+export { afterAll, afterEach, alignByKey, beforeAll, beforeEach, checkIntegrity, computeAccuracy, computeClassificationMetrics, computeF1, computePrecision, computeRecall, createDataset, describe, evalTest, expectStats, extractFieldValues, filterComplete, it, loadDataset, runModel, runModelParallel, test, validatePredictions };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map