evalsense 0.2.0

package/dist/chunk-5P7LNNO6.js

@@ -0,0 +1,747 @@
+import { writeFileSync, existsSync } from 'fs';
+import stringify from 'fast-json-stable-stringify';
+import { glob } from 'glob';
+import { resolve } from 'path';
+import { pathToFileURL } from 'url';
+
+// src/core/errors.ts
+var EvalSenseError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "EvalSenseError";
+  }
+};
+var AssertionError = class extends EvalSenseError {
+  expected;
+  actual;
+  field;
+  constructor(message, expected, actual, field) {
+    super(message);
+    this.name = "AssertionError";
+    this.expected = expected;
+    this.actual = actual;
+    this.field = field;
+  }
+};
+var DatasetError = class extends EvalSenseError {
+  source;
+  constructor(message, source) {
+    super(message);
+    this.name = "DatasetError";
+    this.source = source;
+  }
+};
+var IntegrityError = class extends EvalSenseError {
+  missingIds;
+  duplicateIds;
+  constructor(message, missingIds, duplicateIds) {
+    super(message);
+    this.name = "IntegrityError";
+    this.missingIds = missingIds;
+    this.duplicateIds = duplicateIds;
+  }
+};
+var ConfigurationError = class extends EvalSenseError {
+  constructor(message) {
+    super(message);
+    this.name = "ConfigurationError";
+  }
+};
+var TestExecutionError = class extends EvalSenseError {
+  testName;
+  originalError;
+  constructor(message, testName, originalError) {
+    super(message);
+    this.name = "TestExecutionError";
+    this.testName = testName;
+    this.originalError = originalError;
+  }
+};
+
+// src/statistics/confusion-matrix.ts
+function buildConfusionMatrix(actual, expected) {
+  if (actual.length !== expected.length) {
+    throw new Error(
+      `Array length mismatch: actual has ${actual.length} elements, expected has ${expected.length}`
+    );
+  }
+  const labelSet = /* @__PURE__ */ new Set();
+  for (const val of actual) {
+    if (val !== void 0 && val !== null) {
+      labelSet.add(String(val));
+    }
+  }
+  for (const val of expected) {
+    if (val !== void 0 && val !== null) {
+      labelSet.add(String(val));
+    }
+  }
+  const labels = Array.from(labelSet).sort();
+  const labelIndex = /* @__PURE__ */ new Map();
+  labels.forEach((label, idx) => labelIndex.set(label, idx));
+  const matrix = labels.map(() => labels.map(() => 0));
+  for (let i = 0; i < actual.length; i++) {
+    const actualVal = actual[i];
+    const expectedVal = expected[i];
+    if (actualVal === void 0 || actualVal === null) continue;
+    if (expectedVal === void 0 || expectedVal === null) continue;
+    const actualIdx = labelIndex.get(String(actualVal));
+    const expectedIdx = labelIndex.get(String(expectedVal));
+    if (actualIdx !== void 0 && expectedIdx !== void 0) {
+      matrix[expectedIdx][actualIdx]++;
+    }
+  }
+  const total = actual.filter(
+    (v, i) => v !== void 0 && v !== null && expected[i] !== void 0 && expected[i] !== null
+  ).length;
+  return { matrix, labels, total };
+}
+function getCount(cm, expectedLabel, actualLabel) {
+  const expectedIdx = cm.labels.indexOf(expectedLabel);
+  const actualIdx = cm.labels.indexOf(actualLabel);
+  if (expectedIdx === -1 || actualIdx === -1) {
+    return 0;
+  }
+  return cm.matrix[expectedIdx]?.[actualIdx] ?? 0;
+}
+function getTruePositives(cm, label) {
+  return getCount(cm, label, label);
+}
+function getFalsePositives(cm, label) {
+  const labelIdx = cm.labels.indexOf(label);
+  if (labelIdx === -1) return 0;
+  let fp = 0;
+  for (let i = 0; i < cm.labels.length; i++) {
+    if (i !== labelIdx) {
+      fp += cm.matrix[i]?.[labelIdx] ?? 0;
+    }
+  }
+  return fp;
+}
+function getFalseNegatives(cm, label) {
+  const labelIdx = cm.labels.indexOf(label);
+  if (labelIdx === -1) return 0;
+  let fn = 0;
+  for (let j = 0; j < cm.labels.length; j++) {
+    if (j !== labelIdx) {
+      fn += cm.matrix[labelIdx]?.[j] ?? 0;
+    }
+  }
+  return fn;
+}
+function getSupport(cm, label) {
+  const labelIdx = cm.labels.indexOf(label);
+  if (labelIdx === -1) return 0;
+  let support = 0;
+  for (let j = 0; j < cm.labels.length; j++) {
+    support += cm.matrix[labelIdx]?.[j] ?? 0;
+  }
+  return support;
+}
+function formatConfusionMatrix(cm) {
+  const maxLabelLen = Math.max(...cm.labels.map((l) => l.length), 8);
+  const colWidth = Math.max(
+    ...cm.matrix.flat().map((n) => String(n).length),
+    maxLabelLen
+  );
+  const header = " ".repeat(maxLabelLen + 2) + cm.labels.map((l) => l.padStart(colWidth)).join(" ");
+  const rows = cm.labels.map((label, i) => {
+    const rowData = cm.matrix[i].map((n) => String(n).padStart(colWidth)).join(" ");
+    return label.padEnd(maxLabelLen) + "  " + rowData;
+  });
+  return [header, ...rows].join("\n");
+}
+var JsonReporter = class {
+  /**
+   * Formats a report as deterministic JSON
+   */
+  format(report) {
+    const serializable = this.toSerializable(report);
+    return stringify(serializable) ?? "{}";
+  }
+  /**
+   * Writes report to a file
+   */
+  writeToFile(report, path) {
+    const json = this.format(report);
+    writeFileSync(path, json, "utf-8");
+  }
+  /**
+   * Converts report to a JSON-serializable format
+   */
+  toSerializable(report) {
+    return {
+      version: report.version,
+      timestamp: report.timestamp,
+      summary: report.summary,
+      suites: report.suites.map((suite) => ({
+        name: suite.name,
+        passed: suite.passed,
+        failed: suite.failed,
+        errors: suite.errors,
+        skipped: suite.skipped,
+        duration: suite.duration,
+        tests: suite.tests.map((test) => ({
+          name: test.name,
+          status: test.status,
+          duration: test.duration,
+          error: test.error ? {
+            name: test.error.name,
+            message: test.error.message
+          } : void 0,
+          assertions: test.assertions.map((a) => ({
+            type: a.type,
+            passed: a.passed,
+            message: a.message,
+            expected: a.expected,
+            actual: a.actual,
+            field: a.field,
+            class: a.class
+          })),
+          fieldMetrics: test.fieldMetrics.map((fm) => ({
+            field: fm.field,
+            binarized: fm.binarized,
+            binarizeThreshold: fm.binarizeThreshold,
+            metrics: {
+              accuracy: fm.metrics.accuracy,
+              perClass: fm.metrics.perClass,
+              macroAvg: fm.metrics.macroAvg,
+              weightedAvg: fm.metrics.weightedAvg,
+              confusionMatrix: {
+                labels: fm.metrics.confusionMatrix.labels,
+                matrix: fm.metrics.confusionMatrix.matrix,
+                total: fm.metrics.confusionMatrix.total
+              }
+            }
+          }))
+        }))
+      })),
+      integrity: report.integrity
+    };
+  }
+};
+function parseReport(json) {
+  const data = JSON.parse(json);
+  return data;
+}
+
+// src/report/console-reporter.ts
+var colors = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  red: "\x1B[31m",
+  green: "\x1B[32m",
+  yellow: "\x1B[33m",
+  blue: "\x1B[34m",
+  cyan: "\x1B[36m",
+  gray: "\x1B[90m"
+};
+var symbols = {
+  pass: "\u2713",
+  // ✓
+  fail: "\u2717",
+  // ✗
+  error: "!",
+  skip: "-"
+};
+var ConsoleReporter = class {
+  useColors;
+  constructor(useColors = true) {
+    this.useColors = useColors && process.stdout.isTTY !== false;
+  }
+  /**
+   * Prints the run header
+   */
+  printHeader(fileCount) {
+    this.log("");
+    this.log(this.color("bold", `EvalSense v0.1.0`));
+    this.log(this.color("dim", `Running ${fileCount} eval file(s)...`));
+    this.log("");
+  }
+  /**
+   * Prints the full report
+   */
+  printReport(report) {
+    for (const suite of report.suites) {
+      this.printSuite(suite.name, suite.tests);
+    }
+    this.printSummary(report);
+  }
+  /**
+   * Prints a suite's results
+   */
+  printSuite(name, tests) {
+    this.log(this.color("bold", `  ${name}`));
+    this.log("");
+    for (const test of tests) {
+      this.printTest(test);
+    }
+    this.log("");
+  }
+  /**
+   * Prints a single test result
+   */
+  printTest(test) {
+    const symbol = this.getStatusSymbol(test.status);
+    const statusColor = this.getStatusColor(test.status);
+    const duration = this.color("dim", `(${test.duration}ms)`);
+    this.log(`    ${this.color(statusColor, symbol)} ${test.name} ${duration}`);
+    for (const fm of test.fieldMetrics) {
+      this.printFieldMetrics(fm);
+    }
+    if (test.error && test.status === "failed") {
+      this.log(this.color("red", `      ${test.error.message}`));
+    } else if (test.error && test.status === "error") {
+      this.log(this.color("red", `      Error: ${test.error.message}`));
+    }
+    for (const assertion of test.assertions) {
+      if (!assertion.passed) {
+        this.log(this.color("red", `      ${assertion.message}`));
+      }
+    }
+  }
+  /**
+   * Prints field metrics summary
+   */
+  printFieldMetrics(fm) {
+    const { metrics, field, binarized, binarizeThreshold } = fm;
+    const fieldLabel = binarized ? `${field} (binarized @ ${binarizeThreshold})` : field;
+    this.log(
+      this.color(
+        "cyan",
+        `      Field: ${fieldLabel} | Accuracy: ${this.pct(metrics.accuracy)} | F1: ${this.pct(metrics.macroAvg.f1)}`
+      )
+    );
+    if (Object.keys(metrics.perClass).length > 1) {
+      for (const [cls, classMetrics] of Object.entries(metrics.perClass)) {
+        this.log(
+          this.color(
+            "dim",
+            `        ${cls}: P=${this.pct(classMetrics.precision)} R=${this.pct(classMetrics.recall)} F1=${this.pct(classMetrics.f1)} (n=${classMetrics.support})`
+          )
+        );
+      }
+    }
+  }
+  /**
+   * Prints the summary
+   */
+  printSummary(report) {
+    const { summary } = report;
+    this.log(this.color("bold", "  Summary"));
+    this.log("");
+    const passedStr = this.color("green", `${summary.passed} passed`);
+    const failedStr = summary.failed > 0 ? this.color("red", `${summary.failed} failed`) : `${summary.failed} failed`;
+    const errorsStr = summary.errors > 0 ? this.color("red", `${summary.errors} errors`) : `${summary.errors} errors`;
+    const skippedStr = summary.skipped > 0 ? this.color("yellow", `${summary.skipped} skipped`) : `${summary.skipped} skipped`;
+    this.log(`    Tests:    ${passedStr}, ${failedStr}, ${errorsStr}, ${skippedStr}`);
+    this.log(`    Suites:   ${summary.totalSuites}`);
+    this.log(`    Duration: ${this.formatDuration(summary.duration)}`);
+    this.log("");
+    if (summary.failed === 0 && summary.errors === 0) {
+      this.log(this.color("green", "  All tests passed!"));
+    } else {
+      this.log(this.color("red", "  Some tests failed."));
+    }
+    this.log("");
+  }
+  /**
+   * Prints a confusion matrix
+   */
+  printConfusionMatrix(fm) {
+    this.log("");
+    this.log(this.color("bold", `  Confusion Matrix: ${fm.field}`));
+    this.log("");
+    const matrixStr = formatConfusionMatrix(fm.metrics.confusionMatrix);
+    for (const line of matrixStr.split("\n")) {
+      this.log(`    ${line}`);
+    }
+    this.log("");
+  }
+  /**
+   * Formats a percentage
+   */
+  pct(value) {
+    return `${(value * 100).toFixed(1)}%`;
+  }
+  /**
+   * Formats duration
+   */
+  formatDuration(ms) {
+    if (ms < 1e3) {
+      return `${ms}ms`;
+    }
+    return `${(ms / 1e3).toFixed(2)}s`;
+  }
+  /**
+   * Gets status symbol
+   */
+  getStatusSymbol(status) {
+    switch (status) {
+      case "passed":
+        return symbols.pass;
+      case "failed":
+        return symbols.fail;
+      case "error":
+        return symbols.error;
+      case "skipped":
+        return symbols.skip;
+    }
+  }
+  /**
+   * Gets status color
+   */
+  getStatusColor(status) {
+    switch (status) {
+      case "passed":
+        return "green";
+      case "failed":
+        return "red";
+      case "error":
+        return "red";
+      case "skipped":
+        return "yellow";
+    }
+  }
+  /**
+   * Applies color if enabled
+   */
+  color(colorName, text) {
+    if (!this.useColors) {
+      return text;
+    }
+    return `${colors[colorName]}${text}${colors.reset}`;
+  }
+  /**
+   * Logs a line
+   */
+  log(message) {
+    console.log(message);
+  }
+};
+var DEFAULT_PATTERNS = [
+  "**/*.eval.js",
+  "**/*.eval.ts",
+  "**/*.eval.mjs"
+];
+var DEFAULT_IGNORE = [
+  "**/node_modules/**",
+  "**/dist/**",
+  "**/build/**",
+  "**/.git/**"
+];
+async function discoverEvalFiles(options = {}) {
+  const {
+    patterns = DEFAULT_PATTERNS,
+    ignore = DEFAULT_IGNORE,
+    cwd = process.cwd()
+  } = options;
+  const files = [];
+  for (const pattern of patterns) {
+    const matches = await glob(pattern, {
+      cwd,
+      ignore,
+      absolute: true,
+      nodir: true
+    });
+    files.push(...matches);
+  }
+  const unique = [...new Set(files)].sort();
+  return unique;
+}
+async function discoverFromPath(path, options = {}) {
+  const absolutePath = resolve(process.cwd(), path);
+  if (!existsSync(absolutePath)) {
+    throw new Error(`Path does not exist: ${path}`);
+  }
+  const { statSync } = await import('fs');
+  const stat = statSync(absolutePath);
+  if (stat.isFile()) {
+    return [absolutePath];
+  }
+  return discoverEvalFiles({
+    ...options,
+    cwd: absolutePath
+  });
+}
+function filterFiles(files, filter) {
+  if (!filter) {
+    return files;
+  }
+  const filterLower = filter.toLowerCase();
+  return files.filter((file) => file.toLowerCase().includes(filterLower));
+}
+
+// src/core/types.ts
+var ExitCodes = {
+  SUCCESS: 0,
+  ASSERTION_FAILURE: 1,
+  INTEGRITY_FAILURE: 2,
+  EXECUTION_ERROR: 3,
+  CONFIGURATION_ERROR: 4
+};
+
+// src/core/context.ts
+var globalContext = createEmptyContext();
+var currentTestState = null;
+function createEmptyContext() {
+  return {
+    currentSuite: null,
+    suites: [],
+    results: []
+  };
+}
+function getCurrentSuite() {
+  return globalContext.currentSuite;
+}
+function setCurrentSuite(suite) {
+  globalContext.currentSuite = suite;
+}
+function addSuite(suite) {
+  globalContext.suites.push(suite);
+}
+function addTestToCurrentSuite(test) {
+  if (!globalContext.currentSuite) {
+    throw new Error("Cannot add test outside of a describe() block");
+  }
+  globalContext.currentSuite.tests.push(test);
+}
+function getSuites() {
+  return globalContext.suites;
+}
+function startTestExecution() {
+  currentTestState = {
+    assertions: [],
+    fieldMetrics: []
+  };
+}
+function endTestExecution() {
+  const state = currentTestState;
+  currentTestState = null;
+  return state ?? { assertions: [], fieldMetrics: [] };
+}
+function recordAssertion(result) {
+  if (currentTestState) {
+    currentTestState.assertions.push(result);
+  }
+}
+function recordFieldMetrics(metrics) {
+  if (currentTestState) {
+    currentTestState.fieldMetrics.push(metrics);
+  }
+}
+
+// src/runner/executor.ts
+async function executeEvalFiles(files, options = {}) {
+  const startTime = Date.now();
+  const suiteResults = [];
+  for (const file of files) {
+    try {
+      const fileUrl = pathToFileURL(file).href;
+      await import(fileUrl);
+    } catch (error) {
+      throw new TestExecutionError(`Failed to load eval file: ${file}`, file, error);
+    }
+  }
+  const suites = getSuites();
+  for (const suite of suites) {
+    const result = await executeSuite(suite, options);
+    suiteResults.push(result);
+    if (options.bail && result.failed > 0) {
+      break;
+    }
+  }
+  const report = buildReport(suiteResults, Date.now() - startTime);
+  return report;
+}
+async function executeSuite(suite, options) {
+  const startTime = Date.now();
+  const testResults = [];
+  let passed = 0;
+  let failed = 0;
+  let errors = 0;
+  let skipped = 0;
+  for (const hook of suite.beforeAll ?? []) {
+    try {
+      await hook();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      for (const test of suite.tests) {
+        testResults.push({
+          name: test.name,
+          status: "error",
+          assertions: [],
+          fieldMetrics: [],
+          duration: 0,
+          error: new Error(`beforeAll hook failed: ${message}`)
+        });
+        errors++;
+      }
+      return {
+        name: suite.name,
+        tests: testResults,
+        passed,
+        failed,
+        errors,
+        skipped,
+        duration: Date.now() - startTime
+      };
+    }
+  }
+  for (const test of suite.tests) {
+    if (options.filter && !test.name.toLowerCase().includes(options.filter.toLowerCase())) {
+      testResults.push({
+        name: test.name,
+        status: "skipped",
+        assertions: [],
+        fieldMetrics: [],
+        duration: 0
+      });
+      skipped++;
+      continue;
+    }
+    if (test.name.startsWith("[SKIPPED]")) {
+      testResults.push({
+        name: test.name,
+        status: "skipped",
+        assertions: [],
+        fieldMetrics: [],
+        duration: 0
+      });
+      skipped++;
+      continue;
+    }
+    for (const hook of suite.beforeEach ?? []) {
+      try {
+        await hook();
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        testResults.push({
+          name: test.name,
+          status: "error",
+          assertions: [],
+          fieldMetrics: [],
+          duration: 0,
+          error: new Error(`beforeEach hook failed: ${message}`)
+        });
+        errors++;
+        continue;
+      }
+    }
+    const result = await executeTest(test.name, test.fn, options.timeout);
+    testResults.push(result);
+    if (result.status === "passed") {
+      passed++;
+    } else if (result.status === "failed") {
+      failed++;
+    } else if (result.status === "error") {
+      errors++;
+    }
+    for (const hook of suite.afterEach ?? []) {
+      try {
+        await hook();
+      } catch {
+      }
+    }
+    if (options.bail && (failed > 0 || errors > 0)) {
+      break;
+    }
+  }
+  for (const hook of suite.afterAll ?? []) {
+    try {
+      await hook();
+    } catch {
+    }
+  }
+  return {
+    name: suite.name,
+    tests: testResults,
+    passed,
+    failed,
+    errors,
+    skipped,
+    duration: Date.now() - startTime
+  };
+}
+async function executeTest(name, fn, timeout = 3e4) {
+  const startTime = Date.now();
+  startTestExecution();
+  try {
+    await Promise.race([
+      fn(),
+      new Promise(
+        (_, reject) => setTimeout(() => reject(new Error(`Test timed out after ${timeout}ms`)), timeout)
+      )
+    ]);
+    const { assertions, fieldMetrics } = endTestExecution();
+    return {
+      name,
+      status: "passed",
+      assertions,
+      fieldMetrics,
+      duration: Date.now() - startTime
+    };
+  } catch (error) {
+    const { assertions, fieldMetrics } = endTestExecution();
+    if (error instanceof AssertionError) {
+      return {
+        name,
+        status: "failed",
+        assertions,
+        fieldMetrics,
+        duration: Date.now() - startTime,
+        error
+      };
+    }
+    return {
+      name,
+      status: "error",
+      assertions,
+      fieldMetrics,
+      duration: Date.now() - startTime,
+      error: error instanceof Error ? error : new Error(String(error))
+    };
+  }
+}
+function buildReport(suiteResults, totalDuration) {
+  let totalTests = 0;
+  let totalPassed = 0;
+  let totalFailed = 0;
+  let totalErrors = 0;
+  let totalSkipped = 0;
+  for (const suite of suiteResults) {
+    totalTests += suite.tests.length;
+    totalPassed += suite.passed;
+    totalFailed += suite.failed;
+    totalErrors += suite.errors;
+    totalSkipped += suite.skipped;
+  }
+  return {
+    version: "1.0.0",
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    suites: suiteResults,
+    summary: {
+      totalSuites: suiteResults.length,
+      totalTests,
+      passed: totalPassed,
+      failed: totalFailed,
+      errors: totalErrors,
+      skipped: totalSkipped,
+      duration: totalDuration
+    }
+  };
+}
+function getExitCode(report) {
+  if (report.summary.errors > 0) {
+    return ExitCodes.EXECUTION_ERROR;
+  }
+  if (report.summary.failed > 0) {
+    return ExitCodes.ASSERTION_FAILURE;
+  }
+  return ExitCodes.SUCCESS;
+}
+
+export { AssertionError, ConfigurationError, ConsoleReporter, DatasetError, EvalSenseError, ExitCodes, IntegrityError, JsonReporter, TestExecutionError, addSuite, addTestToCurrentSuite, buildConfusionMatrix, discoverEvalFiles, discoverFromPath, executeEvalFiles, filterFiles, formatConfusionMatrix, getCurrentSuite, getExitCode, getFalseNegatives, getFalsePositives, getSupport, getTruePositives, parseReport, recordAssertion, recordFieldMetrics, setCurrentSuite };
+//# sourceMappingURL=chunk-5P7LNNO6.js.map
+//# sourceMappingURL=chunk-5P7LNNO6.js.map