evalsense 0.2.0

package/dist/types-C71p0wzM.d.cts

@@ -0,0 +1,265 @@
+/**
+ * Core type definitions for EvalSense
+ */
+/**
+ * A loaded dataset with records and metadata
+ */
+interface Dataset<T = Record<string, unknown>> {
+    records: T[];
+    metadata: DatasetMetadata;
+}
+interface DatasetMetadata {
+    source: string;
+    count: number;
+    loadedAt: Date;
+}
+/**
+ * A record aligned between actual (model output) and expected (ground truth)
+ */
+interface AlignedRecord {
+    id: string;
+    actual: Record<string, unknown>;
+    expected: Record<string, unknown>;
+}
+/**
+ * Output from runModel() - predictions with IDs for alignment
+ */
+interface Prediction {
+    id: string;
+    [field: string]: unknown;
+}
+/**
+ * JSON Schema for structured LLM outputs
+ */
+interface JSONSchema {
+    type: string;
+    properties?: Record<string, unknown>;
+    required?: string[];
+    [key: string]: unknown;
+}
+/**
+ * LLM client interface for metric evaluation
+ */
+interface LLMClient {
+    /**
+     * Generate a text completion from a prompt
+     */
+    complete(prompt: string): Promise<string>;
+    /**
+     * Generate a structured JSON completion (optional)
+     */
+    completeStructured?<T>(prompt: string, schema: JSONSchema): Promise<T>;
+}
+/**
+ * Output from an LLM metric evaluation
+ */
+interface MetricOutput {
+    id: string;
+    metric: string;
+    score: number;
+    label?: string;
+    /** LLM's reasoning/explanation (for LLM-based metrics) */
+    reasoning?: string;
+    /** Evaluation mode used (for LLM-based metrics) */
+    evaluationMode?: "per-row" | "batch";
+}
+/**
+ * Configuration for a metric function
+ */
+interface MetricConfig {
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    context?: string[];
+    query?: string[];
+    source?: string[];
+    /** LLM client override (defaults to global client) */
+    llmClient?: LLMClient;
+    /** Evaluation mode: per-row (accurate, expensive) or batch (cheaper, potentially less accurate) */
+    evaluationMode?: "per-row" | "batch";
+    /** Custom prompt template override */
+    customPrompt?: string;
+    /** LLM temperature (default: 0) */
+    temperature?: number;
+    /** Max tokens per completion */
+    maxTokens?: number;
+    /** Timeout in milliseconds */
+    timeout?: number;
+}
+/**
+ * A metric function that evaluates outputs
+ */
+type MetricFn = (config: MetricConfig) => Promise<MetricOutput[]>;
+/**
+ * Confusion matrix with labels
+ */
+interface ConfusionMatrix {
+    matrix: number[][];
+    labels: string[];
+    total: number;
+}
+/**
+ * Per-class classification metrics
+ */
+interface ClassMetrics {
+    precision: number;
+    recall: number;
+    f1: number;
+    support: number;
+}
+/**
+ * Full classification metrics result
+ */
+interface ClassificationMetrics {
+    accuracy: number;
+    perClass: Record<string, ClassMetrics>;
+    macroAvg: {
+        precision: number;
+        recall: number;
+        f1: number;
+    };
+    weightedAvg: {
+        precision: number;
+        recall: number;
+        f1: number;
+    };
+    confusionMatrix: ConfusionMatrix;
+}
+/**
+ * Regression metrics result
+ */
+interface RegressionMetrics {
+    mae: number;
+    mse: number;
+    rmse: number;
+    r2: number;
+}
+/**
+ * Result of evaluating a single field across all predictions
+ */
+interface FieldMetricResult {
+    field: string;
+    metrics: ClassificationMetrics;
+    binarized: boolean;
+    binarizeThreshold?: number;
+}
+/**
+ * Test function signature
+ */
+type TestFn = () => Promise<void> | void;
+/**
+ * An individual eval test
+ */
+interface EvalTest {
+    name: string;
+    fn: TestFn;
+}
+/**
+ * A test suite (describe block)
+ */
+interface Suite {
+    name: string;
+    tests: EvalTest[];
+    beforeAll?: TestFn[];
+    afterAll?: TestFn[];
+    beforeEach?: TestFn[];
+    afterEach?: TestFn[];
+}
+/**
+ * Current test execution context
+ */
+interface TestContext {
+    currentSuite: Suite | null;
+    suites: Suite[];
+    results: SuiteResult[];
+}
+/**
+ * Result of a single assertion
+ */
+interface AssertionResult {
+    type: string;
+    passed: boolean;
+    message: string;
+    expected?: unknown;
+    actual?: unknown;
+    field?: string;
+    class?: string;
+}
+/**
+ * Result of a single test
+ */
+interface TestResult {
+    name: string;
+    status: "passed" | "failed" | "error" | "skipped";
+    assertions: AssertionResult[];
+    fieldMetrics: FieldMetricResult[];
+    duration: number;
+    error?: Error;
+}
+/**
+ * Result of a test suite
+ */
+interface SuiteResult {
+    name: string;
+    tests: TestResult[];
+    passed: number;
+    failed: number;
+    errors: number;
+    skipped: number;
+    duration: number;
+}
+/**
+ * Integrity check results for a dataset
+ */
+interface IntegrityResult {
+    valid: boolean;
+    totalRecords: number;
+    missingIds: string[];
+    duplicateIds: string[];
+    missingFields: Array<{
+        id: string;
+        fields: string[];
+    }>;
+}
+/**
+ * Final evaluation report
+ */
+interface EvalReport {
+    version: string;
+    timestamp: string;
+    suites: SuiteResult[];
+    summary: {
+        totalSuites: number;
+        totalTests: number;
+        passed: number;
+        failed: number;
+        errors: number;
+        skipped: number;
+        duration: number;
+    };
+    integrity?: IntegrityResult;
+}
+/**
+ * CLI configuration options
+ */
+interface CLIOptions {
+    filter?: string;
+    output?: string;
+    reporter?: "json" | "console" | "both";
+    bail?: boolean;
+    timeout?: number;
+}
+/**
+ * Exit codes for CI integration
+ */
+declare const ExitCodes: {
+    readonly SUCCESS: 0;
+    readonly ASSERTION_FAILURE: 1;
+    readonly INTEGRITY_FAILURE: 2;
+    readonly EXECUTION_ERROR: 3;
+    readonly CONFIGURATION_ERROR: 4;
+};
+type ExitCode = (typeof ExitCodes)[keyof typeof ExitCodes];
+
+export { type AlignedRecord as A, type ClassificationMetrics as C, type Dataset as D, type EvalReport as E, type FieldMetricResult as F, type IntegrityResult as I, type JSONSchema as J, type LLMClient as L, type MetricFn as M, type Prediction as P, type RegressionMetrics as R, type Suite as S, type TestFn as T, type MetricConfig as a, type MetricOutput as b, type AssertionResult as c, type ConfusionMatrix as d, type CLIOptions as e, type ClassMetrics as f, type DatasetMetadata as g, type EvalTest as h, type ExitCode as i, ExitCodes as j, type SuiteResult as k, type TestContext as l, type TestResult as m };

package/dist/types-C71p0wzM.d.ts

@@ -0,0 +1,265 @@
+/**
+ * Core type definitions for EvalSense
+ */
+/**
+ * A loaded dataset with records and metadata
+ */
+interface Dataset<T = Record<string, unknown>> {
+    records: T[];
+    metadata: DatasetMetadata;
+}
+interface DatasetMetadata {
+    source: string;
+    count: number;
+    loadedAt: Date;
+}
+/**
+ * A record aligned between actual (model output) and expected (ground truth)
+ */
+interface AlignedRecord {
+    id: string;
+    actual: Record<string, unknown>;
+    expected: Record<string, unknown>;
+}
+/**
+ * Output from runModel() - predictions with IDs for alignment
+ */
+interface Prediction {
+    id: string;
+    [field: string]: unknown;
+}
+/**
+ * JSON Schema for structured LLM outputs
+ */
+interface JSONSchema {
+    type: string;
+    properties?: Record<string, unknown>;
+    required?: string[];
+    [key: string]: unknown;
+}
+/**
+ * LLM client interface for metric evaluation
+ */
+interface LLMClient {
+    /**
+     * Generate a text completion from a prompt
+     */
+    complete(prompt: string): Promise<string>;
+    /**
+     * Generate a structured JSON completion (optional)
+     */
+    completeStructured?<T>(prompt: string, schema: JSONSchema): Promise<T>;
+}
+/**
+ * Output from an LLM metric evaluation
+ */
+interface MetricOutput {
+    id: string;
+    metric: string;
+    score: number;
+    label?: string;
+    /** LLM's reasoning/explanation (for LLM-based metrics) */
+    reasoning?: string;
+    /** Evaluation mode used (for LLM-based metrics) */
+    evaluationMode?: "per-row" | "batch";
+}
+/**
+ * Configuration for a metric function
+ */
+interface MetricConfig {
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    context?: string[];
+    query?: string[];
+    source?: string[];
+    /** LLM client override (defaults to global client) */
+    llmClient?: LLMClient;
+    /** Evaluation mode: per-row (accurate, expensive) or batch (cheaper, potentially less accurate) */
+    evaluationMode?: "per-row" | "batch";
+    /** Custom prompt template override */
+    customPrompt?: string;
+    /** LLM temperature (default: 0) */
+    temperature?: number;
+    /** Max tokens per completion */
+    maxTokens?: number;
+    /** Timeout in milliseconds */
+    timeout?: number;
+}
+/**
+ * A metric function that evaluates outputs
+ */
+type MetricFn = (config: MetricConfig) => Promise<MetricOutput[]>;
+/**
+ * Confusion matrix with labels
+ */
+interface ConfusionMatrix {
+    matrix: number[][];
+    labels: string[];
+    total: number;
+}
+/**
+ * Per-class classification metrics
+ */
+interface ClassMetrics {
+    precision: number;
+    recall: number;
+    f1: number;
+    support: number;
+}
+/**
+ * Full classification metrics result
+ */
+interface ClassificationMetrics {
+    accuracy: number;
+    perClass: Record<string, ClassMetrics>;
+    macroAvg: {
+        precision: number;
+        recall: number;
+        f1: number;
+    };
+    weightedAvg: {
+        precision: number;
+        recall: number;
+        f1: number;
+    };
+    confusionMatrix: ConfusionMatrix;
+}
+/**
+ * Regression metrics result
+ */
+interface RegressionMetrics {
+    mae: number;
+    mse: number;
+    rmse: number;
+    r2: number;
+}
+/**
+ * Result of evaluating a single field across all predictions
+ */
+interface FieldMetricResult {
+    field: string;
+    metrics: ClassificationMetrics;
+    binarized: boolean;
+    binarizeThreshold?: number;
+}
+/**
+ * Test function signature
+ */
+type TestFn = () => Promise<void> | void;
+/**
+ * An individual eval test
+ */
+interface EvalTest {
+    name: string;
+    fn: TestFn;
+}
+/**
+ * A test suite (describe block)
+ */
+interface Suite {
+    name: string;
+    tests: EvalTest[];
+    beforeAll?: TestFn[];
+    afterAll?: TestFn[];
+    beforeEach?: TestFn[];
+    afterEach?: TestFn[];
+}
+/**
+ * Current test execution context
+ */
+interface TestContext {
+    currentSuite: Suite | null;
+    suites: Suite[];
+    results: SuiteResult[];
+}
+/**
+ * Result of a single assertion
+ */
+interface AssertionResult {
+    type: string;
+    passed: boolean;
+    message: string;
+    expected?: unknown;
+    actual?: unknown;
+    field?: string;
+    class?: string;
+}
+/**
+ * Result of a single test
+ */
+interface TestResult {
+    name: string;
+    status: "passed" | "failed" | "error" | "skipped";
+    assertions: AssertionResult[];
+    fieldMetrics: FieldMetricResult[];
+    duration: number;
+    error?: Error;
+}
+/**
+ * Result of a test suite
+ */
+interface SuiteResult {
+    name: string;
+    tests: TestResult[];
+    passed: number;
+    failed: number;
+    errors: number;
+    skipped: number;
+    duration: number;
+}
+/**
+ * Integrity check results for a dataset
+ */
+interface IntegrityResult {
+    valid: boolean;
+    totalRecords: number;
+    missingIds: string[];
+    duplicateIds: string[];
+    missingFields: Array<{
+        id: string;
+        fields: string[];
+    }>;
+}
+/**
+ * Final evaluation report
+ */
+interface EvalReport {
+    version: string;
+    timestamp: string;
+    suites: SuiteResult[];
+    summary: {
+        totalSuites: number;
+        totalTests: number;
+        passed: number;
+        failed: number;
+        errors: number;
+        skipped: number;
+        duration: number;
+    };
+    integrity?: IntegrityResult;
+}
+/**
+ * CLI configuration options
+ */
+interface CLIOptions {
+    filter?: string;
+    output?: string;
+    reporter?: "json" | "console" | "both";
+    bail?: boolean;
+    timeout?: number;
+}
+/**
+ * Exit codes for CI integration
+ */
+declare const ExitCodes: {
+    readonly SUCCESS: 0;
+    readonly ASSERTION_FAILURE: 1;
+    readonly INTEGRITY_FAILURE: 2;
+    readonly EXECUTION_ERROR: 3;
+    readonly CONFIGURATION_ERROR: 4;
+};
+type ExitCode = (typeof ExitCodes)[keyof typeof ExitCodes];
+
+export { type AlignedRecord as A, type ClassificationMetrics as C, type Dataset as D, type EvalReport as E, type FieldMetricResult as F, type IntegrityResult as I, type JSONSchema as J, type LLMClient as L, type MetricFn as M, type Prediction as P, type RegressionMetrics as R, type Suite as S, type TestFn as T, type MetricConfig as a, type MetricOutput as b, type AssertionResult as c, type ConfusionMatrix as d, type CLIOptions as e, type ClassMetrics as f, type DatasetMetadata as g, type EvalTest as h, type ExitCode as i, ExitCodes as j, type SuiteResult as k, type TestContext as l, type TestResult as m };

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "evalsense",
+  "version": "0.2.0",
+  "description": "JS-native LLM evaluation framework with Jest-like API and statistical assertions",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./metrics": {
+      "import": {
+        "types": "./dist/metrics/index.d.ts",
+        "default": "./dist/metrics/index.js"
+      },
+      "require": {
+        "types": "./dist/metrics/index.d.cts",
+        "default": "./dist/metrics/index.cjs"
+      }
+    },
+    "./metrics/opinionated": {
+      "import": {
+        "types": "./dist/metrics/opinionated/index.d.ts",
+        "default": "./dist/metrics/opinionated/index.js"
+      },
+      "require": {
+        "types": "./dist/metrics/opinionated/index.d.cts",
+        "default": "./dist/metrics/opinionated/index.cjs"
+      }
+    }
+  },
+  "bin": {
+    "evalsense": "./bin/evalsense.js"
+  },
+  "files": [
+    "dist",
+    "bin"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src tests",
+    "lint:fix": "eslint src tests --fix",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "llm",
+    "evaluation",
+    "testing",
+    "metrics",
+    "machine-learning",
+    "ai",
+    "statistical-testing"
+  ],
+  "author": "Mohit Joshi",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^12.1.0",
+    "glob": "^11.0.0",
+    "fast-json-stable-stringify": "^2.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "@typescript-eslint/eslint-plugin": "^8.18.0",
+    "@typescript-eslint/parser": "^8.18.0",
+    "@vitest/coverage-v8": "^2.1.8",
+    "eslint": "^9.17.0",
+    "prettier": "^3.4.2",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2",
+    "vitest": "^2.1.8"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}