@cogitator-ai/evals 0.1.2 → 0.1.10

package/README.md

@@ -397,7 +397,7 @@ result.report('ci');
 | `Dataset`        | Immutable dataset with from/fromJsonl/fromCsv/filter/sample/shuffle |
 | `loadJsonl`      | Low-level JSONL file loader                                         |
 | `loadCsv`        | Low-level CSV file loader                                           |
-| `isLLMMetric`    | Type guard: checks if a `MetricFn` is an `LLMMetricFn`             |
+| `isLLMMetric`    | Type guard: checks if a `MetricFn` is an `LLMMetricFn`              |
 
 ### Metrics
 

package/dist/assertions/custom.d.ts

@@ -1,4 +1,4 @@
-import type { AssertionFn, AggregatedMetric } from './index';
+import type { AssertionFn, AggregatedMetric } from './index.js';
 export declare function assertion(opts: {
     name: string;
     check: (aggregated: Record<string, AggregatedMetric>, stats: {

package/dist/assertions/index.d.ts

@@ -21,7 +21,7 @@ export type AssertionFn = (aggregated: Record<string, AggregatedMetric>, stats:
     duration: number;
     cost: number;
 }) => AssertionResult;
-export { threshold } from './threshold';
-export { noRegression } from './regression';
-export { assertion } from './custom';
+export { threshold } from './threshold.js';
+export { noRegression } from './regression.js';
+export { assertion } from './custom.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/assertions/index.js

@@ -1,4 +1,4 @@
-export { threshold } from './threshold';
-export { noRegression } from './regression';
-export { assertion } from './custom';
+export { threshold } from './threshold.js';
+export { noRegression } from './regression.js';
+export { assertion } from './custom.js';
 //# sourceMappingURL=index.js.map

package/dist/assertions/regression.d.ts

@@ -1,4 +1,4 @@
-import type { AssertionFn } from './index';
+import type { AssertionFn } from './index.js';
 export declare function noRegression(baselinePath: string, opts?: {
     tolerance?: number;
 }): AssertionFn;

package/dist/assertions/threshold.d.ts

@@ -1,3 +1,3 @@
-import type { AssertionFn } from './index';
+import type { AssertionFn } from './index.js';
 export declare function threshold(metricName: string, value: number): AssertionFn;
 //# sourceMappingURL=threshold.d.ts.map

package/dist/datasets/csv-loader.d.ts

@@ -1,3 +1,3 @@
-import type { EvalCase } from '../schema';
+import type { EvalCase } from '../schema.js';
 export declare function loadCsv(path: string): Promise<EvalCase[]>;
 //# sourceMappingURL=csv-loader.d.ts.map

package/dist/datasets/csv-loader.js

@@ -1,5 +1,5 @@
 import { readFile } from 'node:fs/promises';
-import { EvalCaseSchema } from '../schema';
+import { EvalCaseSchema } from '../schema.js';
 export async function loadCsv(path) {
     let Papa;
     try {

package/dist/datasets/dataset.d.ts

@@ -1,4 +1,4 @@
-import type { EvalCase } from '../schema';
+import type { EvalCase } from '../schema.js';
 export declare class Dataset {
     private readonly _cases;
     private constructor();

package/dist/datasets/dataset.js

@@ -1,6 +1,6 @@
-import { EvalCaseSchema } from '../schema';
-import { loadJsonl } from './jsonl-loader';
-import { loadCsv } from './csv-loader';
+import { EvalCaseSchema } from '../schema.js';
+import { loadJsonl } from './jsonl-loader.js';
+import { loadCsv } from './csv-loader.js';
 export class Dataset {
     _cases;
     constructor(cases) {

package/dist/datasets/index.d.ts

@@ -1,4 +1,4 @@
-export { Dataset } from './dataset';
-export { loadJsonl } from './jsonl-loader';
-export { loadCsv } from './csv-loader';
+export { Dataset } from './dataset.js';
+export { loadJsonl } from './jsonl-loader.js';
+export { loadCsv } from './csv-loader.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/datasets/index.js

@@ -1,4 +1,4 @@
-export { Dataset } from './dataset';
-export { loadJsonl } from './jsonl-loader';
-export { loadCsv } from './csv-loader';
+export { Dataset } from './dataset.js';
+export { loadJsonl } from './jsonl-loader.js';
+export { loadCsv } from './csv-loader.js';
 //# sourceMappingURL=index.js.map

package/dist/datasets/jsonl-loader.d.ts

@@ -1,3 +1,3 @@
-import type { EvalCase } from '../schema';
+import type { EvalCase } from '../schema.js';
 export declare function loadJsonl(path: string): Promise<EvalCase[]>;
 //# sourceMappingURL=jsonl-loader.d.ts.map

package/dist/datasets/jsonl-loader.js

@@ -1,6 +1,6 @@
 import { createReadStream } from 'node:fs';
 import { createInterface } from 'node:readline';
-import { EvalCaseSchema } from '../schema';
+import { EvalCaseSchema } from '../schema.js';
 export async function loadJsonl(path) {
     const cases = [];
     let lineNumber = 0;

package/dist/eval-builder.d.ts

@@ -1,9 +1,9 @@
-import { EvalSuite } from './eval-suite';
-import type { EvalTarget, EvalSuiteOptions } from './eval-suite';
-import { Dataset } from './datasets';
-import type { MetricFn, StatisticalMetricFn } from './metrics/types';
-import type { JudgeConfig } from './schema';
-import type { AssertionFn } from './assertions';
+import { EvalSuite } from './eval-suite.js';
+import type { EvalTarget, EvalSuiteOptions } from './eval-suite.js';
+import { Dataset } from './datasets/index.js';
+import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
+import type { JudgeConfig } from './schema.js';
+import type { AssertionFn } from './assertions/index.js';
 export declare class EvalBuilder {
     private _dataset?;
     private _target?;

package/dist/eval-builder.js

@@ -1,4 +1,4 @@
-import { EvalSuite, isLLMMetric } from './eval-suite';
+import { EvalSuite, isLLMMetric } from './eval-suite.js';
 export class EvalBuilder {
     _dataset;
     _target;

package/dist/eval-comparison.d.ts

@@ -1,7 +1,7 @@
-import type { EvalTarget, EvalSuiteResult } from './eval-suite';
-import type { MetricFn, StatisticalMetricFn } from './metrics/types';
-import type { JudgeConfig } from './schema';
-import { Dataset } from './datasets';
+import type { EvalTarget, EvalSuiteResult } from './eval-suite.js';
+import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
+import type { JudgeConfig } from './schema.js';
+import { Dataset } from './datasets/index.js';
 export interface EvalComparisonOptions {
     dataset: Dataset;
     targets: {

package/dist/eval-comparison.js

@@ -1,7 +1,7 @@
-import { EvalSuite } from './eval-suite';
-import { pairedTTest } from './stats/t-test';
-import { mcnemarsTest } from './stats/mcnemar';
-import { mean } from './stats/percentiles';
+import { EvalSuite } from './eval-suite.js';
+import { pairedTTest } from './stats/t-test.js';
+import { mcnemarsTest } from './stats/mcnemar.js';
+import { mean } from './stats/percentiles.js';
 function isBinary(scores) {
     return scores.every((s) => s === 0 || s === 1);
 }

package/dist/eval-suite.d.ts

@@ -1,10 +1,10 @@
-import { Dataset } from './datasets';
-import type { EvalCase } from './schema';
-import type { JudgeConfig } from './schema';
-import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types';
-import type { LLMMetricFn } from './metrics/llm-judge';
-import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions';
-import type { ReporterType, ReporterOptions } from './reporters';
+import { Dataset } from './datasets/index.js';
+import type { EvalCase } from './schema.js';
+import type { JudgeConfig } from './schema.js';
+import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
+import type { LLMMetricFn } from './metrics/llm-judge.js';
+import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions/index.js';
+import type { ReporterType, ReporterOptions } from './reporters/index.js';
 export interface EvalTarget {
     agent?: unknown;
     cogitator?: unknown;

package/dist/eval-suite.js

@@ -1,8 +1,8 @@
 import { writeFileSync } from 'node:fs';
-import { EvalSuiteConfigSchema } from './schema';
-import { bindJudgeContext } from './metrics/llm-judge';
-import { aggregate } from './stats';
-import { report } from './reporters';
+import { EvalSuiteConfigSchema } from './schema.js';
+import { bindJudgeContext } from './metrics/llm-judge.js';
+import { aggregate } from './stats/index.js';
+import { report } from './reporters/index.js';
 export function isLLMMetric(m) {
     return 'requiresJudge' in m && m.requiresJudge === true;
 }

package/dist/index.d.ts

@@ -1,31 +1,31 @@
 export declare const VERSION = "0.1.0";
-export { EvalSuite, isLLMMetric } from './eval-suite';
-export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite';
-export { EvalComparison } from './eval-comparison';
-export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison';
-export { EvalBuilder } from './eval-builder';
-export { Dataset } from './datasets';
-export { loadJsonl } from './datasets';
-export { loadCsv } from './datasets';
-export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge';
-export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge';
-export { latency, cost, tokenUsage } from './metrics/statistical';
-export { metric } from './metrics/custom';
-export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types';
-export type { CustomMetricConfig } from './metrics/custom';
-export { threshold } from './assertions';
-export { noRegression } from './assertions';
-export { assertion } from './assertions';
-export type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions';
-export { report } from './reporters';
-export type { ReporterType, ReporterOptions } from './reporters';
-export { pairedTTest } from './stats/t-test';
-export type { TTestResult } from './stats/t-test';
-export { mcnemarsTest } from './stats/mcnemar';
-export type { McNemarResult } from './stats/mcnemar';
-export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles';
-export type { EvalCase, EvalSuiteConfig, JudgeConfig, EvalComparisonConfig } from './schema';
-export { createRunEvalTool, evalTools } from './tools';
-export type { EvalTool } from './tools';
+export { EvalSuite, isLLMMetric } from './eval-suite.js';
+export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite.js';
+export { EvalComparison } from './eval-comparison.js';
+export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison.js';
+export { EvalBuilder } from './eval-builder.js';
+export { Dataset } from './datasets/index.js';
+export { loadJsonl } from './datasets/index.js';
+export { loadCsv } from './datasets/index.js';
+export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
+export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { metric } from './metrics/custom.js';
+export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
+export type { CustomMetricConfig } from './metrics/custom.js';
+export { threshold } from './assertions/index.js';
+export { noRegression } from './assertions/index.js';
+export { assertion } from './assertions/index.js';
+export type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions/index.js';
+export { report } from './reporters/index.js';
+export type { ReporterType, ReporterOptions } from './reporters/index.js';
+export { pairedTTest } from './stats/t-test.js';
+export type { TTestResult } from './stats/t-test.js';
+export { mcnemarsTest } from './stats/mcnemar.js';
+export type { McNemarResult } from './stats/mcnemar.js';
+export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
+export type { EvalCase, EvalSuiteConfig, JudgeConfig, EvalComparisonConfig } from './schema.js';
+export { createRunEvalTool, evalTools } from './tools.js';
+export type { EvalTool } from './tools.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,20 +1,20 @@
 export const VERSION = '0.1.0';
-export { EvalSuite, isLLMMetric } from './eval-suite';
-export { EvalComparison } from './eval-comparison';
-export { EvalBuilder } from './eval-builder';
-export { Dataset } from './datasets';
-export { loadJsonl } from './datasets';
-export { loadCsv } from './datasets';
-export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge';
-export { latency, cost, tokenUsage } from './metrics/statistical';
-export { metric } from './metrics/custom';
-export { threshold } from './assertions';
-export { noRegression } from './assertions';
-export { assertion } from './assertions';
-export { report } from './reporters';
-export { pairedTTest } from './stats/t-test';
-export { mcnemarsTest } from './stats/mcnemar';
-export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles';
-export { createRunEvalTool, evalTools } from './tools';
+export { EvalSuite, isLLMMetric } from './eval-suite.js';
+export { EvalComparison } from './eval-comparison.js';
+export { EvalBuilder } from './eval-builder.js';
+export { Dataset } from './datasets/index.js';
+export { loadJsonl } from './datasets/index.js';
+export { loadCsv } from './datasets/index.js';
+export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { metric } from './metrics/custom.js';
+export { threshold } from './assertions/index.js';
+export { noRegression } from './assertions/index.js';
+export { assertion } from './assertions/index.js';
+export { report } from './reporters/index.js';
+export { pairedTTest } from './stats/t-test.js';
+export { mcnemarsTest } from './stats/mcnemar.js';
+export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
+export { createRunEvalTool, evalTools } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/metrics/custom.d.ts

@@ -1,4 +1,4 @@
-import type { MetricFn } from './types';
+import type { MetricFn } from './types.js';
 export interface CustomMetricConfig {
     name: string;
     evaluate: (data: {

package/dist/metrics/deterministic.d.ts

@@ -1,5 +1,5 @@
 import type { ZodType } from 'zod';
-import type { MetricFn } from './types';
+import type { MetricFn } from './types.js';
 interface MatchOptions {
     caseSensitive?: boolean;
 }

package/dist/metrics/index.d.ts

@@ -1,8 +1,8 @@
-export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn } from './types';
-export { exactMatch, contains, regex, jsonSchema } from './deterministic';
-export { metric } from './custom';
-export type { CustomMetricConfig } from './custom';
-export { latency, cost, tokenUsage } from './statistical';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge';
-export type { JudgeContext, LLMMetricFn } from './llm-judge';
+export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn } from './types.js';
+export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
+export { metric } from './custom.js';
+export type { CustomMetricConfig } from './custom.js';
+export { latency, cost, tokenUsage } from './statistical.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
+export type { JudgeContext, LLMMetricFn } from './llm-judge.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.js

@@ -1,5 +1,5 @@
-export { exactMatch, contains, regex, jsonSchema } from './deterministic';
-export { metric } from './custom';
-export { latency, cost, tokenUsage } from './statistical';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge';
+export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
+export { metric } from './custom.js';
+export { latency, cost, tokenUsage } from './statistical.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
 //# sourceMappingURL=index.js.map

package/dist/metrics/llm-judge.d.ts

@@ -1,5 +1,5 @@
-import type { JudgeConfig } from '../schema';
-import type { MetricFn } from './types';
+import type { JudgeConfig } from '../schema.js';
+import type { MetricFn } from './types.js';
 export interface JudgeContext {
     cogitator: {
         run: (opts: {

package/dist/metrics/statistical.d.ts

@@ -1,4 +1,4 @@
-import type { StatisticalMetricFn } from './types';
+import type { StatisticalMetricFn } from './types.js';
 export declare function latency(): StatisticalMetricFn;
 export declare function cost(): StatisticalMetricFn;
 export declare function tokenUsage(): StatisticalMetricFn;

package/dist/metrics/statistical.js

@@ -1,4 +1,4 @@
-import { aggregate, mean } from '../stats';
+import { aggregate, mean } from '../stats/index.js';
 function createStatisticalFn(name, fn) {
     const statFn = fn;
     statFn.metricName = name;

package/dist/metrics/types.d.ts

@@ -1,4 +1,4 @@
-import type { EvalCase } from '../schema';
+import type { EvalCase } from '../schema.js';
 export interface MetricScore {
     name: string;
     score: number;

package/dist/reporters/ci.d.ts

@@ -1,3 +1,3 @@
-import type { EvalSuiteResult } from './index';
+import type { EvalSuiteResult } from './index.js';
 export declare function ciReport(result: EvalSuiteResult): void;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporters/console.d.ts

@@ -1,3 +1,3 @@
-import type { EvalSuiteResult } from './index';
+import type { EvalSuiteResult } from './index.js';
 export declare function consoleReport(result: EvalSuiteResult): void;
 //# sourceMappingURL=console.d.ts.map

package/dist/reporters/csv.d.ts

@@ -1,4 +1,4 @@
-import type { EvalSuiteResult } from './index';
+import type { EvalSuiteResult } from './index.js';
 export declare function csvReport(result: EvalSuiteResult, options: {
     path: string;
 }): void;

package/dist/reporters/index.d.ts

@@ -43,8 +43,8 @@ export type ReporterOptions = {
     path?: string;
 };
 export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
-export { consoleReport } from './console';
-export { jsonReport } from './json';
-export { csvReport } from './csv';
-export { ciReport } from './ci';
+export { consoleReport } from './console.js';
+export { jsonReport } from './json.js';
+export { csvReport } from './csv.js';
+export { ciReport } from './ci.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.js

@@ -1,7 +1,7 @@
-import { consoleReport } from './console';
-import { jsonReport } from './json';
-import { csvReport } from './csv';
-import { ciReport } from './ci';
+import { consoleReport } from './console.js';
+import { jsonReport } from './json.js';
+import { csvReport } from './csv.js';
+import { ciReport } from './ci.js';
 export function report(result, type, options) {
     const types = Array.isArray(type) ? type : [type];
     for (const t of types) {
@@ -21,8 +21,8 @@ export function report(result, type, options) {
         }
     }
 }
-export { consoleReport } from './console';
-export { jsonReport } from './json';
-export { csvReport } from './csv';
-export { ciReport } from './ci';
+export { consoleReport } from './console.js';
+export { jsonReport } from './json.js';
+export { csvReport } from './csv.js';
+export { ciReport } from './ci.js';
 //# sourceMappingURL=index.js.map

package/dist/reporters/json.d.ts

@@ -1,4 +1,4 @@
-import type { EvalSuiteResult } from './index';
+import type { EvalSuiteResult } from './index.js';
 export declare function jsonReport(result: EvalSuiteResult, options: {
     path: string;
 }): void;

package/dist/stats/index.d.ts

@@ -1,6 +1,6 @@
-export { percentile, mean, median, stdDev, aggregate } from './percentiles';
-export { pairedTTest } from './t-test';
-export type { TTestResult } from './t-test';
-export { mcnemarsTest } from './mcnemar';
-export type { McNemarResult } from './mcnemar';
+export { percentile, mean, median, stdDev, aggregate } from './percentiles.js';
+export { pairedTTest } from './t-test.js';
+export type { TTestResult } from './t-test.js';
+export { mcnemarsTest } from './mcnemar.js';
+export type { McNemarResult } from './mcnemar.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/stats/index.js

@@ -1,4 +1,4 @@
-export { percentile, mean, median, stdDev, aggregate } from './percentiles';
-export { pairedTTest } from './t-test';
-export { mcnemarsTest } from './mcnemar';
+export { percentile, mean, median, stdDev, aggregate } from './percentiles.js';
+export { pairedTTest } from './t-test.js';
+export { mcnemarsTest } from './mcnemar.js';
 //# sourceMappingURL=index.js.map

package/dist/stats/t-test.js

@@ -1,4 +1,4 @@
-import { mean, stdDev } from './percentiles';
+import { mean, stdDev } from './percentiles.js';
 function lnGamma(z) {
     const g = 7;
     const coefficients = [

package/dist/tools.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import type { EvalSuite } from './eval-suite';
+import type { EvalSuite } from './eval-suite.js';
 export interface EvalTool<TParams = unknown> {
     name: string;
     description: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/evals",
-  "version": "0.1.2",
+  "version": "0.1.10",
   "description": "Evaluation framework for Cogitator AI agents",
   "type": "module",
   "main": "./dist/index.js",