@elsium-ai/testing 0.8.0 → 0.9.0

package/dist/dataset.d.ts

@@ -0,0 +1,16 @@
+import type { EvalCase } from './eval';
+export interface EvalDataset {
+    name: string;
+    version?: string;
+    cases: EvalCase[];
+}
+export interface DatasetLoaderOptions {
+    inputField?: string;
+    expectedField?: string;
+    nameField?: string;
+    tagsField?: string;
+}
+export declare function loadDatasetFromJSON(path: string, options?: DatasetLoaderOptions): Promise<EvalDataset>;
+export declare function loadDatasetFromCSV(path: string, options?: DatasetLoaderOptions): Promise<EvalDataset>;
+export declare function loadDataset(path: string, options?: DatasetLoaderOptions): Promise<EvalDataset>;
+//# sourceMappingURL=dataset.d.ts.map

package/dist/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../src/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEtC,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,QAAQ,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AA8BD,wBAAsB,mBAAmB,CACxC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAkBtB;AAgCD,wBAAsB,kBAAkB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAqBtB;AAYD,wBAAsB,WAAW,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAatB"}

package/dist/eval-compare.d.ts

@@ -0,0 +1,34 @@
+import type { EvalSuiteResult } from './eval';
+export interface EvalBaseline {
+    name: string;
+    timestamp: number;
+    score: number;
+    results: Array<{
+        name: string;
+        passed: boolean;
+        score: number;
+    }>;
+}
+export interface EvalComparison {
+    baselineName: string;
+    currentName: string;
+    baselineScore: number;
+    currentScore: number;
+    delta: number;
+    regressions: Array<{
+        name: string;
+        baselineScore: number;
+        currentScore: number;
+    }>;
+    improvements: Array<{
+        name: string;
+        baselineScore: number;
+        currentScore: number;
+    }>;
+    regression: boolean;
+}
+export declare function saveBaseline(result: EvalSuiteResult, dir: string): Promise<string>;
+export declare function loadBaseline(name: string, dir: string): Promise<EvalBaseline | null>;
+export declare function compareResults(baseline: EvalBaseline, current: EvalSuiteResult): EvalComparison;
+export declare function formatComparison(comparison: EvalComparison): string;
+//# sourceMappingURL=eval-compare.d.ts.map

package/dist/eval-compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-compare.d.ts","sourceRoot":"","sources":["../src/eval-compare.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AAE7C,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAChE;AAED,MAAM,WAAW,cAAc;IAC9B,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACjF,YAAY,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAClF,UAAU,EAAE,OAAO,CAAA;CACnB;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiBxF;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAQ1F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,GAAG,cAAc,CAyC/F;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,cAAc,GAAG,MAAM,CAkCnE"}

package/dist/index.d.ts

@@ -16,4 +16,8 @@ export { createPinStore, pinOutput } from './pinning';
 export type { Pin, PinStore, PinResult } from './pinning';
 export { assertDeterministic, assertStable } from './determinism';
 export type { DeterminismResult, StabilityResult } from './determinism';
+export { loadDataset, loadDatasetFromJSON, loadDatasetFromCSV } from './dataset';
+export type { EvalDataset, DatasetLoaderOptions } from './dataset';
+export { saveBaseline, loadBaseline, compareResults, formatComparison } from './eval-compare';
+export type { EvalBaseline, EvalComparison } from './eval-compare';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAG5F,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACvE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAGxE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAA;AACvD,YAAY,EACX,QAAQ,EACR,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,eAAe,EACf,QAAQ,GACR,MAAM,QAAQ,CAAA;AAGf,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC1E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAGnF,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAC9D,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAGvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,YAAY,EACX,kBAAkB,EAClB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GACf,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AACnE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAGzE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrD,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAGzD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAG5F,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACvE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAGxE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAA;AACvD,YAAY,EACX,QAAQ,EACR,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,eAAe,EACf,QAAQ,GACR,MAAM,QAAQ,CAAA;AAGf,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC1E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAGnF,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAC9D,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAGvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,YAAY,EACX,kBAAkB,EAClB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GACf,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AACnE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAGzE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrD,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAGzD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAGvE,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAA;AAChF,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAGlE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AAC7F,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA"}

package/dist/index.js

@@ -1288,14 +1288,217 @@ async function assertStable(fn, options) {
     variance
   };
 }
+// src/dataset.ts
+import { readFile } from "node:fs/promises";
+import { extname } from "node:path";
+function mapRecordToCase(record, options) {
+  const inputField = options?.inputField ?? "input";
+  const expectedField = options?.expectedField ?? "expected";
+  const nameField = options?.nameField ?? "name";
+  const tagsField = options?.tagsField ?? "tags";
+  const tags = record[tagsField];
+  let parsedTags;
+  if (typeof tags === "string") {
+    parsedTags = tags.split(",").map((t) => t.trim()).filter(Boolean);
+  } else if (Array.isArray(tags)) {
+    parsedTags = tags.map(String);
+  }
+  return {
+    name: String(record[nameField] ?? ""),
+    input: String(record[inputField] ?? ""),
+    expected: record[expectedField] !== undefined ? String(record[expectedField]) : undefined,
+    tags: parsedTags
+  };
+}
+async function loadDatasetFromJSON(path, options) {
+  const content = await readFile(path, "utf-8");
+  const parsed = JSON.parse(content);
+  if (Array.isArray(parsed)) {
+    return {
+      name: "",
+      cases: parsed.map((record) => mapRecordToCase(record, options))
+    };
+  }
+  return {
+    name: parsed.name ?? "",
+    version: parsed.version,
+    cases: (parsed.cases ?? []).map((record) => mapRecordToCase(record, options))
+  };
+}
+function parseCSVLine(line) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0;i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"' && line[i + 1] === '"') {
+        current += '"';
+        i++;
+      } else if (char === '"') {
+        inQuotes = false;
+      } else {
+        current += char;
+      }
+    } else if (char === '"') {
+      inQuotes = true;
+    } else if (char === ",") {
+      fields.push(current.trim());
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  fields.push(current.trim());
+  return fields;
+}
+async function loadDatasetFromCSV(path, options) {
+  const content = await readFile(path, "utf-8");
+  const lines = content.split(`
+`).filter((line) => line.trim().length > 0);
+  if (lines.length < 2) {
+    return { name: "", cases: [] };
+  }
+  const headers = parseCSVLine(lines[0]);
+  const cases = [];
+  for (let i = 1;i < lines.length; i++) {
+    const values = parseCSVLine(lines[i]);
+    const record = {};
+    for (let j = 0;j < headers.length; j++) {
+      record[headers[j]] = values[j] ?? "";
+    }
+    cases.push(mapRecordToCase(record, options));
+  }
+  return { name: "", cases };
+}
+async function loadDatasetFromJSONL(path, options) {
+  const content = await readFile(path, "utf-8");
+  const lines = content.split(`
+`).filter((line) => line.trim().length > 0);
+  const cases = lines.map((line) => mapRecordToCase(JSON.parse(line), options));
+  return { name: "", cases };
+}
+async function loadDataset(path, options) {
+  const ext = extname(path).toLowerCase();
+  switch (ext) {
+    case ".json":
+      return loadDatasetFromJSON(path, options);
+    case ".csv":
+      return loadDatasetFromCSV(path, options);
+    case ".jsonl":
+      return loadDatasetFromJSONL(path, options);
+    default:
+      throw new Error(`Unsupported dataset format: ${ext}`);
+  }
+}
+// src/eval-compare.ts
+import { mkdir, readFile as readFile2, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+async function saveBaseline(result, dir) {
+  await mkdir(dir, { recursive: true });
+  const baseline = {
+    name: result.name,
+    timestamp: Date.now(),
+    score: result.score,
+    results: result.results.map((r) => ({
+      name: r.name,
+      passed: r.passed,
+      score: r.score
+    }))
+  };
+  const filePath = join(dir, `${result.name}.baseline.json`);
+  await writeFile(filePath, JSON.stringify(baseline, null, "\t"), "utf-8");
+  return filePath;
+}
+async function loadBaseline(name, dir) {
+  const filePath = join(dir, `${name}.baseline.json`);
+  try {
+    const content = await readFile2(filePath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+function compareResults(baseline, current) {
+  const baselineMap = new Map(baseline.results.map((r) => [r.name, r]));
+  const regressions = [];
+  const improvements = [];
+  for (const result of current.results) {
+    const baselineResult = baselineMap.get(result.name);
+    if (!baselineResult)
+      continue;
+    if (result.score < baselineResult.score) {
+      regressions.push({
+        name: result.name,
+        baselineScore: baselineResult.score,
+        currentScore: result.score
+      });
+    } else if (result.score > baselineResult.score) {
+      improvements.push({
+        name: result.name,
+        baselineScore: baselineResult.score,
+        currentScore: result.score
+      });
+    }
+  }
+  const delta = current.score - baseline.score;
+  const hasFailedRegression = current.results.some((r) => {
+    const base = baselineMap.get(r.name);
+    return base?.passed && !r.passed;
+  });
+  return {
+    baselineName: baseline.name,
+    currentName: current.name,
+    baselineScore: baseline.score,
+    currentScore: current.score,
+    delta,
+    regressions,
+    improvements,
+    regression: delta < 0 || hasFailedRegression
+  };
+}
+function formatComparison(comparison) {
+  const lines = [];
+  const deltaSign = comparison.delta >= 0 ? "+" : "";
+  const deltaPercent = `${deltaSign}${(comparison.delta * 100).toFixed(1)}%`;
+  lines.push(`
+  Comparison: ${comparison.baselineName} -> ${comparison.currentName}`);
+  lines.push(`  ${"─".repeat(50)}`);
+  lines.push(`  Baseline: ${(comparison.baselineScore * 100).toFixed(1)}% | Current: ${(comparison.currentScore * 100).toFixed(1)}% | Delta: ${deltaPercent}`);
+  if (comparison.regressions.length > 0) {
+    lines.push(`
+  Regressions (${comparison.regressions.length}):`);
+    for (const r of comparison.regressions) {
+      lines.push(`    - ${r.name}: ${(r.baselineScore * 100).toFixed(1)}% -> ${(r.currentScore * 100).toFixed(1)}%`);
+    }
+  }
+  if (comparison.improvements.length > 0) {
+    lines.push(`
+  Improvements (${comparison.improvements.length}):`);
+    for (const imp of comparison.improvements) {
+      lines.push(`    + ${imp.name}: ${(imp.baselineScore * 100).toFixed(1)}% -> ${(imp.currentScore * 100).toFixed(1)}%`);
+    }
+  }
+  lines.push(`  ${"─".repeat(50)}`);
+  lines.push(`  Result: ${comparison.regression ? "REGRESSION DETECTED" : "OK"}`);
+  lines.push("");
+  return lines.join(`
+`);
+}
 export {
   testSnapshot,
+  saveBaseline,
   runEvalSuite,
   pinOutput,
   mockProvider,
   loadFixture,
+  loadDatasetFromJSON,
+  loadDatasetFromCSV,
+  loadDataset,
+  loadBaseline,
   hashOutput,
   formatEvalReport,
+  formatComparison,
   definePrompt,
   createSnapshotStore,
   createReplayRecorder,
@@ -1305,6 +1508,7 @@ export {
   createPromptRegistry,
   createPinStore,
   createFixture,
+  compareResults,
   assertStable,
   assertDeterministic
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@elsium-ai/testing",
-	"version": "0.8.0",
+	"version": "0.9.0",
 	"description": "Testing utilities, mock providers, fixtures, and eval framework for ElsiumAI",
 	"license": "MIT",
 	"author": "Eric Utrera <ebutrera9103@gmail.com>",
@@ -26,10 +26,10 @@
 		"dev": "bun --watch src/index.ts"
 	},
 	"dependencies": {
-		"@elsium-ai/core": "^0.8.0",
-		"@elsium-ai/gateway": "^0.8.0",
-		"@elsium-ai/agents": "^0.8.0",
-		"@elsium-ai/tools": "^0.8.0"
+		"@elsium-ai/core": "^0.9.0",
+		"@elsium-ai/gateway": "^0.9.0",
+		"@elsium-ai/agents": "^0.9.0",
+		"@elsium-ai/tools": "^0.9.0"
 	},
 	"devDependencies": {
 		"typescript": "^5.7.0"