@graphorin/evals 0.5.0

package/dist/scorers/trajectory/recovery-after-error.js

@@ -0,0 +1,37 @@
+//#region src/scorers/trajectory/recovery-after-error.ts
+/** @stable */
+function recoveryAfterError(options = {}) {
+	return {
+		name: options.name ?? "recovery-after-error",
+		async score({ output }) {
+			const calls = output.calls;
+			const errorIdx = [];
+			for (let i = 0; i < calls.length; i++) if (calls[i]?.status === "error") errorIdx.push(i);
+			if (errorIdx.length === 0) return {
+				pass: true,
+				score: 1,
+				reason: "no tool errors to recover from."
+			};
+			const unrecovered = errorIdx.filter((i) => {
+				for (let j = i + 1; j < calls.length; j++) if (calls[j]?.status === "ok") return false;
+				return true;
+			});
+			const pass = unrecovered.length === 0;
+			const score = (errorIdx.length - unrecovered.length) / errorIdx.length;
+			if (pass) return {
+				pass,
+				score,
+				reason: `recovered from ${errorIdx.length} tool error(s).`
+			};
+			return {
+				pass,
+				score,
+				reason: `${unrecovered.length}/${errorIdx.length} tool error(s) had no successful follow-up call.`
+			};
+		}
+	};
+}
+
+//#endregion
+export { recoveryAfterError };
+//# sourceMappingURL=recovery-after-error.js.map

package/dist/scorers/trajectory/recovery-after-error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery-after-error.js","names":["errorIdx: number[]"],"sources":["../../../src/scorers/trajectory/recovery-after-error.ts"],"sourcesContent":["/**\n * `recoveryAfterError` — passes when the harness recovered from every\n * tool error: each `'error'` call must be followed by at least one later\n * `'ok'` call (the run did not dead-end on a failure). A trajectory with\n * no errors passes trivially. This measures harness resilience — that a\n * surfaced `ToolError` re-enters the loop as a tool message and the agent\n * makes forward progress afterwards.\n *\n * @packageDocumentation\n */\n\nimport type { Scorer } from '@graphorin/observability/eval';\nimport type { Trajectory } from './types.js';\n\n/** @stable */\nexport interface RecoveryAfterErrorOptions {\n  /** Optional name override. */\n  readonly name?: string;\n}\n\n/** @stable */\nexport function recoveryAfterError<I = unknown>(\n  options: RecoveryAfterErrorOptions = {},\n): Scorer<I, Trajectory> {\n  const name = options.name ?? 'recovery-after-error';\n  return {\n    name,\n    async score({ output }) {\n      const calls = output.calls;\n      const errorIdx: number[] = [];\n      for (let i = 0; i < calls.length; i++) {\n        if (calls[i]?.status === 'error') errorIdx.push(i);\n      }\n      if (errorIdx.length === 0) {\n        return { pass: true, score: 1, reason: 'no tool errors to recover from.' };\n      }\n      const unrecovered = errorIdx.filter((i) => {\n        for (let j = i + 1; j < calls.length; j++) {\n          if (calls[j]?.status === 'ok') return false;\n        }\n        return true;\n      });\n      const pass = unrecovered.length === 0;\n      const score = (errorIdx.length - unrecovered.length) / errorIdx.length;\n      if (pass) {\n        return { pass, score, reason: `recovered from ${errorIdx.length} tool error(s).` };\n      }\n      return {\n        pass,\n        score,\n        reason: `${unrecovered.length}/${errorIdx.length} tool error(s) had no successful follow-up call.`,\n      };\n    },\n  };\n}\n"],"mappings":";;AAqBA,SAAgB,mBACd,UAAqC,EAAE,EAChB;AAEvB,QAAO;EACL,MAFW,QAAQ,QAAQ;EAG3B,MAAM,MAAM,EAAE,UAAU;GACtB,MAAM,QAAQ,OAAO;GACrB,MAAMA,WAAqB,EAAE;AAC7B,QAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,IAChC,KAAI,MAAM,IAAI,WAAW,QAAS,UAAS,KAAK,EAAE;AAEpD,OAAI,SAAS,WAAW,EACtB,QAAO;IAAE,MAAM;IAAM,OAAO;IAAG,QAAQ;IAAmC;GAE5E,MAAM,cAAc,SAAS,QAAQ,MAAM;AACzC,SAAK,IAAI,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,IACpC,KAAI,MAAM,IAAI,WAAW,KAAM,QAAO;AAExC,WAAO;KACP;GACF,MAAM,OAAO,YAAY,WAAW;GACpC,MAAM,SAAS,SAAS,SAAS,YAAY,UAAU,SAAS;AAChE,OAAI,KACF,QAAO;IAAE;IAAM;IAAO,QAAQ,kBAAkB,SAAS,OAAO;IAAkB;AAEpF,UAAO;IACL;IACA;IACA,QAAQ,GAAG,YAAY,OAAO,GAAG,SAAS,OAAO;IAClD;;EAEJ"}

package/dist/scorers/trajectory/redundant-call-detection.d.ts

@@ -0,0 +1,19 @@
+import { Trajectory } from "./types.js";
+import { Scorer } from "@graphorin/observability/eval";
+
+//#region src/scorers/trajectory/redundant-call-detection.d.ts
+
+/** @stable */
+interface RedundantCallDetectionOptions {
+  /** Maximum tolerated redundant repeats before the scorer fails. Default `0`. */
+  readonly maxRedundant?: number;
+  /** Tool names exempt from the check (legitimately repeatable). */
+  readonly ignore?: ReadonlyArray<string>;
+  /** Optional name override. */
+  readonly name?: string;
+}
+/** @stable */
+declare function redundantCallDetection<I = unknown>(options?: RedundantCallDetectionOptions): Scorer<I, Trajectory>;
+//#endregion
+export { RedundantCallDetectionOptions, redundantCallDetection };
+//# sourceMappingURL=redundant-call-detection.d.ts.map

package/dist/scorers/trajectory/redundant-call-detection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redundant-call-detection.d.ts","names":[],"sources":["../../../src/scorers/trajectory/redundant-call-detection.ts"],"sourcesContent":[],"mappings":";;;;;;UAeiB,6BAAA;;;;oBAIG;;;;;iBAMJ,8CACL,gCACR,OAAO,GAAG"}

package/dist/scorers/trajectory/redundant-call-detection.js

@@ -0,0 +1,39 @@
+import { canonicalize } from "./util.js";
+
+//#region src/scorers/trajectory/redundant-call-detection.ts
+/** @stable */
+function redundantCallDetection(options = {}) {
+	const maxRedundant = options.maxRedundant ?? 0;
+	const ignore = new Set(options.ignore ?? []);
+	return {
+		name: options.name ?? "redundant-call-detection",
+		async score({ output }) {
+			const seen = /* @__PURE__ */ new Set();
+			const redundant = [];
+			let total = 0;
+			for (const call of output.calls) {
+				if (ignore.has(call.toolName)) continue;
+				total++;
+				if (call.status !== "ok") continue;
+				const key = `${call.toolName}(${canonicalize(call.args)})`;
+				if (seen.has(key)) redundant.push(call.toolName);
+				else seen.add(key);
+			}
+			const pass = redundant.length <= maxRedundant;
+			const score = total > 0 ? (total - redundant.length) / total : 1;
+			if (pass) return {
+				pass,
+				score
+			};
+			return {
+				pass,
+				score,
+				reason: `${redundant.length} redundant repeat call(s) (max ${maxRedundant}): [${redundant.join(", ")}].`
+			};
+		}
+	};
+}
+
+//#endregion
+export { redundantCallDetection };
+//# sourceMappingURL=redundant-call-detection.js.map

package/dist/scorers/trajectory/redundant-call-detection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redundant-call-detection.js","names":["redundant: string[]"],"sources":["../../../src/scorers/trajectory/redundant-call-detection.ts"],"sourcesContent":["/**\n * `redundantCallDetection` — passes when the harness made no more than\n * `maxRedundant` (default `0`) redundant repeat calls. A call is redundant\n * when an *earlier successful* call with the same name and deep-equal\n * arguments already produced that result. Only prior `'ok'` calls seed the\n * set, so a retry after an error is never flagged as redundant.\n *\n * @packageDocumentation\n */\n\nimport type { Scorer } from '@graphorin/observability/eval';\nimport type { Trajectory } from './types.js';\nimport { canonicalize } from './util.js';\n\n/** @stable */\nexport interface RedundantCallDetectionOptions {\n  /** Maximum tolerated redundant repeats before the scorer fails. Default `0`. */\n  readonly maxRedundant?: number;\n  /** Tool names exempt from the check (legitimately repeatable). */\n  readonly ignore?: ReadonlyArray<string>;\n  /** Optional name override. */\n  readonly name?: string;\n}\n\n/** @stable */\nexport function redundantCallDetection<I = unknown>(\n  options: RedundantCallDetectionOptions = {},\n): Scorer<I, Trajectory> {\n  const maxRedundant = options.maxRedundant ?? 0;\n  const ignore = new Set(options.ignore ?? []);\n  const name = options.name ?? 'redundant-call-detection';\n  return {\n    name,\n    async score({ output }) {\n      const seen = new Set<string>();\n      const redundant: string[] = [];\n      let total = 0;\n      for (const call of output.calls) {\n        if (ignore.has(call.toolName)) continue;\n        total++;\n        // Only a successful call establishes a \"result already obtained\"\n        // fact; a retry after an error is expected, not redundant.\n        if (call.status !== 'ok') continue;\n        const key = `${call.toolName}(${canonicalize(call.args)})`;\n        if (seen.has(key)) redundant.push(call.toolName);\n        else seen.add(key);\n      }\n      const pass = redundant.length <= maxRedundant;\n      const score = total > 0 ? (total - redundant.length) / total : 1;\n      if (pass) return { pass, score };\n      return {\n        pass,\n        score,\n        reason: `${redundant.length} redundant repeat call(s) (max ${maxRedundant}): [${redundant.join(', ')}].`,\n      };\n    },\n  };\n}\n"],"mappings":";;;;AAyBA,SAAgB,uBACd,UAAyC,EAAE,EACpB;CACvB,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,SAAS,IAAI,IAAI,QAAQ,UAAU,EAAE,CAAC;AAE5C,QAAO;EACL,MAFW,QAAQ,QAAQ;EAG3B,MAAM,MAAM,EAAE,UAAU;GACtB,MAAM,uBAAO,IAAI,KAAa;GAC9B,MAAMA,YAAsB,EAAE;GAC9B,IAAI,QAAQ;AACZ,QAAK,MAAM,QAAQ,OAAO,OAAO;AAC/B,QAAI,OAAO,IAAI,KAAK,SAAS,CAAE;AAC/B;AAGA,QAAI,KAAK,WAAW,KAAM;IAC1B,MAAM,MAAM,GAAG,KAAK,SAAS,GAAG,aAAa,KAAK,KAAK,CAAC;AACxD,QAAI,KAAK,IAAI,IAAI,CAAE,WAAU,KAAK,KAAK,SAAS;QAC3C,MAAK,IAAI,IAAI;;GAEpB,MAAM,OAAO,UAAU,UAAU;GACjC,MAAM,QAAQ,QAAQ,KAAK,QAAQ,UAAU,UAAU,QAAQ;AAC/D,OAAI,KAAM,QAAO;IAAE;IAAM;IAAO;AAChC,UAAO;IACL;IACA;IACA,QAAQ,GAAG,UAAU,OAAO,iCAAiC,aAAa,MAAM,UAAU,KAAK,KAAK,CAAC;IACtG;;EAEJ"}

package/dist/scorers/trajectory/types.d.ts

@@ -0,0 +1,49 @@
+//#region src/scorers/trajectory/types.d.ts
+/**
+ * Shared types for the trajectory scorer family. A {@link Trajectory} is
+ * the recorded sequence of tool calls a harness made while attempting a
+ * task, plus an optional goal-state snapshot and the final text output.
+ *
+ * The scorers in this folder are pure functions over a `Trajectory` — no
+ * network, no model — so they can gate harness reliability in CI. Build a
+ * `Trajectory` by folding an agent's `AgentEvent` stream (correlate
+ * `tool.call.start` / `tool.call.end` with `tool.execute.end` /
+ * `tool.execute.error` by `toolCallId`).
+ *
+ * @packageDocumentation
+ */
+/**
+ * One executed tool call as observed on the `AgentEvent` stream.
+ *
+ * @stable
+ */
+interface TrajectoryToolCall {
+  readonly toolCallId: string;
+  readonly toolName: string;
+  /** The arguments the model emitted (the resolved `tool.call.end.finalArgs`). */
+  readonly args: unknown;
+  /** `'ok'` when the call returned; `'error'` when the executor surfaced a `ToolError`. */
+  readonly status: 'ok' | 'error';
+  /** The tool output, present when `status === 'ok'`. */
+  readonly result?: unknown;
+  /** The surfaced error, present when `status === 'error'`. */
+  readonly error?: {
+    readonly kind?: string;
+    readonly message?: string;
+  };
+}
+/**
+ * The full record of a single harness attempt at a task.
+ *
+ * @stable
+ */
+interface Trajectory {
+  readonly calls: ReadonlyArray<TrajectoryToolCall>;
+  /** Goal-state snapshot compared by {@link finalStateCorrect}. */
+  readonly finalState?: unknown;
+  /** The assistant's final text, when the run completed. */
+  readonly finalOutput?: string;
+}
+//#endregion
+export { Trajectory, TrajectoryToolCall };
+//# sourceMappingURL=types.d.ts.map

package/dist/scorers/trajectory/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../../../src/scorers/trajectory/types.ts"],"sourcesContent":[],"mappings":";;AAmBA;AAkBA;;;;;;;;;;;;;;;;UAlBiB,kBAAA;;;;;;;;;;;;;;;;;;;;UAkBA,UAAA;kBACC,cAAc"}

package/dist/scorers/trajectory/util.js

@@ -0,0 +1,71 @@
+//#region src/scorers/trajectory/util.ts
+/**
+* Internal helpers shared by the trajectory scorers: structural deep
+* equality, dot-path reads, and an order-insensitive canonical key used to
+* de-duplicate tool calls by `(name, args)`. Private to the family — not
+* exported from the package surface.
+*
+* @packageDocumentation
+*/
+/** Structural deep-equality with order-insensitive object key comparison. */
+function deepEqual(a, b) {
+	if (a === b) return true;
+	if (typeof a !== typeof b) return false;
+	if (a === null || b === null) return a === b;
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+		return true;
+	}
+	if (typeof a === "object" && typeof b === "object") {
+		const aObj = a;
+		const bObj = b;
+		const aKeys = Object.keys(aObj).sort();
+		const bKeys = Object.keys(bObj).sort();
+		if (aKeys.length !== bKeys.length) return false;
+		for (let i = 0; i < aKeys.length; i++) {
+			const key = aKeys[i];
+			if (key === void 0 || key !== bKeys[i]) return false;
+			if (!deepEqual(aObj[key], bObj[key])) return false;
+		}
+		return true;
+	}
+	return false;
+}
+/** Read a dot-separated path (`'orders.0.status'`) out of a value. */
+function readPath(root, path) {
+	if (path.length === 0) return root;
+	let cursor = root;
+	for (const segment of path.split(".")) {
+		if (cursor === null || cursor === void 0) return void 0;
+		if (Array.isArray(cursor)) {
+			const idx = Number.parseInt(segment, 10);
+			if (!Number.isFinite(idx)) return void 0;
+			cursor = cursor[idx];
+		} else if (typeof cursor === "object") cursor = cursor[segment];
+		else return;
+	}
+	return cursor;
+}
+/** Order-insensitive canonical JSON, so `{a,b}` and `{b,a}` key alike. */
+function canonicalize(value) {
+	if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+	if (Array.isArray(value)) return `[${value.map(canonicalize).join(",")}]`;
+	const obj = value;
+	return `{${Object.keys(obj).sort().map((k) => `${JSON.stringify(k)}:${canonicalize(obj[k])}`).join(",")}}`;
+}
+function truncate(s, max = 80) {
+	return s.length > max ? `${s.slice(0, max)}…` : s;
+}
+function stringifySafe(value) {
+	if (value === void 0) return "undefined";
+	try {
+		return JSON.stringify(value) ?? String(value);
+	} catch {
+		return String(value);
+	}
+}
+
+//#endregion
+export { canonicalize, deepEqual, readPath, stringifySafe, truncate };
+//# sourceMappingURL=util.js.map

package/dist/scorers/trajectory/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","names":["cursor: unknown"],"sources":["../../../src/scorers/trajectory/util.ts"],"sourcesContent":["/**\n * Internal helpers shared by the trajectory scorers: structural deep\n * equality, dot-path reads, and an order-insensitive canonical key used to\n * de-duplicate tool calls by `(name, args)`. Private to the family — not\n * exported from the package surface.\n *\n * @packageDocumentation\n */\n\n/** Structural deep-equality with order-insensitive object key comparison. */\nexport function deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true;\n  if (typeof a !== typeof b) return false;\n  if (a === null || b === null) return a === b;\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    for (let i = 0; i < a.length; i++) {\n      if (!deepEqual(a[i], b[i])) return false;\n    }\n    return true;\n  }\n  if (typeof a === 'object' && typeof b === 'object') {\n    const aObj = a as Record<string, unknown>;\n    const bObj = b as Record<string, unknown>;\n    const aKeys = Object.keys(aObj).sort();\n    const bKeys = Object.keys(bObj).sort();\n    if (aKeys.length !== bKeys.length) return false;\n    for (let i = 0; i < aKeys.length; i++) {\n      const key = aKeys[i];\n      if (key === undefined || key !== bKeys[i]) return false;\n      if (!deepEqual(aObj[key], bObj[key])) return false;\n    }\n    return true;\n  }\n  return false;\n}\n\n/** Read a dot-separated path (`'orders.0.status'`) out of a value. */\nexport function readPath(root: unknown, path: string): unknown {\n  if (path.length === 0) return root;\n  let cursor: unknown = root;\n  for (const segment of path.split('.')) {\n    if (cursor === null || cursor === undefined) return undefined;\n    if (Array.isArray(cursor)) {\n      const idx = Number.parseInt(segment, 10);\n      if (!Number.isFinite(idx)) return undefined;\n      cursor = cursor[idx];\n    } else if (typeof cursor === 'object') {\n      cursor = (cursor as Record<string, unknown>)[segment];\n    } else {\n      return undefined;\n    }\n  }\n  return cursor;\n}\n\n/** Order-insensitive canonical JSON, so `{a,b}` and `{b,a}` key alike. */\nexport function canonicalize(value: unknown): string {\n  if (value === null || typeof value !== 'object') {\n    return JSON.stringify(value) ?? 'null';\n  }\n  if (Array.isArray(value)) {\n    return `[${value.map(canonicalize).join(',')}]`;\n  }\n  const obj = value as Record<string, unknown>;\n  const keys = Object.keys(obj).sort();\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalize(obj[k])}`).join(',')}}`;\n}\n\nexport function truncate(s: string, max = 80): string {\n  return s.length > max ? `${s.slice(0, max)}…` : s;\n}\n\nexport function stringifySafe(value: unknown): string {\n  if (value === undefined) return 'undefined';\n  try {\n    return JSON.stringify(value) ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n"],"mappings":";;;;;;;;;;AAUA,SAAgB,UAAU,GAAY,GAAqB;AACzD,KAAI,MAAM,EAAG,QAAO;AACpB,KAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAClC,KAAI,MAAM,QAAQ,MAAM,KAAM,QAAO,MAAM;AAC3C,KAAI,MAAM,QAAQ,EAAE,IAAI,MAAM,QAAQ,EAAE,EAAE;AACxC,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,OAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,IAC5B,KAAI,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAE,QAAO;AAErC,SAAO;;AAET,KAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;EAClD,MAAM,OAAO;EACb,MAAM,OAAO;EACb,MAAM,QAAQ,OAAO,KAAK,KAAK,CAAC,MAAM;EACtC,MAAM,QAAQ,OAAO,KAAK,KAAK,CAAC,MAAM;AACtC,MAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,MAAM,MAAM;AAClB,OAAI,QAAQ,UAAa,QAAQ,MAAM,GAAI,QAAO;AAClD,OAAI,CAAC,UAAU,KAAK,MAAM,KAAK,KAAK,CAAE,QAAO;;AAE/C,SAAO;;AAET,QAAO;;;AAIT,SAAgB,SAAS,MAAe,MAAuB;AAC7D,KAAI,KAAK,WAAW,EAAG,QAAO;CAC9B,IAAIA,SAAkB;AACtB,MAAK,MAAM,WAAW,KAAK,MAAM,IAAI,EAAE;AACrC,MAAI,WAAW,QAAQ,WAAW,OAAW,QAAO;AACpD,MAAI,MAAM,QAAQ,OAAO,EAAE;GACzB,MAAM,MAAM,OAAO,SAAS,SAAS,GAAG;AACxC,OAAI,CAAC,OAAO,SAAS,IAAI,CAAE,QAAO;AAClC,YAAS,OAAO;aACP,OAAO,WAAW,SAC3B,UAAU,OAAmC;MAE7C;;AAGJ,QAAO;;;AAIT,SAAgB,aAAa,OAAwB;AACnD,KAAI,UAAU,QAAQ,OAAO,UAAU,SACrC,QAAO,KAAK,UAAU,MAAM,IAAI;AAElC,KAAI,MAAM,QAAQ,MAAM,CACtB,QAAO,IAAI,MAAM,IAAI,aAAa,CAAC,KAAK,IAAI,CAAC;CAE/C,MAAM,MAAM;AAEZ,QAAO,IADM,OAAO,KAAK,IAAI,CAAC,MAAM,CACpB,KAAK,MAAM,GAAG,KAAK,UAAU,EAAE,CAAC,GAAG,aAAa,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC;;AAGvF,SAAgB,SAAS,GAAW,MAAM,IAAY;AACpD,QAAO,EAAE,SAAS,MAAM,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC,KAAK;;AAGlD,SAAgB,cAAc,OAAwB;AACpD,KAAI,UAAU,OAAW,QAAO;AAChC,KAAI;AACF,SAAO,KAAK,UAAU,MAAM,IAAI,OAAO,MAAM;SACvC;AACN,SAAO,OAAO,MAAM"}

package/dist/types.d.ts

@@ -0,0 +1,98 @@
+import { Case as Case$1, Dataset as Dataset$1, EvalCaseResult, EvalReport, EvalReport as EvalReport$1, RunEvalOptions, ScoreResult as ScoreResult$1, Scorer, Scorer as Scorer$1 } from "@graphorin/observability/eval";
+
+//#region src/types.d.ts
+
+/**
+ * Agent shape consumed by the runner. Anything with a `run(input)`
+ * method satisfies the contract — the framework's own `Agent` type
+ * matches by structural typing.
+ *
+ * @stable
+ */
+interface AgentLike<I, O> {
+  run(input: I, ctx?: {
+    signal?: AbortSignal;
+  }): Promise<O>;
+}
+/**
+ * Options accepted by the parallel runner.
+ *
+ * @stable
+ */
+interface RunOptions<I, O> {
+  readonly agent: AgentLike<I, O>;
+  readonly dataset: {
+    readonly cases: ReadonlyArray<{
+      readonly id?: string;
+      readonly input: I;
+      readonly expected?: O;
+      readonly metadata?: Readonly<Record<string, unknown>>;
+    }>;
+  };
+  readonly scorers: ReadonlyArray<Scorer<I, O>>;
+  /** Default `1`. */
+  readonly iterations?: number;
+  /** Default `1` (sequential). Set higher for parallel evaluation. */
+  readonly concurrency?: number;
+  readonly signal?: AbortSignal;
+  /**
+   * How to handle `signal` abort. Default (`false`): stop dispatching and
+   * resolve with a **partial** {@link EvalReport} (`aborted: true`) covering
+   * the cases that finished — a long judged run shouldn't lose all completed
+   * work to a Ctrl+C. Set `true` to throw the abort reason instead.
+   */
+  readonly throwOnAbort?: boolean;
+  /**
+   * Optional progress hook invoked after every case. Useful for
+   * terminal reporters that want a per-case heartbeat.
+   */
+  readonly onProgress?: (event: ProgressEvent) => void;
+}
+/** @stable */
+interface ProgressEvent {
+  readonly index: number;
+  readonly total: number;
+  readonly caseId: string;
+  readonly durationMs: number;
+  readonly passed: boolean;
+}
+/**
+ * Regression-detection options.
+ *
+ * @stable
+ */
+interface RegressionOptions {
+  /** Minimum drop in pass-rate (in percentage points) that counts as a regression. */
+  readonly maxPassRateDropPct?: number;
+  /** Minimum drop in average score per scorer that counts as a regression. */
+  readonly maxAvgScoreDrop?: number;
+  /**
+   * Maximum allowed increase in `avgDurationMs` before it counts as a
+   * regression. **Opt-in: defaults to `Infinity` (gate off)** because absolute
+   * wall-clock budgets are environment-sensitive (workstation baseline vs CI
+   * runner, real LLM-latency jitter). Pass a finite ms budget to enable an
+   * absolute duration gate; leave unset to ignore duration entirely.
+   */
+  readonly maxAvgDurationIncreaseMs?: number;
+}
+/**
+ * Result of {@link detectRegressions}.
+ *
+ * @stable
+ */
+interface RegressionReport<I, O> {
+  readonly hasRegressions: boolean;
+  readonly findings: ReadonlyArray<RegressionFinding>;
+  readonly current: EvalReport<I, O>;
+  readonly baseline: EvalReport<I, O>;
+}
+/** @stable */
+interface RegressionFinding {
+  readonly kind: 'pass-rate-drop' | 'avg-score-drop' | 'avg-duration-increase' | 'scorer-removed';
+  readonly scorer?: string;
+  readonly message: string;
+  readonly delta: number;
+}
+//#endregion
+export { AgentLike, type Case$1 as Case, type Dataset$1 as Dataset, type EvalCaseResult, type EvalReport$1 as EvalReport, ProgressEvent, RegressionFinding, RegressionOptions, RegressionReport, type RunEvalOptions, RunOptions, type ScoreResult$1 as ScoreResult, type Scorer$1 as Scorer };
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;AAiD4C,UAnB3B,SAmB2B,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EAAV,GAAA,CAAA,KAAA,EAlBrB,CAkBqB,EAAA,GAKd,CALc,EAAA;IAAd,MAAA,CAAA,EAlBa,WAkBb;EAKA,CAAA,CAAA,EAvB6B,OAuB7B,CAvBqC,CAuBrC,CAAA;;;AAgBpB;AAaA;AAoBA;;AAEqB,UAlEJ,UAkEI,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EACU,SAAA,KAAA,EAlEb,SAkEa,CAlEH,CAkEG,EAlEA,CAkEA,CAAA;EAAG,SAAA,OAAA,EAAA;IAAd,SAAA,KAAA,EAhEA,aAgEA,CAAA;MACY,SAAA,EAAA,CAAA,EAAA,MAAA;MAAG,SAAA,KAAA,EA/Db,CA+Da;MAAd,SAAA,QAAA,CAAA,EA9DK,CA8DL;MAAU,SAAA,QAAA,CAAA,EA7DL,QA6DK,CA7DI,MA6DJ,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA;IAId,CAAA,CAAA;;oBA9DG,cAAc,OAAO,GAAG;;;;;oBAKxB;;;;;;;;;;;;gCAYY;;;UAIf,aAAA;;;;;;;;;;;;UAaA,iBAAA;;;;;;;;;;;;;;;;;;;UAoBA;;qBAEI,cAAc;oBACf,WAAW,GAAG;qBACb,WAAW,GAAG;;;UAIlB,iBAAA"}

package/package.json

@@ -0,0 +1,81 @@
+{
+  "name": "@graphorin/evals",
+  "version": "0.5.0",
+  "description": "Eval framework for the Graphorin framework. Ships scorer libraries (`code/` exact-match, regex, JSON path, predicates; `llm/` LLM-judge; `prebuilt/` toxicity / factuality / helpfulness), dataset loaders (JSONL, CSV, from-traces, generic iterable), reporters (terminal, markdown, JSON, JUnit XML), a parallel runner with `concurrency` config, and regression detection that compares the current run against a stored baseline. Builds on the eval primitives shipped from `@graphorin/observability` (RB-17 / DEC-152 — full orchestrator lives here, post-MVP, decoupled from the core observability bundle so consumers do not pay the dataset / reporter cost when only the inline runner is needed). Created and maintained by Oleksiy Stepurenko.",
+  "license": "MIT",
+  "author": "Oleksiy Stepurenko",
+  "homepage": "https://github.com/o-stepper/graphorin/tree/main/packages/evals",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/o-stepper/graphorin.git",
+    "directory": "packages/evals"
+  },
+  "bugs": {
+    "url": "https://github.com/o-stepper/graphorin/issues"
+  },
+  "keywords": [
+    "graphorin",
+    "ai",
+    "agents",
+    "framework",
+    "evals",
+    "evaluation",
+    "scorers",
+    "llm-judge",
+    "datasets",
+    "reporters",
+    "regression-detection"
+  ],
+  "type": "module",
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./scorers": {
+      "types": "./dist/scorers/index.d.ts",
+      "import": "./dist/scorers/index.js"
+    },
+    "./loaders": {
+      "types": "./dist/loaders/index.d.ts",
+      "import": "./dist/loaders/index.js"
+    },
+    "./reporters": {
+      "types": "./dist/reporters/index.d.ts",
+      "import": "./dist/reporters/index.js"
+    },
+    "./cli": {
+      "types": "./dist/cli/index.d.ts",
+      "import": "./dist/cli/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "@graphorin/core": "0.5.0",
+    "@graphorin/observability": "0.5.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "scripts": {
+    "build": "tsdown",
+    "typecheck": "tsc --noEmit && tsc -p tsconfig.tests.json",
+    "test": "vitest run",
+    "lint": "biome check .",
+    "clean": "rimraf dist .turbo *.tsbuildinfo"
+  }
+}