@b4run/evals 0.8.28

package/dist/types.d.ts

@@ -0,0 +1,74 @@
+import type { AgentRunResult, FixtureSet, ScriptBuilder } from "@b4run/testing";
+/** One dataset row. `input` is the user message for agent routes (v1). */
+export interface EvalCase {
+    readonly name?: string;
+    readonly input: unknown;
+    readonly expected?: unknown;
+    /** Per-case aimock fixtures for replay mode; ignored under --live. */
+    readonly fixtures?: FixtureSet | ScriptBuilder;
+    readonly metadata?: Record<string, unknown>;
+}
+/** Inline cases, a path to a committed .json/.jsonl, or a (sync/async) factory. */
+export type Dataset = readonly EvalCase[] | string | (() => EvalCase[] | Promise<EvalCase[]>);
+/** A scorer may return a 0..1 number, a boolean, or a rich verdict. */
+export type Score = number | boolean | {
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+};
+export interface Scorer {
+    readonly name: string;
+    /** This scorer's own pass bar (used by gate.perScorer and case-pass). */
+    readonly threshold?: number;
+    readonly score: (run: AgentRunResult, testCase: EvalCase) => Score | Promise<Score>;
+}
+export interface EvalDefinition {
+    readonly name: string;
+    /** Route key like "/chat#agent"; defaults to the co-located route at load time. */
+    readonly route?: string;
+    readonly dataset: Dataset;
+    readonly scorers: readonly Scorer[];
+    /** Sugar for gate.mean(threshold). Ignored if `gate` is set. */
+    readonly threshold?: number;
+    readonly gate?: GatePolicy;
+}
+/** Normalized score for one (case, scorer) pair. */
+export interface CaseScore {
+    readonly scorer: string;
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+}
+export interface CaseResult {
+    readonly name: string;
+    readonly scores: readonly CaseScore[];
+    readonly mean: number;
+    /** Every scorer met its bar (scorer.threshold ?? DEFAULT_CASE_BAR). */
+    readonly passed: boolean;
+}
+export interface ScorerAggregate {
+    readonly scorer: string;
+    readonly mean: number;
+    readonly threshold?: number;
+}
+/** Pre-gate report fed to gate policies. */
+export interface ScoredReport {
+    readonly name: string;
+    readonly cases: readonly CaseResult[];
+    readonly byScorer: readonly ScorerAggregate[];
+    readonly mean: number;
+}
+export interface EvalReport extends ScoredReport {
+    /** Whether a gate/threshold was configured (informational evals are false). */
+    readonly gated: boolean;
+    readonly passed: boolean;
+    readonly reason?: string;
+}
+export type GateResult = {
+    readonly passed: boolean;
+    readonly reason?: string;
+};
+export type GatePolicy = (report: ScoredReport) => GateResult;
+/** A case "passes" when every scorer ≥ its threshold, defaulting to this bar. */
+export declare const DEFAULT_CASE_BAR = 0.5;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAE/E,0EAA0E;AAC1E,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC5C;AAED,mFAAmF;AACnF,MAAM,MAAM,OAAO,GAAG,SAAS,QAAQ,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;AAE7F,uEAAuE;AACvE,MAAM,MAAM,KAAK,GACb,MAAM,GACN,OAAO,GACP;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAEjF,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;CACpF;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,mFAAmF;IACnF,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IACnC,gEAAgE;IAChE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAA;CAC3B;AAED,oDAAoD;AACpD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAA;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,MAAM,UAAU,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAC/E,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAA;AAE7D,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,MAAM,CAAA"}

package/dist/types.js

@@ -0,0 +1,2 @@
+/** A case "passes" when every scorer ≥ its threshold, defaulting to this bar. */
+export const DEFAULT_CASE_BAR = 0.5;

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@b4run/evals",
+  "version": "0.8.28",
+  "private": false,
+  "type": "module",
+  "license": "MIT",
+  "description": "Evaluation definitions, scorers, datasets, and runners for B4.run agents.",
+  "keywords": [
+    "b4",
+    "typescript",
+    "ai-agents",
+    "evals",
+    "testing",
+    "llm"
+  ],
+  "homepage": "https://github.com/cacheplane/b4run/tree/main/packages/evals#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cacheplane/b4run.git",
+    "directory": "packages/evals"
+  },
+  "bugs": {
+    "url": "https://github.com/cacheplane/b4run/issues"
+  },
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "@b4run/testing": "0.8.28"
+  },
+  "devDependencies": {
+    "@types/node": "26.1.2",
+    "@b4run/config-typescript": "0.8.28",
+    "@b4run/testing": "0.8.28"
+  },
+  "scripts": {
+    "build": "tsc -b tsconfig.json",
+    "lint": "biome check --config-path ../config-biome/biome.json package.json src test tsconfig.json vitest.config.ts",
+    "test": "vitest --run --config vitest.config.ts --passWithNoTests",
+    "typecheck": "tsc --noEmit && tsc -p tsconfig.test.json"
+  }
+}