@databricks/appkit 0.72.0 → 0.73.0

package/dist/evals/run-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;UAsBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAAU;EAElB,MAAA;AAAA;;;;;AAQF;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}
+{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAuBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;AAAA;;;;;AAQR;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}

package/dist/evals/run-eval.js

@@ -43,14 +43,13 @@ async function runEval(def, options) {
 				return handle;
 			},
 			atLeast(threshold) {
-				result.severity = "soft";
 				result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;
 				return handle;
 			}
 		};
 		return handle;
 	};
-	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();
+	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);
 	const t = {
 		async send(message) {
 			lastInput = message;
@@ -61,6 +60,9 @@ async function runEval(def, options) {
 			lastSucceeded = r.succeeded;
 			if (r.traceId) lastTraceId = r.traceId;
 		},
+		reset() {
+			options.driver.reset?.();
+		},
 		get reply() {
 			return reply;
 		},
@@ -70,6 +72,12 @@ async function runEval(def, options) {
 		get sessionId() {
 			return sessionId;
 		},
+		get input() {
+			return options.row?.inputs ?? {};
+		},
+		get expected() {
+			return options.row?.expectations;
+		},
 		succeeded() {
 			return record("succeeded", lastSucceeded, void 0, lastSucceeded ? void 0 : "agent turn did not complete successfully");
 		},

package/dist/evals/run-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        result.severity = \"soft\";\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and soft by default; the caller chains\n  // `.atLeast(n)` to set the pass threshold or `.gate()` to promote.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAYA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAkBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AACzB,WAAO,WAAW;AAClB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAKT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU,CAAC,MAAM;CAE1E,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAaA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAoBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}

package/dist/evals/run-evals.d.ts

@@ -1,3 +1,5 @@
+import { WorkspaceClient } from "../workspace-client/index.js";
+import "./dataset.js";
 import { EvalResult } from "./types.js";
 import { ReportOutcome } from "./mlflow-report.js";
 import { FinishOutcome } from "./mlflow-run.js";
@@ -43,6 +45,13 @@ interface RunEvalsOptions {
     token: string;
     model: string;
   };
+  /**
+   * Workspace client used to read managed evaluation datasets (for evals that
+   * declare `dataset`). Required alongside {@link warehouseId} for those evals.
+   */
+  workspaceClient?: WorkspaceClient;
+  /** SQL warehouse id used to read managed evaluation datasets. */
+  warehouseId?: string;
   /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
   now?: number;
   /** Progress callback, invoked as evals are discovered, started, and finished. */

package/dist/evals/run-evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;UAYiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAoC8B;EAlC9B,MAAA;EAFA;EAIA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;EAEV,SAAA;EAOA;;;;;;EAAA,WAAA;EAiBU;;;;;EAXV,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeQ;IAbR,cAAA;EAAA;EAiBoC;;;;EAXtC,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAWnC;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAqIrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAciB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAMU;EAJV,MAAA;EAyCkB;EAvClB,MAAA;EAuC8B;EArC9B,OAAA,GAAU,MAAA;EANV;EAQA,SAAA;EAJA;;;;;;EAWA,WAAA;EAQE;;;;;EAFF,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeF;IAbE,cAAA;EAAA;EAiBgB;;;;EAXlB,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAef;;;;EAVxB,eAAA,GAAkB,eAAA;EAYa;EAV/B,WAAA;EAWI;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAoOrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}

package/dist/evals/run-evals.js

@@ -1,4 +1,5 @@
 import { MlflowClient } from "../connectors/mlflow/client.js";
+import { readEvalDataset } from "./dataset.js";
 import { discoverEvalFiles } from "./discover.js";
 import { createHttpDriver } from "./http-driver.js";
 import { configureJudge, teardownJudge } from "./judge.js";
@@ -41,12 +42,13 @@ function resolveEvalDefault(mod) {
 	}
 }
 /**
-* Load and run a single discovered eval. Never throws — a load/run failure
-* becomes a non-passing {@link EvalResult} so one bad eval can't abort the run.
+* Run one eval turn against a fresh driver. Never throws — a run failure becomes
+* a non-passing {@link EvalResult} so one bad eval can't abort the run. `row`
+* binds the current managed-dataset row (see {@link resolveDatasetRows}), or is
+* `undefined` for a plain single-run eval.
 */
-async function runOne(d, id, runId, options) {
+async function runOne(d, id, def, row, runId, options) {
 	try {
-		const def = await loadEval(d.file);
 		return await runEval(def, {
 			id,
 			driver: createHttpDriver({
@@ -56,7 +58,8 @@ async function runOne(d, id, runId, options) {
 				mlflowRunId: runId,
 				timeoutMs: options.timeoutMs
 			}),
-			strict: options.strict
+			strict: options.strict,
+			row
 		});
 	} catch (err) {
 		return {
@@ -67,6 +70,94 @@ async function runOne(d, id, runId, options) {
 		};
 	}
 }
+/**
+* Resolve the rows a (possibly dataset-driven) eval runs over. A plain eval
+* yields a single `undefined` row; a dataset eval reads its Unity Catalog table
+* via {@link readEvalDataset}. On misconfiguration or read failure, returns a
+* single `undefined` row plus an `error`, so the eval still surfaces one result.
+*/
+async function resolveDatasetRows(def, options) {
+	if (!def.dataset) return { rows: [void 0] };
+	if (!options.workspaceClient || !options.warehouseId) return {
+		rows: [void 0],
+		error: "dataset eval requires a workspace client and warehouse (pass --warehouse-id)"
+	};
+	try {
+		const rows = await readEvalDataset(options.workspaceClient, {
+			table: def.dataset.table,
+			warehouseId: options.warehouseId,
+			limit: def.dataset.limit
+		});
+		if (rows.length === 0) return {
+			rows: [void 0],
+			error: `dataset "${def.dataset.table}" returned no rows`
+		};
+		return { rows };
+	} catch (err) {
+		return {
+			rows: [void 0],
+			error: err instanceof Error ? err.message : String(err)
+		};
+	}
+}
+/**
+* Load one discovered eval and run it, expanding a dataset-driven eval into one
+* run per row. Appends one result per row to `results`, emitting `start`/
+* `result` around each. Never throws: a load or dataset-read failure surfaces as
+* a non-passing result. `total` counts eval files, not rows — per-row detail is
+* carried in the result id (`[row i/n]`).
+*/
+async function runDiscovered(d, index, total, runId, options, emit, results) {
+	const id = `${d.agent}/${d.id}`;
+	let def;
+	try {
+		def = await loadEval(d.file);
+	} catch (err) {
+		emit({
+			type: "start",
+			id,
+			index,
+			total
+		});
+		const result = {
+			id,
+			assertions: [],
+			passed: false,
+			error: err instanceof Error ? err.message : String(err)
+		};
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+		return;
+	}
+	const { rows, error: datasetError } = await resolveDatasetRows(def, options);
+	for (let r = 0; r < rows.length; r++) {
+		const rowId = def.dataset && rows.length > 1 ? `${id} [row ${r + 1}/${rows.length}]` : id;
+		emit({
+			type: "start",
+			id: rowId,
+			index,
+			total
+		});
+		const result = datasetError ? {
+			id: rowId,
+			assertions: [],
+			passed: false,
+			error: datasetError
+		} : await runOne(d, rowId, def, rows[r], runId, options);
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+	}
+}
 /** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */
 async function maybeConfigureJudge(options) {
 	if (!options.judge) return;
@@ -147,23 +238,11 @@ async function runEvalsInDir(options) {
 				runId
 			});
 		}
-		const results = await mapPool(discovered, options.concurrency ?? DEFAULT_CONCURRENCY, async (d, index) => {
-			const id = `${d.agent}/${d.id}`;
-			emit({
-				type: "start",
-				id,
-				index,
-				total
-			});
-			const result = await runOne(d, id, runId, options);
-			emit({
-				type: "result",
-				result,
-				index,
-				total
-			});
-			return result;
-		});
+		const results = (await mapPool(discovered, options.concurrency ?? DEFAULT_CONCURRENCY, async (d, index) => {
+			const fileResults = [];
+			await runDiscovered(d, index, total, runId, options, emit, fileResults);
+			return fileResults;
+		})).flat();
 		const summary = { results };
 		const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);
 		if (mlflow) summary.mlflow = mlflow;

package/dist/evals/run-evals.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-evals.js","names":[],"sources":["../../src/evals/run-evals.ts"],"sourcesContent":["import { pathToFileURL } from \"node:url\";\n\nimport { MlflowClient } from \"../connectors/mlflow\";\nimport { type DiscoveredEval, discoverEvalFiles } from \"./discover\";\nimport { createHttpDriver } from \"./http-driver\";\nimport { configureJudge, teardownJudge } from \"./judge\";\nimport { type ReportOutcome, reportToMlflow } from \"./mlflow-report\";\nimport { createEvalRun, type FinishOutcome, finishEvalRun } from \"./mlflow-run\";\nimport { mapPool } from \"./pool\";\nimport { runEval } from \"./run-eval\";\nimport type { EvalDefinition, EvalResult } from \"./types\";\n\nexport interface RunEvalsOptions {\n  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */\n  rootDir?: string;\n  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Substring filter on `<agent>/<id>` (or an exact agent id). */\n  filter?: string;\n  /** Soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Extra request headers for the driver (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */\n  timeoutMs?: number;\n  /**\n   * Max evals to drive concurrently. Each eval opens one stream to the app as\n   * the same user, so keep this at or below the app's\n   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the\n   * 429 guard. Defaults to 4; clamped to `[1, total]`.\n   */\n  concurrency?: number;\n  /**\n   * When set, create a native MLflow \"Evaluation run\": each eval's trace is\n   * linked to the run, pass/fail is written as feedback, and aggregate metrics\n   * are logged. Requires Databricks creds + the target experiment.\n   */\n  mlflow?: {\n    host: string;\n    token: string;\n    experimentId: string;\n    /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */\n    sqlWarehouseId?: string;\n  };\n  /**\n   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a\n   * Databricks serving endpoint (`model`).\n   */\n  judge?: { host: string; token: string; model: string };\n  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */\n  now?: number;\n  /** Progress callback, invoked as evals are discovered, started, and finished. */\n  onEvent?: (event: EvalProgress) => void;\n}\n\nexport type EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: EvalResult; index: number; total: number };\n\nexport interface EvalRunSummary {\n  results: EvalResult[];\n  /** Present when an MLflow evaluation run was created. */\n  mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome };\n}\n\n/**\n * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.\n * Uses tsx's programmatic loader so TypeScript eval files run without a build\n * step. The specifier is indirected so the type checker doesn't try to resolve\n * tsx's internal entry.\n */\nasync function loadEval(file: string): Promise<EvalDefinition> {\n  const tsxApi = \"tsx/esm/api\";\n  let tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n  try {\n    ({ tsImport } = (await import(tsxApi)) as {\n      tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n    });\n  } catch {\n    throw new Error(\n      \"Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).\",\n    );\n  }\n\n  const mod = await tsImport(pathToFileURL(file).href, import.meta.url);\n  const def = resolveEvalDefault(mod);\n  if (!def) {\n    throw new Error(`${file}: must default-export defineEval({ test })`);\n  }\n  return def;\n}\n\n/**\n * Unwrap the eval default export across module-interop shapes. Depending on\n * whether the eval file is treated as ESM or CJS, the value lands at\n * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or\n * `mod` itself. Returns the first candidate that looks like an eval.\n */\nexport function resolveEvalDefault(mod: unknown): EvalDefinition | undefined {\n  let candidate: unknown = mod;\n  for (let i = 0; i < 4 && candidate; i++) {\n    if (typeof (candidate as EvalDefinition).test === \"function\") {\n      return candidate as EvalDefinition;\n    }\n    candidate = (candidate as { default?: unknown }).default;\n  }\n  return undefined;\n}\n\n/**\n * Load and run a single discovered eval. Never throws — a load/run failure\n * becomes a non-passing {@link EvalResult} so one bad eval can't abort the run.\n */\nasync function runOne(\n  d: DiscoveredEval,\n  id: string,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n): Promise<EvalResult> {\n  try {\n    const def = await loadEval(d.file);\n    const driver = createHttpDriver({\n      baseUrl: options.baseUrl,\n      agent: def.agent ?? d.agent,\n      headers: options.headers,\n      mlflowRunId: runId,\n      timeoutMs: options.timeoutMs,\n    });\n    return await runEval(def, { id, driver, strict: options.strict });\n  } catch (err) {\n    return {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */\nasync function maybeConfigureJudge(options: RunEvalsOptions): Promise<void> {\n  if (!options.judge) return;\n  await configureJudge({\n    client: new MlflowClient(options.judge.host, options.judge.token),\n    token: options.judge.token,\n    model: options.judge.model,\n  });\n}\n\n/**\n * Report per-eval assessments and finish the MLflow run, when one was created.\n * Returns the run summary, or `undefined` when there was no run to finalize.\n */\nasync function finalizeMlflow(\n  client: MlflowClient | undefined,\n  runId: string | undefined,\n  results: EvalResult[],\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary[\"mlflow\"]> {\n  if (!client || !runId) return undefined;\n  // reportToMlflow is not supposed to throw, but if it ever does the run must\n  // still be finished — otherwise it hangs in RUNNING forever.\n  let report: ReportOutcome = { written: 0, skipped: 0, failures: [] };\n  try {\n    report = await reportToMlflow(\n      client,\n      results,\n      options.mlflow?.sqlWarehouseId,\n    );\n  } catch (err) {\n    report.failures.push({\n      traceId: \"(report)\",\n      error: err instanceof Error ? err.message : String(err),\n    });\n  }\n  const finish = await finishEvalRun(client, {\n    runId,\n    results,\n    endTime: options.now ?? Date.now(),\n  });\n  return { runId, report, finish };\n}\n\n/**\n * Default max evals in flight. Each eval opens one stream to the app as the\n * same user; the server caps concurrent streams per user at 5 by default\n * (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.\n */\nconst DEFAULT_CONCURRENCY = 4;\n\n/**\n * Discover, load, and run every eval under each agent's `evals/` dir, driving\n * the agents on a running app. Never throws for an individual eval — load/run\n * failures become non-passing {@link EvalResult}s.\n */\nexport async function runEvalsInDir(\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary> {\n  const root = options.rootDir ?? process.cwd();\n  const now = options.now ?? Date.now();\n  let discovered = discoverEvalFiles(root);\n\n  if (options.filter) {\n    const f = options.filter;\n    discovered = discovered.filter(\n      (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f),\n    );\n  }\n\n  const emit = options.onEvent ?? (() => {});\n  const total = discovered.length;\n  emit({ type: \"discovered\", total });\n\n  // The judge sets OPENAI_* env vars globally (autoevals reads them per call),\n  // so tear them down in `finally` once the run is over — pass or throw — so\n  // the bearer doesn't linger in process.env.\n  await maybeConfigureJudge(options);\n  try {\n    // Create the MLflow evaluation run up front so each eval's trace can be\n    // linked to it as it runs. One client is shared by run create/finish and\n    // the per-trace assessment writes.\n    let runId: string | undefined;\n    let mlflowClient: MlflowClient | undefined;\n    if (options.mlflow) {\n      mlflowClient = new MlflowClient(\n        options.mlflow.host,\n        options.mlflow.token,\n      );\n      runId = await createEvalRun(mlflowClient, {\n        experimentId: options.mlflow.experimentId,\n        runName: `appkit-eval ${new Date(now).toISOString()}`,\n        startTime: now,\n      });\n      emit({ type: \"run-created\", runId });\n    }\n\n    // Run evals through a bounded pool so independent turns overlap instead of\n    // summing their latencies. runOne never throws, so a pool worker never\n    // rejects; results preserve discovery order (mapPool writes by index).\n    const results = await mapPool(\n      discovered,\n      options.concurrency ?? DEFAULT_CONCURRENCY,\n      async (d, index) => {\n        const id = `${d.agent}/${d.id}`;\n        emit({ type: \"start\", id, index, total });\n        const result = await runOne(d, id, runId, options);\n        emit({ type: \"result\", result, index, total });\n        return result;\n      },\n    );\n\n    const summary: EvalRunSummary = { results };\n    const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);\n    if (mlflow) summary.mlflow = mlflow;\n    return summary;\n  } finally {\n    teardownJudge();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAyEA,eAAe,SAAS,MAAuC;CAC7D,MAAM,SAAS;CACf,IAAI;AACJ,KAAI;AACF,GAAC,CAAE,YAAc,MAAM,OAAO;SAGxB;AACN,QAAM,IAAI,MACR,6FACD;;CAIH,MAAM,MAAM,mBADA,MAAM,SAAS,cAAc,KAAK,CAAC,MAAM,OAAO,KAAK,IAAI,CAClC;AACnC,KAAI,CAAC,IACH,OAAM,IAAI,MAAM,GAAG,KAAK,4CAA4C;AAEtE,QAAO;;;;;;;;AAST,SAAgB,mBAAmB,KAA0C;CAC3E,IAAI,YAAqB;AACzB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,WAAW,KAAK;AACvC,MAAI,OAAQ,UAA6B,SAAS,WAChD,QAAO;AAET,cAAa,UAAoC;;;;;;;AASrD,eAAe,OACb,GACA,IACA,OACA,SACqB;AACrB,KAAI;EACF,MAAM,MAAM,MAAM,SAAS,EAAE,KAAK;AAQlC,SAAO,MAAM,QAAQ,KAAK;GAAE;GAAI,QAPjB,iBAAiB;IAC9B,SAAS,QAAQ;IACjB,OAAO,IAAI,SAAS,EAAE;IACtB,SAAS,QAAQ;IACjB,aAAa;IACb,WAAW,QAAQ;IACpB,CAAC;GACsC,QAAQ,QAAQ;GAAQ,CAAC;UAC1D,KAAK;AACZ,SAAO;GACL;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;AAKL,eAAe,oBAAoB,SAAyC;AAC1E,KAAI,CAAC,QAAQ,MAAO;AACpB,OAAM,eAAe;EACnB,QAAQ,IAAI,aAAa,QAAQ,MAAM,MAAM,QAAQ,MAAM,MAAM;EACjE,OAAO,QAAQ,MAAM;EACrB,OAAO,QAAQ,MAAM;EACtB,CAAC;;;;;;AAOJ,eAAe,eACb,QACA,OACA,SACA,SACmC;AACnC,KAAI,CAAC,UAAU,CAAC,MAAO,QAAO;CAG9B,IAAI,SAAwB;EAAE,SAAS;EAAG,SAAS;EAAG,UAAU,EAAE;EAAE;AACpE,KAAI;AACF,WAAS,MAAM,eACb,QACA,SACA,QAAQ,QAAQ,eACjB;UACM,KAAK;AACZ,SAAO,SAAS,KAAK;GACnB,SAAS;GACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD,CAAC;;CAEJ,MAAM,SAAS,MAAM,cAAc,QAAQ;EACzC;EACA;EACA,SAAS,QAAQ,OAAO,KAAK,KAAK;EACnC,CAAC;AACF,QAAO;EAAE;EAAO;EAAQ;EAAQ;;;;;;;AAQlC,MAAM,sBAAsB;;;;;;AAO5B,eAAsB,cACpB,SACyB;CACzB,MAAM,OAAO,QAAQ,WAAW,QAAQ,KAAK;CAC7C,MAAM,MAAM,QAAQ,OAAO,KAAK,KAAK;CACrC,IAAI,aAAa,kBAAkB,KAAK;AAExC,KAAI,QAAQ,QAAQ;EAClB,MAAM,IAAI,QAAQ;AAClB,eAAa,WAAW,QACrB,MAAM,EAAE,UAAU,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,SAAS,EAAE,CACzD;;CAGH,MAAM,OAAO,QAAQ,kBAAkB;CACvC,MAAM,QAAQ,WAAW;AACzB,MAAK;EAAE,MAAM;EAAc;EAAO,CAAC;AAKnC,OAAM,oBAAoB,QAAQ;AAClC,KAAI;EAIF,IAAI;EACJ,IAAI;AACJ,MAAI,QAAQ,QAAQ;AAClB,kBAAe,IAAI,aACjB,QAAQ,OAAO,MACf,QAAQ,OAAO,MAChB;AACD,WAAQ,MAAM,cAAc,cAAc;IACxC,cAAc,QAAQ,OAAO;IAC7B,SAAS,eAAe,IAAI,KAAK,IAAI,CAAC,aAAa;IACnD,WAAW;IACZ,CAAC;AACF,QAAK;IAAE,MAAM;IAAe;IAAO,CAAC;;EAMtC,MAAM,UAAU,MAAM,QACpB,YACA,QAAQ,eAAe,qBACvB,OAAO,GAAG,UAAU;GAClB,MAAM,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE;AAC3B,QAAK;IAAE,MAAM;IAAS;IAAI;IAAO;IAAO,CAAC;GACzC,MAAM,SAAS,MAAM,OAAO,GAAG,IAAI,OAAO,QAAQ;AAClD,QAAK;IAAE,MAAM;IAAU;IAAQ;IAAO;IAAO,CAAC;AAC9C,UAAO;IAEV;EAED,MAAM,UAA0B,EAAE,SAAS;EAC3C,MAAM,SAAS,MAAM,eAAe,cAAc,OAAO,SAAS,QAAQ;AAC1E,MAAI,OAAQ,SAAQ,SAAS;AAC7B,SAAO;WACC;AACR,iBAAe"}
+{"version":3,"file":"run-evals.js","names":[],"sources":["../../src/evals/run-evals.ts"],"sourcesContent":["import { pathToFileURL } from \"node:url\";\n\nimport { MlflowClient } from \"../connectors/mlflow\";\nimport type { WorkspaceClient } from \"../workspace-client\";\nimport { type DatasetRow, readEvalDataset } from \"./dataset\";\nimport { type DiscoveredEval, discoverEvalFiles } from \"./discover\";\nimport { createHttpDriver } from \"./http-driver\";\nimport { configureJudge, teardownJudge } from \"./judge\";\nimport { type ReportOutcome, reportToMlflow } from \"./mlflow-report\";\nimport { createEvalRun, type FinishOutcome, finishEvalRun } from \"./mlflow-run\";\nimport { mapPool } from \"./pool\";\nimport { runEval } from \"./run-eval\";\nimport type { EvalDefinition, EvalResult } from \"./types\";\n\nexport interface RunEvalsOptions {\n  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */\n  rootDir?: string;\n  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Substring filter on `<agent>/<id>` (or an exact agent id). */\n  filter?: string;\n  /** Soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Extra request headers for the driver (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */\n  timeoutMs?: number;\n  /**\n   * Max evals to drive concurrently. Each eval opens one stream to the app as\n   * the same user, so keep this at or below the app's\n   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the\n   * 429 guard. Defaults to 4; clamped to `[1, total]`.\n   */\n  concurrency?: number;\n  /**\n   * When set, create a native MLflow \"Evaluation run\": each eval's trace is\n   * linked to the run, pass/fail is written as feedback, and aggregate metrics\n   * are logged. Requires Databricks creds + the target experiment.\n   */\n  mlflow?: {\n    host: string;\n    token: string;\n    experimentId: string;\n    /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */\n    sqlWarehouseId?: string;\n  };\n  /**\n   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a\n   * Databricks serving endpoint (`model`).\n   */\n  judge?: { host: string; token: string; model: string };\n  /**\n   * Workspace client used to read managed evaluation datasets (for evals that\n   * declare `dataset`). Required alongside {@link warehouseId} for those evals.\n   */\n  workspaceClient?: WorkspaceClient;\n  /** SQL warehouse id used to read managed evaluation datasets. */\n  warehouseId?: string;\n  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */\n  now?: number;\n  /** Progress callback, invoked as evals are discovered, started, and finished. */\n  onEvent?: (event: EvalProgress) => void;\n}\n\nexport type EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: EvalResult; index: number; total: number };\n\nexport interface EvalRunSummary {\n  results: EvalResult[];\n  /** Present when an MLflow evaluation run was created. */\n  mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome };\n}\n\n/**\n * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.\n * Uses tsx's programmatic loader so TypeScript eval files run without a build\n * step. The specifier is indirected so the type checker doesn't try to resolve\n * tsx's internal entry.\n */\nasync function loadEval(file: string): Promise<EvalDefinition> {\n  const tsxApi = \"tsx/esm/api\";\n  let tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n  try {\n    ({ tsImport } = (await import(tsxApi)) as {\n      tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n    });\n  } catch {\n    throw new Error(\n      \"Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).\",\n    );\n  }\n\n  const mod = await tsImport(pathToFileURL(file).href, import.meta.url);\n  const def = resolveEvalDefault(mod);\n  if (!def) {\n    throw new Error(`${file}: must default-export defineEval({ test })`);\n  }\n  return def;\n}\n\n/**\n * Unwrap the eval default export across module-interop shapes. Depending on\n * whether the eval file is treated as ESM or CJS, the value lands at\n * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or\n * `mod` itself. Returns the first candidate that looks like an eval.\n */\nexport function resolveEvalDefault(mod: unknown): EvalDefinition | undefined {\n  let candidate: unknown = mod;\n  for (let i = 0; i < 4 && candidate; i++) {\n    if (typeof (candidate as EvalDefinition).test === \"function\") {\n      return candidate as EvalDefinition;\n    }\n    candidate = (candidate as { default?: unknown }).default;\n  }\n  return undefined;\n}\n\n/**\n * Run one eval turn against a fresh driver. Never throws — a run failure becomes\n * a non-passing {@link EvalResult} so one bad eval can't abort the run. `row`\n * binds the current managed-dataset row (see {@link resolveDatasetRows}), or is\n * `undefined` for a plain single-run eval.\n */\nasync function runOne(\n  d: DiscoveredEval,\n  id: string,\n  def: EvalDefinition,\n  row: DatasetRow | undefined,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n): Promise<EvalResult> {\n  try {\n    // A fresh driver per row: each row is an independent conversation whose\n    // thread must not carry over the previous row's history. (Multiple\n    // `t.send`s within one row still share the thread — the driver's behavior.)\n    const driver = createHttpDriver({\n      baseUrl: options.baseUrl,\n      agent: def.agent ?? d.agent,\n      headers: options.headers,\n      mlflowRunId: runId,\n      timeoutMs: options.timeoutMs,\n    });\n    return await runEval(def, { id, driver, strict: options.strict, row });\n  } catch (err) {\n    return {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/**\n * Resolve the rows a (possibly dataset-driven) eval runs over. A plain eval\n * yields a single `undefined` row; a dataset eval reads its Unity Catalog table\n * via {@link readEvalDataset}. On misconfiguration or read failure, returns a\n * single `undefined` row plus an `error`, so the eval still surfaces one result.\n */\nexport async function resolveDatasetRows(\n  def: EvalDefinition,\n  options: RunEvalsOptions,\n): Promise<{ rows: Array<DatasetRow | undefined>; error?: string }> {\n  if (!def.dataset) return { rows: [undefined] };\n  if (!options.workspaceClient || !options.warehouseId) {\n    return {\n      rows: [undefined],\n      error:\n        \"dataset eval requires a workspace client and warehouse (pass --warehouse-id)\",\n    };\n  }\n  try {\n    const rows = await readEvalDataset(options.workspaceClient, {\n      table: def.dataset.table,\n      warehouseId: options.warehouseId,\n      limit: def.dataset.limit,\n    });\n    if (rows.length === 0) {\n      return {\n        rows: [undefined],\n        error: `dataset \"${def.dataset.table}\" returned no rows`,\n      };\n    }\n    return { rows };\n  } catch (err) {\n    return {\n      rows: [undefined],\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/**\n * Load one discovered eval and run it, expanding a dataset-driven eval into one\n * run per row. Appends one result per row to `results`, emitting `start`/\n * `result` around each. Never throws: a load or dataset-read failure surfaces as\n * a non-passing result. `total` counts eval files, not rows — per-row detail is\n * carried in the result id (`[row i/n]`).\n */\nasync function runDiscovered(\n  d: DiscoveredEval,\n  index: number,\n  total: number,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n  emit: (event: EvalProgress) => void,\n  results: EvalResult[],\n): Promise<void> {\n  const id = `${d.agent}/${d.id}`;\n\n  let def: EvalDefinition;\n  try {\n    def = await loadEval(d.file);\n  } catch (err) {\n    emit({ type: \"start\", id, index, total });\n    const result: EvalResult = {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n    results.push(result);\n    emit({ type: \"result\", result, index, total });\n    return;\n  }\n\n  const { rows, error: datasetError } = await resolveDatasetRows(def, options);\n\n  for (let r = 0; r < rows.length; r++) {\n    const rowId =\n      def.dataset && rows.length > 1\n        ? `${id} [row ${r + 1}/${rows.length}]`\n        : id;\n    emit({ type: \"start\", id: rowId, index, total });\n    const result: EvalResult = datasetError\n      ? { id: rowId, assertions: [], passed: false, error: datasetError }\n      : await runOne(d, rowId, def, rows[r], runId, options);\n    results.push(result);\n    emit({ type: \"result\", result, index, total });\n  }\n}\n\n/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */\nasync function maybeConfigureJudge(options: RunEvalsOptions): Promise<void> {\n  if (!options.judge) return;\n  await configureJudge({\n    client: new MlflowClient(options.judge.host, options.judge.token),\n    token: options.judge.token,\n    model: options.judge.model,\n  });\n}\n\n/**\n * Report per-eval assessments and finish the MLflow run, when one was created.\n * Returns the run summary, or `undefined` when there was no run to finalize.\n */\nasync function finalizeMlflow(\n  client: MlflowClient | undefined,\n  runId: string | undefined,\n  results: EvalResult[],\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary[\"mlflow\"]> {\n  if (!client || !runId) return undefined;\n  // reportToMlflow is not supposed to throw, but if it ever does the run must\n  // still be finished — otherwise it hangs in RUNNING forever.\n  let report: ReportOutcome = { written: 0, skipped: 0, failures: [] };\n  try {\n    report = await reportToMlflow(\n      client,\n      results,\n      options.mlflow?.sqlWarehouseId,\n    );\n  } catch (err) {\n    report.failures.push({\n      traceId: \"(report)\",\n      error: err instanceof Error ? err.message : String(err),\n    });\n  }\n  const finish = await finishEvalRun(client, {\n    runId,\n    results,\n    endTime: options.now ?? Date.now(),\n  });\n  return { runId, report, finish };\n}\n\n/**\n * Default max evals in flight. Each eval opens one stream to the app as the\n * same user; the server caps concurrent streams per user at 5 by default\n * (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.\n */\nconst DEFAULT_CONCURRENCY = 4;\n\n/**\n * Discover, load, and run every eval under each agent's `evals/` dir, driving\n * the agents on a running app. Never throws for an individual eval — load/run\n * failures become non-passing {@link EvalResult}s.\n */\nexport async function runEvalsInDir(\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary> {\n  const root = options.rootDir ?? process.cwd();\n  const now = options.now ?? Date.now();\n  let discovered = discoverEvalFiles(root);\n\n  if (options.filter) {\n    const f = options.filter;\n    discovered = discovered.filter(\n      (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f),\n    );\n  }\n\n  const emit = options.onEvent ?? (() => {});\n  const total = discovered.length;\n  emit({ type: \"discovered\", total });\n\n  // The judge sets OPENAI_* env vars globally (autoevals reads them per call),\n  // so tear them down in `finally` once the run is over — pass or throw — so\n  // the bearer doesn't linger in process.env.\n  await maybeConfigureJudge(options);\n  try {\n    // Create the MLflow evaluation run up front so each eval's trace can be\n    // linked to it as it runs. One client is shared by run create/finish and\n    // the per-trace assessment writes.\n    let runId: string | undefined;\n    let mlflowClient: MlflowClient | undefined;\n    if (options.mlflow) {\n      mlflowClient = new MlflowClient(\n        options.mlflow.host,\n        options.mlflow.token,\n      );\n      runId = await createEvalRun(mlflowClient, {\n        experimentId: options.mlflow.experimentId,\n        runName: `appkit-eval ${new Date(now).toISOString()}`,\n        startTime: now,\n      });\n      emit({ type: \"run-created\", runId });\n    }\n\n    // Run each eval through the bounded pool — one in-flight stream per eval, so\n    // the pool respects the server's per-user stream cap (see mapPool/concurrency).\n    // A dataset eval expands into per-row runs that execute serially within its\n    // slot; results preserve discovery order (mapPool writes by index) and row\n    // order within each file. `total` counts eval files, not dataset rows — per-row\n    // detail is carried in the result id (`[row i/n]`).\n    const perFile = await mapPool(\n      discovered,\n      options.concurrency ?? DEFAULT_CONCURRENCY,\n      async (d, index) => {\n        const fileResults: EvalResult[] = [];\n        await runDiscovered(d, index, total, runId, options, emit, fileResults);\n        return fileResults;\n      },\n    );\n    const results = perFile.flat();\n\n    const summary: EvalRunSummary = { results };\n    const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);\n    if (mlflow) summary.mlflow = mlflow;\n    return summary;\n  } finally {\n    teardownJudge();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAkFA,eAAe,SAAS,MAAuC;CAC7D,MAAM,SAAS;CACf,IAAI;AACJ,KAAI;AACF,GAAC,CAAE,YAAc,MAAM,OAAO;SAGxB;AACN,QAAM,IAAI,MACR,6FACD;;CAIH,MAAM,MAAM,mBADA,MAAM,SAAS,cAAc,KAAK,CAAC,MAAM,OAAO,KAAK,IAAI,CAClC;AACnC,KAAI,CAAC,IACH,OAAM,IAAI,MAAM,GAAG,KAAK,4CAA4C;AAEtE,QAAO;;;;;;;;AAST,SAAgB,mBAAmB,KAA0C;CAC3E,IAAI,YAAqB;AACzB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,WAAW,KAAK;AACvC,MAAI,OAAQ,UAA6B,SAAS,WAChD,QAAO;AAET,cAAa,UAAoC;;;;;;;;;AAWrD,eAAe,OACb,GACA,IACA,KACA,KACA,OACA,SACqB;AACrB,KAAI;AAWF,SAAO,MAAM,QAAQ,KAAK;GAAE;GAAI,QAPjB,iBAAiB;IAC9B,SAAS,QAAQ;IACjB,OAAO,IAAI,SAAS,EAAE;IACtB,SAAS,QAAQ;IACjB,aAAa;IACb,WAAW,QAAQ;IACpB,CAAC;GACsC,QAAQ,QAAQ;GAAQ;GAAK,CAAC;UAC/D,KAAK;AACZ,SAAO;GACL;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;;;;;;AAUL,eAAsB,mBACpB,KACA,SACkE;AAClE,KAAI,CAAC,IAAI,QAAS,QAAO,EAAE,MAAM,CAAC,OAAU,EAAE;AAC9C,KAAI,CAAC,QAAQ,mBAAmB,CAAC,QAAQ,YACvC,QAAO;EACL,MAAM,CAAC,OAAU;EACjB,OACE;EACH;AAEH,KAAI;EACF,MAAM,OAAO,MAAM,gBAAgB,QAAQ,iBAAiB;GAC1D,OAAO,IAAI,QAAQ;GACnB,aAAa,QAAQ;GACrB,OAAO,IAAI,QAAQ;GACpB,CAAC;AACF,MAAI,KAAK,WAAW,EAClB,QAAO;GACL,MAAM,CAAC,OAAU;GACjB,OAAO,YAAY,IAAI,QAAQ,MAAM;GACtC;AAEH,SAAO,EAAE,MAAM;UACR,KAAK;AACZ,SAAO;GACL,MAAM,CAAC,OAAU;GACjB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;;;;;;;AAWL,eAAe,cACb,GACA,OACA,OACA,OACA,SACA,MACA,SACe;CACf,MAAM,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE;CAE3B,IAAI;AACJ,KAAI;AACF,QAAM,MAAM,SAAS,EAAE,KAAK;UACrB,KAAK;AACZ,OAAK;GAAE,MAAM;GAAS;GAAI;GAAO;GAAO,CAAC;EACzC,MAAM,SAAqB;GACzB;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;AACD,UAAQ,KAAK,OAAO;AACpB,OAAK;GAAE,MAAM;GAAU;GAAQ;GAAO;GAAO,CAAC;AAC9C;;CAGF,MAAM,EAAE,MAAM,OAAO,iBAAiB,MAAM,mBAAmB,KAAK,QAAQ;AAE5E,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,QACJ,IAAI,WAAW,KAAK,SAAS,IACzB,GAAG,GAAG,QAAQ,IAAI,EAAE,GAAG,KAAK,OAAO,KACnC;AACN,OAAK;GAAE,MAAM;GAAS,IAAI;GAAO;GAAO;GAAO,CAAC;EAChD,MAAM,SAAqB,eACvB;GAAE,IAAI;GAAO,YAAY,EAAE;GAAE,QAAQ;GAAO,OAAO;GAAc,GACjE,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,IAAI,OAAO,QAAQ;AACxD,UAAQ,KAAK,OAAO;AACpB,OAAK;GAAE,MAAM;GAAU;GAAQ;GAAO;GAAO,CAAC;;;;AAKlD,eAAe,oBAAoB,SAAyC;AAC1E,KAAI,CAAC,QAAQ,MAAO;AACpB,OAAM,eAAe;EACnB,QAAQ,IAAI,aAAa,QAAQ,MAAM,MAAM,QAAQ,MAAM,MAAM;EACjE,OAAO,QAAQ,MAAM;EACrB,OAAO,QAAQ,MAAM;EACtB,CAAC;;;;;;AAOJ,eAAe,eACb,QACA,OACA,SACA,SACmC;AACnC,KAAI,CAAC,UAAU,CAAC,MAAO,QAAO;CAG9B,IAAI,SAAwB;EAAE,SAAS;EAAG,SAAS;EAAG,UAAU,EAAE;EAAE;AACpE,KAAI;AACF,WAAS,MAAM,eACb,QACA,SACA,QAAQ,QAAQ,eACjB;UACM,KAAK;AACZ,SAAO,SAAS,KAAK;GACnB,SAAS;GACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD,CAAC;;CAEJ,MAAM,SAAS,MAAM,cAAc,QAAQ;EACzC;EACA;EACA,SAAS,QAAQ,OAAO,KAAK,KAAK;EACnC,CAAC;AACF,QAAO;EAAE;EAAO;EAAQ;EAAQ;;;;;;;AAQlC,MAAM,sBAAsB;;;;;;AAO5B,eAAsB,cACpB,SACyB;CACzB,MAAM,OAAO,QAAQ,WAAW,QAAQ,KAAK;CAC7C,MAAM,MAAM,QAAQ,OAAO,KAAK,KAAK;CACrC,IAAI,aAAa,kBAAkB,KAAK;AAExC,KAAI,QAAQ,QAAQ;EAClB,MAAM,IAAI,QAAQ;AAClB,eAAa,WAAW,QACrB,MAAM,EAAE,UAAU,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,SAAS,EAAE,CACzD;;CAGH,MAAM,OAAO,QAAQ,kBAAkB;CACvC,MAAM,QAAQ,WAAW;AACzB,MAAK;EAAE,MAAM;EAAc;EAAO,CAAC;AAKnC,OAAM,oBAAoB,QAAQ;AAClC,KAAI;EAIF,IAAI;EACJ,IAAI;AACJ,MAAI,QAAQ,QAAQ;AAClB,kBAAe,IAAI,aACjB,QAAQ,OAAO,MACf,QAAQ,OAAO,MAChB;AACD,WAAQ,MAAM,cAAc,cAAc;IACxC,cAAc,QAAQ,OAAO;IAC7B,SAAS,eAAe,IAAI,KAAK,IAAI,CAAC,aAAa;IACnD,WAAW;IACZ,CAAC;AACF,QAAK;IAAE,MAAM;IAAe;IAAO,CAAC;;EAkBtC,MAAM,WATU,MAAM,QACpB,YACA,QAAQ,eAAe,qBACvB,OAAO,GAAG,UAAU;GAClB,MAAM,cAA4B,EAAE;AACpC,SAAM,cAAc,GAAG,OAAO,OAAO,OAAO,SAAS,MAAM,YAAY;AACvE,UAAO;IAEV,EACuB,MAAM;EAE9B,MAAM,UAA0B,EAAE,SAAS;EAC3C,MAAM,SAAS,MAAM,eAAe,cAAc,OAAO,SAAS,QAAQ;AAC1E,MAAI,OAAQ,SAAQ,SAAS;AAC7B,SAAO;WACC;AACR,iBAAe"}

package/dist/evals/types.d.ts

@@ -38,7 +38,11 @@ interface AssertionHandle {
   gate(): AssertionHandle;
   /** Demote to a tracked metric — doesn't fail unless running with `strict`. */
   soft(): AssertionHandle;
-  /** Soft assertion that passes only when the score is at least `threshold`. */
+  /**
+   * Set the pass threshold for a scored assertion: it passes only when the
+   * score is at least `threshold`. Keeps the current severity (gate unless also
+   * chained with `.soft()`).
+   */
   atLeast(threshold: number): AssertionHandle;
 }
 /** What a driver returns for a single `t.send`. */
@@ -60,17 +64,38 @@ interface DriveResult {
  */
 interface EvalDriver {
   send(message: string): Promise<DriveResult>;
+  /**
+   * Drop the current conversation so the next `send` starts a fresh thread.
+   * Optional: drivers without a session concept omit it.
+   */
+  reset?(): void;
 }
 /** The `t` context passed to an eval's `test` function. */
 interface TestContext {
   /** Send a user message to the agent and capture its response. */
   send(message: string): Promise<void>;
+  /**
+   * Start a fresh conversation: the next `send` opens a new thread with no
+   * history. Use to run several independent one-shot checks in one test.
+   * Consecutive `send`s (without a `reset`) stay in one multi-turn conversation.
+   */
+  reset(): void;
   /** The last assistant reply. */
   readonly reply: string;
   /** Tools called during the last turn. */
   readonly toolCalls: string[];
   /** The current session/thread id, if any. */
   readonly sessionId: string | undefined;
+  /**
+   * The current dataset row's `inputs` when the eval is dataset-driven (see
+   * {@link EvalDefinition.dataset}); `{}` for a plain single-run eval.
+   */
+  readonly input: Record<string, unknown>;
+  /**
+   * The current dataset row's `expectations` (ground truth / guidelines), or
+   * `undefined` when the row has none or the eval isn't dataset-driven.
+   */
+  readonly expected: Record<string, unknown> | undefined;
   /** Assert the last turn completed successfully (gate by default). */
   succeeded(): AssertionHandle;
   /** Assert a tool was called during the run (gate by default). */
@@ -79,9 +104,10 @@ interface TestContext {
   check(value: string, matcher: Matcher): AssertionHandle;
   /**
    * LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge
-   * model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)`
-   * to set the pass threshold or `.gate()` to make it a hard gate. Requires the
-   * judge to be configured (`--judge-model`).
+   * model). Each returns a scored assertion that gates by default (a miss fails
+   * the eval); chain `.atLeast(n)` to change the pass threshold or `.soft()` to
+   * demote to a tracked-only metric. Requires the judge to be configured
+   * (`--judge-model`).
    */
   judge: {
     /** Score factuality of the reply against an expected reference. */factuality(expected: string): Promise<AssertionHandle>; /** Score whether the reply answers the question, per optional `criteria`. */
@@ -103,6 +129,16 @@ interface EvalDefinition {
   description?: string;
   /** Target agent id. Defaults to the eval's parent `server/agents/<id>` dir. */
   agent?: string;
+  /**
+   * Run this eval once per row of a Databricks managed evaluation dataset (a
+   * Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns).
+   * Each row is bound to `t.input`/`t.expected`. Requires the runner to have a
+   * workspace client + warehouse (`--warehouse-id`). Omit for a single-run eval.
+   */
+  dataset?: {
+    table: string;
+    limit?: number;
+  };
   /** The eval body: drive the agent and assert on its behavior. */
   test(t: TestContext): Promise<void> | void;
 }

package/dist/evals/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":[],"sources":["../../src/evals/types.ts"],"mappings":";;AAWA;;;;;;;;;UAAiB,WAAA;EACf,IAAA;;EAEA,KAAA;EAMkD;EAJlD,MAAA;AAAA;;KAIU,OAAA,IAAW,KAAA,aAAkB,WAAA;;KAG7B,QAAA;;UAGK,eAAA;EACf,KAAA;EACA,QAAA,EAAU,QAAA;EACV,IAAA;EACA,KAAA;EACA,MAAA;AAAA;;;;AAQF;;UAAiB,eAAA;EAEP;EAAR,IAAA,IAAQ,eAAA;EAIoB;EAF5B,IAAA,IAAQ,eAAA;EAEmC;EAA3C,OAAA,CAAQ,SAAA,WAAoB,eAAA;AAAA;;UAIb,WAAA;EAJf;EAMA,KAAA;EAN4B;EAQ5B,SAAA;EAR2C;EAU3C,SAAA;EAN0B;EAQ1B,SAAA;EAR0B;EAU1B,OAAA;AAAA;;;;;UAOe,UAAA;EACf,IAAA,CAAK,OAAA,WAAkB,OAAA,CAAQ,WAAA;AAAA;;UAIhB,WAAA;EAJf;EAMA,IAAA,CAAK,OAAA,WAAkB,OAAA;EANA;EAAA,SAQd,KAAA;EARiC;EAAA,SAUjC,SAAA;EANM;EAAA,SAQN,SAAA;;EAET,SAAA,IAAa,eAAA;EAAA;EAEb,UAAA,CAAW,IAAA,WAAe,eAAA;EAEI;EAA9B,KAAA,CAAM,KAAA,UAAe,OAAA,EAAS,OAAA,GAAU,eAAA;EASA;;;;;;EAFxC,KAAA;IAMwC,mEAJtC,UAAA,CAAW,QAAA,WAAmB,OAAA,CAAQ,eAAA,GArBxC;IAuBE,QAAA,CAAS,QAAA,WAAmB,OAAA,CAAQ,eAAA,GAvBf;IAyBrB,MAAA,CAAO,IAAA,EAAM,eAAA,GAAkB,OAAA,CAAQ,eAAA;EAAA;EAnBhC;EAsBT,IAAA,CAAK,MAAA;AAAA;;UAIU,eAAA;EACf,IAAA;EACA,cAAA;EACA,YAAA,EAAc,MAAA;AAAA;;UAIC,cAAA;EApBf;EAsBA,WAAA;EApBa;EAsBb,KAAA;EAtBwC;EAwBxC,IAAA,CAAK,CAAA,EAAG,WAAA,GAAc,OAAA;AAAA;;UAIP,UAAA;EACf,EAAA;EACA,WAAA;EA1BS;EA4BT,OAAA;IAAY,MAAA;EAAA;EACZ,UAAA,EAAY,eAAA;EA1BQ;EA4BpB,MAAA;EAxBe;EA0Bf,KAAA;;EAEA,OAAA;AAAA"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../../src/evals/types.ts"],"mappings":";;AAWA;;;;;;;;;UAAiB,WAAA;EACf,IAAA;;EAEA,KAAA;EAMkD;EAJlD,MAAA;AAAA;;KAIU,OAAA,IAAW,KAAA,aAAkB,WAAA;;KAG7B,QAAA;;UAGK,eAAA;EACf,KAAA;EACA,QAAA,EAAU,QAAA;EACV,IAAA;EACA,KAAA;EACA,MAAA;AAAA;;;;AAQF;;UAAiB,eAAA;EAEP;EAAR,IAAA,IAAQ,eAAA;EAQoB;EAN5B,IAAA,IAAQ,eAAA;EAMmC;;;;;EAA3C,OAAA,CAAQ,SAAA,WAAoB,eAAA;AAAA;;UAIb,WAAA;EAJ4B;EAM3C,KAAA;EAF0B;EAI1B,SAAA;EAJ0B;EAM1B,SAAA;EAFA;EAIA,SAAA;EAAA;EAEA,OAAA;AAAA;;AAOF;;;UAAiB,UAAA;EACf,IAAA,CAAK,OAAA,WAAkB,OAAA,CAAQ,WAAA;EAA1B;;;;EAKL,KAAA;AAAA;AAIF;AAAA,UAAiB,WAAA;;EAEf,IAAA,CAAK,OAAA,WAAkB,OAAA;EAiBP;;;;;EAXhB,KAAA;EAgCwC;EAAA,SA9B/B,KAAA;EAgC6B;EAAA,SA9B7B,SAAA;EAgCM;EAAA,SA9BN,SAAA;EA8BwB;;;;EAAA,SAzBxB,KAAA,EAAO,MAAA;EAjBO;;;;EAAA,SAsBd,QAAA,EAAU,MAAA;EALV;EAOT,SAAA,IAAa,eAAA;EAFJ;EAIT,UAAA,CAAW,IAAA,WAAe,eAAA;EAF1B;EAIA,KAAA,CAAM,KAAA,UAAe,OAAA,EAAS,OAAA,GAAU,eAAA;EAFxC;;;;;;;EAUA,KAAA;IAAA,mEAEE,UAAA,CAAW,QAAA,WAAmB,OAAA,CAAQ,eAAA,GAA3B;IAEX,QAAA,CAAS,QAAA,WAAmB,OAAA,CAAQ,eAAA,GAFE;IAItC,MAAA,CAAO,IAAA,EAAM,eAAA,GAAkB,OAAA,CAAQ,eAAA;EAAA;EAFX;EAK9B,IAAA,CAAK,MAAA;AAAA;;UAIU,eAAA;EACf,IAAA;EACA,cAAA;EACA,YAAA,EAAc,MAAA;AAAA;;UAIC,cAAA;EAPA;EASf,WAAA;;EAEA,KAAA;EAVA;;;;;;EAiBA,OAAA;IAAY,KAAA;IAAe,KAAA;EAAA;EAT3B;EAWA,IAAA,CAAK,CAAA,EAAG,WAAA,GAAc,OAAA;AAAA;;UAIP,UAAA;EACf,EAAA;EACA,WAAA;EANK;EAQL,OAAA;IAAY,MAAA;EAAA;EACZ,UAAA,EAAY,eAAA;EALa;EAOzB,MAAA;EAF2B;EAI3B,KAAA;EAPA;EASA,OAAA;AAAA"}

package/dist/index.d.ts

@@ -24,6 +24,7 @@ import { AppKitError } from "./errors/base.js";
 import { AuthenticationError } from "./errors/authentication.js";
 import { ConfigurationError } from "./errors/configuration.js";
 import { ConnectionError } from "./errors/connection.js";
+import { DatabaseValidationError, DatabaseValidationIssue } from "./errors/database-validation.js";
 import { ExecutionError } from "./errors/execution.js";
 import { InitializationError } from "./errors/initialization.js";
 import { ServerError } from "./errors/server.js";
@@ -53,4 +54,4 @@ import "./plugins/ga-exports.generated.js";
 import { extractServingEndpoints, findServerFile } from "./type-generator/serving/server-file-extractor.js";
 import { appKitServingTypesPlugin } from "./type-generator/serving/vite-plugin.js";
 import { appKitTypesPlugin } from "./type-generator/vite-plugin.js";
-export { ApiError, AppKitError, AuthenticationError, type BasePluginConfig, type CacheConfig, CacheManager, type ConfigSchema, ConfigurationError, ConnectionError, type Counter, type DatabaseCredential, type DatabaseRegistry, type EndpointConfig, ExecutionError, type ExecutionResult, type FileAction, type FilePolicy, type FilePolicyUser, type FileResource, type GenerateDatabaseCredentialRequest, type Histogram, type IAppRouter, type IJobsConfig, type ITelemetry, InitializationError, type JobAPI, type JobConfig, type JobsConnectorConfig, type JobsExport, type LakebasePool, type LakebasePoolConfig, type LakebasePoolManager, Plugin, type PluginData, type PluginManifest, PolicyDeniedError, READ_ACTIONS, type RequestedClaims, RequestedClaimsPermissionSet, type RequestedResource, type ResourceEntry, type ResourceFieldEntry, type ResourcePermission, ResourceRegistry, type ResourceRequirement, ResourceType, ServerError, type ServingEndpointEntry, type ServingEndpointRegistry, type ServingFactory, SeverityNumber, type Span, SpanStatusCode, type StreamExecutionSettings, type TelemetryConfig, type ToPlugin, TunnelError, ValidationError, type ValidationResult, WRITE_ACTIONS, type WorkspaceClient, type WorkspaceClientOptions, analytics, appKitServingTypesPlugin, appKitTypesPlugin, createApp, createLakebasePool, createLakebasePoolManager, createWorkspaceClient, defineManifest, extractServingEndpoints, files, findServerFile, generateDatabaseCredential, genie, getExecutionContext, getLakebaseOrmConfig, getLakebasePgConfig, getPluginManifest, getResourceRequirements, getUsernameWithApiLookup, getWorkspaceClient, isSQLTypeMarker, jobs, lakebase, server, serving, sql, toPlugin };
+export { ApiError, AppKitError, AuthenticationError, type BasePluginConfig, type CacheConfig, CacheManager, type ConfigSchema, ConfigurationError, ConnectionError, type Counter, type DatabaseCredential, type DatabaseRegistry, DatabaseValidationError, type DatabaseValidationIssue, type EndpointConfig, ExecutionError, type ExecutionResult, type FileAction, type FilePolicy, type FilePolicyUser, type FileResource, type GenerateDatabaseCredentialRequest, type Histogram, type IAppRouter, type IJobsConfig, type ITelemetry, InitializationError, type JobAPI, type JobConfig, type JobsConnectorConfig, type JobsExport, type LakebasePool, type LakebasePoolConfig, type LakebasePoolManager, Plugin, type PluginData, type PluginManifest, PolicyDeniedError, READ_ACTIONS, type RequestedClaims, RequestedClaimsPermissionSet, type RequestedResource, type ResourceEntry, type ResourceFieldEntry, type ResourcePermission, ResourceRegistry, type ResourceRequirement, ResourceType, ServerError, type ServingEndpointEntry, type ServingEndpointRegistry, type ServingFactory, SeverityNumber, type Span, SpanStatusCode, type StreamExecutionSettings, type TelemetryConfig, type ToPlugin, TunnelError, ValidationError, type ValidationResult, WRITE_ACTIONS, type WorkspaceClient, type WorkspaceClientOptions, analytics, appKitServingTypesPlugin, appKitTypesPlugin, createApp, createLakebasePool, createLakebasePoolManager, createWorkspaceClient, defineManifest, extractServingEndpoints, files, findServerFile, generateDatabaseCredential, genie, getExecutionContext, getLakebaseOrmConfig, getLakebasePgConfig, getPluginManifest, getResourceRequirements, getUsernameWithApiLookup, getWorkspaceClient, isSQLTypeMarker, jobs, lakebase, server, serving, sql, toPlugin };

package/dist/index.js

@@ -6,6 +6,7 @@ import { AppKitError } from "./errors/base.js";
 import { AuthenticationError } from "./errors/authentication.js";
 import { ConfigurationError } from "./errors/configuration.js";
 import { ConnectionError } from "./errors/connection.js";
+import { DatabaseValidationError } from "./errors/database-validation.js";
 import { ExecutionError } from "./errors/execution.js";
 import { InitializationError } from "./errors/initialization.js";
 import { ServerError } from "./errors/server.js";
@@ -39,4 +40,4 @@ import { server } from "./plugins/server/index.js";
 import { serving } from "./plugins/serving/serving.js";
 import "./plugins/ga-exports.generated.js";
 
-export { ApiError, AppKitError, AuthenticationError, CacheManager, ConfigurationError, ConnectionError, ExecutionError, InitializationError, Plugin, PolicyDeniedError, READ_ACTIONS, RequestedClaimsPermissionSet, ResourceRegistry, ResourceType, ServerError, SeverityNumber, SpanStatusCode, TunnelError, ValidationError, WRITE_ACTIONS, analytics, appKitServingTypesPlugin, appKitTypesPlugin, createApp, createLakebasePool, createLakebasePoolManager, createWorkspaceClient, defineManifest, extractServingEndpoints, files, findServerFile, generateDatabaseCredential, genie, getExecutionContext, getLakebaseOrmConfig, getLakebasePgConfig, getPluginManifest, getResourceRequirements, getUsernameWithApiLookup, getWorkspaceClient, isSQLTypeMarker, jobs, lakebase, server, serving, sql, toPlugin };
+export { ApiError, AppKitError, AuthenticationError, CacheManager, ConfigurationError, ConnectionError, DatabaseValidationError, ExecutionError, InitializationError, Plugin, PolicyDeniedError, READ_ACTIONS, RequestedClaimsPermissionSet, ResourceRegistry, ResourceType, ServerError, SeverityNumber, SpanStatusCode, TunnelError, ValidationError, WRITE_ACTIONS, analytics, appKitServingTypesPlugin, appKitTypesPlugin, createApp, createLakebasePool, createLakebasePoolManager, createWorkspaceClient, defineManifest, extractServingEndpoints, files, findServerFile, generateDatabaseCredential, genie, getExecutionContext, getLakebaseOrmConfig, getLakebasePgConfig, getPluginManifest, getResourceRequirements, getUsernameWithApiLookup, getWorkspaceClient, isSQLTypeMarker, jobs, lakebase, server, serving, sql, toPlugin };

package/dist/plugin/plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.d.ts","names":[],"sources":["../../src/plugin/plugin.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBA0NsB,MAAA,iBACJ,gBAAA,GAAmB,gBAAA,aACxB,UAAA;EAAA,UA4BW,MAAA,EAAQ,OAAA;EAAA,UA3BpB,OAAA;EAAA,UACA,KAAA,EAAQ,YAAA;EAAA,UACR,GAAA,EAAK,UAAA;EAAA,UACL,aAAA,EAAe,aAAA;EAAA,UACf,aAAA,EAAe,aAAA;EAAA,UACf,SAAA,EAAY,UAAA;EAAA,UACZ,OAAA,GAAU,aAAA;EAwSO;EAAA,QArSnB,mBAAA;EAsSG;EAAA,QAnSH,oBAAA;EAoSN;;;;;;EAAA,OA5RK,KAAA,EAAO,WAAA;EA6W0B;;;EAxWxC,IAAA;cAEsB,MAAA,EAAQ,OAAA;EAAA,QAoBtB,gBAAA;EAqVG;;;;;;;;EAhUX,aAAA,CACE,IAAA;IACE,OAAA;IACA,eAAA,GAAkB,gBAAA;EAAA;EAgBtB,YAAA,CAAa,CAAA,EAAG,OAAA,CAAQ,MAAA;EAIlB,KAAA,CAAA,GAAK,OAAA;EAEX,YAAA,CAAA,GAAgB,iBAAA;EAIhB,uBAAA,CAAA,GAA2B,WAAA;EAI3B,qBAAA,CAAA;EA8ayB;;;;;;;;;;;;;;;;;;;;;;;;EAlZzB,OAAA,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiDA,YAAA,CAAA,GAAgB,MAAA;;;;;;;;;;;YAcN,aAAA,CAAc,GAAA,EAAK,OAAA,CAAQ,OAAA;;;;;;;;;;;EAiBrC,MAAA,CAAO,GAAA,EAAK,OAAA,CAAQ,OAAA;;;;;;;;;;;;;;UAyDZ,kBAAA;EAAA,UAkCQ,aAAA,GAAA,CACd,GAAA,EAAK,YAAA,EACL,EAAA,EAAI,oBAAA,CAAqB,CAAA,GACzB,OAAA,EAAS,uBAAA,EACT,OAAA,YAAgB,OAAA;;;;;;;;;;YAgFF,OAAA,GAAA,CACd,EAAA,GAAK,MAAA,GAAS,WAAA,KAAgB,OAAA,CAAQ,CAAA,GACtC,OAAA,EAAS,uBAAA,EACT,OAAA,YACC,OAAA,CAAQ,eAAA,CAAgB,CAAA;EAAA,UAmDjB,gBAAA,CAAiB,IAAA,UAAc,IAAA;EAAA,UAI/B,KAAA,YAAA,CACR,MAAA,EAAQ,OAAA,CAAQ,MAAA,EAChB,MAAA,EAAQ,WAAA;EAAA,QAeF,qBAAA;EAAA,QAaA,kBAAA;EAAA,QAqCM,wBAAA;EAAA,QAqBN,iBAAA;AAAA"}
+{"version":3,"file":"plugin.d.ts","names":[],"sources":["../../src/plugin/plugin.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBA0NsB,MAAA,iBACJ,gBAAA,GAAmB,gBAAA,aACxB,UAAA;EAAA,UA4BW,MAAA,EAAQ,OAAA;EAAA,UA3BpB,OAAA;EAAA,UACA,KAAA,EAAQ,YAAA;EAAA,UACR,GAAA,EAAK,UAAA;EAAA,UACL,aAAA,EAAe,aAAA;EAAA,UACf,aAAA,EAAe,aAAA;EAAA,UACf,SAAA,EAAY,UAAA;EAAA,UACZ,OAAA,GAAU,aAAA;EA0SO;EAAA,QAvSnB,mBAAA;EAwSG;EAAA,QArSH,oBAAA;EAsSN;;;;;;EAAA,OA9RK,KAAA,EAAO,WAAA;EA+W0B;;;EA1WxC,IAAA;cAEsB,MAAA,EAAQ,OAAA;EAAA,QAoBtB,gBAAA;EAuVG;;;;;;;;EAlUX,aAAA,CACE,IAAA;IACE,OAAA;IACA,eAAA,GAAkB,gBAAA;EAAA;EAgBtB,YAAA,CAAa,CAAA,EAAG,OAAA,CAAQ,MAAA;EAIlB,KAAA,CAAA,GAAK,OAAA;EAEX,YAAA,CAAA,GAAgB,iBAAA;EAIhB,uBAAA,CAAA,GAA2B,WAAA;EAI3B,qBAAA,CAAA;EAgbyB;;;;;;;;;;;;;;;;;;;;;;;;EApZzB,OAAA,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiDA,YAAA,CAAA,GAAgB,MAAA;;;;;;;;;;;YAcN,aAAA,CAAc,GAAA,EAAK,OAAA,CAAQ,OAAA;;;;;;;;;;;EAiBrC,MAAA,CAAO,GAAA,EAAK,OAAA,CAAQ,OAAA;;;;;;;;;;;;;;UAyDZ,kBAAA;EAAA,UAoCQ,aAAA,GAAA,CACd,GAAA,EAAK,YAAA,EACL,EAAA,EAAI,oBAAA,CAAqB,CAAA,GACzB,OAAA,EAAS,uBAAA,EACT,OAAA,YAAgB,OAAA;;;;;;;;;;YAgFF,OAAA,GAAA,CACd,EAAA,GAAK,MAAA,GAAS,WAAA,KAAgB,OAAA,CAAQ,CAAA,GACtC,OAAA,EAAS,uBAAA,EACT,OAAA,YACC,OAAA,CAAQ,eAAA,CAAgB,CAAA;EAAA,UAmDjB,gBAAA,CAAiB,IAAA,UAAc,IAAA;EAAA,UAI/B,KAAA,YAAA,CACR,MAAA,EAAQ,OAAA,CAAQ,MAAA,EAChB,MAAA,EAAQ,WAAA;EAAA,QAeF,qBAAA;EAAA,QAaA,kBAAA;EAAA,QAqCM,wBAAA;EAAA,QAqBN,iBAAA;AAAA"}

package/dist/plugin/plugin.js

@@ -382,7 +382,7 @@ var Plugin = class {
 				const raw = value.call(target);
 				if (raw == null) return {};
 				if (typeof raw === "function") return raw;
-				if (isPlainObject(raw)) return wrapExportFunctions(raw, wrapCall);
+				if (isPlainObject(raw)) return wrapExportFunctions(raw, (fn) => wrapCall(fn.bind(target)));
 				return raw;
 			};
 			return wrapCall(value.bind(target));