@databricks/appkit 0.71.0 → 0.73.0

package/dist/evals/judge.d.ts

@@ -0,0 +1,27 @@
+import { MlflowClient } from "../connectors/mlflow/client.js";
+import "../connectors/mlflow/index.js";
+
+//#region src/evals/judge.d.ts
+interface JudgeConfig {
+  /** Client for the workspace hosting the judge serving endpoint. */
+  client: MlflowClient;
+  /** Bearer token for the serving endpoint. */
+  token: string;
+  /** Serving endpoint name used as the judge model. */
+  model: string;
+}
+/** A normalized judge result. `score` is 0..1. */
+interface JudgeScore {
+  score: number;
+  rationale?: string;
+}
+/**
+ * Configure the judge once. Sets the OpenAI-compatible client env autoevals
+ * reads and the default judge model. No-op-safe: on failure, judging stays
+ * disabled and {@link isJudgeConfigured} returns false.
+ */
+declare function configureJudge(config: JudgeConfig): Promise<void>;
+declare function isJudgeConfigured(): boolean;
+//#endregion
+export { JudgeConfig, JudgeScore, configureJudge, isJudgeConfigured };
+//# sourceMappingURL=judge.d.ts.map

package/dist/evals/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","names":[],"sources":["../../src/evals/judge.ts"],"mappings":";;;;UAuBiB,WAAA;;EAEf,MAAA,EAAQ,YAAA;EAFO;EAIf,KAAA;;EAEA,KAAA;AAAA;;UAIe,UAAA;EACf,KAAA;EACA,SAAA;AAAA;AAFF;;;;;AAAA,iBAUsB,cAAA,CAAe,MAAA,EAAQ,WAAA,GAAc,OAAA;AAAA,iBAe3C,iBAAA,CAAA"}

package/dist/evals/judge.js

@@ -0,0 +1,77 @@
+//#region src/evals/judge.ts
+let mod;
+let enabled = false;
+let configured = false;
+let prevBaseUrl;
+let prevApiKey;
+/**
+* Configure the judge once. Sets the OpenAI-compatible client env autoevals
+* reads and the default judge model. No-op-safe: on failure, judging stays
+* disabled and {@link isJudgeConfigured} returns false.
+*/
+async function configureJudge(config) {
+	try {
+		mod = await import("autoevals");
+		prevBaseUrl = process.env.OPENAI_BASE_URL;
+		prevApiKey = process.env.OPENAI_API_KEY;
+		process.env.OPENAI_BASE_URL = config.client.servingEndpointsUrl();
+		process.env.OPENAI_API_KEY = config.token;
+		configured = true;
+		mod.init({ defaultModel: config.model });
+		enabled = true;
+	} catch {
+		enabled = false;
+	}
+}
+function isJudgeConfigured() {
+	return enabled;
+}
+/**
+* Restore the `OPENAI_*` env vars {@link configureJudge} set, so the judge
+* bearer doesn't linger in `process.env` (readable by any imported eval code)
+* after the run. Call once the run is done; safe when the judge was never
+* configured. Also disables judging so a late `t.judge.*` call fails cleanly
+* rather than hitting a torn-down client.
+*/
+function teardownJudge() {
+	if (!configured) return;
+	restoreEnv("OPENAI_BASE_URL", prevBaseUrl);
+	restoreEnv("OPENAI_API_KEY", prevApiKey);
+	configured = false;
+	enabled = false;
+}
+function restoreEnv(key, prev) {
+	if (prev === void 0) delete process.env[key];
+	else process.env[key] = prev;
+}
+/** Normalize an autoevals `Score` into a `JudgeScore`. */
+function toJudgeScore(s) {
+	const rationale = s.metadata?.rationale;
+	return {
+		score: typeof s.score === "number" ? s.score : 0,
+		rationale: typeof rationale === "string" ? rationale : void 0
+	};
+}
+function ensure() {
+	if (!enabled || !mod) throw new Error("LLM judge is not configured. Pass --judge-model and authenticate via --profile (or DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*");
+	return mod;
+}
+/** Factuality of `output` vs an `expected` reference. */
+async function judgeFactuality(args) {
+	return toJudgeScore(await ensure().Factuality(args));
+}
+/** Whether `output` answers the question in `input`, optionally constrained by `criteria`. */
+async function judgeClosedQA(args) {
+	return toJudgeScore(await ensure().ClosedQA(args));
+}
+/**
+* A custom LLM judge defined by a prompt template + choice→score map — the
+* TypeScript analog of MLflow's custom `@scorer`.
+*/
+async function judgeCustom(spec, args) {
+	return toJudgeScore(await ensure().LLMClassifierFromTemplate(spec)(args));
+}
+
+//#endregion
+export { configureJudge, isJudgeConfigured, judgeClosedQA, judgeCustom, judgeFactuality, teardownJudge };
+//# sourceMappingURL=judge.js.map

package/dist/evals/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","names":[],"sources":["../../src/evals/judge.ts"],"sourcesContent":["import type { MlflowClient } from \"../connectors/mlflow\";\n\n/**\n * LLM-as-judge scoring via the `autoevals` library (the same scorers eve uses),\n * pointed at a Databricks serving endpoint. autoevals talks to an\n * OpenAI-compatible API; Databricks Model Serving exposes one at\n * `<host>/serving-endpoints`, so we set `OPENAI_BASE_URL`/`OPENAI_API_KEY` and\n * use the judge endpoint name as the model.\n *\n * There is no public REST to call Databricks' built-in judges directly (they're\n * Python/SDK-only and the rubric prompts live in the mlflow package), so we run\n * autoevals' equivalent scorers against a Databricks judge model.\n */\ntype AutoEvals = typeof import(\"autoevals\");\n\nlet mod: AutoEvals | undefined;\nlet enabled = false;\n// Whether configureJudge overwrote the OPENAI_* env vars and they still need\n// restoring, plus the values to restore them to.\nlet configured = false;\nlet prevBaseUrl: string | undefined;\nlet prevApiKey: string | undefined;\n\nexport interface JudgeConfig {\n  /** Client for the workspace hosting the judge serving endpoint. */\n  client: MlflowClient;\n  /** Bearer token for the serving endpoint. */\n  token: string;\n  /** Serving endpoint name used as the judge model. */\n  model: string;\n}\n\n/** A normalized judge result. `score` is 0..1. */\nexport interface JudgeScore {\n  score: number;\n  rationale?: string;\n}\n\n/**\n * Configure the judge once. Sets the OpenAI-compatible client env autoevals\n * reads and the default judge model. No-op-safe: on failure, judging stays\n * disabled and {@link isJudgeConfigured} returns false.\n */\nexport async function configureJudge(config: JudgeConfig): Promise<void> {\n  try {\n    mod = await import(\"autoevals\");\n    prevBaseUrl = process.env.OPENAI_BASE_URL;\n    prevApiKey = process.env.OPENAI_API_KEY;\n    process.env.OPENAI_BASE_URL = config.client.servingEndpointsUrl();\n    process.env.OPENAI_API_KEY = config.token;\n    configured = true;\n    mod.init({ defaultModel: config.model });\n    enabled = true;\n  } catch {\n    enabled = false;\n  }\n}\n\nexport function isJudgeConfigured(): boolean {\n  return enabled;\n}\n\n/**\n * Restore the `OPENAI_*` env vars {@link configureJudge} set, so the judge\n * bearer doesn't linger in `process.env` (readable by any imported eval code)\n * after the run. Call once the run is done; safe when the judge was never\n * configured. Also disables judging so a late `t.judge.*` call fails cleanly\n * rather than hitting a torn-down client.\n */\nexport function teardownJudge(): void {\n  if (!configured) return;\n  restoreEnv(\"OPENAI_BASE_URL\", prevBaseUrl);\n  restoreEnv(\"OPENAI_API_KEY\", prevApiKey);\n  configured = false;\n  enabled = false;\n}\n\nfunction restoreEnv(key: string, prev: string | undefined): void {\n  if (prev === undefined) delete process.env[key];\n  else process.env[key] = prev;\n}\n\n/** Normalize an autoevals `Score` into a `JudgeScore`. */\nexport function toJudgeScore(s: {\n  score?: number | null;\n  metadata?: Record<string, unknown>;\n}): JudgeScore {\n  const rationale = s.metadata?.rationale;\n  return {\n    score: typeof s.score === \"number\" ? s.score : 0,\n    rationale: typeof rationale === \"string\" ? rationale : undefined,\n  };\n}\n\nfunction ensure(): AutoEvals {\n  if (!enabled || !mod) {\n    throw new Error(\n      \"LLM judge is not configured. Pass --judge-model and authenticate via --profile (or DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*\",\n    );\n  }\n  return mod;\n}\n\n/** Factuality of `output` vs an `expected` reference. */\nexport async function judgeFactuality(args: {\n  input: string;\n  output: string;\n  expected: string;\n}): Promise<JudgeScore> {\n  return toJudgeScore(await ensure().Factuality(args));\n}\n\n/** Whether `output` answers the question in `input`, optionally constrained by `criteria`. */\nexport async function judgeClosedQA(args: {\n  input: string;\n  output: string;\n  criteria: string;\n}): Promise<JudgeScore> {\n  return toJudgeScore(await ensure().ClosedQA(args));\n}\n\n/**\n * A custom LLM judge defined by a prompt template + choice→score map — the\n * TypeScript analog of MLflow's custom `@scorer`.\n */\nexport async function judgeCustom(\n  spec: {\n    name: string;\n    promptTemplate: string;\n    choiceScores: Record<string, number>;\n  },\n  args: { input: string; output: string },\n): Promise<JudgeScore> {\n  const scorer = ensure().LLMClassifierFromTemplate(spec);\n  return toJudgeScore(await scorer(args));\n}\n"],"mappings":";AAeA,IAAI;AACJ,IAAI,UAAU;AAGd,IAAI,aAAa;AACjB,IAAI;AACJ,IAAI;;;;;;AAsBJ,eAAsB,eAAe,QAAoC;AACvE,KAAI;AACF,QAAM,MAAM,OAAO;AACnB,gBAAc,QAAQ,IAAI;AAC1B,eAAa,QAAQ,IAAI;AACzB,UAAQ,IAAI,kBAAkB,OAAO,OAAO,qBAAqB;AACjE,UAAQ,IAAI,iBAAiB,OAAO;AACpC,eAAa;AACb,MAAI,KAAK,EAAE,cAAc,OAAO,OAAO,CAAC;AACxC,YAAU;SACJ;AACN,YAAU;;;AAId,SAAgB,oBAA6B;AAC3C,QAAO;;;;;;;;;AAUT,SAAgB,gBAAsB;AACpC,KAAI,CAAC,WAAY;AACjB,YAAW,mBAAmB,YAAY;AAC1C,YAAW,kBAAkB,WAAW;AACxC,cAAa;AACb,WAAU;;AAGZ,SAAS,WAAW,KAAa,MAAgC;AAC/D,KAAI,SAAS,OAAW,QAAO,QAAQ,IAAI;KACtC,SAAQ,IAAI,OAAO;;;AAI1B,SAAgB,aAAa,GAGd;CACb,MAAM,YAAY,EAAE,UAAU;AAC9B,QAAO;EACL,OAAO,OAAO,EAAE,UAAU,WAAW,EAAE,QAAQ;EAC/C,WAAW,OAAO,cAAc,WAAW,YAAY;EACxD;;AAGH,SAAS,SAAoB;AAC3B,KAAI,CAAC,WAAW,CAAC,IACf,OAAM,IAAI,MACR,wIACD;AAEH,QAAO;;;AAIT,eAAsB,gBAAgB,MAId;AACtB,QAAO,aAAa,MAAM,QAAQ,CAAC,WAAW,KAAK,CAAC;;;AAItD,eAAsB,cAAc,MAIZ;AACtB,QAAO,aAAa,MAAM,QAAQ,CAAC,SAAS,KAAK,CAAC;;;;;;AAOpD,eAAsB,YACpB,MAKA,MACqB;AAErB,QAAO,aAAa,MADL,QAAQ,CAAC,0BAA0B,KAAK,CACtB,KAAK,CAAC"}

package/dist/evals/matchers.d.ts

@@ -0,0 +1,12 @@
+import { Matcher } from "./types.js";
+
+//#region src/evals/matchers.d.ts
+/** Passes when the value contains `substring`. */
+declare function includes(substring: string): Matcher;
+/** Passes when the value equals `expected` exactly. */
+declare function equals(expected: string): Matcher;
+/** Passes when the value matches `pattern`. */
+declare function matches(pattern: RegExp): Matcher;
+//#endregion
+export { equals, includes, matches };
+//# sourceMappingURL=matchers.d.ts.map

package/dist/evals/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","names":[],"sources":["../../src/evals/matchers.ts"],"mappings":";;;;iBAGgB,QAAA,CAAS,SAAA,WAAoB,OAAA;AAA7C;AAAA,iBAQgB,MAAA,CAAO,QAAA,WAAmB,OAAA;;iBAQ1B,OAAA,CAAQ,OAAA,EAAS,MAAA,GAAS,OAAA"}

package/dist/evals/matchers.js

@@ -0,0 +1,26 @@
+//#region src/evals/matchers.ts
+/** Passes when the value contains `substring`. */
+function includes(substring) {
+	return (value) => ({
+		pass: value.includes(substring),
+		detail: `expected to include ${JSON.stringify(substring)}`
+	});
+}
+/** Passes when the value equals `expected` exactly. */
+function equals(expected) {
+	return (value) => ({
+		pass: value === expected,
+		detail: `expected to equal ${JSON.stringify(expected)}`
+	});
+}
+/** Passes when the value matches `pattern`. */
+function matches(pattern) {
+	return (value) => ({
+		pass: pattern.test(value),
+		detail: `expected to match ${pattern}`
+	});
+}
+
+//#endregion
+export { equals, includes, matches };
+//# sourceMappingURL=matchers.js.map

package/dist/evals/matchers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.js","names":[],"sources":["../../src/evals/matchers.ts"],"sourcesContent":["import type { Matcher } from \"./types\";\n\n/** Passes when the value contains `substring`. */\nexport function includes(substring: string): Matcher {\n  return (value) => ({\n    pass: value.includes(substring),\n    detail: `expected to include ${JSON.stringify(substring)}`,\n  });\n}\n\n/** Passes when the value equals `expected` exactly. */\nexport function equals(expected: string): Matcher {\n  return (value) => ({\n    pass: value === expected,\n    detail: `expected to equal ${JSON.stringify(expected)}`,\n  });\n}\n\n/** Passes when the value matches `pattern`. */\nexport function matches(pattern: RegExp): Matcher {\n  return (value) => ({\n    pass: pattern.test(value),\n    detail: `expected to match ${pattern}`,\n  });\n}\n"],"mappings":";;AAGA,SAAgB,SAAS,WAA4B;AACnD,SAAQ,WAAW;EACjB,MAAM,MAAM,SAAS,UAAU;EAC/B,QAAQ,uBAAuB,KAAK,UAAU,UAAU;EACzD;;;AAIH,SAAgB,OAAO,UAA2B;AAChD,SAAQ,WAAW;EACjB,MAAM,UAAU;EAChB,QAAQ,qBAAqB,KAAK,UAAU,SAAS;EACtD;;;AAIH,SAAgB,QAAQ,SAA0B;AAChD,SAAQ,WAAW;EACjB,MAAM,QAAQ,KAAK,MAAM;EACzB,QAAQ,qBAAqB;EAC9B"}

package/dist/evals/mlflow-report.d.ts

@@ -0,0 +1,37 @@
+import { MlflowClient } from "../connectors/mlflow/client.js";
+import "../connectors/mlflow/index.js";
+import { EvalResult } from "./types.js";
+
+//#region src/evals/mlflow-report.d.ts
+/** A Feedback assessment in the MLflow REST proto-JSON shape. */
+interface Assessment {
+  trace_id: string;
+  assessment_name: string;
+  source: {
+    source_type: "CODE" | "HUMAN" | "LLM_JUDGE";
+    source_id: string;
+  };
+  feedback: {
+    value: unknown;
+  };
+  rationale?: string;
+  metadata?: Record<string, string>;
+}
+interface ReportOutcome {
+  written: number;
+  skipped: number;
+  failures: Array<{
+    traceId: string;
+    status?: number;
+    error?: string;
+  }>;
+}
+declare function buildAssessments(result: EvalResult): Assessment[];
+/**
+ * Write one pass/fail assessment per eval result to the Databricks MLflow REST
+ * API. Never throws — failures are collected so the run still reports.
+ */
+declare function reportToMlflow(client: MlflowClient, results: EvalResult[], sqlWarehouseId?: string): Promise<ReportOutcome>;
+//#endregion
+export { Assessment, ReportOutcome, buildAssessments, reportToMlflow };
+//# sourceMappingURL=mlflow-report.d.ts.map

package/dist/evals/mlflow-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlflow-report.d.ts","names":[],"sources":["../../src/evals/mlflow-report.ts"],"mappings":";;;;;;UAqBiB,UAAA;EACf,QAAA;EACA,eAAA;EACA,MAAA;IAAU,WAAA;IAA6C,SAAA;EAAA;EACvD,QAAA;IAAY,KAAA;EAAA;EACZ,SAAA;EACA,QAAA,GAAW,MAAA;AAAA;AAAA,UAGI,aAAA;EACf,OAAA;EACA,OAAA;EACA,QAAA,EAAU,KAAA;IAAQ,OAAA;IAAiB,MAAA;IAAiB,KAAA;EAAA;AAAA;AAAA,iBA2DtC,gBAAA,CAAiB,MAAA,EAAQ,UAAA,GAAa,UAAA;;;;;iBA6FhC,cAAA,CACpB,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,UAAA,IACT,cAAA,YACC,OAAA,CAAQ,aAAA"}

package/dist/evals/mlflow-report.js

@@ -0,0 +1,161 @@
+import { mapPool } from "./pool.js";
+
+//#region src/evals/mlflow-report.ts
+/**
+* Max assessment writes in flight. Independent per-trace REST POSTs to the
+* Databricks MLflow API; bounded to stay well under its rate limits.
+*/
+const ASSESSMENT_WRITE_CONCURRENCY = 8;
+/**
+* Retry budget for a 404 on an assessment write. The app exports each turn's
+* trace asynchronously, so a write issued right after the run can beat the
+* trace into the store (404 "trace not found"); linear backoff rides out that
+* ingestion lag. Only 404s retry — a 4xx/5xx won't resolve by waiting.
+*/
+const ASSESSMENT_WRITE_RETRIES = 5;
+const ASSESSMENT_RETRY_BASE_MS = 500;
+const TRACE_NOT_FOUND = 404;
+/**
+* MLflow assessment names reject `.` (and we avoid spaces/parens too), so map
+* anything outside `[A-Za-z0-9_-]` to `_`. The judge check on the raw label
+* (`judge.`-prefixed) is unaffected — it runs before sanitization.
+*/
+function sanitizeName(label) {
+	return label.replace(/[^A-Za-z0-9_-]/g, "_");
+}
+/**
+* Build the Feedback assessments for an eval result: one per assertion (judge
+* assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they
+* render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail.
+* Returns [] when there's no trace to attach to or the eval was skipped.
+*/
+/** One Feedback assessment for a single assertion (judge assertions tagged `LLM_JUDGE`). */
+function assertionAssessment(a, traceId, name, evalId) {
+	return {
+		trace_id: traceId,
+		assessment_name: name,
+		source: a.label.startsWith("judge.") ? {
+			source_type: "LLM_JUDGE",
+			source_id: "appkit-judge"
+		} : {
+			source_type: "CODE",
+			source_id: "appkit-eval"
+		},
+		feedback: { value: a.score ?? a.pass },
+		rationale: a.detail,
+		metadata: {
+			eval_id: evalId,
+			severity: a.severity
+		}
+	};
+}
+/** The overall `appkit_eval` pass/fail assessment for an eval result. */
+function overallAssessment(result, traceId) {
+	return {
+		trace_id: traceId,
+		assessment_name: "appkit_eval",
+		source: {
+			source_type: "CODE",
+			source_id: "appkit-eval"
+		},
+		feedback: { value: result.passed },
+		rationale: result.error ? "eval errored" : result.passed ? "all gates passed" : "one or more gates failed",
+		metadata: { eval_id: result.id }
+	};
+}
+function buildAssessments(result) {
+	if (!result.traceId || result.skipped) return [];
+	const traceId = result.traceId;
+	const out = [];
+	const used = /* @__PURE__ */ new Map();
+	for (const a of result.assertions) {
+		const base = sanitizeName(a.label);
+		const seen = used.get(base) ?? 0;
+		used.set(base, seen + 1);
+		const name = seen === 0 ? base : `${base}_${seen + 1}`;
+		out.push(assertionAssessment(a, traceId, name, result.id));
+	}
+	out.push(overallAssessment(result, traceId));
+	return out;
+}
+/**
+* A Unity Catalog V4 trace id: `trace:/<catalog.schema[.prefix]>/<otel_hex>`.
+* Databricks addresses these with the location and the bare hex as *separate*
+* path segments (mirrors MLflow's `parse_trace_id_v4`).
+*/
+const V4_TRACE_ID = /^trace:\/([^/]+)\/([^/]+)$/;
+/**
+* Build the REST request to create one assessment, dispatching on the trace-id
+* form:
+* - **V4 UC** (`trace:/<location>/<hex>`) → `POST /api/4.0/mlflow/traces/
+*   {location}/{hex}/assessments`, body is the bare assessment (the Databricks
+*   RPC maps `location_id` from the path). This is the path UC-backed
+*   experiments require — the V3 endpoint 400s on a V4 id.
+* - **V3** (`tr-...`, classic experiments) → `POST /api/3.0/mlflow/traces/
+*   {trace_id}/assessments`, body wraps the assessment in `{ assessment }`.
+*/
+function assessmentRequest(assessment, sqlWarehouseId) {
+	const v4 = V4_TRACE_ID.exec(assessment.trace_id);
+	if (v4) {
+		const [, location, id] = v4;
+		const query = sqlWarehouseId ? `?sql_warehouse_id=${encodeURIComponent(sqlWarehouseId)}` : "";
+		return {
+			path: `/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${id}/assessments${query}`,
+			body: assessment
+		};
+	}
+	return {
+		path: `/api/3.0/mlflow/traces/${encodeURIComponent(assessment.trace_id)}/assessments`,
+		body: { assessment }
+	};
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* Write one assessment, retrying on a 404 (trace not yet ingested) with linear
+* backoff. Returns the final {@link PostResult}; non-404 failures return on the
+* first attempt.
+*/
+async function writeAssessment(client, assessment, sqlWarehouseId) {
+	const { path, body } = assessmentRequest(assessment, sqlWarehouseId);
+	let res = await client.postResult(path, body);
+	for (let attempt = 1; attempt <= ASSESSMENT_WRITE_RETRIES && !res.ok && res.status === TRACE_NOT_FOUND; attempt++) {
+		await sleep(ASSESSMENT_RETRY_BASE_MS * attempt);
+		res = await client.postResult(path, body);
+	}
+	return res;
+}
+/**
+* Write one pass/fail assessment per eval result to the Databricks MLflow REST
+* API. Never throws — failures are collected so the run still reports.
+*/
+async function reportToMlflow(client, results, sqlWarehouseId) {
+	const outcome = {
+		written: 0,
+		skipped: 0,
+		failures: []
+	};
+	const assessments = [];
+	for (const result of results) {
+		const built = buildAssessments(result);
+		if (built.length === 0) {
+			outcome.skipped++;
+			continue;
+		}
+		assessments.push(...built);
+	}
+	const posts = await mapPool(assessments, ASSESSMENT_WRITE_CONCURRENCY, (a) => writeAssessment(client, a, sqlWarehouseId));
+	for (let i = 0; i < posts.length; i++) {
+		const res = posts[i];
+		if (res.ok) outcome.written++;
+		else outcome.failures.push({
+			traceId: assessments[i].trace_id,
+			status: res.status,
+			error: res.error
+		});
+	}
+	return outcome;
+}
+
+//#endregion
+export { buildAssessments, reportToMlflow };
+//# sourceMappingURL=mlflow-report.js.map

package/dist/evals/mlflow-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlflow-report.js","names":[],"sources":["../../src/evals/mlflow-report.ts"],"sourcesContent":["import type { MlflowClient, PostResult } from \"../connectors/mlflow\";\nimport { mapPool } from \"./pool\";\nimport type { AssertionResult, EvalResult } from \"./types\";\n\n/**\n * Max assessment writes in flight. Independent per-trace REST POSTs to the\n * Databricks MLflow API; bounded to stay well under its rate limits.\n */\nconst ASSESSMENT_WRITE_CONCURRENCY = 8;\n\n/**\n * Retry budget for a 404 on an assessment write. The app exports each turn's\n * trace asynchronously, so a write issued right after the run can beat the\n * trace into the store (404 \"trace not found\"); linear backoff rides out that\n * ingestion lag. Only 404s retry — a 4xx/5xx won't resolve by waiting.\n */\nconst ASSESSMENT_WRITE_RETRIES = 5;\nconst ASSESSMENT_RETRY_BASE_MS = 500;\nconst TRACE_NOT_FOUND = 404;\n\n/** A Feedback assessment in the MLflow REST proto-JSON shape. */\nexport interface Assessment {\n  trace_id: string;\n  assessment_name: string;\n  source: { source_type: \"CODE\" | \"HUMAN\" | \"LLM_JUDGE\"; source_id: string };\n  feedback: { value: unknown };\n  rationale?: string;\n  metadata?: Record<string, string>;\n}\n\nexport interface ReportOutcome {\n  written: number;\n  skipped: number;\n  failures: Array<{ traceId: string; status?: number; error?: string }>;\n}\n\n/**\n * MLflow assessment names reject `.` (and we avoid spaces/parens too), so map\n * anything outside `[A-Za-z0-9_-]` to `_`. The judge check on the raw label\n * (`judge.`-prefixed) is unaffected — it runs before sanitization.\n */\nfunction sanitizeName(label: string): string {\n  return label.replace(/[^A-Za-z0-9_-]/g, \"_\");\n}\n\n/**\n * Build the Feedback assessments for an eval result: one per assertion (judge\n * assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they\n * render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail.\n * Returns [] when there's no trace to attach to or the eval was skipped.\n */\n/** One Feedback assessment for a single assertion (judge assertions tagged `LLM_JUDGE`). */\nfunction assertionAssessment(\n  a: AssertionResult,\n  traceId: string,\n  name: string,\n  evalId: string,\n): Assessment {\n  const isJudge = a.label.startsWith(\"judge.\");\n  return {\n    trace_id: traceId,\n    assessment_name: name,\n    source: isJudge\n      ? { source_type: \"LLM_JUDGE\", source_id: \"appkit-judge\" }\n      : { source_type: \"CODE\", source_id: \"appkit-eval\" },\n    // Judges report a numeric score; deterministic assertions a boolean.\n    feedback: { value: a.score ?? a.pass },\n    rationale: a.detail,\n    metadata: { eval_id: evalId, severity: a.severity },\n  };\n}\n\n/** The overall `appkit_eval` pass/fail assessment for an eval result. */\nfunction overallAssessment(result: EvalResult, traceId: string): Assessment {\n  return {\n    trace_id: traceId,\n    assessment_name: \"appkit_eval\",\n    source: { source_type: \"CODE\", source_id: \"appkit-eval\" },\n    feedback: { value: result.passed },\n    // Persist only a generic marker, never `result.error` itself: this rationale\n    // is POSTed to MLflow and readable by anyone with access to the experiment,\n    // a broader audience than the runner. The full error stays on the operator\n    // console (printed via the reporter's per-eval detail line).\n    rationale: result.error\n      ? \"eval errored\"\n      : result.passed\n        ? \"all gates passed\"\n        : \"one or more gates failed\",\n    metadata: { eval_id: result.id },\n  };\n}\n\nexport function buildAssessments(result: EvalResult): Assessment[] {\n  if (!result.traceId || result.skipped) return [];\n  const traceId = result.traceId;\n  const out: Assessment[] = [];\n  const used = new Map<string, number>();\n\n  for (const a of result.assertions) {\n    const base = sanitizeName(a.label);\n    const seen = used.get(base) ?? 0;\n    used.set(base, seen + 1);\n    const name = seen === 0 ? base : `${base}_${seen + 1}`;\n    out.push(assertionAssessment(a, traceId, name, result.id));\n  }\n\n  out.push(overallAssessment(result, traceId));\n  return out;\n}\n\n/**\n * A Unity Catalog V4 trace id: `trace:/<catalog.schema[.prefix]>/<otel_hex>`.\n * Databricks addresses these with the location and the bare hex as *separate*\n * path segments (mirrors MLflow's `parse_trace_id_v4`).\n */\nconst V4_TRACE_ID = /^trace:\\/([^/]+)\\/([^/]+)$/;\n\n/**\n * Build the REST request to create one assessment, dispatching on the trace-id\n * form:\n * - **V4 UC** (`trace:/<location>/<hex>`) → `POST /api/4.0/mlflow/traces/\n *   {location}/{hex}/assessments`, body is the bare assessment (the Databricks\n *   RPC maps `location_id` from the path). This is the path UC-backed\n *   experiments require — the V3 endpoint 400s on a V4 id.\n * - **V3** (`tr-...`, classic experiments) → `POST /api/3.0/mlflow/traces/\n *   {trace_id}/assessments`, body wraps the assessment in `{ assessment }`.\n */\nfunction assessmentRequest(\n  assessment: Assessment,\n  sqlWarehouseId?: string,\n): {\n  path: string;\n  body: unknown;\n} {\n  const v4 = V4_TRACE_ID.exec(assessment.trace_id);\n  if (v4) {\n    const [, location, id] = v4;\n    // UC trace assessments are backed by a SQL warehouse; Databricks requires\n    // its id as a query param (mlflow's `_append_sql_warehouse_id_param`).\n    const query = sqlWarehouseId\n      ? `?sql_warehouse_id=${encodeURIComponent(sqlWarehouseId)}`\n      : \"\";\n    return {\n      path: `/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${id}/assessments${query}`,\n      body: assessment,\n    };\n  }\n  return {\n    path: `/api/3.0/mlflow/traces/${encodeURIComponent(assessment.trace_id)}/assessments`,\n    body: { assessment },\n  };\n}\n\nconst sleep = (ms: number): Promise<void> =>\n  new Promise((resolve) => setTimeout(resolve, ms));\n\n/**\n * Write one assessment, retrying on a 404 (trace not yet ingested) with linear\n * backoff. Returns the final {@link PostResult}; non-404 failures return on the\n * first attempt.\n */\nasync function writeAssessment(\n  client: MlflowClient,\n  assessment: Assessment,\n  sqlWarehouseId?: string,\n): Promise<PostResult> {\n  const { path, body } = assessmentRequest(assessment, sqlWarehouseId);\n  let res = await client.postResult(path, body);\n  for (\n    let attempt = 1;\n    attempt <= ASSESSMENT_WRITE_RETRIES &&\n    !res.ok &&\n    res.status === TRACE_NOT_FOUND;\n    attempt++\n  ) {\n    await sleep(ASSESSMENT_RETRY_BASE_MS * attempt);\n    res = await client.postResult(path, body);\n  }\n  return res;\n}\n\n/**\n * Write one pass/fail assessment per eval result to the Databricks MLflow REST\n * API. Never throws — failures are collected so the run still reports.\n */\nexport async function reportToMlflow(\n  client: MlflowClient,\n  results: EvalResult[],\n  sqlWarehouseId?: string,\n): Promise<ReportOutcome> {\n  const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] };\n\n  // Build every assessment first (pure, no I/O). `skipped` counts results with\n  // no trace to attach to; `written`/`failures` are counted per assessment.\n  const assessments: Assessment[] = [];\n  for (const result of results) {\n    const built = buildAssessments(result);\n    if (built.length === 0) {\n      outcome.skipped++;\n      continue;\n    }\n    assessments.push(...built);\n  }\n\n  // The writes are independent per-trace REST calls, so run them through a\n  // bounded pool instead of strictly serial. postResult never throws.\n  const posts = await mapPool(assessments, ASSESSMENT_WRITE_CONCURRENCY, (a) =>\n    writeAssessment(client, a, sqlWarehouseId),\n  );\n  for (let i = 0; i < posts.length; i++) {\n    const res = posts[i];\n    if (res.ok) {\n      outcome.written++;\n    } else {\n      outcome.failures.push({\n        traceId: assessments[i].trace_id,\n        status: res.status,\n        error: res.error,\n      });\n    }\n  }\n  return outcome;\n}\n"],"mappings":";;;;;;;AAQA,MAAM,+BAA+B;;;;;;;AAQrC,MAAM,2BAA2B;AACjC,MAAM,2BAA2B;AACjC,MAAM,kBAAkB;;;;;;AAuBxB,SAAS,aAAa,OAAuB;AAC3C,QAAO,MAAM,QAAQ,mBAAmB,IAAI;;;;;;;;;AAU9C,SAAS,oBACP,GACA,SACA,MACA,QACY;AAEZ,QAAO;EACL,UAAU;EACV,iBAAiB;EACjB,QAJc,EAAE,MAAM,WAAW,SAAS,GAKtC;GAAE,aAAa;GAAa,WAAW;GAAgB,GACvD;GAAE,aAAa;GAAQ,WAAW;GAAe;EAErD,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM;EACtC,WAAW,EAAE;EACb,UAAU;GAAE,SAAS;GAAQ,UAAU,EAAE;GAAU;EACpD;;;AAIH,SAAS,kBAAkB,QAAoB,SAA6B;AAC1E,QAAO;EACL,UAAU;EACV,iBAAiB;EACjB,QAAQ;GAAE,aAAa;GAAQ,WAAW;GAAe;EACzD,UAAU,EAAE,OAAO,OAAO,QAAQ;EAKlC,WAAW,OAAO,QACd,iBACA,OAAO,SACL,qBACA;EACN,UAAU,EAAE,SAAS,OAAO,IAAI;EACjC;;AAGH,SAAgB,iBAAiB,QAAkC;AACjE,KAAI,CAAC,OAAO,WAAW,OAAO,QAAS,QAAO,EAAE;CAChD,MAAM,UAAU,OAAO;CACvB,MAAM,MAAoB,EAAE;CAC5B,MAAM,uBAAO,IAAI,KAAqB;AAEtC,MAAK,MAAM,KAAK,OAAO,YAAY;EACjC,MAAM,OAAO,aAAa,EAAE,MAAM;EAClC,MAAM,OAAO,KAAK,IAAI,KAAK,IAAI;AAC/B,OAAK,IAAI,MAAM,OAAO,EAAE;EACxB,MAAM,OAAO,SAAS,IAAI,OAAO,GAAG,KAAK,GAAG,OAAO;AACnD,MAAI,KAAK,oBAAoB,GAAG,SAAS,MAAM,OAAO,GAAG,CAAC;;AAG5D,KAAI,KAAK,kBAAkB,QAAQ,QAAQ,CAAC;AAC5C,QAAO;;;;;;;AAQT,MAAM,cAAc;;;;;;;;;;;AAYpB,SAAS,kBACP,YACA,gBAIA;CACA,MAAM,KAAK,YAAY,KAAK,WAAW,SAAS;AAChD,KAAI,IAAI;EACN,MAAM,GAAG,UAAU,MAAM;EAGzB,MAAM,QAAQ,iBACV,qBAAqB,mBAAmB,eAAe,KACvD;AACJ,SAAO;GACL,MAAM,0BAA0B,mBAAmB,SAAS,CAAC,GAAG,GAAG,cAAc;GACjF,MAAM;GACP;;AAEH,QAAO;EACL,MAAM,0BAA0B,mBAAmB,WAAW,SAAS,CAAC;EACxE,MAAM,EAAE,YAAY;EACrB;;AAGH,MAAM,SAAS,OACb,IAAI,SAAS,YAAY,WAAW,SAAS,GAAG,CAAC;;;;;;AAOnD,eAAe,gBACb,QACA,YACA,gBACqB;CACrB,MAAM,EAAE,MAAM,SAAS,kBAAkB,YAAY,eAAe;CACpE,IAAI,MAAM,MAAM,OAAO,WAAW,MAAM,KAAK;AAC7C,MACE,IAAI,UAAU,GACd,WAAW,4BACX,CAAC,IAAI,MACL,IAAI,WAAW,iBACf,WACA;AACA,QAAM,MAAM,2BAA2B,QAAQ;AAC/C,QAAM,MAAM,OAAO,WAAW,MAAM,KAAK;;AAE3C,QAAO;;;;;;AAOT,eAAsB,eACpB,QACA,SACA,gBACwB;CACxB,MAAM,UAAyB;EAAE,SAAS;EAAG,SAAS;EAAG,UAAU,EAAE;EAAE;CAIvE,MAAM,cAA4B,EAAE;AACpC,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,QAAQ,iBAAiB,OAAO;AACtC,MAAI,MAAM,WAAW,GAAG;AACtB,WAAQ;AACR;;AAEF,cAAY,KAAK,GAAG,MAAM;;CAK5B,MAAM,QAAQ,MAAM,QAAQ,aAAa,+BAA+B,MACtE,gBAAgB,QAAQ,GAAG,eAAe,CAC3C;AACD,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,MAAM,MAAM;AAClB,MAAI,IAAI,GACN,SAAQ;MAER,SAAQ,SAAS,KAAK;GACpB,SAAS,YAAY,GAAG;GACxB,QAAQ,IAAI;GACZ,OAAO,IAAI;GACZ,CAAC;;AAGN,QAAO"}

package/dist/evals/mlflow-run.d.ts

@@ -0,0 +1,13 @@
+import "../connectors/mlflow/index.js";
+
+//#region src/evals/mlflow-run.d.ts
+interface FinishOutcome {
+  finished: boolean;
+  /** Metric logging is best-effort; set when it failed (the run is still finished). */
+  metricsError?: string;
+  /** Set when the FINISHED update itself failed (run may be left RUNNING). */
+  finishError?: string;
+}
+//#endregion
+export { FinishOutcome };
+//# sourceMappingURL=mlflow-run.d.ts.map

package/dist/evals/mlflow-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlflow-run.d.ts","names":[],"sources":["../../src/evals/mlflow-run.ts"],"mappings":";;;UA6EiB,aAAA;EACf,QAAA;;EAEA,YAAA;;EAEA,WAAA;AAAA"}

package/dist/evals/mlflow-run.js

@@ -0,0 +1,101 @@
+//#region src/evals/mlflow-run.ts
+/** Run tag value that makes a run appear under the experiment's "Evaluation runs". */
+const GENAI_EVALUATE_RUN_TYPE = "genai_evaluate";
+/**
+* Source tags on the eval run. Traces linked via `mlflow.sourceRun` surface the
+* run's source in the traces-table "Source" column (mirrors how Python's
+* `evaluate()` shows the script name), so tagging the run is what populates it.
+*/
+const EVAL_SOURCE_NAME = "appkit agent eval";
+/**
+* Create an MLflow run tagged as a GenAI evaluation, so it shows under the
+* experiment's "Evaluation runs". Returns the run id; link traces to it via the
+* `mlflow.sourceRun` trace metadata and log results before finishing.
+*/
+async function createEvalRun(client, options) {
+	const created = await client.post("/api/2.0/mlflow/runs/create", {
+		experiment_id: options.experimentId,
+		start_time: options.startTime,
+		...options.runName ? { run_name: options.runName } : {},
+		tags: [
+			{
+				key: "mlflow.runType",
+				value: GENAI_EVALUATE_RUN_TYPE
+			},
+			{
+				key: "mlflow.source.name",
+				value: EVAL_SOURCE_NAME
+			},
+			{
+				key: "mlflow.source.type",
+				value: "LOCAL"
+			}
+		]
+	});
+	const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid;
+	if (!runId) throw new Error("runs/create returned no run id");
+	return runId;
+}
+/** Aggregate per-eval results into MLflow run metrics. */
+function aggregateMetrics(results, timestamp) {
+	const scored = results.filter((r) => !r.skipped);
+	const passed = scored.filter((r) => r.passed).length;
+	return [
+		{
+			key: "eval/total",
+			value: results.length,
+			timestamp,
+			step: 0
+		},
+		{
+			key: "eval/scored",
+			value: scored.length,
+			timestamp,
+			step: 0
+		},
+		{
+			key: "eval/passed",
+			value: passed,
+			timestamp,
+			step: 0
+		},
+		{
+			key: "eval/pass_rate",
+			value: scored.length ? passed / scored.length : 0,
+			timestamp,
+			step: 0
+		}
+	];
+}
+/**
+* Log aggregate metrics (best-effort) and mark the eval run FINISHED. Never
+* throws — a metric-logging failure must not prevent the run from being closed,
+* or it would be left stuck in RUNNING forever.
+*/
+async function finishEvalRun(client, options) {
+	const outcome = { finished: false };
+	const metrics = aggregateMetrics(options.results, options.endTime);
+	if (metrics.length) try {
+		await client.post("/api/2.0/mlflow/runs/log-batch", {
+			run_id: options.runId,
+			metrics
+		});
+	} catch (err) {
+		outcome.metricsError = err instanceof Error ? err.message : String(err);
+	}
+	try {
+		await client.post("/api/2.0/mlflow/runs/update", {
+			run_id: options.runId,
+			status: "FINISHED",
+			end_time: options.endTime
+		});
+		outcome.finished = true;
+	} catch (err) {
+		outcome.finishError = err instanceof Error ? err.message : String(err);
+	}
+	return outcome;
+}
+
+//#endregion
+export { createEvalRun, finishEvalRun };
+//# sourceMappingURL=mlflow-run.js.map

package/dist/evals/mlflow-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlflow-run.js","names":[],"sources":["../../src/evals/mlflow-run.ts"],"sourcesContent":["import type { MlflowClient } from \"../connectors/mlflow\";\nimport type { EvalResult } from \"./types\";\n\n/** Run tag value that makes a run appear under the experiment's \"Evaluation runs\". */\nconst GENAI_EVALUATE_RUN_TYPE = \"genai_evaluate\";\n\n/**\n * Source tags on the eval run. Traces linked via `mlflow.sourceRun` surface the\n * run's source in the traces-table \"Source\" column (mirrors how Python's\n * `evaluate()` shows the script name), so tagging the run is what populates it.\n */\nconst EVAL_SOURCE_NAME = \"appkit agent eval\";\n\ninterface CreateRunResponse {\n  run: { info: { run_id?: string; run_uuid?: string } };\n}\n\ninterface MlflowMetric {\n  key: string;\n  value: number;\n  timestamp: number;\n  step: number;\n}\n\n/**\n * Create an MLflow run tagged as a GenAI evaluation, so it shows under the\n * experiment's \"Evaluation runs\". Returns the run id; link traces to it via the\n * `mlflow.sourceRun` trace metadata and log results before finishing.\n */\nexport async function createEvalRun(\n  client: MlflowClient,\n  options: {\n    experimentId: string;\n    runName?: string;\n    startTime: number;\n  },\n): Promise<string> {\n  const created = await client.post<CreateRunResponse>(\n    \"/api/2.0/mlflow/runs/create\",\n    {\n      experiment_id: options.experimentId,\n      start_time: options.startTime,\n      ...(options.runName ? { run_name: options.runName } : {}),\n      tags: [\n        { key: \"mlflow.runType\", value: GENAI_EVALUATE_RUN_TYPE },\n        { key: \"mlflow.source.name\", value: EVAL_SOURCE_NAME },\n        { key: \"mlflow.source.type\", value: \"LOCAL\" },\n      ],\n    },\n  );\n  const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid;\n  if (!runId) {\n    throw new Error(\"runs/create returned no run id\");\n  }\n  return runId;\n}\n\n/** Aggregate per-eval results into MLflow run metrics. */\nexport function aggregateMetrics(\n  results: EvalResult[],\n  timestamp: number,\n): MlflowMetric[] {\n  const scored = results.filter((r) => !r.skipped);\n  const passed = scored.filter((r) => r.passed).length;\n  return [\n    { key: \"eval/total\", value: results.length, timestamp, step: 0 },\n    { key: \"eval/scored\", value: scored.length, timestamp, step: 0 },\n    { key: \"eval/passed\", value: passed, timestamp, step: 0 },\n    {\n      key: \"eval/pass_rate\",\n      value: scored.length ? passed / scored.length : 0,\n      timestamp,\n      step: 0,\n    },\n  ];\n}\n\nexport interface FinishOutcome {\n  finished: boolean;\n  /** Metric logging is best-effort; set when it failed (the run is still finished). */\n  metricsError?: string;\n  /** Set when the FINISHED update itself failed (run may be left RUNNING). */\n  finishError?: string;\n}\n\n/**\n * Log aggregate metrics (best-effort) and mark the eval run FINISHED. Never\n * throws — a metric-logging failure must not prevent the run from being closed,\n * or it would be left stuck in RUNNING forever.\n */\nexport async function finishEvalRun(\n  client: MlflowClient,\n  options: {\n    runId: string;\n    results: EvalResult[];\n    endTime: number;\n  },\n): Promise<FinishOutcome> {\n  const outcome: FinishOutcome = { finished: false };\n\n  const metrics = aggregateMetrics(options.results, options.endTime);\n  if (metrics.length) {\n    try {\n      await client.post(\"/api/2.0/mlflow/runs/log-batch\", {\n        run_id: options.runId,\n        metrics,\n      });\n    } catch (err) {\n      outcome.metricsError = err instanceof Error ? err.message : String(err);\n    }\n  }\n\n  try {\n    await client.post(\"/api/2.0/mlflow/runs/update\", {\n      run_id: options.runId,\n      status: \"FINISHED\",\n      end_time: options.endTime,\n    });\n    outcome.finished = true;\n  } catch (err) {\n    outcome.finishError = err instanceof Error ? err.message : String(err);\n  }\n\n  return outcome;\n}\n"],"mappings":";;AAIA,MAAM,0BAA0B;;;;;;AAOhC,MAAM,mBAAmB;;;;;;AAkBzB,eAAsB,cACpB,QACA,SAKiB;CACjB,MAAM,UAAU,MAAM,OAAO,KAC3B,+BACA;EACE,eAAe,QAAQ;EACvB,YAAY,QAAQ;EACpB,GAAI,QAAQ,UAAU,EAAE,UAAU,QAAQ,SAAS,GAAG,EAAE;EACxD,MAAM;GACJ;IAAE,KAAK;IAAkB,OAAO;IAAyB;GACzD;IAAE,KAAK;IAAsB,OAAO;IAAkB;GACtD;IAAE,KAAK;IAAsB,OAAO;IAAS;GAC9C;EACF,CACF;CACD,MAAM,QAAQ,QAAQ,KAAK,MAAM,UAAU,QAAQ,KAAK,MAAM;AAC9D,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iCAAiC;AAEnD,QAAO;;;AAIT,SAAgB,iBACd,SACA,WACgB;CAChB,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ;CAChD,MAAM,SAAS,OAAO,QAAQ,MAAM,EAAE,OAAO,CAAC;AAC9C,QAAO;EACL;GAAE,KAAK;GAAc,OAAO,QAAQ;GAAQ;GAAW,MAAM;GAAG;EAChE;GAAE,KAAK;GAAe,OAAO,OAAO;GAAQ;GAAW,MAAM;GAAG;EAChE;GAAE,KAAK;GAAe,OAAO;GAAQ;GAAW,MAAM;GAAG;EACzD;GACE,KAAK;GACL,OAAO,OAAO,SAAS,SAAS,OAAO,SAAS;GAChD;GACA,MAAM;GACP;EACF;;;;;;;AAgBH,eAAsB,cACpB,QACA,SAKwB;CACxB,MAAM,UAAyB,EAAE,UAAU,OAAO;CAElD,MAAM,UAAU,iBAAiB,QAAQ,SAAS,QAAQ,QAAQ;AAClE,KAAI,QAAQ,OACV,KAAI;AACF,QAAM,OAAO,KAAK,kCAAkC;GAClD,QAAQ,QAAQ;GAChB;GACD,CAAC;UACK,KAAK;AACZ,UAAQ,eAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;;AAI3E,KAAI;AACF,QAAM,OAAO,KAAK,+BAA+B;GAC/C,QAAQ,QAAQ;GAChB,QAAQ;GACR,UAAU,QAAQ;GACnB,CAAC;AACF,UAAQ,WAAW;UACZ,KAAK;AACZ,UAAQ,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;;AAGxE,QAAO"}

package/dist/evals/pool.js

@@ -0,0 +1,24 @@
+//#region src/evals/pool.ts
+/**
+* Run `fn` over `items` with at most `concurrency` calls in flight, preserving
+* input order in the result array. `concurrency` is clamped to `[1, length]`.
+* `fn` must not reject — a rejection abandons the other in-flight items.
+*/
+async function mapPool(items, concurrency, fn) {
+	const results = new Array(items.length);
+	const n = Number.isFinite(concurrency) ? concurrency : 1;
+	const limit = Math.max(1, Math.min(n, items.length));
+	let cursor = 0;
+	const worker = async () => {
+		while (cursor < items.length) {
+			const index = cursor++;
+			results[index] = await fn(items[index], index);
+		}
+	};
+	await Promise.all(Array.from({ length: limit }, () => worker()));
+	return results;
+}
+
+//#endregion
+export { mapPool };
+//# sourceMappingURL=pool.js.map

package/dist/evals/pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pool.js","names":[],"sources":["../../src/evals/pool.ts"],"sourcesContent":["/**\n * Run `fn` over `items` with at most `concurrency` calls in flight, preserving\n * input order in the result array. `concurrency` is clamped to `[1, length]`.\n * `fn` must not reject — a rejection abandons the other in-flight items.\n */\nexport async function mapPool<T, R>(\n  items: T[],\n  concurrency: number,\n  fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results = new Array<R>(items.length);\n  // Coerce a non-finite concurrency (e.g. NaN from a bad CLI `--concurrency`)\n  // to 1: otherwise Math.min(NaN, len) is NaN, Array.from({length: NaN}) is [],\n  // and zero workers spawn — silently skipping every item and leaving `undefined`\n  // holes in the result.\n  const n = Number.isFinite(concurrency) ? concurrency : 1;\n  const limit = Math.max(1, Math.min(n, items.length));\n  let cursor = 0;\n  // Each worker pulls the next index off the shared cursor until exhausted.\n  // `cursor++` is atomic between awaits (single-threaded), so no index is\n  // handed to two workers.\n  const worker = async (): Promise<void> => {\n    while (cursor < items.length) {\n      const index = cursor++;\n      results[index] = await fn(items[index], index);\n    }\n  };\n  await Promise.all(Array.from({ length: limit }, () => worker()));\n  return results;\n}\n"],"mappings":";;;;;;AAKA,eAAsB,QACpB,OACA,aACA,IACc;CACd,MAAM,UAAU,IAAI,MAAS,MAAM,OAAO;CAK1C,MAAM,IAAI,OAAO,SAAS,YAAY,GAAG,cAAc;CACvD,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,OAAO,CAAC;CACpD,IAAI,SAAS;CAIb,MAAM,SAAS,YAA2B;AACxC,SAAO,SAAS,MAAM,QAAQ;GAC5B,MAAM,QAAQ;AACd,WAAQ,SAAS,MAAM,GAAG,MAAM,QAAQ,MAAM;;;AAGlD,OAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,OAAO,QAAQ,QAAQ,CAAC,CAAC;AAChE,QAAO"}

package/dist/evals/report.d.ts

@@ -0,0 +1,25 @@
+import { EvalResult } from "./types.js";
+
+//#region src/evals/report.d.ts
+interface EvalSummary {
+  total: number;
+  passed: number;
+  failed: number;
+  skipped: number;
+  /** True when no eval failed (skips don't count as failures). */
+  allPassed: boolean;
+}
+declare function summarize(results: EvalResult[]): EvalSummary;
+/** Status glyph for a single eval result. */
+declare function evalGlyph(result: EvalResult): string;
+/** The one-line header for a single eval result (no failure detail). */
+declare function formatEvalHeadline(result: EvalResult): string;
+/** Indented detail lines for a failing eval (error + failing assertions). */
+declare function formatEvalDetail(result: EvalResult): string[];
+/** The final PASS/FAIL summary line. */
+declare function formatSummaryLine(results: EvalResult[]): string;
+/** Render all results as a human-readable console report (non-streaming). */
+declare function formatEvalResults(results: EvalResult[]): string;
+//#endregion
+export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+//# sourceMappingURL=report.d.ts.map

package/dist/evals/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAmBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;AAzB3C;AAAA,iBAqCgB,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA"}

package/dist/evals/report.js

@@ -0,0 +1,57 @@
+//#region src/evals/report.ts
+function summarize(results) {
+	let passed = 0;
+	let failed = 0;
+	let skipped = 0;
+	for (const r of results) if (r.skipped) skipped++;
+	else if (r.passed) passed++;
+	else failed++;
+	return {
+		total: results.length,
+		passed,
+		failed,
+		skipped,
+		allPassed: failed === 0
+	};
+}
+/** Status glyph for a single eval result. */
+function evalGlyph(result) {
+	if (result.skipped) return "−";
+	return result.passed ? "✓" : "✗";
+}
+/** The one-line header for a single eval result (no failure detail). */
+function formatEvalHeadline(result) {
+	if (result.skipped) return `− ${result.id} (skipped${result.skipped.reason ? `: ${result.skipped.reason}` : ""})`;
+	return `${evalGlyph(result)} ${result.id}${result.description ? ` — ${result.description}` : ""}`;
+}
+/** Indented detail lines for a failing eval (error + failing assertions). */
+function formatEvalDetail(result) {
+	const lines = [];
+	if (result.error) lines.push(`    error: ${result.error}`);
+	for (const a of result.assertions) {
+		if (a.pass) continue;
+		const tag = a.severity === "soft" ? "soft" : "gate";
+		lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : ""}`);
+	}
+	return lines;
+}
+/** The final PASS/FAIL summary line. */
+function formatSummaryLine(results) {
+	const s = summarize(results);
+	return `${s.allPassed ? "PASS" : "FAIL"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;
+}
+/** Render all results as a human-readable console report (non-streaming). */
+function formatEvalResults(results) {
+	const lines = [];
+	for (const r of results) {
+		lines.push(formatEvalHeadline(r));
+		lines.push(...formatEvalDetail(r));
+	}
+	lines.push("");
+	lines.push(formatSummaryLine(results));
+	return lines.join("\n");
+}
+
+//#endregion
+export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+//# sourceMappingURL=report.js.map