@voltro/ai 0.85.0 → 0.86.0

package/dist/callEvidence.js

@@ -0,0 +1,51 @@
+import { gatewayCostUsd as e } from "./cost.js";
+import { Schema as t } from "effect";
+//#region src/callEvidence.ts
+var n = t.Number.pipe(t.check(t.isInt()), t.check(t.isGreaterThanOrEqualTo(0)), t.check(t.isLessThanOrEqualTo(2 ** 53 - 1))), r = t.Struct({
+	provider: t.String,
+	model: t.String,
+	usage: t.Struct({
+		inputTokens: t.optional(n),
+		outputTokens: t.optional(n),
+		totalTokens: t.optional(n),
+		completeness: t.Literals([
+			"complete",
+			"partial",
+			"unknown"
+		])
+	}),
+	cost: t.Union([t.Struct({
+		status: t.Literal("actual"),
+		source: t.Literal("gateway"),
+		usd: t.Number.pipe(t.check(t.isFinite()), t.check(t.isGreaterThanOrEqualTo(0)))
+	}), t.Struct({
+		status: t.Literal("unknown"),
+		source: t.Literal("unreported")
+	})])
+}), i = (e) => typeof e == "number" && Number.isSafeInteger(e) && e >= 0 ? e : void 0, a = (t, n, r, a) => {
+	let o = typeof r == "object" && r ? r : {}, s = i("inputTokens" in o ? o.inputTokens : void 0), c = i("outputTokens" in o ? o.outputTokens : void 0), l = i("totalTokens" in o ? o.totalTokens : void 0), u = [
+		s,
+		c,
+		l
+	].filter((e) => e !== void 0).length, d = e(a);
+	return {
+		provider: t,
+		model: n,
+		usage: {
+			...s === void 0 ? {} : { inputTokens: s },
+			...c === void 0 ? {} : { outputTokens: c },
+			...l === void 0 ? {} : { totalTokens: l },
+			completeness: u === 3 ? "complete" : u === 0 ? "unknown" : "partial"
+		},
+		cost: d === void 0 ? {
+			status: "unknown",
+			source: "unreported"
+		} : {
+			status: "actual",
+			source: "gateway",
+			usd: d
+		}
+	};
+};
+//#endregion
+export { r as AiCallEvidenceSchema, a as normalizeAiCallEvidence };

package/dist/copilot.js

@@ -0,0 +1,126 @@
+import { Schema as e } from "effect";
+import { and as t, contains as n, eq as r, gt as i, gte as a, inSet as o, lt as s, lte as c, neq as l } from "@voltro/database";
+//#region src/copilot.ts
+var u = class extends e.TaggedError()("CopilotRejected", {
+	reason: e.Literals([
+		"unknown-table",
+		"unknown-column",
+		"bad-op",
+		"bad-aggregate",
+		"empty"
+	]),
+	detail: e.String
+}) {}, d = 200, f = 50, p = /* @__PURE__ */ new Set([
+	"eq",
+	"neq",
+	"gt",
+	"gte",
+	"lt",
+	"lte",
+	"contains",
+	"in"
+]), m = (e) => {
+	switch (e.op) {
+		case "eq": return r(e.column, e.value);
+		case "neq": return l(e.column, e.value);
+		case "gt": return i(e.column, e.value);
+		case "gte": return a(e.column, e.value);
+		case "lt": return s(e.column, e.value);
+		case "lte": return c(e.column, e.value);
+		case "contains": return n(e.column, e.value);
+		case "in": return o(e.column, Array.isArray(e.value) ? e.value : [e.value]);
+	}
+}, h = (e, t) => ({
+	ok: !1,
+	rejection: new u({
+		reason: e,
+		detail: t
+	})
+}), g = (e, n) => {
+	let r = n.tables.find((t) => t.name === e.table);
+	if (r === void 0) return h("unknown-table", e.table);
+	let i = new Set(r.columns.map((e) => e.name));
+	for (let t of e.columns ?? []) if (!i.has(t)) return h("unknown-column", `${e.table}.${t}`);
+	for (let t of e.filters ?? []) {
+		if (!p.has(t.op)) return h("bad-op", String(t.op));
+		if (!i.has(t.column)) return h("unknown-column", `${e.table}.${t.column}`);
+	}
+	if (e.orderBy !== void 0 && !i.has(e.orderBy.column)) return h("unknown-column", `${e.table}.${e.orderBy.column}`);
+	if (e.aggregate !== void 0) {
+		let { fn: t, column: n } = e.aggregate;
+		if (t !== "count" && (n === void 0 || !i.has(n))) return h("bad-aggregate", `${t}(${n ?? "∅"})`);
+	}
+	let a = e.filters ?? [], o = a.length === 0 ? void 0 : a.length === 1 ? m(a[0]) : t(...a.map(m)), s = Math.min(Math.max(1, e.limit ?? f), 200), c = {
+		table: e.table,
+		predicate: o,
+		order: e.orderBy ? [{
+			column: e.orderBy.column,
+			direction: e.orderBy.direction ?? "asc"
+		}] : [],
+		take: s,
+		skip: void 0,
+		projection: e.columns && e.columns.length > 0 ? [...e.columns] : void 0
+	};
+	return e.aggregate ? {
+		ok: !0,
+		descriptor: c,
+		table: e.table,
+		aggregate: e.aggregate
+	} : {
+		ok: !0,
+		descriptor: c,
+		table: e.table
+	};
+}, _ = e.Struct({
+	table: e.String,
+	columns: e.optional(e.Array(e.String)),
+	filters: e.optional(e.Array(e.Struct({
+		column: e.String,
+		op: e.Literals([
+			"eq",
+			"neq",
+			"gt",
+			"gte",
+			"lt",
+			"lte",
+			"contains",
+			"in"
+		]),
+		value: e.Unknown
+	}))),
+	orderBy: e.optional(e.Struct({
+		column: e.String,
+		direction: e.optional(e.Literals(["asc", "desc"]))
+	})),
+	limit: e.optional(e.Number),
+	aggregate: e.optional(e.Struct({
+		fn: e.Literals([
+			"count",
+			"sum",
+			"avg",
+			"min",
+			"max"
+		]),
+		column: e.optional(e.String)
+	}))
+}), v = (e) => [
+	"You are a data copilot. Translate the user question into a JSON query proposal.",
+	"",
+	"You may ONLY reference these tables and columns:",
+	e.tables.map((e) => `- ${e.name}(${e.columns.map((e) => e.name).join(", ")})`).join("\n"),
+	"",
+	"Rules:",
+	"- Pick EXACTLY ONE table from the list as `table`.",
+	"- Every column in `columns`, `filters`, `orderBy`, and `aggregate` MUST be one of that table's listed columns.",
+	"- NEVER invent a table or column. If the question cannot be answered from the listed schema, pick the closest table and leave filters empty.",
+	"- The query is READ-ONLY. You cannot insert, update, or delete.",
+	"- Keep the result small (limit ≤ 200)."
+].join("\n"), y = async (e, t, n) => {
+	let r = v(t);
+	return g(await n.propose({
+		system: r,
+		prompt: e
+	}), t);
+};
+//#endregion
+export { d as COPILOT_MAX_LIMIT, _ as CopilotProposalSchema, u as CopilotRejected, v as buildCopilotSystemPrompt, y as runDataCopilot, g as validateCopilotProposal };

package/dist/cost.js

@@ -0,0 +1,242 @@
+import { AI_BUDGET_RESERVATIONS_TABLE as e, AI_BUDGET_TABLE as t, AiBudgetExceeded as n, AiBudgetInvalidInput as r, AiBudgetReservationConflict as i, aiBudgetReservationsTable as a, aiBudgetTable as o, releaseAiBudget as s, requireAiBudget as c } from "./budget.js";
+import { Data as l, Effect as u } from "effect";
+import { and as d, eq as f, generateId as p, gte as m, id as h, integer as g, queryFor as _, registerRetention as v, retentionTtlMsFromEnv as y, table as b, text as x, timestamp as S } from "@voltro/database";
+//#region src/cost.ts
+var C = {
+	"claude-opus-4-8": {
+		inputPer1M: 15,
+		outputPer1M: 75
+	},
+	"claude-opus-4-6": {
+		inputPer1M: 15,
+		outputPer1M: 75
+	},
+	"claude-sonnet-4-6": {
+		inputPer1M: 3,
+		outputPer1M: 15
+	},
+	"claude-haiku-4-5": {
+		inputPer1M: 1,
+		outputPer1M: 5
+	},
+	"claude-haiku-4-5-20251001": {
+		inputPer1M: 1,
+		outputPer1M: 5
+	},
+	"gpt-5.5": {
+		inputPer1M: 5,
+		outputPer1M: 30
+	},
+	"gpt-4o": {
+		inputPer1M: 2.5,
+		outputPer1M: 10
+	},
+	"gpt-4o-mini": {
+		inputPer1M: .15,
+		outputPer1M: .6
+	},
+	"gemini-2.5-pro": {
+		inputPer1M: 1.25,
+		outputPer1M: 10
+	},
+	"gemini-2.5-flash": {
+		inputPer1M: .3,
+		outputPer1M: 2.5
+	},
+	"gemini-2.0-flash": {
+		inputPer1M: .1,
+		outputPer1M: .4
+	}
+}, w = {}, T = (e) => {
+	let t = w;
+	return w = { ...e }, t;
+}, E = () => ({
+	...C,
+	...w
+}), D = (e) => {
+	let t = E();
+	if (t[e] !== void 0) return t[e];
+	let n = e.indexOf("/");
+	if (n > 0) {
+		let r = e.slice(n + 1);
+		if (t[r] !== void 0) return t[r];
+	}
+}, O = (e) => typeof e.unit == "string", k = (e) => O(e) ? {
+	unit: e.unit,
+	input: e.input ?? 0,
+	output: e.output ?? 0
+} : {
+	unit: "tokens",
+	input: e.inputTokens ?? 0,
+	output: e.outputTokens ?? 0
+}, A = (e, t) => {
+	let { unit: n, input: r, output: i } = k(e);
+	if (t.actualCostUsd !== void 0) {
+		if (!Number.isFinite(t.actualCostUsd) || t.actualCostUsd < 0) throw RangeError("estimateCostUsd: actualCostUsd must be a finite, non-negative USD amount.");
+		let e = r + i, a = e > 0 ? r / e : 0;
+		return {
+			inputTokens: r,
+			outputTokens: i,
+			inputCostUsd: t.actualCostUsd * a,
+			outputCostUsd: t.actualCostUsd * (1 - a),
+			totalCostUsd: t.actualCostUsd,
+			costSource: "gateway",
+			unit: n
+		};
+	}
+	if (n !== "tokens" && t.price === void 0) return {
+		inputTokens: r,
+		outputTokens: i,
+		inputCostUsd: 0,
+		outputCostUsd: 0,
+		totalCostUsd: 0,
+		costSource: "unpriced",
+		unit: n
+	};
+	let a = t.price ?? D(t.model) ?? {
+		inputPer1M: 0,
+		outputPer1M: 0
+	}, o = r / 1e6 * a.inputPer1M, s = i / 1e6 * a.outputPer1M;
+	return {
+		inputTokens: r,
+		outputTokens: i,
+		inputCostUsd: o,
+		outputCostUsd: s,
+		totalCostUsd: o + s,
+		costSource: "estimated",
+		unit: n
+	};
+}, j = "_voltro_ai_usage", M = b(j, {
+	id: h({ prefix: "aiu" }),
+	tenantId: x().nullable(),
+	provider: x(),
+	model: x(),
+	operation: x(),
+	agent: x().nullable(),
+	inputTokens: g().default(0),
+	outputTokens: g().default(0),
+	costMicroUsd: g().default(0),
+	costSource: x().default("estimated"),
+	unit: x().default("tokens"),
+	promptId: x().nullable(),
+	promptDigest: x().nullable(),
+	promptRevision: g().nullable(),
+	requestId: x().maxLength(191).nullable(),
+	attempt: g().nullable(),
+	receiptKey: x().maxLength(191).backfill((e) => `aiu-pre-receipt-${String(e.id)}`),
+	calledAt: S().default("now")
+}).index(["tenantId"]).index(["promptDigest"]).unique("byAiUsageReceipt", ["receiptKey"]), N = (e) => {
+	let t = typeof e == "string" ? e.trim() : void 0, n = typeof e == "number" ? e : t !== void 0 && /^(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?$/i.test(t) ? Number(t) : NaN;
+	return Number.isFinite(n) && n >= 0 ? n : void 0;
+}, P = (e) => {
+	let t = e?.gateway, n = N(t?.cost);
+	if (n === void 0) return;
+	let r = (Array.isArray(t?.routing?.modelAttempts) ? t.routing.modelAttempts : []).findLast((e) => e?.success === !0);
+	if ((Array.isArray(r?.providerAttempts) ? r.providerAttempts : []).findLast((e) => e?.success === !0)?.credentialType !== "byok") return n;
+	let i = N(t?.marketCost);
+	return i === void 0 ? void 0 : n + i;
+}, F = class extends l.TaggedError("AiUsageReceiptConflict") {
+	get message() {
+		let e = Object.keys(this.recorded).filter((e) => this.recorded[e] !== this.attempted[e]).map((e) => `${e} ${String(this.recorded[e])} → ${String(this.attempted[e])}`);
+		return `AI usage receipt ${this.receiptKey} was recorded again with different values (${e.join(", ")}). A replay of the same paid call has the same values; the row keeps what was recorded first.`;
+	}
+}, I = class extends l.TaggedError("AiUsageLedgerError") {
+	get message() {
+		return `AI usage receipt ${this.receiptKey} could not be written: ${this.cause instanceof Error ? this.cause.message : String(this.cause)}`;
+	}
+}, L = (e) => ({
+	tenantId: e.tenantId ?? null,
+	provider: String(e.provider),
+	model: String(e.model),
+	operation: String(e.operation),
+	inputTokens: Number(e.inputTokens),
+	outputTokens: Number(e.outputTokens),
+	costMicroUsd: Number(e.costMicroUsd)
+}), R = (e, t) => (v({
+	source: "plugin",
+	table: j,
+	timeColumn: "calledAt",
+	ttlMs: y(process.env.VOLTRO_AI_USAGE_TTL_HOURS, 8760)
+}), u.gen(function* () {
+	let n = A(t.usage, {
+		model: t.model,
+		...t.price === void 0 ? {} : { price: t.price },
+		...t.actualCostUsd === void 0 ? {} : { actualCostUsd: t.actualCostUsd }
+	}), r = t.receiptKey ?? (t.requestId === void 0 ? p({
+		kind: "typeid",
+		prefix: "aiu"
+	}, "_voltro_ai_usage") : `${t.requestId}#${t.attempt ?? 0}`), i = {
+		tenantId: t.tenantId ?? null,
+		provider: t.provider,
+		model: t.model,
+		operation: t.operation,
+		agent: t.agent ?? null,
+		inputTokens: n.inputTokens,
+		outputTokens: n.outputTokens,
+		costMicroUsd: Math.round(n.totalCostUsd * 1e6),
+		costSource: n.costSource,
+		unit: n.unit,
+		promptId: t.prompt?.promptId ?? null,
+		promptDigest: t.prompt?.digest ?? null,
+		promptRevision: t.prompt?.revision ?? null,
+		requestId: t.requestId ?? null,
+		attempt: t.attempt ?? null,
+		receiptKey: r,
+		calledAt: t.calledAt ?? /* @__PURE__ */ new Date()
+	}, a = yield* u.tryPromise({
+		try: () => e.insertIgnoreWithOutcome(j, i, { conflictColumns: ["receiptKey"] }),
+		catch: (e) => new I({
+			receiptKey: String(r),
+			cause: e
+		})
+	});
+	if (a.outcome === "inserted") return {
+		...n,
+		receipt: "recorded"
+	};
+	let o = L(a.row), s = L(i);
+	return Object.keys(o).some((e) => o[e] !== s[e]) ? yield* u.fail(new F({
+		receiptKey: String(r),
+		recorded: o,
+		attempted: s
+	})) : {
+		...z(a.row, n),
+		receipt: "replayed"
+	};
+})), z = (e, t) => {
+	let n = Number(e.costMicroUsd) / 1e6;
+	return {
+		...t,
+		inputTokens: Number(e.inputTokens),
+		outputTokens: Number(e.outputTokens),
+		totalCostUsd: n,
+		costSource: e.costSource ?? t.costSource
+	};
+}, B = (e, t) => R(e, t).pipe(u.asVoid, u.catchCause((e) => u.logError("AI usage NOT booked as recorded — the call itself succeeded and was returned", e).pipe(u.annotateLogs({
+	scope: "voltro:ai:usage",
+	provider: t.provider,
+	model: t.model,
+	operation: t.operation,
+	...O(t.usage) ? {
+		unit: t.usage.unit,
+		input: t.usage.input ?? null,
+		output: t.usage.output ?? null
+	} : {
+		inputTokens: t.usage.inputTokens ?? null,
+		outputTokens: t.usage.outputTokens ?? null
+	},
+	...t.requestId === void 0 ? {} : {
+		requestId: t.requestId,
+		attempt: t.attempt ?? 0
+	},
+	...t.actualCostUsd === void 0 ? {} : { actualCostUsd: t.actualCostUsd }
+})))), V = (e, t = {}) => u.promise(async () => {
+	let n = [
+		...t.tenantId === void 0 ? [] : [f("tenantId", t.tenantId)],
+		...t.since === void 0 ? [] : [m("calledAt", t.since)],
+		...t.promptDigest === void 0 ? [] : [f("promptDigest", t.promptDigest)]
+	], r = _(M), i = n.length === 0 ? r : r.where(n.length === 1 ? n[0] : d(...n));
+	return (await e.query(i.descriptor)).reduce((e, t) => e + (Number(t.costMicroUsd) || 0), 0) / 1e6;
+});
+//#endregion
+export { e as AI_BUDGET_RESERVATIONS_TABLE, t as AI_BUDGET_TABLE, j as AI_USAGE_TABLE, n as AiBudgetExceeded, r as AiBudgetInvalidInput, i as AiBudgetReservationConflict, I as AiUsageLedgerError, F as AiUsageReceiptConflict, C as MODEL_PRICING_DEFAULTS, a as aiBudgetReservationsTable, o as aiBudgetTable, V as aiSpendUsd, M as aiUsageTable, A as estimateCostUsd, P as gatewayCostUsd, O as isMeteredUsage, E as modelPricing, k as normaliseUsage, D as priceForModel, R as recordAiUsage, B as recordAiUsageBestEffort, s as releaseAiBudget, c as requireAiBudget, T as setModelPricing };

package/dist/embed.js

@@ -0,0 +1,41 @@
+import { aiGenerationError as e } from "./api.js";
+import { embeddingProviderFromEnv as t, resolveEmbeddingModel as n } from "./embeddingProvider.js";
+import { cosineSimilarity as r, embed as i, embedMany as a } from "ai";
+import { Effect as o } from "effect";
+//#region src/embed.ts
+var s = (r, a) => o.tryPromise({
+	try: async (e) => {
+		let o = a ?? t();
+		l(o.expectedDimensions);
+		let s = await n(o), c = await i({
+			model: s,
+			value: r,
+			abortSignal: e,
+			...o.providerOptions === void 0 ? {} : { providerOptions: o.providerOptions }
+		});
+		return u(c.embedding, o.expectedDimensions, 0), c.embedding;
+	},
+	catch: (t) => e(t)
+}), c = (r, i) => o.tryPromise({
+	try: async (e) => {
+		let o = i ?? t();
+		l(o.expectedDimensions);
+		let s = await n(o), c = await a({
+			model: s,
+			values: [...r],
+			abortSignal: e,
+			...o.providerOptions === void 0 ? {} : { providerOptions: o.providerOptions }
+		});
+		if (c.embeddings.length !== r.length) throw Error(`embedMany: expected ${r.length} vectors, received ${c.embeddings.length}`);
+		return c.embeddings.forEach((e, t) => u(e, o.expectedDimensions, t)), c.embeddings;
+	},
+	catch: (t) => e(t)
+}), l = (e) => {
+	if (e !== void 0 && (!Number.isSafeInteger(e) || e <= 0)) throw Error("embedding expectedDimensions must be a positive safe integer");
+}, u = (e, t, n) => {
+	if (!Array.isArray(e)) throw Error(`embedding ${n}: expected a vector array`);
+	if (t !== void 0 && e.length !== t) throw Error(`embedding ${n}: expected ${t} dimensions, received ${e.length}`);
+	if (e.length === 0 || e.some((e) => !Number.isFinite(e))) throw Error(`embedding ${n}: expected a non-empty vector of finite numbers`);
+}, d = (e, t) => r([...e], [...t]);
+//#endregion
+export { d as cosineSimilarity, s as embed, c as embedMany };

package/dist/embeddingProvider.js

@@ -0,0 +1,51 @@
+import { createGateway as e, gateway as t } from "@ai-sdk/gateway";
+//#region src/embeddingProvider.ts
+var n = (e, t) => ({
+	specificationVersion: "v4",
+	provider: "mock",
+	modelId: e,
+	maxEmbeddingsPerCall: 2048,
+	supportsParallelCalls: !0,
+	doEmbed: async ({ values: e }) => ({
+		embeddings: e.map((e) => r(e, t)),
+		usage: { tokens: e.reduce((e, t) => e + t.length, 0) },
+		warnings: []
+	})
+}), r = (e, t) => {
+	let n = Array(t);
+	for (let r = 0; r < t; r++) {
+		let t = 2166136261 ^ r * 16777619;
+		for (let n = 0; n < e.length; n++) t ^= e.charCodeAt(n), t = Math.imul(t, 16777619);
+		n[r] = (t >>> 0) / 4294967295 * 2 - 1;
+	}
+	return n;
+}, i = async (r) => {
+	switch (r.name) {
+		case "mock": return n(r.model, r.mockDim ?? 8);
+		case "openai": return (await a("@ai-sdk/openai", "openai")).openai.textEmbeddingModel(r.model);
+		case "voyage": return (await a("voyage-ai-provider", "voyage")).voyage.textEmbeddingModel(r.model);
+		case "cohere": return (await a("@ai-sdk/cohere", "cohere")).cohere.textEmbeddingModel(r.model);
+		case "gateway": return (r.apiKey !== void 0 || r.baseURL !== void 0 ? e({
+			...r.apiKey === void 0 ? {} : { apiKey: r.apiKey },
+			...r.baseURL === void 0 ? {} : { baseURL: r.baseURL }
+		}) : t).embeddingModel(r.model);
+	}
+}, a = async (e, t) => {
+	try {
+		return await import(
+			/* @vite-ignore */
+			e
+);
+	} catch {
+		throw Error(`embedding provider '${t}' needs the '${e}' package — install it (pnpm add ${e}) or use AI_EMBED_PROVIDER=mock for tests/demos.`);
+	}
+}, o = (e = process.env) => {
+	let t = (e.AI_EMBED_PROVIDER ?? "mock").toLowerCase(), n = t === "openai" ? "text-embedding-3-small" : t === "voyage" ? "voyage-3" : t === "cohere" ? "embed-english-v3.0" : t === "gateway" ? "openai/text-embedding-3-small" : "mock", r = e.AI_EMBED_MODEL ?? n, i = e.AI_EMBED_MOCK_DIM === void 0 ? void 0 : Number(e.AI_EMBED_MOCK_DIM);
+	return {
+		name: t,
+		model: r,
+		...i !== void 0 && Number.isFinite(i) ? { mockDim: i } : {}
+	};
+};
+//#endregion
+export { o as embeddingProviderFromEnv, r as hashToVector, i as resolveEmbeddingModel };

package/dist/eval.js

@@ -0,0 +1,130 @@
+import { providerFromEnv as e } from "./provider.js";
+import { generateObject as t } from "./api.js";
+import { Effect as n, Schema as r } from "effect";
+//#region src/eval.ts
+var i = (e) => {
+	if (e.name.trim() === "") throw Error("defineEval: name must be non-empty");
+	if (e.cases.length === 0) throw Error(`defineEval "${e.name}": at least one case is required`);
+	let t = /* @__PURE__ */ new Set();
+	for (let n of e.cases) {
+		if (n.id.trim() === "") throw Error(`defineEval "${e.name}": every case needs a non-empty id`);
+		if (t.has(n.id)) throw Error(`defineEval "${e.name}": duplicate case id "${n.id}"`);
+		t.add(n.id);
+	}
+	let n = e.judge?.threshold;
+	if (n !== void 0 && (n < 0 || n > 1)) throw Error(`defineEval "${e.name}": judge.threshold must be within 0..1 (got ${n})`);
+	return {
+		...e,
+		__eval: !0
+	};
+}, a = (e) => typeof e == "object" && !!e && "__eval" in e && e.__eval === !0, o = (e, t) => {
+	let n = [], r = e.text;
+	for (let i of t) switch (i.kind) {
+		case "contains": {
+			let e = i.caseInsensitive ? r.toLowerCase() : r, t = i.caseInsensitive ? i.value.toLowerCase() : i.value;
+			e.includes(t) || n.push({
+				assertion: i,
+				detail: `output does not contain "${i.value}"`
+			});
+			break;
+		}
+		case "notContains": {
+			let e = i.caseInsensitive ? r.toLowerCase() : r, t = i.caseInsensitive ? i.value.toLowerCase() : i.value;
+			e.includes(t) && n.push({
+				assertion: i,
+				detail: `output unexpectedly contains "${i.value}"`
+			});
+			break;
+		}
+		case "matches": {
+			let e;
+			try {
+				e = new RegExp(i.pattern, i.flags);
+			} catch (e) {
+				n.push({
+					assertion: i,
+					detail: `invalid pattern /${i.pattern}/: ${e.message}`
+				});
+				break;
+			}
+			e.test(r) || n.push({
+				assertion: i,
+				detail: `output does not match /${i.pattern}/`
+			});
+			break;
+		}
+		case "equals":
+			(i.trim ? r.trim() : r) !== (i.trim ? i.value.trim() : i.value) && n.push({
+				assertion: i,
+				detail: "output is not equal to the expected value"
+			});
+			break;
+		case "maxLatencyMs":
+			e.latencyMs > i.value && n.push({
+				assertion: i,
+				detail: `latency ${e.latencyMs}ms exceeds ${i.value}ms`
+			});
+			break;
+		case "nonEmpty": r.trim() === "" && n.push({
+			assertion: i,
+			detail: "output is empty"
+		});
+	}
+	return n;
+}, s = .7, c = (e, t, n, r, i) => {
+	let a = o(t, [...n, ...e.assert ?? []]), s = r?.threshold ?? .7, c = i === void 0 || i.score >= s;
+	return {
+		id: e.id,
+		pass: a.length === 0 && c,
+		assertionFailures: a,
+		...i === void 0 ? {} : { judge: i },
+		output: t
+	};
+}, l = async (e, t) => {
+	let n = [];
+	for (let r of e.cases) {
+		let i = await t.replay({
+			case: r,
+			def: e
+		}), a = e.judge !== void 0 && t.judge !== void 0 ? await t.judge({
+			case: r,
+			output: i,
+			config: e.judge
+		}) : void 0;
+		n.push(c(r, i, e.assert ?? [], e.judge, a));
+	}
+	let r = n.filter((e) => !e.pass).length;
+	return {
+		name: e.name,
+		ok: r === 0,
+		passed: n.length - r,
+		failed: r,
+		cases: n,
+		...t.branchId === void 0 ? {} : { branch: { id: t.branchId } }
+	};
+}, u = r.Struct({
+	pass: r.Boolean,
+	score: r.Number,
+	reason: r.String
+}), d = (e, t, n) => [
+	"You are grading an AI assistant response against a rubric. Respond with a JSON",
+	"object { pass: boolean, score: number (0..1), reason: string }. Score 1 is a",
+	"perfect response, 0 is unusable. Be strict: a plausible-sounding but wrong or",
+	"incomplete answer scores low.",
+	"",
+	`RUBRIC:\n${n.rubric}`,
+	...e.golden === void 0 ? [] : ["", `REFERENCE (the previously-accepted answer):\n${e.golden}`],
+	...e.input.prompt === "" ? [] : ["", `USER PROMPT:\n${e.input.prompt}`],
+	"",
+	`CANDIDATE RESPONSE (to grade):\n${t.text}`
+].join("\n"), f = async ({ case: r, output: i, config: a }) => await n.runPromise(t({
+	schema: u,
+	prompt: d(r, i, a),
+	provider: e()
+}).pipe(n.map((e) => e.object), n.catch((e) => n.succeed({
+	pass: !1,
+	score: 0,
+	reason: `judge failed: ${e.message}`
+}))));
+//#endregion
+export { s as DEFAULT_JUDGE_THRESHOLD, i as defineEval, o as evaluateAssertions, a as isEvalDefinition, f as llmJudge, l as runEval, c as scoreCase };

package/dist/events.d.ts

@@ -17,6 +17,8 @@ export declare const AgentEvent: Schema.Union<readonly [Schema.Struct<{
     readonly id: Schema.String;
     readonly name: Schema.String;
     readonly input: Schema.Unknown;
+    /** Present when the caller must execute this pending call. */
+    readonly execution: Schema.optional<Schema.Literal<"client">>;
 }>, Schema.Struct<{
     readonly _tag: Schema.Literal<"toolResult">;
     readonly id: Schema.String;
@@ -169,6 +171,8 @@ export declare const SeqEvent: Schema.Struct<{
         readonly id: Schema.String;
         readonly name: Schema.String;
         readonly input: Schema.Unknown;
+        /** Present when the caller must execute this pending call. */
+        readonly execution: Schema.optional<Schema.Literal<"client">>;
     }>, Schema.Struct<{
         readonly _tag: Schema.Literal<"toolResult">;
         readonly id: Schema.String;