@llmops/sdk 1.0.0-beta.2 → 1.0.0-beta.23

package/dist/agents.cjs

@@ -1,3 +1,3 @@
-const require_agents_exporter = require('./agents-exporter-BuTq2n2y.cjs');
+const require_agents_exporter = require('./agents-exporter-DizRE7CQ.cjs');
 
 exports.createLLMOpsAgentsExporter = require_agents_exporter.createLLMOpsAgentsExporter;

package/dist/agents.d.cts

@@ -1,2 +1,2 @@
-import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-uzN3bkth.cjs";
+import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-DkqkCcIx.cjs";
 export { AgentsSpan, AgentsSpanData, AgentsSpanError, AgentsTrace, AgentsTracingExporter, LLMOpsAgentsExporterConfig, createLLMOpsAgentsExporter };

package/dist/agents.d.mts

@@ -1,2 +1,2 @@
-import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-BZHCcFSd.mjs";
+import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-CehKIArI.mjs";
 export { AgentsSpan, AgentsSpanData, AgentsSpanError, AgentsTrace, AgentsTracingExporter, LLMOpsAgentsExporterConfig, createLLMOpsAgentsExporter };

package/dist/agents.mjs

@@ -1,3 +1,3 @@
-import { t as createLLMOpsAgentsExporter } from "./agents-exporter-vcpgCF69.mjs";
+import { t as createLLMOpsAgentsExporter } from "./agents-exporter-CGxTzDeQ.mjs";
 
 export { createLLMOpsAgentsExporter };

package/dist/chunk-CxwUPGYo.mjs

@@ -0,0 +1,21 @@
+import { createRequire } from "node:module";
+
+//#region rolldown:runtime
+var __defProp = Object.defineProperty;
+var __export = (all, symbols) => {
+	let target = {};
+	for (var name in all) {
+		__defProp(target, name, {
+			get: all[name],
+			enumerable: true
+		});
+	}
+	if (symbols) {
+		__defProp(target, Symbol.toStringTag, { value: "Module" });
+	}
+	return target;
+};
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+//#endregion
+export { __require as n, __export as t };

package/dist/constants--ywcWP7q.cjs

@@ -0,0 +1,18 @@
+
+//#region src/telemetry/constants.ts
+const COST_SUMMARY_GROUP_BY = [
+	"day",
+	"hour",
+	"model",
+	"provider",
+	"endpoint",
+	"tags"
+];
+
+//#endregion
+Object.defineProperty(exports, 'COST_SUMMARY_GROUP_BY', {
+  enumerable: true,
+  get: function () {
+    return COST_SUMMARY_GROUP_BY;
+  }
+});

package/dist/constants-BvnYU_pl.mjs

@@ -0,0 +1,12 @@
+//#region src/telemetry/constants.ts
+const COST_SUMMARY_GROUP_BY = [
+	"day",
+	"hour",
+	"model",
+	"provider",
+	"endpoint",
+	"tags"
+];
+
+//#endregion
+export { COST_SUMMARY_GROUP_BY as t };

package/dist/eval.cjs

@@ -0,0 +1,464 @@
+let node_crypto = require("node:crypto");
+let node_fs = require("node:fs");
+let node_path = require("node:path");
+
+//#region src/eval/dataset.ts
+/**
+* Wraps a plain array as an EvaluationDataset.
+*/
+var InlineDataset = class {
+	constructor(items) {
+		this.items = items;
+	}
+	size() {
+		return this.items.length;
+	}
+	get(index) {
+		return this.items[index];
+	}
+	slice(start, end) {
+		return this.items.slice(start, end);
+	}
+};
+
+//#endregion
+//#region src/eval/evaluate.ts
+const RESET = "\x1B[0m";
+const DIM = "\x1B[2m";
+const BOLD = "\x1B[1m";
+const CYAN = "\x1B[36m";
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+async function pool(items, concurrency, fn) {
+	const executing = [];
+	for (const item of items) {
+		const p = fn(item).then(() => {
+			executing.splice(executing.indexOf(p), 1);
+		});
+		executing.push(p);
+		if (executing.length >= concurrency) await Promise.race(executing);
+	}
+	await Promise.all(executing);
+}
+function computeStats(values) {
+	const valid = values.filter((v) => !Number.isNaN(v));
+	if (valid.length === 0) return {
+		mean: 0,
+		min: 0,
+		max: 0,
+		median: 0,
+		count: 0
+	};
+	const sorted = [...valid].sort((a, b) => a - b);
+	const sum = sorted.reduce((a, b) => a + b, 0);
+	const mid = Math.floor(sorted.length / 2);
+	const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+	return {
+		mean: sum / sorted.length,
+		min: sorted[0],
+		max: sorted[sorted.length - 1],
+		median,
+		count: sorted.length
+	};
+}
+const isSilent = process.env.LLMOPS_EVAL_OUTPUT === "json";
+const w = process.stderr;
+function printHeader(name, total) {
+	if (isSilent) return;
+	w.write("\n");
+	w.write(`  ${BOLD}${name}${RESET}  ${DIM}(${total} datapoints)${RESET}\n`);
+	w.write(`  ${DIM}${"─".repeat(50)}${RESET}\n`);
+}
+function printDatapointResult(idx, total, dp) {
+	if (isSilent) return;
+	const label = typeof dp.data === "object" && dp.data !== null ? JSON.stringify(dp.data).slice(0, 50) : String(dp.data).slice(0, 50);
+	if (dp.error) {
+		w.write(`  ${RED}✗${RESET} ${DIM}[${idx + 1}/${total}]${RESET} ${label}  ${RED}ERROR${RESET} ${DIM}${dp.error.slice(0, 60)}${RESET}\n`);
+		return;
+	}
+	const scoreStr = Object.entries(dp.scores).map(([name, val]) => {
+		if (Number.isNaN(val)) return `${DIM}${name}=NaN${RESET}`;
+		return `${val >= .8 ? GREEN : val >= .5 ? YELLOW : RED}${name}=${val.toFixed(2)}${RESET}`;
+	}).join("  ");
+	w.write(`  ${GREEN}✓${RESET} ${DIM}[${idx + 1}/${total}]${RESET} ${label}  ${scoreStr}  ${DIM}${dp.durationMs}ms${RESET}\n`);
+}
+function scoreBar(score, width = 20) {
+	const filled = Math.round(score * width);
+	const empty = width - filled;
+	return "█".repeat(filled) + "░".repeat(empty);
+}
+function scoreColor(score) {
+	if (score >= .8) return GREEN;
+	if (score >= .5) return YELLOW;
+	return RED;
+}
+function printSummary(result) {
+	if (isSilent) return;
+	w.write("\n");
+	const entries = Object.entries(result.scores);
+	if (entries.length > 0) {
+		const maxNameLen = Math.max(...entries.map(([n]) => n.length), 10);
+		w.write(`  ${DIM}${"Evaluator".padEnd(maxNameLen)}  ${"Mean".padStart(6)}  ${"Bar".padEnd(20)}  ${"Min".padStart(5)}  ${"Max".padStart(5)}  ${"Med".padStart(5)}${RESET}\n`);
+		w.write(`  ${DIM}${"─".repeat(maxNameLen + 50)}${RESET}\n`);
+		for (const [name, stats] of entries) {
+			const color = scoreColor(stats.mean);
+			const bar = scoreBar(stats.mean);
+			w.write(`  ${name.padEnd(maxNameLen)}  ${color}${stats.mean.toFixed(2).padStart(6)}${RESET}  ${DIM}${bar}${RESET}  ${stats.min.toFixed(2).padStart(5)}  ${stats.max.toFixed(2).padStart(5)}  ${stats.median.toFixed(2).padStart(5)}\n`);
+		}
+	}
+	const completed = result.count - result.errors;
+	w.write("\n");
+	w.write(`  ${DIM}Duration${RESET} ${(result.durationMs / 1e3).toFixed(1)}s`);
+	w.write(`    ${DIM}Passed${RESET} ${completed}/${result.count}`);
+	if (result.errors > 0) w.write(`    ${RED}Failed ${result.errors}${RESET}`);
+	w.write(`    ${DIM}Run${RESET} ${CYAN}${result.runId.slice(0, 8)}${RESET}`);
+	w.write("\n\n");
+}
+function saveResult(result, outputDir) {
+	const dir = (0, node_path.join)(outputDir, result.name);
+	(0, node_fs.mkdirSync)(dir, { recursive: true });
+	(0, node_fs.writeFileSync)((0, node_path.join)(dir, `${Date.now()}.json`), JSON.stringify(result, null, 2));
+}
+async function runSingleExecutor(name, dataset, executor, evaluators, concurrency) {
+	const size = await dataset.size();
+	const datapoints = await dataset.slice(0, size);
+	const results = new Array(datapoints.length);
+	const startTime = Date.now();
+	printHeader(name, datapoints.length);
+	await pool(datapoints, concurrency, async (dp) => {
+		const idx = datapoints.indexOf(dp);
+		const dpStart = Date.now();
+		let output = null;
+		let error;
+		const scores = {};
+		try {
+			output = await executor(dp.data);
+		} catch (err) {
+			error = err instanceof Error ? err.message : String(err);
+		}
+		if (!error && output !== null) for (const [evalName, evaluator] of Object.entries(evaluators)) try {
+			const result = await evaluator(output, dp.target, dp.data);
+			if (typeof result === "number") scores[evalName] = result;
+			else for (const [subKey, subScore] of Object.entries(result)) scores[`${evalName}.${subKey}`] = subScore;
+		} catch (evalErr) {
+			scores[evalName] = NaN;
+			const msg = evalErr instanceof Error ? evalErr.message : String(evalErr);
+			if (!isSilent) w.write(`  ${YELLOW}⚠${RESET} ${DIM}evaluator "${evalName}":${RESET} ${msg.slice(0, 80)}\n`);
+		}
+		const dpResult = {
+			data: dp.data,
+			target: dp.target,
+			metadata: dp.metadata,
+			output,
+			scores,
+			durationMs: Date.now() - dpStart,
+			error
+		};
+		results[idx] = dpResult;
+		printDatapointResult(idx, datapoints.length, dpResult);
+	});
+	return {
+		results,
+		durationMs: Date.now() - startTime
+	};
+}
+async function evaluate(options) {
+	const { name, data, executor, variants, evaluators, concurrency = 5, group, metadata, outputDir = process.env.LLMOPS_EVAL_OUTPUT_DIR || "./llmops-evals" } = options;
+	const runId = (0, node_crypto.randomUUID)();
+	if (executor && variants) throw new Error("evaluate(): provide either executor or variants, not both");
+	if (!executor && !variants) throw new Error("evaluate(): provide either executor or variants");
+	const dataset = Array.isArray(data) ? new InlineDataset(data) : data;
+	if (executor) {
+		const { results, durationMs } = await runSingleExecutor(name, dataset, executor, evaluators, concurrency);
+		const scoreNames = /* @__PURE__ */ new Set();
+		for (const r of results) for (const key of Object.keys(r.scores)) scoreNames.add(key);
+		const scores = {};
+		for (const scoreName of scoreNames) scores[scoreName] = computeStats(results.map((r) => r.scores[scoreName] ?? NaN));
+		const result = {
+			name,
+			runId,
+			group,
+			scores,
+			durationMs,
+			count: results.length,
+			errors: results.filter((r) => r.error).length,
+			metadata,
+			results
+		};
+		if (isSilent) process.stdout.write(JSON.stringify(result, null, 2));
+		else printSummary(result);
+		saveResult(result, outputDir);
+		return result;
+	}
+	const variantResults = {};
+	const totalStart = Date.now();
+	for (const [variantName, variantExecutor] of Object.entries(variants)) {
+		const { results, durationMs } = await runSingleExecutor(`${name}/${variantName}`, dataset, variantExecutor, evaluators, concurrency);
+		const scoreNames = /* @__PURE__ */ new Set();
+		for (const r of results) for (const key of Object.keys(r.scores)) scoreNames.add(key);
+		const scores = {};
+		for (const scoreName of scoreNames) scores[scoreName] = computeStats(results.map((r) => r.scores[scoreName] ?? NaN));
+		const variantResult = {
+			name: `${name}/${variantName}`,
+			runId,
+			group,
+			scores,
+			durationMs,
+			count: results.length,
+			errors: results.filter((r) => r.error).length,
+			metadata,
+			results
+		};
+		variantResults[variantName] = variantResult;
+		if (!isSilent) printSummary(variantResult);
+		saveResult(variantResult, outputDir);
+	}
+	const variantEvalResult = {
+		name,
+		runId,
+		group,
+		durationMs: Date.now() - totalStart,
+		metadata,
+		variants: variantResults
+	};
+	if (isSilent) process.stdout.write(JSON.stringify(variantEvalResult, null, 2));
+	return variantEvalResult;
+}
+
+//#endregion
+//#region src/eval/compare.ts
+/**
+* Load an eval result from a JSON file.
+*/
+function loadResult(filePath) {
+	try {
+		const content = (0, node_fs.readFileSync)(filePath, "utf-8");
+		return JSON.parse(content);
+	} catch {
+		throw new Error(`Could not read eval result: ${filePath}`);
+	}
+}
+/**
+* Compare two eval result files. First file is the baseline.
+*
+* Usage with version control:
+* 1. Run eval → results saved to ./llmops-evals/my-eval.eval.json
+* 2. Commit the file
+* 3. Make changes, re-run eval
+* 4. Compare: git stash the new result, compare old vs new
+*
+* Or compare two named eval files:
+* ```ts
+* const diff = await compare({
+*   files: ['./llmops-evals/baseline.eval.json', './llmops-evals/candidate.eval.json'],
+* })
+* ```
+*/
+async function compare(options) {
+	const { files } = options;
+	const baselineRun = loadResult(files[0]);
+	const candidateRun = loadResult(files[1]);
+	const allScoreNames = new Set([...Object.keys(baselineRun.scores), ...Object.keys(candidateRun.scores)]);
+	const scores = {};
+	for (const scoreName of allScoreNames) {
+		const baselineMean = baselineRun.scores[scoreName]?.mean ?? 0;
+		const candidateMean = candidateRun.scores[scoreName]?.mean ?? 0;
+		scores[scoreName] = {
+			baseline: baselineMean,
+			candidate: candidateMean,
+			delta: candidateMean - baselineMean
+		};
+	}
+	const regressions = [];
+	const improvements = [];
+	const minLen = Math.min(baselineRun.results.length, candidateRun.results.length);
+	for (let i = 0; i < minLen; i++) {
+		const baselineResult = baselineRun.results[i];
+		const candidateResult = candidateRun.results[i];
+		for (const scoreName of allScoreNames) {
+			const baselineScore = baselineResult.scores[scoreName] ?? NaN;
+			const candidateScore = candidateResult.scores[scoreName] ?? NaN;
+			if (Number.isNaN(baselineScore) || Number.isNaN(candidateScore)) continue;
+			if (candidateScore < baselineScore) regressions.push({
+				data: baselineResult.data,
+				evaluator: scoreName,
+				baselineScore,
+				candidateScore
+			});
+			else if (candidateScore > baselineScore) improvements.push({
+				data: baselineResult.data,
+				evaluator: scoreName,
+				baselineScore,
+				candidateScore
+			});
+		}
+	}
+	const result = {
+		baseline: baselineRun.runId,
+		candidate: candidateRun.runId,
+		scores,
+		regressions,
+		improvements
+	};
+	const w$1 = process.stderr;
+	const RESET$1 = "\x1B[0m";
+	const DIM$1 = "\x1B[2m";
+	const BOLD$1 = "\x1B[1m";
+	const GREEN$1 = "\x1B[32m";
+	const RED$1 = "\x1B[31m";
+	const CYAN$1 = "\x1B[36m";
+	w$1.write("\n");
+	w$1.write(`  ${BOLD$1}Compare${RESET$1}  ${DIM$1}${baselineRun.name} → ${candidateRun.name}${RESET$1}\n`);
+	w$1.write(`  ${DIM$1}${"─".repeat(50)}${RESET$1}\n\n`);
+	const scoreEntries = Object.entries(scores);
+	if (scoreEntries.length > 0) {
+		const maxNameLen = Math.max(...scoreEntries.map(([n]) => n.length), 10);
+		w$1.write(`  ${DIM$1}${"Evaluator".padEnd(maxNameLen)}  ${"Base".padStart(6)}    ${"New".padStart(6)}  ${"Delta".padStart(7)}${RESET$1}\n`);
+		w$1.write(`  ${DIM$1}${"─".repeat(maxNameLen + 30)}${RESET$1}\n`);
+		for (const [scoreName, delta] of scoreEntries) {
+			const sign = delta.delta >= 0 ? "+" : "";
+			const color = delta.delta >= 0 ? GREEN$1 : RED$1;
+			const icon = delta.delta > 0 ? "▲" : delta.delta < 0 ? "▼" : "=";
+			w$1.write(`  ${scoreName.padEnd(maxNameLen)}  ${delta.baseline.toFixed(2).padStart(6)}  ${DIM$1}→${RESET$1}  ${delta.candidate.toFixed(2).padStart(6)}  ${color}${sign}${delta.delta.toFixed(2).padStart(5)} ${icon}${RESET$1}\n`);
+		}
+		w$1.write("\n");
+	}
+	if (regressions.length > 0) {
+		w$1.write(`  ${RED$1}▼ ${regressions.length} regression${regressions.length > 1 ? "s" : ""}${RESET$1}\n`);
+		for (const r of regressions.slice(0, 5)) {
+			const dataStr = typeof r.data === "string" ? r.data : JSON.stringify(r.data).slice(0, 50);
+			w$1.write(`    ${DIM$1}${dataStr}${RESET$1}  ${r.evaluator}: ${r.baselineScore.toFixed(2)} → ${RED$1}${r.candidateScore.toFixed(2)}${RESET$1}\n`);
+		}
+		if (regressions.length > 5) w$1.write(`    ${DIM$1}... and ${regressions.length - 5} more${RESET$1}\n`);
+		w$1.write("\n");
+	}
+	if (improvements.length > 0) {
+		w$1.write(`  ${GREEN$1}▲ ${improvements.length} improvement${improvements.length > 1 ? "s" : ""}${RESET$1}\n`);
+		for (const imp of improvements.slice(0, 5)) {
+			const dataStr = typeof imp.data === "string" ? imp.data : JSON.stringify(imp.data).slice(0, 50);
+			w$1.write(`    ${DIM$1}${dataStr}${RESET$1}  ${imp.evaluator}: ${imp.baselineScore.toFixed(2)} → ${GREEN$1}${imp.candidateScore.toFixed(2)}${RESET$1}\n`);
+		}
+		if (improvements.length > 5) w$1.write(`    ${DIM$1}... and ${improvements.length - 5} more${RESET$1}\n`);
+		w$1.write("\n");
+	}
+	if (regressions.length === 0 && improvements.length === 0) w$1.write(`  ${CYAN$1}No changes between runs${RESET$1}\n\n`);
+	return result;
+}
+
+//#endregion
+//#region src/eval/judge.ts
+function interpolate(template, vars) {
+	return template.replace(/\{\{(\w+(?:\.\w+)*)\}\}/g, (_, path) => {
+		const value = path.split(".").reduce((obj, key) => obj?.[key], vars);
+		if (value === void 0 || value === null) return "";
+		return typeof value === "string" ? value : JSON.stringify(value, null, 2);
+	});
+}
+function buildVars(output, target, data) {
+	const vars = {
+		output: typeof output === "string" ? output : JSON.stringify(output, null, 2),
+		target: typeof target === "string" ? target : JSON.stringify(target, null, 2),
+		data: typeof data === "string" ? data : JSON.stringify(data, null, 2)
+	};
+	if (target && typeof target === "object") for (const [k, v] of Object.entries(target)) vars[`target.${k}`] = v;
+	if (data && typeof data === "object") for (const [k, v] of Object.entries(data)) vars[`data.${k}`] = v;
+	return vars;
+}
+const DEFAULT_SYSTEM = `You are an expert evaluator. Your job is to grade an AI system's output.
+
+Instructions:
+- Read the grading criteria in the user message carefully.
+- Evaluate the output objectively.
+- Return ONLY valid JSON. No markdown, no explanation outside the JSON.
+- The JSON must contain a "score" field with a number between 0.0 and 1.0.
+- You may optionally include a "reasoning" field with a brief explanation.
+
+Example response:
+{"score": 0.85, "reasoning": "The response is mostly accurate but misses one detail."}`;
+function defaultParse(response) {
+	let cleaned = response.replace(/```(?:json)?\n?/g, "").replace(/```$/g, "").trim();
+	const jsonMatch = cleaned.match(/\{[\s\S]*\}/);
+	if (jsonMatch) cleaned = jsonMatch[0];
+	const parsed = JSON.parse(cleaned);
+	if (typeof parsed?.score === "number") return Math.max(0, Math.min(1, parsed.score));
+	if (typeof parsed === "number") return Math.max(0, Math.min(1, parsed));
+	if (typeof parsed === "object" && parsed !== null) {
+		const entries = Object.entries(parsed).filter(([k, v]) => typeof v === "number" && k !== "reasoning");
+		if (entries.length > 0) return Object.fromEntries(entries.map(([k, v]) => [k, Math.max(0, Math.min(1, v))]));
+	}
+	throw new Error(`Could not extract score from judge response: ${response.slice(0, 300)}`);
+}
+async function callLLM(client, model, system, userMessage, temperature) {
+	const providerConfig = client.provider();
+	const response = await providerConfig.fetch(`${providerConfig.baseURL}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: `Bearer ${providerConfig.apiKey}`
+		},
+		body: JSON.stringify({
+			model,
+			temperature,
+			messages: [{
+				role: "system",
+				content: system
+			}, {
+				role: "user",
+				content: userMessage
+			}]
+		})
+	});
+	if (!response.ok) {
+		const errorBody = await response.text().catch(() => "unknown error");
+		throw new Error(`Judge LLM call failed (${response.status}): ${errorBody.slice(0, 300)}`);
+	}
+	const content = (await response.json()).choices?.[0]?.message?.content;
+	if (!content) throw new Error("Judge LLM returned empty response");
+	return content;
+}
+/**
+* Factory that returns an Evaluator which uses an LLM to score output.
+*
+* The judge:
+* - Uses a system message that instructs the LLM to return JSON scores
+* - Interpolates {{output}}, {{target}}, {{data}} and their fields in the prompt
+* - Uses temperature 0 by default for deterministic scoring
+* - Retries on parse failure (configurable)
+* - Clamps scores to [0, 1]
+*
+* Usage:
+* ```ts
+* import { llmops } from '@llmops/sdk'
+*
+* const client = llmops()
+* const accuracy = judgeScorer({
+*   model: '@openai/gpt-4o',
+*   prompt: `Rate the accuracy of this response.
+* Expected: {{target.answer}}
+* Actual: {{output}}`,
+*   client,
+* })
+* ```
+*/
+function judgeScorer(options) {
+	const { model, prompt, client, system = DEFAULT_SYSTEM, temperature = 0, maxRetries = 1, parse = defaultParse } = options;
+	return async (output, target, data) => {
+		const userMessage = interpolate(prompt, buildVars(output, target, data));
+		let lastError = null;
+		const attempts = 1 + maxRetries;
+		for (let attempt = 0; attempt < attempts; attempt++) try {
+			return parse(await callLLM(client, model, system, userMessage, temperature));
+		} catch (err) {
+			lastError = err instanceof Error ? err : new Error(String(err));
+			if (lastError.message.includes("Judge LLM call failed")) throw lastError;
+		}
+		throw lastError ?? /* @__PURE__ */ new Error("Judge scoring failed");
+	};
+}
+
+//#endregion
+exports.InlineDataset = InlineDataset;
+exports.compare = compare;
+exports.evaluate = evaluate;
+exports.judgeScorer = judgeScorer;