@llmops/sdk 1.0.0-beta.19 → 1.0.0-beta.22

package/dist/agents.d.cts

@@ -1,2 +1,2 @@
-import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-YNq4HFTK.cjs";
+import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-DkqkCcIx.cjs";
 export { AgentsSpan, AgentsSpanData, AgentsSpanError, AgentsTrace, AgentsTracingExporter, LLMOpsAgentsExporterConfig, createLLMOpsAgentsExporter };

package/dist/agents.d.mts

@@ -1,2 +1,2 @@
-import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-CI29gyrT.mjs";
+import { a as AgentsTracingExporter, i as AgentsTrace, n as AgentsSpanData, o as LLMOpsAgentsExporterConfig, r as AgentsSpanError, s as createLLMOpsAgentsExporter, t as AgentsSpan } from "./agents-exporter-CehKIArI.mjs";
 export { AgentsSpan, AgentsSpanData, AgentsSpanError, AgentsTrace, AgentsTracingExporter, LLMOpsAgentsExporterConfig, createLLMOpsAgentsExporter };

package/dist/agents.mjs

@@ -1,3 +1,3 @@
-import { t as createLLMOpsAgentsExporter } from "./agents-exporter-BY7BXquG.mjs";
+import { t as createLLMOpsAgentsExporter } from "./agents-exporter-CGxTzDeQ.mjs";
 
 export { createLLMOpsAgentsExporter };

package/dist/eval.cjs

@@ -0,0 +1,367 @@
+let node_crypto = require("node:crypto");
+let node_fs = require("node:fs");
+let node_path = require("node:path");
+
+//#region src/eval/dataset.ts
+/**
+* Wraps a plain array as an EvaluationDataset.
+*/
+var InlineDataset = class {
+	constructor(items) {
+		this.items = items;
+	}
+	size() {
+		return this.items.length;
+	}
+	get(index) {
+		return this.items[index];
+	}
+	slice(start, end) {
+		return this.items.slice(start, end);
+	}
+};
+
+//#endregion
+//#region src/eval/evaluate.ts
+async function pool(items, concurrency, fn) {
+	const executing = [];
+	for (const item of items) {
+		const p = fn(item).then(() => {
+			executing.splice(executing.indexOf(p), 1);
+		});
+		executing.push(p);
+		if (executing.length >= concurrency) await Promise.race(executing);
+	}
+	await Promise.all(executing);
+}
+function computeStats(values) {
+	const valid = values.filter((v) => !Number.isNaN(v));
+	if (valid.length === 0) return {
+		mean: 0,
+		min: 0,
+		max: 0,
+		median: 0,
+		count: 0
+	};
+	const sorted = [...valid].sort((a, b) => a - b);
+	const sum = sorted.reduce((a, b) => a + b, 0);
+	const mid = Math.floor(sorted.length / 2);
+	const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+	return {
+		mean: sum / sorted.length,
+		min: sorted[0],
+		max: sorted[sorted.length - 1],
+		median,
+		count: sorted.length
+	};
+}
+async function runSingleExecutor(dataset, executor, evaluators, concurrency) {
+	const size = await dataset.size();
+	const datapoints = await dataset.slice(0, size);
+	const results = new Array(datapoints.length);
+	const startTime = Date.now();
+	await pool(datapoints, concurrency, async (dp) => {
+		const idx = datapoints.indexOf(dp);
+		const dpStart = Date.now();
+		let output = null;
+		let error;
+		const scores = {};
+		try {
+			output = await executor(dp.data);
+		} catch (err) {
+			error = err instanceof Error ? err.message : String(err);
+		}
+		if (!error && output !== null) for (const [name, evaluator] of Object.entries(evaluators)) try {
+			const result = await evaluator(output, dp.target, dp.data);
+			if (typeof result === "number") scores[name] = result;
+			else for (const [subKey, subScore] of Object.entries(result)) scores[`${name}.${subKey}`] = subScore;
+		} catch {
+			scores[name] = NaN;
+		}
+		results[idx] = {
+			data: dp.data,
+			target: dp.target,
+			metadata: dp.metadata,
+			output,
+			scores,
+			durationMs: Date.now() - dpStart,
+			error
+		};
+	});
+	return {
+		results,
+		durationMs: Date.now() - startTime
+	};
+}
+function printSummary(result) {
+	const lines = [];
+	lines.push("");
+	lines.push(` ${result.name}`);
+	lines.push("");
+	const completed = result.count - result.errors;
+	lines.push(` ✓ ${completed}/${result.count} completed${result.errors > 0 ? `  ✗ ${result.errors} errors` : ""}`);
+	lines.push("");
+	lines.push(" Scores:");
+	for (const [name, stats] of Object.entries(result.scores)) lines.push(`   ${name.padEnd(16)} mean=${stats.mean.toFixed(2)}  min=${stats.min.toFixed(2)}  max=${stats.max.toFixed(2)}  median=${stats.median.toFixed(2)}`);
+	lines.push("");
+	lines.push(` Duration: ${(result.durationMs / 1e3).toFixed(1)}s`);
+	lines.push(` Run ID:   ${result.runId}`);
+	lines.push("");
+	process.stderr.write(lines.join("\n"));
+}
+function saveResult(result, outputDir) {
+	const dir = (0, node_path.join)(outputDir, result.name);
+	(0, node_fs.mkdirSync)(dir, { recursive: true });
+	(0, node_fs.writeFileSync)((0, node_path.join)(dir, `${result.runId}.json`), JSON.stringify(result, null, 2));
+}
+async function evaluate(options) {
+	const { name, data, executor, variants, evaluators, concurrency = 5, group, metadata, outputDir = process.env.LLMOPS_EVAL_OUTPUT_DIR || "./llmops-evals" } = options;
+	const runId = (0, node_crypto.randomUUID)();
+	if (executor && variants) throw new Error("evaluate(): provide either executor or variants, not both");
+	if (!executor && !variants) throw new Error("evaluate(): provide either executor or variants");
+	const dataset = Array.isArray(data) ? new InlineDataset(data) : data;
+	if (executor) {
+		const { results, durationMs } = await runSingleExecutor(dataset, executor, evaluators, concurrency);
+		const scoreNames = /* @__PURE__ */ new Set();
+		for (const r of results) for (const key of Object.keys(r.scores)) scoreNames.add(key);
+		const scores = {};
+		for (const scoreName of scoreNames) scores[scoreName] = computeStats(results.map((r) => r.scores[scoreName] ?? NaN));
+		const result = {
+			name,
+			runId,
+			group,
+			scores,
+			durationMs,
+			count: results.length,
+			errors: results.filter((r) => r.error).length,
+			metadata,
+			results
+		};
+		if (process.env.LLMOPS_EVAL_OUTPUT === "json") process.stdout.write(JSON.stringify(result, null, 2));
+		else printSummary(result);
+		saveResult(result, outputDir);
+		return result;
+	}
+	const variantResults = {};
+	const totalStart = Date.now();
+	for (const [variantName, variantExecutor] of Object.entries(variants)) {
+		const { results, durationMs } = await runSingleExecutor(dataset, variantExecutor, evaluators, concurrency);
+		const scoreNames = /* @__PURE__ */ new Set();
+		for (const r of results) for (const key of Object.keys(r.scores)) scoreNames.add(key);
+		const scores = {};
+		for (const scoreName of scoreNames) scores[scoreName] = computeStats(results.map((r) => r.scores[scoreName] ?? NaN));
+		const variantResult = {
+			name: `${name}/${variantName}`,
+			runId,
+			group,
+			scores,
+			durationMs,
+			count: results.length,
+			errors: results.filter((r) => r.error).length,
+			metadata,
+			results
+		};
+		variantResults[variantName] = variantResult;
+		if (process.env.LLMOPS_EVAL_OUTPUT !== "json") printSummary(variantResult);
+		saveResult(variantResult, outputDir);
+	}
+	const variantEvalResult = {
+		name,
+		runId,
+		group,
+		durationMs: Date.now() - totalStart,
+		metadata,
+		variants: variantResults
+	};
+	if (process.env.LLMOPS_EVAL_OUTPUT === "json") process.stdout.write(JSON.stringify(variantEvalResult, null, 2));
+	return variantEvalResult;
+}
+
+//#endregion
+//#region src/eval/compare.ts
+/**
+* Load an eval run from the filesystem.
+*/
+function loadRun(outputDir, name, runId) {
+	const dir = (0, node_path.join)(outputDir, name);
+	const filePath = (0, node_path.join)(dir, `${runId}.json`);
+	try {
+		const content = (0, node_fs.readFileSync)(filePath, "utf-8");
+		return JSON.parse(content);
+	} catch {
+		try {
+			const match = (0, node_fs.readdirSync)(dir).find((f) => f.startsWith(runId) && f.endsWith(".json"));
+			if (match) {
+				const content = (0, node_fs.readFileSync)((0, node_path.join)(dir, match), "utf-8");
+				return JSON.parse(content);
+			}
+		} catch {}
+		throw new Error(`Eval run "${runId}" not found for "${name}" in ${outputDir}. Expected file: ${filePath}`);
+	}
+}
+/**
+* Compare two eval runs. First run ID is the baseline.
+*
+* Usage:
+* ```ts
+* const diff = await compare({
+*   name: 'support-bot',
+*   runs: [run1.runId, run2.runId],
+* })
+* ```
+*/
+async function compare(options) {
+	const { runs, name, outputDir = "./llmops-evals" } = options;
+	if (runs.length < 2) throw new Error("compare() requires at least 2 run IDs");
+	const baselineRun = loadRun(outputDir, name, runs[0]);
+	const candidateRun = loadRun(outputDir, name, runs[1]);
+	const allScoreNames = new Set([...Object.keys(baselineRun.scores), ...Object.keys(candidateRun.scores)]);
+	const scores = {};
+	for (const scoreName of allScoreNames) {
+		const baselineMean = baselineRun.scores[scoreName]?.mean ?? 0;
+		const candidateMean = candidateRun.scores[scoreName]?.mean ?? 0;
+		scores[scoreName] = {
+			baseline: baselineMean,
+			candidate: candidateMean,
+			delta: candidateMean - baselineMean
+		};
+	}
+	const regressions = [];
+	const improvements = [];
+	const minLen = Math.min(baselineRun.results.length, candidateRun.results.length);
+	for (let i = 0; i < minLen; i++) {
+		const baselineResult = baselineRun.results[i];
+		const candidateResult = candidateRun.results[i];
+		for (const scoreName of allScoreNames) {
+			const baselineScore = baselineResult.scores[scoreName] ?? NaN;
+			const candidateScore = candidateResult.scores[scoreName] ?? NaN;
+			if (Number.isNaN(baselineScore) || Number.isNaN(candidateScore)) continue;
+			if (candidateScore < baselineScore) regressions.push({
+				data: baselineResult.data,
+				evaluator: scoreName,
+				baselineScore,
+				candidateScore
+			});
+			else if (candidateScore > baselineScore) improvements.push({
+				data: baselineResult.data,
+				evaluator: scoreName,
+				baselineScore,
+				candidateScore
+			});
+		}
+	}
+	const result = {
+		baseline: runs[0],
+		candidate: runs[1],
+		scores,
+		regressions,
+		improvements
+	};
+	const lines = [];
+	lines.push("");
+	lines.push(` compare: ${runs[0].slice(0, 8)} → ${runs[1].slice(0, 8)}`);
+	lines.push("");
+	lines.push(" Scores:");
+	for (const [scoreName, delta] of Object.entries(scores)) {
+		const sign = delta.delta >= 0 ? "+" : "";
+		const marker = delta.delta >= 0 ? "✓" : "✗";
+		lines.push(`   ${scoreName.padEnd(16)} ${delta.baseline.toFixed(2)} → ${delta.candidate.toFixed(2)}  (${sign}${delta.delta.toFixed(2)}) ${marker}`);
+	}
+	if (regressions.length > 0) {
+		lines.push("");
+		lines.push(` Regressions (${regressions.length}):`);
+		for (const r of regressions.slice(0, 5)) {
+			const dataStr = typeof r.data === "string" ? r.data : JSON.stringify(r.data).slice(0, 60);
+			lines.push(`   "${dataStr}"  ${r.evaluator}: ${r.baselineScore.toFixed(2)} → ${r.candidateScore.toFixed(2)}`);
+		}
+		if (regressions.length > 5) lines.push(`   ... and ${regressions.length - 5} more`);
+	}
+	if (improvements.length > 0) {
+		lines.push("");
+		lines.push(` Improvements (${improvements.length}):`);
+		for (const imp of improvements.slice(0, 5)) {
+			const dataStr = typeof imp.data === "string" ? imp.data : JSON.stringify(imp.data).slice(0, 60);
+			lines.push(`   "${dataStr}"  ${imp.evaluator}: ${imp.baselineScore.toFixed(2)} → ${imp.candidateScore.toFixed(2)}`);
+		}
+		if (improvements.length > 5) lines.push(`   ... and ${improvements.length - 5} more`);
+	}
+	lines.push("");
+	process.stderr.write(lines.join("\n"));
+	return result;
+}
+
+//#endregion
+//#region src/eval/judge.ts
+/**
+* Simple mustache-style template interpolation.
+*/
+function interpolate(template, vars) {
+	return template.replace(/\{\{(\w+(?:\.\w+)*)\}\}/g, (_, path) => {
+		const value = path.split(".").reduce((obj, key) => obj?.[key], vars);
+		return typeof value === "string" ? value : JSON.stringify(value);
+	});
+}
+/**
+* Default parser: expects JSON with a `score` field, a bare number,
+* or an object of number values (multi-score).
+*/
+function defaultParse(response) {
+	const cleaned = response.replace(/```json\n?|```/g, "").trim();
+	const parsed = JSON.parse(cleaned);
+	if (typeof parsed === "number") return parsed;
+	if (typeof parsed?.score === "number") return parsed.score;
+	if (typeof parsed === "object" && parsed !== null) {
+		const entries = Object.entries(parsed).filter(([, v]) => typeof v === "number");
+		if (entries.length > 0) return Object.fromEntries(entries);
+	}
+	throw new Error(`Could not extract score from judge response: ${response.slice(0, 200)}`);
+}
+/**
+* Factory that returns an Evaluator which uses an LLM to score output.
+*
+* Usage:
+* ```ts
+* const accuracy = judgeScorer({
+*   model: '@openai/gpt-4o',
+*   prompt: 'Rate accuracy 0-1. Expected: {{target.answer}} Actual: {{output}}',
+*   ops,
+* })
+* ```
+*/
+function judgeScorer(options) {
+	const { model, prompt, ops, parse = defaultParse } = options;
+	return async (output, target) => {
+		const vars = {
+			output: typeof output === "string" ? output : JSON.stringify(output),
+			target
+		};
+		if (target && typeof target === "object") for (const [k, v] of Object.entries(target)) vars[`target.${k}`] = v;
+		const renderedPrompt = interpolate(prompt, vars);
+		const providerConfig = ops.provider();
+		const response = await providerConfig.fetch(`${providerConfig.baseURL}/chat/completions`, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${providerConfig.apiKey}`
+			},
+			body: JSON.stringify({
+				model,
+				messages: [{
+					role: "user",
+					content: renderedPrompt
+				}],
+				response_format: { type: "json_object" }
+			})
+		});
+		if (!response.ok) throw new Error(`Judge LLM call failed: ${response.status} ${await response.text()}`);
+		const content = (await response.json()).choices?.[0]?.message?.content;
+		if (!content) throw new Error("Judge LLM returned empty response");
+		return parse(content);
+	};
+}
+
+//#endregion
+exports.InlineDataset = InlineDataset;
+exports.compare = compare;
+exports.evaluate = evaluate;
+exports.judgeScorer = judgeScorer;

package/dist/eval.d.cts

@@ -0,0 +1,200 @@
+import "./agents-exporter-DkqkCcIx.cjs";
+import { t as LLMOpsClient } from "./index-lgspeSNr.cjs";
+
+//#region src/eval/dataset.d.ts
+
+/**
+ * Interface for custom dataset sources.
+ * Built-in: inline arrays are wrapped in InlineDataset automatically.
+ * Future: CSVDataset, JSONLDataset, S3Dataset.
+ */
+interface EvaluationDataset<D = Record<string, unknown>, T = Record<string, unknown>> {
+  size(): number | Promise<number>;
+  get(index: number): Datapoint<D, T> | Promise<Datapoint<D, T>>;
+  slice(start: number, end: number): Datapoint<D, T>[] | Promise<Datapoint<D, T>[]>;
+}
+/**
+ * Wraps a plain array as an EvaluationDataset.
+ */
+declare class InlineDataset<D, T> implements EvaluationDataset<D, T> {
+  private items;
+  constructor(items: Datapoint<D, T>[]);
+  size(): number;
+  get(index: number): Datapoint<D, T>;
+  slice(start: number, end: number): Datapoint<D, T>[];
+}
+//#endregion
+//#region src/eval/types.d.ts
+/**
+ * A single datapoint in a dataset.
+ */
+interface Datapoint<D = Record<string, unknown>, T = Record<string, unknown>> {
+  data: D;
+  target?: T;
+  metadata?: Record<string, unknown>;
+}
+/**
+ * An evaluator scores executor output.
+ * Returns a single number (0-1) or an object of named scores.
+ */
+type Evaluator<O = unknown, T = unknown, D = unknown> = (output: O, target?: T, data?: D) => number | Record<string, number> | Promise<number | Record<string, number>>;
+/**
+ * An executor is the function under test.
+ */
+type Executor<D = Record<string, unknown>, O = unknown> = (data: D) => O | Promise<O>;
+/**
+ * Configuration for evaluate().
+ */
+interface EvaluateOptions<D, T, O> {
+  /** Name of this evaluation run. Required. */
+  name: string;
+  /** Dataset — inline array of datapoints or an EvaluationDataset */
+  data: Datapoint<D, T>[] | EvaluationDataset<D, T>;
+  /** The function under test. Provide either executor or variants, not both. */
+  executor?: Executor<D, O>;
+  /** Named variants for side-by-side comparison. Keys become variant labels. */
+  variants?: Record<string, Executor<D, O>>;
+  /** Named evaluator functions. Keys become score names. */
+  evaluators: Record<string, Evaluator<O, T>>;
+  /** Maximum concurrent datapoints. Default: 5 */
+  concurrency?: number;
+  /** Group name for tracking score progression across runs. */
+  group?: string;
+  /** Metadata attached to the entire run. */
+  metadata?: Record<string, unknown>;
+  /** Output directory for JSON results. Default: './llmops-evals' */
+  outputDir?: string;
+}
+/**
+ * Result for a single datapoint.
+ */
+interface DatapointResult<D = unknown, O = unknown> {
+  data: D;
+  target?: unknown;
+  metadata?: Record<string, unknown>;
+  output: O;
+  scores: Record<string, number>;
+  durationMs: number;
+  error?: string;
+}
+/**
+ * Aggregated score statistics for one evaluator.
+ */
+interface ScoreStats {
+  mean: number;
+  min: number;
+  max: number;
+  median: number;
+  count: number;
+}
+/**
+ * Summary of an evaluation run.
+ */
+interface EvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  scores: Record<string, ScoreStats>;
+  durationMs: number;
+  count: number;
+  errors: number;
+  metadata?: Record<string, unknown>;
+  results: DatapointResult<D, O>[];
+}
+/**
+ * When variants are used, wraps per-variant results.
+ */
+interface VariantEvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  durationMs: number;
+  metadata?: Record<string, unknown>;
+  variants: Record<string, EvaluateResult<D, O>>;
+}
+/**
+ * Options for compare().
+ */
+interface CompareOptions {
+  /** Run IDs to compare. First is baseline. */
+  runs: string[];
+  /** Directory where eval results are stored. Default: './llmops-evals' */
+  outputDir?: string;
+  /** Eval name to search within. Required. */
+  name: string;
+}
+/**
+ * Per-evaluator delta between two runs.
+ */
+interface ScoreDelta {
+  baseline: number;
+  candidate: number;
+  delta: number;
+}
+/**
+ * Result of comparing two runs.
+ */
+interface CompareResult {
+  baseline: string;
+  candidate: string;
+  scores: Record<string, ScoreDelta>;
+  regressions: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+  improvements: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+}
+/**
+ * Options for judgeScorer().
+ */
+interface JudgeScorerOptions {
+  /** Model identifier — routed through the gateway. e.g. '@openai/gpt-4o' */
+  model: string;
+  /** Prompt template. Supports {{output}}, {{target}}, {{target.*}} placeholders. */
+  prompt: string;
+  /** The llmops client instance. Judge call routed through gateway. */
+  ops: LLMOpsClient;
+  /** Custom parser for extracting score from LLM response. */
+  parse?: (response: string) => number | Record<string, number>;
+}
+//#endregion
+//#region src/eval/evaluate.d.ts
+declare function evaluate<D = Record<string, unknown>, T = Record<string, unknown>, O = unknown>(options: EvaluateOptions<D, T, O>): Promise<EvaluateResult<D, O> | VariantEvaluateResult<D, O>>;
+//#endregion
+//#region src/eval/compare.d.ts
+/**
+ * Compare two eval runs. First run ID is the baseline.
+ *
+ * Usage:
+ * ```ts
+ * const diff = await compare({
+ *   name: 'support-bot',
+ *   runs: [run1.runId, run2.runId],
+ * })
+ * ```
+ */
+declare function compare(options: CompareOptions): Promise<CompareResult>;
+//#endregion
+//#region src/eval/judge.d.ts
+/**
+ * Factory that returns an Evaluator which uses an LLM to score output.
+ *
+ * Usage:
+ * ```ts
+ * const accuracy = judgeScorer({
+ *   model: '@openai/gpt-4o',
+ *   prompt: 'Rate accuracy 0-1. Expected: {{target.answer}} Actual: {{output}}',
+ *   ops,
+ * })
+ * ```
+ */
+declare function judgeScorer(options: JudgeScorerOptions): Evaluator;
+//#endregion
+export { type CompareOptions, type CompareResult, type Datapoint, type DatapointResult, type EvaluateOptions, type EvaluateResult, type EvaluationDataset, type Evaluator, type Executor, InlineDataset, type JudgeScorerOptions, type ScoreDelta, type ScoreStats, type VariantEvaluateResult, compare, evaluate, judgeScorer };