@llmops/sdk 1.0.0-beta.2 → 1.0.0-beta.23

package/dist/eval.d.cts

@@ -0,0 +1,240 @@
+import "./agents-exporter-DkqkCcIx.cjs";
+import { t as LLMOpsClient } from "./index-lgspeSNr.cjs";
+
+//#region src/eval/dataset.d.ts
+
+/**
+ * Interface for custom dataset sources.
+ * Built-in: inline arrays are wrapped in InlineDataset automatically.
+ * Future: CSVDataset, JSONLDataset, S3Dataset.
+ */
+interface EvaluationDataset<D = Record<string, unknown>, T = Record<string, unknown>> {
+  size(): number | Promise<number>;
+  get(index: number): Datapoint<D, T> | Promise<Datapoint<D, T>>;
+  slice(start: number, end: number): Datapoint<D, T>[] | Promise<Datapoint<D, T>[]>;
+}
+/**
+ * Wraps a plain array as an EvaluationDataset.
+ */
+declare class InlineDataset<D, T> implements EvaluationDataset<D, T> {
+  private items;
+  constructor(items: Datapoint<D, T>[]);
+  size(): number;
+  get(index: number): Datapoint<D, T>;
+  slice(start: number, end: number): Datapoint<D, T>[];
+}
+//#endregion
+//#region src/eval/types.d.ts
+/**
+ * A single datapoint in a dataset.
+ */
+interface Datapoint<D = Record<string, unknown>, T = Record<string, unknown>> {
+  data: D;
+  target?: T;
+  metadata?: Record<string, unknown>;
+}
+/**
+ * An evaluator scores executor output.
+ * Returns a single number (0-1) or an object of named scores.
+ */
+type Evaluator<O = unknown, T = unknown, D = unknown> = (output: O, target?: T, data?: D) => number | Record<string, number> | Promise<number | Record<string, number>>;
+/**
+ * An executor is the function under test.
+ */
+type Executor<D = Record<string, unknown>, O = unknown> = (data: D) => O | Promise<O>;
+/**
+ * Configuration for evaluate().
+ */
+interface EvaluateOptions<D, T, O> {
+  /** Name of this evaluation run. Required. */
+  name: string;
+  /** Dataset — inline array of datapoints or an EvaluationDataset */
+  data: Datapoint<D, T>[] | EvaluationDataset<D, T>;
+  /** The function under test. Provide either executor or variants, not both. */
+  executor?: Executor<D, O>;
+  /** Named variants for side-by-side comparison. Keys become variant labels. */
+  variants?: Record<string, Executor<D, O>>;
+  /** Named evaluator functions. Keys become score names. */
+  evaluators: Record<string, Evaluator<O, T>>;
+  /** Maximum concurrent datapoints. Default: 5 */
+  concurrency?: number;
+  /** Group name for tracking score progression across runs. */
+  group?: string;
+  /** Metadata attached to the entire run. */
+  metadata?: Record<string, unknown>;
+  /** Output directory for JSON results. Default: './llmops-evals' */
+  outputDir?: string;
+}
+/**
+ * Result for a single datapoint.
+ */
+interface DatapointResult<D = unknown, O = unknown> {
+  data: D;
+  target?: unknown;
+  metadata?: Record<string, unknown>;
+  output: O;
+  scores: Record<string, number>;
+  durationMs: number;
+  error?: string;
+}
+/**
+ * Aggregated score statistics for one evaluator.
+ */
+interface ScoreStats {
+  mean: number;
+  min: number;
+  max: number;
+  median: number;
+  count: number;
+}
+/**
+ * Summary of an evaluation run.
+ */
+interface EvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  scores: Record<string, ScoreStats>;
+  durationMs: number;
+  count: number;
+  errors: number;
+  metadata?: Record<string, unknown>;
+  results: DatapointResult<D, O>[];
+}
+/**
+ * When variants are used, wraps per-variant results.
+ */
+interface VariantEvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  durationMs: number;
+  metadata?: Record<string, unknown>;
+  variants: Record<string, EvaluateResult<D, O>>;
+}
+/**
+ * Options for compare().
+ */
+interface CompareOptions {
+  /** Paths to eval result JSON files. First is baseline, second is candidate. */
+  files: [string, string];
+}
+/**
+ * Per-evaluator delta between two runs.
+ */
+interface ScoreDelta {
+  baseline: number;
+  candidate: number;
+  delta: number;
+}
+/**
+ * Result of comparing two runs.
+ */
+interface CompareResult {
+  baseline: string;
+  candidate: string;
+  scores: Record<string, ScoreDelta>;
+  regressions: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+  improvements: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+}
+/**
+ * Options for judgeScorer().
+ */
+interface JudgeScorerOptions {
+  /** Model identifier — routed through the gateway. e.g. '@openai/gpt-4o' */
+  model: string;
+  /**
+   * Grading prompt. Supports {{output}}, {{target}}, {{target.*}},
+   * {{data}}, {{data.*}} placeholders.
+   *
+   * This becomes the user message. A system message is added automatically
+   * that instructs the LLM to return a JSON score.
+   */
+  prompt: string;
+  /**
+   * The llmops client instance. The judge call is routed through the
+   * gateway and traced like any other LLM call.
+   *
+   * ```ts
+   * const client = llmops({ telemetry: pgStore(url) })
+   * judgeScorer({ model: '@openai/gpt-4o', prompt: '...', client })
+   * ```
+   */
+  client: LLMOpsClient;
+  /**
+   * Custom system message. Overrides the default grading instructions.
+   * If omitted, a default system message is used that instructs
+   * the LLM to return JSON with a "score" field (0-1).
+   */
+  system?: string;
+  /** Temperature for the judge LLM. Default: 0 (deterministic). */
+  temperature?: number;
+  /** Max retries on parse failure. Default: 1. */
+  maxRetries?: number;
+  /**
+   * Custom parser for extracting score from LLM response.
+   * Default: expects JSON with a `score` field.
+   */
+  parse?: (response: string) => number | Record<string, number>;
+}
+//#endregion
+//#region src/eval/evaluate.d.ts
+declare function evaluate<D = Record<string, unknown>, T = Record<string, unknown>, O = unknown>(options: EvaluateOptions<D, T, O>): Promise<EvaluateResult<D, O> | VariantEvaluateResult<D, O>>;
+//#endregion
+//#region src/eval/compare.d.ts
+/**
+ * Compare two eval result files. First file is the baseline.
+ *
+ * Usage with version control:
+ * 1. Run eval → results saved to ./llmops-evals/my-eval.eval.json
+ * 2. Commit the file
+ * 3. Make changes, re-run eval
+ * 4. Compare: git stash the new result, compare old vs new
+ *
+ * Or compare two named eval files:
+ * ```ts
+ * const diff = await compare({
+ *   files: ['./llmops-evals/baseline.eval.json', './llmops-evals/candidate.eval.json'],
+ * })
+ * ```
+ */
+declare function compare(options: CompareOptions): Promise<CompareResult>;
+//#endregion
+//#region src/eval/judge.d.ts
+/**
+ * Factory that returns an Evaluator which uses an LLM to score output.
+ *
+ * The judge:
+ * - Uses a system message that instructs the LLM to return JSON scores
+ * - Interpolates {{output}}, {{target}}, {{data}} and their fields in the prompt
+ * - Uses temperature 0 by default for deterministic scoring
+ * - Retries on parse failure (configurable)
+ * - Clamps scores to [0, 1]
+ *
+ * Usage:
+ * ```ts
+ * import { llmops } from '@llmops/sdk'
+ *
+ * const client = llmops()
+ * const accuracy = judgeScorer({
+ *   model: '@openai/gpt-4o',
+ *   prompt: `Rate the accuracy of this response.
+ * Expected: {{target.answer}}
+ * Actual: {{output}}`,
+ *   client,
+ * })
+ * ```
+ */
+declare function judgeScorer(options: JudgeScorerOptions): Evaluator;
+//#endregion
+export { type CompareOptions, type CompareResult, type Datapoint, type DatapointResult, type EvaluateOptions, type EvaluateResult, type EvaluationDataset, type Evaluator, type Executor, InlineDataset, type JudgeScorerOptions, type ScoreDelta, type ScoreStats, type VariantEvaluateResult, compare, evaluate, judgeScorer };

package/dist/eval.d.mts

@@ -0,0 +1,240 @@
+import "./agents-exporter-CehKIArI.mjs";
+import { t as LLMOpsClient } from "./index-BZLzywwb.mjs";
+
+//#region src/eval/dataset.d.ts
+
+/**
+ * Interface for custom dataset sources.
+ * Built-in: inline arrays are wrapped in InlineDataset automatically.
+ * Future: CSVDataset, JSONLDataset, S3Dataset.
+ */
+interface EvaluationDataset<D = Record<string, unknown>, T = Record<string, unknown>> {
+  size(): number | Promise<number>;
+  get(index: number): Datapoint<D, T> | Promise<Datapoint<D, T>>;
+  slice(start: number, end: number): Datapoint<D, T>[] | Promise<Datapoint<D, T>[]>;
+}
+/**
+ * Wraps a plain array as an EvaluationDataset.
+ */
+declare class InlineDataset<D, T> implements EvaluationDataset<D, T> {
+  private items;
+  constructor(items: Datapoint<D, T>[]);
+  size(): number;
+  get(index: number): Datapoint<D, T>;
+  slice(start: number, end: number): Datapoint<D, T>[];
+}
+//#endregion
+//#region src/eval/types.d.ts
+/**
+ * A single datapoint in a dataset.
+ */
+interface Datapoint<D = Record<string, unknown>, T = Record<string, unknown>> {
+  data: D;
+  target?: T;
+  metadata?: Record<string, unknown>;
+}
+/**
+ * An evaluator scores executor output.
+ * Returns a single number (0-1) or an object of named scores.
+ */
+type Evaluator<O = unknown, T = unknown, D = unknown> = (output: O, target?: T, data?: D) => number | Record<string, number> | Promise<number | Record<string, number>>;
+/**
+ * An executor is the function under test.
+ */
+type Executor<D = Record<string, unknown>, O = unknown> = (data: D) => O | Promise<O>;
+/**
+ * Configuration for evaluate().
+ */
+interface EvaluateOptions<D, T, O> {
+  /** Name of this evaluation run. Required. */
+  name: string;
+  /** Dataset — inline array of datapoints or an EvaluationDataset */
+  data: Datapoint<D, T>[] | EvaluationDataset<D, T>;
+  /** The function under test. Provide either executor or variants, not both. */
+  executor?: Executor<D, O>;
+  /** Named variants for side-by-side comparison. Keys become variant labels. */
+  variants?: Record<string, Executor<D, O>>;
+  /** Named evaluator functions. Keys become score names. */
+  evaluators: Record<string, Evaluator<O, T>>;
+  /** Maximum concurrent datapoints. Default: 5 */
+  concurrency?: number;
+  /** Group name for tracking score progression across runs. */
+  group?: string;
+  /** Metadata attached to the entire run. */
+  metadata?: Record<string, unknown>;
+  /** Output directory for JSON results. Default: './llmops-evals' */
+  outputDir?: string;
+}
+/**
+ * Result for a single datapoint.
+ */
+interface DatapointResult<D = unknown, O = unknown> {
+  data: D;
+  target?: unknown;
+  metadata?: Record<string, unknown>;
+  output: O;
+  scores: Record<string, number>;
+  durationMs: number;
+  error?: string;
+}
+/**
+ * Aggregated score statistics for one evaluator.
+ */
+interface ScoreStats {
+  mean: number;
+  min: number;
+  max: number;
+  median: number;
+  count: number;
+}
+/**
+ * Summary of an evaluation run.
+ */
+interface EvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  scores: Record<string, ScoreStats>;
+  durationMs: number;
+  count: number;
+  errors: number;
+  metadata?: Record<string, unknown>;
+  results: DatapointResult<D, O>[];
+}
+/**
+ * When variants are used, wraps per-variant results.
+ */
+interface VariantEvaluateResult<D = unknown, O = unknown> {
+  name: string;
+  runId: string;
+  group?: string;
+  durationMs: number;
+  metadata?: Record<string, unknown>;
+  variants: Record<string, EvaluateResult<D, O>>;
+}
+/**
+ * Options for compare().
+ */
+interface CompareOptions {
+  /** Paths to eval result JSON files. First is baseline, second is candidate. */
+  files: [string, string];
+}
+/**
+ * Per-evaluator delta between two runs.
+ */
+interface ScoreDelta {
+  baseline: number;
+  candidate: number;
+  delta: number;
+}
+/**
+ * Result of comparing two runs.
+ */
+interface CompareResult {
+  baseline: string;
+  candidate: string;
+  scores: Record<string, ScoreDelta>;
+  regressions: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+  improvements: Array<{
+    data: unknown;
+    evaluator: string;
+    baselineScore: number;
+    candidateScore: number;
+  }>;
+}
+/**
+ * Options for judgeScorer().
+ */
+interface JudgeScorerOptions {
+  /** Model identifier — routed through the gateway. e.g. '@openai/gpt-4o' */
+  model: string;
+  /**
+   * Grading prompt. Supports {{output}}, {{target}}, {{target.*}},
+   * {{data}}, {{data.*}} placeholders.
+   *
+   * This becomes the user message. A system message is added automatically
+   * that instructs the LLM to return a JSON score.
+   */
+  prompt: string;
+  /**
+   * The llmops client instance. The judge call is routed through the
+   * gateway and traced like any other LLM call.
+   *
+   * ```ts
+   * const client = llmops({ telemetry: pgStore(url) })
+   * judgeScorer({ model: '@openai/gpt-4o', prompt: '...', client })
+   * ```
+   */
+  client: LLMOpsClient;
+  /**
+   * Custom system message. Overrides the default grading instructions.
+   * If omitted, a default system message is used that instructs
+   * the LLM to return JSON with a "score" field (0-1).
+   */
+  system?: string;
+  /** Temperature for the judge LLM. Default: 0 (deterministic). */
+  temperature?: number;
+  /** Max retries on parse failure. Default: 1. */
+  maxRetries?: number;
+  /**
+   * Custom parser for extracting score from LLM response.
+   * Default: expects JSON with a `score` field.
+   */
+  parse?: (response: string) => number | Record<string, number>;
+}
+//#endregion
+//#region src/eval/evaluate.d.ts
+declare function evaluate<D = Record<string, unknown>, T = Record<string, unknown>, O = unknown>(options: EvaluateOptions<D, T, O>): Promise<EvaluateResult<D, O> | VariantEvaluateResult<D, O>>;
+//#endregion
+//#region src/eval/compare.d.ts
+/**
+ * Compare two eval result files. First file is the baseline.
+ *
+ * Usage with version control:
+ * 1. Run eval → results saved to ./llmops-evals/my-eval.eval.json
+ * 2. Commit the file
+ * 3. Make changes, re-run eval
+ * 4. Compare: git stash the new result, compare old vs new
+ *
+ * Or compare two named eval files:
+ * ```ts
+ * const diff = await compare({
+ *   files: ['./llmops-evals/baseline.eval.json', './llmops-evals/candidate.eval.json'],
+ * })
+ * ```
+ */
+declare function compare(options: CompareOptions): Promise<CompareResult>;
+//#endregion
+//#region src/eval/judge.d.ts
+/**
+ * Factory that returns an Evaluator which uses an LLM to score output.
+ *
+ * The judge:
+ * - Uses a system message that instructs the LLM to return JSON scores
+ * - Interpolates {{output}}, {{target}}, {{data}} and their fields in the prompt
+ * - Uses temperature 0 by default for deterministic scoring
+ * - Retries on parse failure (configurable)
+ * - Clamps scores to [0, 1]
+ *
+ * Usage:
+ * ```ts
+ * import { llmops } from '@llmops/sdk'
+ *
+ * const client = llmops()
+ * const accuracy = judgeScorer({
+ *   model: '@openai/gpt-4o',
+ *   prompt: `Rate the accuracy of this response.
+ * Expected: {{target.answer}}
+ * Actual: {{output}}`,
+ *   client,
+ * })
+ * ```
+ */
+declare function judgeScorer(options: JudgeScorerOptions): Evaluator;
+//#endregion
+export { type CompareOptions, type CompareResult, type Datapoint, type DatapointResult, type EvaluateOptions, type EvaluateResult, type EvaluationDataset, type Evaluator, type Executor, InlineDataset, type JudgeScorerOptions, type ScoreDelta, type ScoreStats, type VariantEvaluateResult, compare, evaluate, judgeScorer };