@fabricorg/experiments-evals 0.1.0

package/dist/model-client-B7yk_i1s.d.ts

@@ -0,0 +1,601 @@
+import { z } from 'zod';
+
+declare const EvaluatorKind: z.ZodEnum<["llm-judge", "code", "heuristic"]>;
+type EvaluatorKind = z.infer<typeof EvaluatorKind>;
+declare const TokenUsage: z.ZodObject<{
+    inputTokens: z.ZodNumber;
+    outputTokens: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    inputTokens: number;
+    outputTokens: number;
+}, {
+    inputTokens: number;
+    outputTokens: number;
+}>;
+type TokenUsage = z.infer<typeof TokenUsage>;
+/**
+ * One evaluator's verdict on one example (or, later, one span). `score` is
+ * normalized to [0, 1] where defined; `label` carries the categorical verdict
+ * ("hallucinated", "correct", …). Provenance fields are mandatory — every
+ * score must be traceable to evaluator name+version and, for judges, the
+ * prompt hash and model endpoint (governance: evals-judge-provenance).
+ */
+declare const EvalScore: z.ZodObject<{
+    evaluatorName: z.ZodString;
+    evaluatorVersion: z.ZodString;
+    label: z.ZodOptional<z.ZodString>;
+    score: z.ZodNullable<z.ZodNumber>;
+    explanation: z.ZodOptional<z.ZodString>;
+    /** SHA-256 of the rendered judge prompt; absent for code/heuristic evaluators. */
+    promptHash: z.ZodOptional<z.ZodString>;
+    /** Model Serving endpoint name; absent for code/heuristic evaluators. */
+    modelEndpoint: z.ZodOptional<z.ZodString>;
+    tokens: z.ZodOptional<z.ZodObject<{
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        inputTokens: number;
+        outputTokens: number;
+    }, {
+        inputTokens: number;
+        outputTokens: number;
+    }>>;
+    latencyMs: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    evaluatorName: string;
+    evaluatorVersion: string;
+    score: number | null;
+    latencyMs: number;
+    label?: string | undefined;
+    explanation?: string | undefined;
+    promptHash?: string | undefined;
+    modelEndpoint?: string | undefined;
+    tokens?: {
+        inputTokens: number;
+        outputTokens: number;
+    } | undefined;
+}, {
+    evaluatorName: string;
+    evaluatorVersion: string;
+    score: number | null;
+    latencyMs: number;
+    label?: string | undefined;
+    explanation?: string | undefined;
+    promptHash?: string | undefined;
+    modelEndpoint?: string | undefined;
+    tokens?: {
+        inputTokens: number;
+        outputTokens: number;
+    } | undefined;
+}>;
+type EvalScore = z.infer<typeof EvalScore>;
+declare const EvalRunStatus: z.ZodEnum<["pending", "running", "succeeded", "failed", "cancelled"]>;
+type EvalRunStatus = z.infer<typeof EvalRunStatus>;
+/** Target of a run: a frozen dataset version, or a live experiment's variants. */
+declare const EvalRunTarget: z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+    kind: z.ZodLiteral<"dataset">;
+    datasetId: z.ZodString;
+    version: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    kind: "dataset";
+    datasetId: string;
+    version: number;
+}, {
+    kind: "dataset";
+    datasetId: string;
+    version: number;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"experiment">;
+    experimentId: z.ZodString;
+    /** Optional metric event to correlate scored outputs with conversions. */
+    metricEventName: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    kind: "experiment";
+    experimentId: string;
+    metricEventName?: string | undefined;
+}, {
+    kind: "experiment";
+    experimentId: string;
+    metricEventName?: string | undefined;
+}>]>;
+type EvalRunTarget = z.infer<typeof EvalRunTarget>;
+/** One rendered conversation the generation phase completes through the model. */
+declare const GenerationSignalItem: z.ZodObject<{
+    itemId: z.ZodString;
+    messages: z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["system", "user", "assistant"]>;
+        content: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }, {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    itemId: string;
+    messages: {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }[];
+}, {
+    itemId: string;
+    messages: {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }[];
+}>;
+type GenerationSignalItem = z.infer<typeof GenerationSignalItem>;
+/**
+ * Generation phase of a playground-replay run (rendered prompt version per
+ * example). Rendering happens at dispatch time (pure); the model calls
+ * happen in the durable executor, and their tokens count inside
+ * `maxJudgeTokens` — the run's SHARED budget (review fix: generation must
+ * not occur outside the budget or the request-side lifecycle).
+ */
+declare const GenerationSignal: z.ZodObject<{
+    promptId: z.ZodString;
+    promptVersion: z.ZodNumber;
+    /** Worst-case tokens reserved per generation call under the shared budget. */
+    reserveTokensPerCall: z.ZodNumber;
+    /** Provider-enforced generation completion ceiling from the prompt version. */
+    maxOutputTokens: z.ZodNumber;
+    /** Sampling temperature frozen from the prompt version. */
+    temperature: z.ZodNumber;
+    items: z.ZodArray<z.ZodObject<{
+        itemId: z.ZodString;
+        messages: z.ZodArray<z.ZodObject<{
+            role: z.ZodEnum<["system", "user", "assistant"]>;
+            content: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }, {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        itemId: string;
+        messages: {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+    }, {
+        itemId: string;
+        messages: {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    promptId: string;
+    promptVersion: number;
+    reserveTokensPerCall: number;
+    maxOutputTokens: number;
+    temperature: number;
+    items: {
+        itemId: string;
+        messages: {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+    }[];
+}, {
+    promptId: string;
+    promptVersion: number;
+    reserveTokensPerCall: number;
+    maxOutputTokens: number;
+    temperature: number;
+    items: {
+        itemId: string;
+        messages: {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+    }[];
+}>;
+type GenerationSignal = z.infer<typeof GenerationSignal>;
+declare const EvalRunSpec: z.ZodObject<{
+    runId: z.ZodString;
+    tenantId: z.ZodString;
+    target: z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+        kind: z.ZodLiteral<"dataset">;
+        datasetId: z.ZodString;
+        version: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    }, {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"experiment">;
+        experimentId: z.ZodString;
+        /** Optional metric event to correlate scored outputs with conversions. */
+        metricEventName: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    }, {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    }>]>;
+    /** Present on playground-replay runs; executors generate before judging. */
+    generation: z.ZodOptional<z.ZodObject<{
+        promptId: z.ZodString;
+        promptVersion: z.ZodNumber;
+        /** Worst-case tokens reserved per generation call under the shared budget. */
+        reserveTokensPerCall: z.ZodNumber;
+        /** Provider-enforced generation completion ceiling from the prompt version. */
+        maxOutputTokens: z.ZodNumber;
+        /** Sampling temperature frozen from the prompt version. */
+        temperature: z.ZodNumber;
+        items: z.ZodArray<z.ZodObject<{
+            itemId: z.ZodString;
+            messages: z.ZodArray<z.ZodObject<{
+                role: z.ZodEnum<["system", "user", "assistant"]>;
+                content: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }, {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }, {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    }, {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    }>>;
+    evaluatorNames: z.ZodArray<z.ZodString, "many">;
+    /** Judge-token budget for the whole run; runner aborts when exceeded. */
+    maxJudgeTokens: z.ZodOptional<z.ZodNumber>;
+    /**
+     * Worst-case tokens reserved per judge call under the hard budget (see
+     * ExecuteOptions.reserveTokensPerCall). Derived at run creation from the
+     * selected judges' prompt template sizes plus a completion cap, and
+     * threaded through the dispatch signal so the durable runner enforces the
+     * same reservation discipline as the inline executor.
+     */
+    reserveTokensPerCall: z.ZodOptional<z.ZodNumber>;
+    createdAtIso: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    runId: string;
+    tenantId: string;
+    target: {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    } | {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    };
+    evaluatorNames: string[];
+    createdAtIso: string;
+    reserveTokensPerCall?: number | undefined;
+    generation?: {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    } | undefined;
+    maxJudgeTokens?: number | undefined;
+}, {
+    runId: string;
+    tenantId: string;
+    target: {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    } | {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    };
+    evaluatorNames: string[];
+    createdAtIso: string;
+    reserveTokensPerCall?: number | undefined;
+    generation?: {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    } | undefined;
+    maxJudgeTokens?: number | undefined;
+}>;
+type EvalRunSpec = z.infer<typeof EvalRunSpec>;
+declare const EvalRun: z.ZodObject<{
+    runId: z.ZodString;
+    tenantId: z.ZodString;
+    target: z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+        kind: z.ZodLiteral<"dataset">;
+        datasetId: z.ZodString;
+        version: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    }, {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"experiment">;
+        experimentId: z.ZodString;
+        /** Optional metric event to correlate scored outputs with conversions. */
+        metricEventName: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    }, {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    }>]>;
+    /** Present on playground-replay runs; executors generate before judging. */
+    generation: z.ZodOptional<z.ZodObject<{
+        promptId: z.ZodString;
+        promptVersion: z.ZodNumber;
+        /** Worst-case tokens reserved per generation call under the shared budget. */
+        reserveTokensPerCall: z.ZodNumber;
+        /** Provider-enforced generation completion ceiling from the prompt version. */
+        maxOutputTokens: z.ZodNumber;
+        /** Sampling temperature frozen from the prompt version. */
+        temperature: z.ZodNumber;
+        items: z.ZodArray<z.ZodObject<{
+            itemId: z.ZodString;
+            messages: z.ZodArray<z.ZodObject<{
+                role: z.ZodEnum<["system", "user", "assistant"]>;
+                content: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }, {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }, {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    }, {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    }>>;
+    evaluatorNames: z.ZodArray<z.ZodString, "many">;
+    /** Judge-token budget for the whole run; runner aborts when exceeded. */
+    maxJudgeTokens: z.ZodOptional<z.ZodNumber>;
+    /**
+     * Worst-case tokens reserved per judge call under the hard budget (see
+     * ExecuteOptions.reserveTokensPerCall). Derived at run creation from the
+     * selected judges' prompt template sizes plus a completion cap, and
+     * threaded through the dispatch signal so the durable runner enforces the
+     * same reservation discipline as the inline executor.
+     */
+    reserveTokensPerCall: z.ZodOptional<z.ZodNumber>;
+    createdAtIso: z.ZodString;
+} & {
+    status: z.ZodEnum<["pending", "running", "succeeded", "failed", "cancelled"]>;
+    scoredExamples: z.ZodNumber;
+    totalExamples: z.ZodNumber;
+    tokensSpent: z.ZodNumber;
+    costUsd: z.ZodNullable<z.ZodNumber>;
+    startedAtIso: z.ZodOptional<z.ZodString>;
+    finishedAtIso: z.ZodOptional<z.ZodString>;
+    error: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    status: "pending" | "running" | "succeeded" | "failed" | "cancelled";
+    runId: string;
+    tenantId: string;
+    target: {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    } | {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    };
+    evaluatorNames: string[];
+    createdAtIso: string;
+    scoredExamples: number;
+    totalExamples: number;
+    tokensSpent: number;
+    costUsd: number | null;
+    reserveTokensPerCall?: number | undefined;
+    generation?: {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    } | undefined;
+    maxJudgeTokens?: number | undefined;
+    startedAtIso?: string | undefined;
+    finishedAtIso?: string | undefined;
+    error?: string | undefined;
+}, {
+    status: "pending" | "running" | "succeeded" | "failed" | "cancelled";
+    runId: string;
+    tenantId: string;
+    target: {
+        kind: "dataset";
+        datasetId: string;
+        version: number;
+    } | {
+        kind: "experiment";
+        experimentId: string;
+        metricEventName?: string | undefined;
+    };
+    evaluatorNames: string[];
+    createdAtIso: string;
+    scoredExamples: number;
+    totalExamples: number;
+    tokensSpent: number;
+    costUsd: number | null;
+    reserveTokensPerCall?: number | undefined;
+    generation?: {
+        promptId: string;
+        promptVersion: number;
+        reserveTokensPerCall: number;
+        maxOutputTokens: number;
+        temperature: number;
+        items: {
+            itemId: string;
+            messages: {
+                role: "system" | "user" | "assistant";
+                content: string;
+            }[];
+        }[];
+    } | undefined;
+    maxJudgeTokens?: number | undefined;
+    startedAtIso?: string | undefined;
+    finishedAtIso?: string | undefined;
+    error?: string | undefined;
+}>;
+type EvalRun = z.infer<typeof EvalRun>;
+
+interface ChatMessage {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+interface ModelResponse {
+    text: string;
+    tokens: TokenUsage;
+}
+interface ModelCompletionOptions {
+    /** Judges must be deterministic unless a caller explicitly builds another evaluator. */
+    temperature: number;
+    /** Provider-enforced completion ceiling used by the run budget reservation. */
+    maxOutputTokens: number;
+}
+/**
+ * The only surface through which evaluators reach a model. Production ships
+ * a Databricks Model Serving adapter at the explicit `/databricks` package
+ * subpath; the provider-neutral root never imports a provider client. Tests
+ * use `StaticModelClient`.
+ */
+interface ModelClient {
+    /** Identifier recorded in EvalScore.modelEndpoint for provenance. */
+    readonly endpoint: string;
+    complete(messages: readonly ChatMessage[], options: ModelCompletionOptions): Promise<ModelResponse>;
+}
+/**
+ * Deterministic client for unit tests and `fx dev`: responses are matched by
+ * substring of the last user message, mirroring the frozen-vector discipline
+ * in `@fabricorg/experiments-testkit`.
+ */
+declare class StaticModelClient implements ModelClient {
+    private readonly rules;
+    private readonly fallback;
+    readonly endpoint = "static-test-endpoint";
+    constructor(rules: Array<{
+        match: string;
+        respond: string;
+    }>, fallback?: string);
+    complete(messages: readonly ChatMessage[], options: ModelCompletionOptions): Promise<ModelResponse>;
+}
+/**
+ * Conservative provider-independent upper bound used before a judge call.
+ * A tokenizer cannot emit more tokens than the UTF-8 bytes representing the
+ * message content; the fixed allowance covers chat framing and role markers.
+ */
+declare function maxCompletionTokenUsage(messages: readonly ChatMessage[], options: ModelCompletionOptions): number;
+
+export { type ChatMessage as C, EvaluatorKind as E, GenerationSignal as G, type ModelClient as M, StaticModelClient as S, TokenUsage as T, EvalScore as a, EvalRun as b, EvalRunSpec as c, EvalRunStatus as d, EvalRunTarget as e, GenerationSignalItem as f, type ModelCompletionOptions as g, type ModelResponse as h, maxCompletionTokenUsage as m };

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@fabricorg/experiments-evals",
+  "version": "0.1.0",
+  "description": "LLM and code evaluators for Fabric Experiments: judge library, ModelClient contract, run aggregation.",
+  "license": "MIT",
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.cjs",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
+      "require": { "types": "./dist/index.d.cts", "default": "./dist/index.cjs" }
+    },
+    "./databricks": {
+      "import": { "types": "./dist/databricks.d.ts", "default": "./dist/databricks.js" },
+      "require": { "types": "./dist/databricks.d.cts", "default": "./dist/databricks.cjs" }
+    }
+  },
+  "files": ["dist"],
+  "scripts": {
+    "build": "tsup",
+    "type-check": "tsc --noEmit",
+    "test": "vitest run",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@fabricorg/experiments-datasets": "^0.1.0",
+    "@fabricorg/experiments-stats": "^0.0.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "tsup": "^8.3.5",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.7"
+  }
+}