@atlanai/sdk 0.1.1 → 0.2.2

package/dist/evals.js

@@ -0,0 +1,916 @@
+"use strict";
+/** Evaluation lifecycle helpers and the declarative `Eval()` runner. */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalRun = exports.CONTEXT_MANIFEST_SCHEMA = void 0;
+exports.createContextManifest = createContextManifest;
+exports.resolveDataset = resolveDataset;
+exports.startExperiment = startExperiment;
+exports.Eval = Eval;
+const client_1 = require("./client");
+const errors_1 = require("./errors");
+exports.CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
+const SHA256 = /^sha256:[0-9a-f]{64}$/;
+const TRACE_ID = /^[0-9a-f]{32}$/;
+const CONTEXT_KIND = /^[a-z][a-z0-9_.-]{0,63}$/;
+const MAX_LIST_OFFSET = 10_000;
+function asResources(client) {
+    return client;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function field(value, name) {
+    return isRecord(value) ? value[name] : undefined;
+}
+function canonicalJson(value) {
+    const normalize = (item) => {
+        if (item === null || typeof item === "string" || typeof item === "boolean")
+            return item;
+        if (typeof item === "number") {
+            if (!Number.isFinite(item))
+                throw new Error("context manifest is not canonical JSON");
+            return item;
+        }
+        if (Array.isArray(item))
+            return item.map(normalize);
+        if (isRecord(item)) {
+            return Object.fromEntries(Object.keys(item)
+                .sort()
+                .map((key) => [key, normalize(item[key])]));
+        }
+        throw new Error("context manifest is not canonical JSON");
+    };
+    return JSON.stringify(normalize(value));
+}
+function validText(value, name, maximum) {
+    if (typeof value !== "string" ||
+        value.length === 0 ||
+        value.length > maximum ||
+        /[\r\n]/.test(value)) {
+        throw new Error(`${name} must be 1-${maximum} characters with no newlines`);
+    }
+}
+function manifestItem(item) {
+    if (!CONTEXT_KIND.test(item.kind)) {
+        throw new Error("context item kind must be a lowercase identifier");
+    }
+    validText(item.name, "context item name", 500);
+    if (!SHA256.test(item.digest)) {
+        throw new Error("context item digest must be sha256:<64 lowercase hex>");
+    }
+    validText(item.version, "context item version", 200);
+    if (item.version.toLowerCase() === "latest") {
+        throw new Error("context item version must be pinned; 'latest' is not a version");
+    }
+    if (item.artifactId !== undefined) {
+        validText(item.artifactId, "context item artifactId", 200);
+    }
+    if (item.versionOrdinal !== undefined &&
+        (!Number.isInteger(item.versionOrdinal) || item.versionOrdinal < 1)) {
+        throw new Error("context item versionOrdinal must be an integer of at least 1");
+    }
+    return {
+        kind: item.kind,
+        name: item.name,
+        digest: item.digest,
+        version: item.version,
+        ...(item.artifactId === undefined ? {} : { artifact_id: item.artifactId }),
+        ...(item.versionOrdinal === undefined ? {} : { version_ordinal: item.versionOrdinal }),
+    };
+}
+/** Build the same canonical, order-independent context manifest as the Python SDK. */
+async function createContextManifest(input) {
+    const items = input.map((item) => Object.freeze(manifestItem(item))).sort((left, right) => {
+        const leftJson = canonicalJson(left);
+        const rightJson = canonicalJson(right);
+        return leftJson < rightJson ? -1 : leftJson > rightJson ? 1 : 0;
+    });
+    const identities = new Set(items.map((item) => `${item.kind}\u0000${item.name}`));
+    if (identities.size !== items.length) {
+        throw new Error("context manifest has duplicate kind/name entries");
+    }
+    const body = { schema_version: exports.CONTEXT_MANIFEST_SCHEMA, items };
+    if (!globalThis.crypto?.subtle) {
+        throw new Error("Web Crypto is required to create a context manifest");
+    }
+    const hash = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(canonicalJson(body)));
+    const digest = "sha256:" + [...new Uint8Array(hash)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+    return Object.freeze({ ...body, items: Object.freeze(items), digest });
+}
+/** Resolve one dataset by artifact ID or exact name, never fuzzy matching. */
+async function resolveDataset(client, idOrExactName, workspaceId) {
+    if (!idOrExactName || idOrExactName.trim() !== idOrExactName) {
+        throw new Error("dataset must be a non-empty ID or exact name with no outer whitespace");
+    }
+    const resources = asResources(client);
+    const workspace = workspaceId ?? resources.workspace;
+    if (idOrExactName.startsWith("dataset_")) {
+        const dataset = await resources.datasets.get(idOrExactName);
+        if (workspace !== undefined && field(dataset, "workspaceId") !== workspace) {
+            throw new Error("dataset ID resolved outside the requested workspace");
+        }
+        return dataset;
+    }
+    const matches = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.datasets.list({ limit: 500, offset });
+        const rawItems = field(page, "items");
+        const items = Array.isArray(rawItems) ? rawItems : [];
+        matches.push(...items.filter((item) => field(item, "name") === idOrExactName &&
+            (workspace === undefined || field(item, "workspaceId") === workspace)));
+        if (matches.length > 1)
+            break;
+        offset += items.length;
+        const total = field(field(page, "page"), "total");
+        if (items.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET) {
+            throw new Error("dataset name resolution exceeded the list limit; pass the dataset ID");
+        }
+    }
+    if (matches.length === 0) {
+        const scope = workspace === undefined ? "" : ` in workspace '${workspace}'`;
+        throw new Error(`no dataset named '${idOrExactName}'${scope}`);
+    }
+    if (matches.length > 1) {
+        throw new Error(`multiple datasets are named '${idOrExactName}'; pass the dataset ID`);
+    }
+    return matches[0];
+}
+class EvalRun {
+    experiment;
+    dataset;
+    contextManifest;
+    constructor(experiment, dataset, contextManifest) {
+        this.experiment = experiment;
+        this.dataset = dataset;
+        this.contextManifest = contextManifest;
+    }
+    get id() {
+        const id = field(this.experiment, "id");
+        if (typeof id !== "string" || id.length === 0) {
+            throw new Error("experiment create response did not contain an ID");
+        }
+        return id;
+    }
+    get experimentId() {
+        return this.id;
+    }
+    /** Options for `propagateAttributes` from `@atlanai/sdk/tracing`. */
+    get traceOptions() {
+        return {
+            experimentId: this.id,
+            ...(this.contextManifest === undefined
+                ? {}
+                : { metadata: { context_manifest_digest: this.contextManifest.digest } }),
+        };
+    }
+}
+exports.EvalRun = EvalRun;
+/** Create a running experiment over a dataset ID or exact dataset name. */
+async function startExperiment(client, dataset, body, options = {}) {
+    const resources = asResources(client);
+    const payload = { ...body };
+    const workspaceId = payload.workspaceId ?? resources.workspace;
+    if (typeof workspaceId !== "string" || workspaceId.length === 0) {
+        throw new Error("workspaceId is required in the body or on the client");
+    }
+    if (resources.workspace !== undefined &&
+        payload.workspaceId !== undefined &&
+        payload.workspaceId !== resources.workspace) {
+        throw new Error("body workspaceId differs from the client's workspace");
+    }
+    payload.workspaceId = workspaceId;
+    const resolved = await resolveDataset(client, dataset, workspaceId);
+    const resolvedId = field(resolved, "id");
+    if (typeof resolvedId !== "string" || resolvedId.length === 0) {
+        throw new Error("dataset response did not contain an ID");
+    }
+    if (payload.datasetId !== undefined && payload.datasetId !== resolvedId) {
+        throw new Error("body datasetId differs from the resolved dataset");
+    }
+    payload.datasetId = resolvedId;
+    if (payload.experimentStatus !== undefined && payload.experimentStatus !== "running") {
+        throw new Error("startExperiment only creates running experiments");
+    }
+    payload.experimentStatus = "running";
+    const rawConfig = payload.config ?? {};
+    if (!isRecord(rawConfig))
+        throw new Error("experiment config must be an object");
+    const config = { ...rawConfig };
+    const datasetPin = Object.fromEntries(Object.entries({
+        id: resolvedId,
+        version_ordinal: field(resolved, "versionOrdinal"),
+        content_hash_hex: field(resolved, "contentHashHex"),
+    }).filter(([, value]) => value !== undefined && value !== null));
+    if (config.registry_dataset !== undefined &&
+        canonicalJson(config.registry_dataset) !== canonicalJson(datasetPin)) {
+        throw new Error("config registry_dataset differs from the resolved dataset version");
+    }
+    config.registry_dataset = datasetPin;
+    const contextManifest = options.contextManifest;
+    if (contextManifest !== undefined) {
+        if (config.context_manifest !== undefined &&
+            canonicalJson(config.context_manifest) !== canonicalJson(contextManifest)) {
+            throw new Error("config context_manifest differs from the supplied manifest");
+        }
+        if (config.context_manifest_digest !== undefined &&
+            config.context_manifest_digest !== contextManifest.digest) {
+            throw new Error("config context_manifest_digest differs from the supplied manifest");
+        }
+        config.context_manifest = contextManifest;
+        config.context_manifest_digest = contextManifest.digest;
+    }
+    payload.config = config;
+    const experiment = await resources.experiments.create(payload);
+    const run = new EvalRun(experiment, resolved, contextManifest);
+    void run.id;
+    if (field(experiment, "workspaceId") !== workspaceId) {
+        throw new Error("experiment resolved outside the requested workspace");
+    }
+    const responseDatasetId = field(experiment, "datasetId");
+    if (responseDatasetId !== undefined && responseDatasetId !== resolvedId) {
+        throw new Error("experiment create response contains a different dataset ID");
+    }
+    return run;
+}
+// ---------------------------------------------------------------------------
+// Declarative evaluator
+const DEFAULT_GATEWAY_ORIGIN = "https://agentgateway.atlan.engineering";
+function tracingModule() {
+    // The root client is CommonJS while tracing has its own NodeNext build. A
+    // runtime require keeps the two compiler targets independent and resolves to
+    // dist/tracing/index.js in the published package.
+    return require("./tracing/index.js");
+}
+function environment(name) {
+    const processLike = globalThis
+        .process;
+    const value = processLike?.env?.[name];
+    return value && value.length > 0 ? value : undefined;
+}
+function nonEmpty(value, message) {
+    if (value === undefined || value.trim().length === 0)
+        throw new Error(message);
+    return value;
+}
+function slug(value, fallback) {
+    const normalized = value
+        .normalize("NFKD")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 80);
+    return normalized || fallback;
+}
+function runName(displayName) {
+    const suffix = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+    return `${slug(displayName, "eval")}-${suffix}`;
+}
+function registryObject(value) {
+    if (value === undefined)
+        return undefined;
+    if (value === null)
+        return null;
+    return isRecord(value) ? value : { value };
+}
+function safeError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return message.replace(/[\r\n]+/g, " ").slice(0, 2_000) || "evaluation failed";
+}
+async function collectData(data) {
+    const produced = typeof data === "function" ? data() : data;
+    const resolved = await produced;
+    const cases = [];
+    if (Symbol.asyncIterator in Object(resolved)) {
+        for await (const item of resolved) {
+            cases.push(item);
+        }
+    }
+    else if (Symbol.iterator in Object(resolved)) {
+        for (const item of resolved)
+            cases.push(item);
+    }
+    else {
+        throw new Error("Eval data must be an iterable or async iterable");
+    }
+    for (const item of cases) {
+        if (!isRecord(item) || !("input" in item))
+            throw new Error("every Eval case needs input");
+        if (item.metadata !== undefined && !isRecord(item.metadata)) {
+            throw new Error("Eval case metadata must be an object");
+        }
+        if (item.tags !== undefined &&
+            (!Array.isArray(item.tags) || item.tags.some((tag) => typeof tag !== "string"))) {
+            throw new Error("Eval case tags must be an array of strings");
+        }
+        if (item.id !== undefined && (typeof item.id !== "string" || item.id.length === 0)) {
+            throw new Error("Eval case id must be a non-empty string");
+        }
+    }
+    return cases;
+}
+async function datasetCases(resources, datasetId) {
+    const records = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.datasets.records.list({ datasetId, limit: 500, offset });
+        const items = field(page, "items");
+        const batch = Array.isArray(items) ? items : [];
+        for (const item of batch) {
+            const recordId = field(item, "id");
+            if (typeof recordId !== "string" || recordId.length === 0) {
+                throw new Error("dataset record response did not contain an ID");
+            }
+            const rawMetadata = field(item, "extra");
+            if (rawMetadata !== undefined && rawMetadata !== null && !isRecord(rawMetadata)) {
+                throw new Error("dataset record response contained invalid metadata");
+            }
+            const rawTags = field(item, "categories");
+            if (rawTags !== undefined &&
+                rawTags !== null &&
+                (!Array.isArray(rawTags) || rawTags.some((tag) => typeof tag !== "string"))) {
+                throw new Error("dataset record response contained invalid categories");
+            }
+            records.push({
+                id: recordId,
+                input: field(item, "input"),
+                expected: field(item, "expected"),
+                metadata: (rawMetadata ?? {}),
+                tags: (rawTags ?? []),
+            });
+        }
+        offset += batch.length;
+        const total = field(field(page, "page"), "total");
+        if (batch.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET) {
+            throw new Error("dataset record resolution exceeded 10000 rows");
+        }
+    }
+    return records;
+}
+function snapshotCases(experiment) {
+    const snapshot = field(experiment, "datasetSnapshot");
+    const raw = field(snapshot, "records");
+    if (!Array.isArray(raw))
+        return undefined;
+    return raw.map((record) => {
+        const id = field(record, "id");
+        if (typeof id !== "string" || id.length === 0) {
+            throw new Error("experiment dataset snapshot contains a record without an ID");
+        }
+        const extra = field(record, "extra");
+        const categories = field(record, "categories");
+        return {
+            id,
+            input: field(record, "input"),
+            expected: field(record, "expected"),
+            metadata: (isRecord(extra) ? extra : {}),
+            tags: (Array.isArray(categories) ? categories : []),
+        };
+    });
+}
+async function experimentResults(resources, experimentId) {
+    const results = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.experiments.results.list({
+            experimentId,
+            limit: 500,
+            offset,
+        });
+        const raw = field(page, "items");
+        const items = Array.isArray(raw) ? raw : [];
+        results.push(...items);
+        offset += items.length;
+        const total = field(field(page, "page"), "total");
+        if (items.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET) {
+            throw new Error("experiment result resolution exceeded 10000 rows");
+        }
+    }
+    return results;
+}
+async function existingCaseIds(resources, experimentId) {
+    return new Set((await experimentResults(resources, experimentId))
+        .map((item) => field(item, "caseId"))
+        .filter((caseId) => typeof caseId === "string" && caseId.length > 0));
+}
+async function findScorer(resources, name, workspaceId) {
+    const matches = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.scorers.list({ limit: 500, offset });
+        const items = field(page, "items");
+        const batch = Array.isArray(items) ? items : [];
+        matches.push(...batch.filter((item) => field(item, "name") === name && field(item, "workspaceId") === workspaceId));
+        if (matches.length > 1)
+            break;
+        offset += batch.length;
+        const total = field(field(page, "page"), "total");
+        if (batch.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET)
+            throw new Error("scorer resolution exceeded 10000 rows");
+    }
+    if (matches.length > 1)
+        throw new Error(`multiple scorers are named '${name}'`);
+    return matches[0];
+}
+function scorerIdentity(value, workspaceId) {
+    const id = field(value, "id");
+    const version = field(value, "versionOrdinal");
+    if (field(value, "workspaceId") !== workspaceId) {
+        throw new Error("scorer resolved outside the evaluation workspace");
+    }
+    if (typeof id !== "string" || !id.startsWith("scorer_")) {
+        throw new Error("scorer response did not contain a valid ID");
+    }
+    if (typeof version !== "number" || !Number.isInteger(version) || version < 1) {
+        throw new Error("scorer response did not contain a valid version");
+    }
+    return { id, version };
+}
+async function resolveScorer(resources, workspaceId, evalName, definition) {
+    if (!definition.name || /[\r\n]/.test(definition.name)) {
+        throw new Error("each scorer needs a non-empty single-line name");
+    }
+    if (definition.scorerId !== undefined) {
+        if (!definition.scorerId.startsWith("scorer_")) {
+            throw new Error(`scorer '${definition.name}' has an invalid scorerId`);
+        }
+        const scorer = await resources.scorers.get(definition.scorerId);
+        const identity = scorerIdentity(scorer, workspaceId);
+        if (definition.scorerVersion !== undefined) {
+            if (!Number.isInteger(definition.scorerVersion) || definition.scorerVersion < 1) {
+                throw new Error(`scorer '${definition.name}' has an invalid scorerVersion`);
+            }
+            return { id: identity.id, version: definition.scorerVersion };
+        }
+        return identity;
+    }
+    if (definition.scorerVersion !== undefined) {
+        throw new Error(`scorer '${definition.name}' has scorerVersion without scorerId`);
+    }
+    const artifactName = definition.registryName ?? `${slug(evalName, "eval")}-${slug(definition.name, "score")}`;
+    const existing = await findScorer(resources, artifactName, workspaceId);
+    if (existing !== undefined)
+        return scorerIdentity(existing, workspaceId);
+    try {
+        return scorerIdentity(await resources.scorers.create({
+            workspaceId,
+            name: artifactName,
+            displayName: definition.name,
+            sourceType: "local",
+            sourceId: `atlan-eval:${slug(evalName, "eval")}:${slug(definition.name, "score")}`,
+            scorerKind: definition.scorerKind ?? "code",
+            scope: definition.scope ?? "result",
+            outputs: definition.outputs ?? { [definition.name]: { type: "number", minimum: 0, maximum: 1 } },
+            spec: definition.spec ?? {
+                executor: "external",
+                sdk: "@atlanai/sdk",
+                entrypoint: definition.scorer.name || "anonymous",
+            },
+        }), workspaceId);
+    }
+    catch (error) {
+        if (!(error instanceof errors_1.AtlanAPIError) || error.status !== 409)
+            throw error;
+        const raced = await findScorer(resources, artifactName, workspaceId);
+        if (raced === undefined)
+            throw error;
+        return scorerIdentity(raced, workspaceId);
+    }
+}
+function normalizedScores(value, defaultName) {
+    if (value === null)
+        return [{ name: defaultName, score: null }];
+    if (typeof value === "number" || typeof value === "boolean") {
+        return [{ name: defaultName, score: value }];
+    }
+    const values = Array.isArray(value) ? value : [value];
+    return values.map((item) => {
+        if (!isRecord(item) || !("score" in item)) {
+            throw new Error(`scorer '${defaultName}' returned an unsupported value`);
+        }
+        const score = item.score;
+        if (score !== null &&
+            typeof score !== "boolean" &&
+            !(typeof score === "number" && Number.isFinite(score))) {
+            throw new Error(`scorer '${defaultName}' returned a non-numeric score`);
+        }
+        return {
+            name: typeof item.name === "string" && item.name ? item.name : defaultName,
+            score,
+            ...(isRecord(item.metadata) ? { metadata: item.metadata } : {}),
+        };
+    });
+}
+function numericScore(value) {
+    return typeof value === "boolean" ? (value ? 1 : 0) : value;
+}
+function localSummary(results, durationMs) {
+    const values = new Map();
+    for (const result of results) {
+        const scores = field(result, "scores");
+        if (scores !== undefined && !isRecord(scores)) {
+            throw new Error("experiment result response contained invalid scores");
+        }
+        for (const [name, score] of Object.entries(scores ?? {})) {
+            if (score === null)
+                continue;
+            if (typeof score !== "number" || !Number.isFinite(score)) {
+                throw new Error("experiment result response contained an invalid score value");
+            }
+            const observed = values.get(name) ?? [];
+            observed.push(score);
+            values.set(name, observed);
+        }
+    }
+    return {
+        scores: Object.fromEntries([...values].map(([name, scores]) => [
+            name,
+            {
+                mean: scores.length === 0 ? null : scores.reduce((sum, value) => sum + value, 0) / scores.length,
+                count: scores.length,
+            },
+        ])),
+        metrics: {
+            cases: results.length,
+            successfulCases: results.filter((result) => field(result, "error") == null).length,
+            failedCases: results.filter((result) => field(result, "error") != null).length,
+            durationMs,
+        },
+    };
+}
+async function listedTraceIds(resources, experimentId) {
+    const traceIds = new Set();
+    const seenCursors = new Set();
+    let cursor;
+    while (true) {
+        const page = await resources.experiments.traces.list(experimentId, {
+            limit: 500,
+            ...(cursor === undefined ? {} : { cursor }),
+        });
+        const items = field(page, "items");
+        if (Array.isArray(items)) {
+            for (const item of items) {
+                const traceId = field(item, "traceId");
+                if (typeof traceId === "string")
+                    traceIds.add(traceId);
+            }
+        }
+        const nextCursor = field(field(page, "page"), "nextCursor");
+        if (typeof nextCursor !== "string" || nextCursor.length === 0)
+            return traceIds;
+        if (seenCursors.has(nextCursor)) {
+            throw new Error("experiment trace pagination repeated a cursor");
+        }
+        seenCursors.add(nextCursor);
+        cursor = nextCursor;
+    }
+}
+async function verifyTraceJoins(resources, experimentId, expected, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    let missing = expected;
+    let lastError;
+    while (true) {
+        try {
+            const observed = await listedTraceIds(resources, experimentId);
+            missing = new Set([...expected].filter((traceId) => !observed.has(traceId)));
+            lastError = undefined;
+        }
+        catch (error) {
+            if (!(error instanceof errors_1.AtlanAPIError) ||
+                ![404, 429, 500, 502, 503, 504].includes(error.status)) {
+                throw error;
+            }
+            missing = expected;
+            lastError = error;
+        }
+        if (missing.size === 0)
+            return;
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+            const failure = new Error(`Registry did not expose ${missing.size} of ${expected.size} case traces ` +
+                "before the verification deadline");
+            if (lastError !== undefined)
+                failure.cause = lastError;
+            throw failure;
+        }
+        await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining)));
+    }
+}
+function defaultConnections(name, options) {
+    const apiKey = options.apiKey ?? environment("ATLAN_API_KEY");
+    const workspaceId = nonEmpty(options.workspaceId ?? options.client?.workspace ?? environment("ATLAN_WORKSPACE_ID"), "Eval needs workspaceId, client.workspace, or ATLAN_WORKSPACE_ID");
+    const gatewayOrigin = options.gatewayOrigin ?? environment("ATLAN_BASE_URL") ?? DEFAULT_GATEWAY_ORIGIN;
+    const client = options.client ??
+        new client_1.AtlanClient({
+            gatewayOrigin,
+            bearerToken: nonEmpty(apiKey, "Eval needs apiKey, client, or ATLAN_API_KEY"),
+            workspace: workspaceId,
+        });
+    const tracing = tracingModule();
+    const traceLogger = options.logger ??
+        tracing.initLogger({
+            projectName: name,
+            apiKey: nonEmpty(apiKey, "Eval needs apiKey, logger, or ATLAN_API_KEY for traces"),
+            baseUrl: gatewayOrigin,
+            workspaceId,
+            asyncFlush: true,
+            sampleRate: 1,
+        });
+    if (!traceLogger.client.isEnabled) {
+        throw new Error("Eval tracing is disabled; every case must produce a Registry trace");
+    }
+    const sampleRate = traceLogger.client.config
+        ?.sampleRate;
+    if (sampleRate !== undefined && sampleRate !== 1) {
+        throw new Error("Eval tracing requires sampleRate=1 so no case trace is sampled out");
+    }
+    return { client, logger: traceLogger, workspaceId, tracing };
+}
+/**
+ * Run a complete Registry evaluation from a compact Braintrust-shaped definition.
+ *
+ * Every case is a root OTel trace. Task/provider spans and scorer spans nest
+ * beneath it, verdicts cite immutable Registry scorer versions, result rows
+ * carry the root trace ID, traces flush before summary, and the experiment is
+ * explicitly finalized.
+ */
+async function Eval(name, evaluator, options = {}) {
+    if (!name || /[\r\n]/.test(name))
+        throw new Error("Eval name must be non-empty and single-line");
+    if ((evaluator.data === undefined) === (evaluator.dataset === undefined)) {
+        throw new Error("Eval needs exactly one of data or dataset");
+    }
+    if (evaluator.scores.length === 0)
+        throw new Error("Eval needs at least one scorer");
+    if (options.verifyTraces !== undefined && typeof options.verifyTraces !== "boolean") {
+        throw new Error("verifyTraces must be a boolean");
+    }
+    const traceVerificationTimeoutMs = options.traceVerificationTimeoutMs ?? 30_000;
+    if (!Number.isFinite(traceVerificationTimeoutMs) || traceVerificationTimeoutMs < 0) {
+        throw new Error("traceVerificationTimeoutMs must be a non-negative number");
+    }
+    const { client, logger: traceLogger, workspaceId, tracing } = defaultConnections(name, options);
+    const resources = asResources(client);
+    const displayName = evaluator.experimentName ?? name;
+    const experimentBody = {
+        workspaceId,
+        name: runName(displayName),
+        displayName,
+        description: evaluator.description,
+        projectId: options.projectId,
+        subjectKind: options.subjectKind,
+        subjectId: options.subjectId,
+        baselineExperimentId: options.baselineExperimentId,
+        experimentStatus: "running",
+        config: {
+            runner: "@atlanai/sdk.Eval",
+            ...(options.config ?? {}),
+        },
+    };
+    for (const [key, value] of Object.entries(experimentBody)) {
+        if (value === undefined)
+            delete experimentBody[key];
+    }
+    let run;
+    let cases = [];
+    if (options.resumeExperimentId !== undefined) {
+        const experiment = await resources.experiments.get(options.resumeExperimentId);
+        if (field(experiment, "workspaceId") !== workspaceId) {
+            throw new Error("resume experiment resolved outside the requested workspace");
+        }
+        if (field(experiment, "experimentStatus") !== "running") {
+            throw new Error("resumeExperimentId must name a running experiment");
+        }
+        if (options.contextManifest !== undefined &&
+            field(field(experiment, "config"), "context_manifest_digest") !==
+                options.contextManifest.digest) {
+            throw new Error("resume experiment uses a different context manifest");
+        }
+        let resolvedDataset;
+        if (evaluator.dataset !== undefined) {
+            resolvedDataset = await resolveDataset(client, evaluator.dataset, workspaceId);
+            if (field(experiment, "datasetId") !== field(resolvedDataset, "id")) {
+                throw new Error("resume experiment uses a different dataset");
+            }
+        }
+        else if (field(experiment, "datasetId") !== undefined) {
+            throw new Error("resume experiment expects a Registry dataset");
+        }
+        else {
+            cases = await collectData(evaluator.data);
+        }
+        run = new EvalRun(experiment, resolvedDataset, options.contextManifest);
+    }
+    else if (evaluator.dataset !== undefined) {
+        run = await startExperiment(client, evaluator.dataset, experimentBody, {
+            contextManifest: options.contextManifest,
+        });
+    }
+    else {
+        cases = await collectData(evaluator.data);
+        const config = { ...experimentBody.config };
+        if (options.contextManifest !== undefined) {
+            config.context_manifest = options.contextManifest;
+            config.context_manifest_digest = options.contextManifest.digest;
+        }
+        const experiment = await resources.experiments.create({ ...experimentBody, config });
+        run = new EvalRun(experiment, undefined, options.contextManifest);
+        void run.id;
+        if (field(experiment, "workspaceId") !== workspaceId) {
+            throw new Error("experiment resolved outside the requested workspace");
+        }
+    }
+    const scorerCache = new Map();
+    const started = Date.now();
+    const results = [];
+    try {
+        options.onStart?.({ experimentId: run.id, experiment: run.experiment });
+        if (evaluator.dataset !== undefined) {
+            cases =
+                snapshotCases(run.experiment) ??
+                    (await datasetCases(resources, String(field(run.dataset, "id"))));
+        }
+        const completedCaseIds = options.resumeExperimentId === undefined
+            ? new Set()
+            : await existingCaseIds(resources, run.id);
+        for (const definition of evaluator.scores) {
+            scorerCache.set(definition.name, await resolveScorer(resources, workspaceId, name, definition));
+        }
+        for (const [index, item] of cases.entries()) {
+            const caseId = item.id ?? `case-${index + 1}`;
+            if (completedCaseIds.has(caseId))
+                continue;
+            const caseStarted = Date.now();
+            const scores = {};
+            const failures = [];
+            let output;
+            let traceId = "";
+            const metadata = (item.metadata ?? {});
+            await tracing.propagateAttributes(run.traceOptions, () => traceLogger.client.startAsCurrentSpan(`eval.case.${index + 1}`, {
+                asType: "task",
+                fields: {
+                    input: item.input,
+                    ...(item.expected === undefined ? {} : { metadata: { expected: item.expected } }),
+                    ...(item.tags === undefined ? {} : { tags: item.tags }),
+                },
+            }, async (rootSpan) => {
+                traceId = rootSpan.traceId;
+                try {
+                    output = await traceLogger.client.startAsCurrentSpan("eval.task", { asType: "task", fields: { input: item.input } }, async (taskSpan) => {
+                        const value = await evaluator.task(item.input, {
+                            expected: item.expected,
+                            metadata,
+                            span: taskSpan,
+                            tags: item.tags,
+                            trialIndex: 0,
+                        });
+                        taskSpan.update({ output: value });
+                        return value;
+                    });
+                }
+                catch (error) {
+                    failures.push(safeError(error));
+                    rootSpan.recordFailure(error);
+                    return;
+                }
+                for (const definition of evaluator.scores) {
+                    try {
+                        const returned = await traceLogger.client.startAsCurrentSpan(`scorer.${definition.name}`, {
+                            asType: "function",
+                            fields: {
+                                input: {
+                                    input: item.input,
+                                    output,
+                                    expected: item.expected,
+                                },
+                            },
+                        }, async (scoreSpan) => {
+                            const value = await definition.scorer({ ...item, output: output });
+                            scoreSpan.update({ output: value });
+                            return value;
+                        });
+                        for (const scored of normalizedScores(returned, definition.name)) {
+                            const scoreName = scored.name ?? definition.name;
+                            if (scored.score === null) {
+                                scores[scoreName] = null;
+                                continue;
+                            }
+                            let registered = scorerCache.get(scoreName);
+                            if (registered === undefined) {
+                                registered = await resolveScorer(resources, workspaceId, name, {
+                                    ...definition,
+                                    name: scoreName,
+                                });
+                                scorerCache.set(scoreName, registered);
+                            }
+                            const numeric = numericScore(scored.score);
+                            scores[scoreName] = numeric;
+                            rootSpan.score(scoreName, numeric, {
+                                scorerId: registered.id,
+                                scorerVersion: registered.version,
+                                ...(scored.metadata === undefined
+                                    ? {}
+                                    : { comment: JSON.stringify(scored.metadata).slice(0, 2_000) }),
+                            });
+                        }
+                    }
+                    catch (error) {
+                        failures.push(`${definition.name}: ${safeError(error)}`);
+                        rootSpan.recordFailure(error);
+                    }
+                }
+                rootSpan.update({ output, metadata });
+            }));
+            if (!TRACE_ID.test(traceId) || /^0+$/.test(traceId)) {
+                throw new Error(`case ${index + 1} did not produce a trace ID`);
+            }
+            results.push({
+                ...item,
+                caseId,
+                ...(output === undefined ? {} : { output }),
+                ...(failures.length === 0 ? {} : { error: failures.join("; ").slice(0, 2_000) }),
+                scores,
+                traceId,
+                durationMs: Date.now() - caseStarted,
+            });
+        }
+        // Score aggregation reads spans, so make the async exporter observable first.
+        await traceLogger.flush();
+        if (options.verifyTraces !== false && results.length > 0) {
+            await verifyTraceJoins(resources, run.id, new Set(results.map((result) => result.traceId)), traceVerificationTimeoutMs);
+        }
+        const resultBodies = results.map((result, index) => ({
+            workspaceId,
+            name: `${slug(run.id, "experiment")}-case-${index + 1}`,
+            caseId: result.caseId,
+            datasetRecordId: result.id || undefined,
+            input: registryObject(result.input),
+            expected: registryObject(result.expected),
+            output: registryObject(result.output),
+            error: result.error,
+            scores: result.scores,
+            traceId: result.traceId,
+            durationMs: result.durationMs,
+            sourceType: "local",
+        }));
+        for (let offset = 0; offset < resultBodies.length; offset += 100) {
+            const response = await resources.experiments.results.createBulk(run.id, {
+                items: resultBodies.slice(offset, offset + 100),
+            });
+            const slots = field(response, "items");
+            if (Array.isArray(slots)) {
+                const failed = slots.filter((slot) => {
+                    const status = field(slot, "statusCode");
+                    return typeof status === "number" && status >= 400;
+                });
+                if (failed.length > 0)
+                    throw new Error(`${failed.length} Eval results were rejected`);
+            }
+        }
+        const persistedResults = await experimentResults(resources, run.id);
+        const local = localSummary(persistedResults, Date.now() - started);
+        const experiment = await resources.experiments.update(run.id, {
+            experimentStatus: "completed",
+            summary: local,
+        });
+        const remote = field(experiment, "summary");
+        const summary = isRecord(remote)
+            ? remote
+            : local;
+        return {
+            experimentId: run.id,
+            experiment,
+            dataset: run.dataset,
+            summary,
+            results: options.returnResults === false ? [] : results,
+        };
+    }
+    catch (error) {
+        // A custom exporter's failure must not replace the task, scorer, or
+        // Registry error that caused the evaluation to fail.
+        try {
+            await traceLogger.flush();
+        }
+        catch {
+            // Preserve the original runner/upload failure.
+        }
+        try {
+            await resources.experiments.update(run.id, {
+                experimentStatus: "failed",
+                summary: localSummary(results, Date.now() - started),
+            });
+        }
+        catch {
+            // Preserve the original runner/upload failure.
+        }
+        throw error;
+    }
+}