@atlanai/sdk 0.2.1 → 0.2.3

package/dist/evals.js

@@ -1,12 +1,16 @@
 "use strict";
-/** Small lifecycle helpers for evaluation runners. */
+/** Evaluation lifecycle helpers and the declarative `Eval()` runner. */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EvalRun = exports.CONTEXT_MANIFEST_SCHEMA = void 0;
 exports.createContextManifest = createContextManifest;
 exports.resolveDataset = resolveDataset;
 exports.startExperiment = startExperiment;
+exports.Eval = Eval;
+const client_1 = require("./client");
+const errors_1 = require("./errors");
 exports.CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
 const SHA256 = /^sha256:[0-9a-f]{64}$/;
+const TRACE_ID = /^[0-9a-f]{32}$/;
 const CONTEXT_KIND = /^[a-z][a-z0-9_.-]{0,63}$/;
 const MAX_LIST_OFFSET = 10_000;
 function asResources(client) {
@@ -58,6 +62,9 @@ function manifestItem(item) {
     if (item.version.toLowerCase() === "latest") {
         throw new Error("context item version must be pinned; 'latest' is not a version");
     }
+    if (item.artifactKind !== undefined && !CONTEXT_KIND.test(item.artifactKind)) {
+        throw new Error("context item artifactKind must be a lowercase identifier");
+    }
     if (item.artifactId !== undefined) {
         validText(item.artifactId, "context item artifactId", 200);
     }
@@ -65,11 +72,18 @@ function manifestItem(item) {
         (!Number.isInteger(item.versionOrdinal) || item.versionOrdinal < 1)) {
         throw new Error("context item versionOrdinal must be an integer of at least 1");
     }
+    if ((item.artifactId === undefined) !== (item.versionOrdinal === undefined)) {
+        throw new Error("context item Registry pins require artifactId and versionOrdinal");
+    }
+    if (item.artifactKind !== undefined && item.artifactId === undefined) {
+        throw new Error("context item artifactKind requires a Registry artifact pin");
+    }
     return {
         kind: item.kind,
         name: item.name,
         digest: item.digest,
         version: item.version,
+        ...(item.artifactKind === undefined ? {} : { artifact_kind: item.artifactKind }),
         ...(item.artifactId === undefined ? {} : { artifact_id: item.artifactId }),
         ...(item.versionOrdinal === undefined ? {} : { version_ordinal: item.versionOrdinal }),
     };
@@ -222,9 +236,896 @@ async function startExperiment(client, dataset, body, options = {}) {
     const experiment = await resources.experiments.create(payload);
     const run = new EvalRun(experiment, resolved, contextManifest);
     void run.id;
+    if (field(experiment, "workspaceId") !== workspaceId) {
+        throw new Error("experiment resolved outside the requested workspace");
+    }
     const responseDatasetId = field(experiment, "datasetId");
     if (responseDatasetId !== undefined && responseDatasetId !== resolvedId) {
         throw new Error("experiment create response contains a different dataset ID");
     }
     return run;
 }
+// ---------------------------------------------------------------------------
+// Declarative evaluator
+const DEFAULT_GATEWAY_ORIGIN = "https://api.atlan.com";
+function tracingModule() {
+    // The root client is CommonJS while tracing has its own NodeNext build. A
+    // runtime require keeps the two compiler targets independent and resolves to
+    // dist/tracing/index.js in the published package.
+    return require("./tracing/index.js");
+}
+function environment(name) {
+    const processLike = globalThis
+        .process;
+    const value = processLike?.env?.[name];
+    return value && value.length > 0 ? value : undefined;
+}
+function nonEmpty(value, message) {
+    if (value === undefined || value.trim().length === 0)
+        throw new Error(message);
+    return value;
+}
+function slug(value, fallback) {
+    const normalized = value
+        .normalize("NFKD")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 80);
+    return normalized || fallback;
+}
+function runName(displayName) {
+    const suffix = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+    return `${slug(displayName, "eval")}-${suffix}`;
+}
+function registryObject(value) {
+    if (value === undefined)
+        return undefined;
+    if (value === null)
+        return null;
+    return isRecord(value) ? value : { value };
+}
+function safeError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return message.replace(/[\r\n]+/g, " ").slice(0, 2_000) || "evaluation failed";
+}
+async function collectData(data) {
+    const produced = typeof data === "function" ? data() : data;
+    const resolved = await produced;
+    const cases = [];
+    if (Symbol.asyncIterator in Object(resolved)) {
+        for await (const item of resolved) {
+            cases.push(item);
+        }
+    }
+    else if (Symbol.iterator in Object(resolved)) {
+        for (const item of resolved)
+            cases.push(item);
+    }
+    else {
+        throw new Error("Eval data must be an iterable or async iterable");
+    }
+    for (const item of cases) {
+        if (!isRecord(item) || !("input" in item))
+            throw new Error("every Eval case needs input");
+        if (item.metadata !== undefined && !isRecord(item.metadata)) {
+            throw new Error("Eval case metadata must be an object");
+        }
+        if (item.tags !== undefined &&
+            (!Array.isArray(item.tags) || item.tags.some((tag) => typeof tag !== "string"))) {
+            throw new Error("Eval case tags must be an array of strings");
+        }
+        if (item.id !== undefined && (typeof item.id !== "string" || item.id.length === 0)) {
+            throw new Error("Eval case id must be a non-empty string");
+        }
+    }
+    return cases;
+}
+async function casesDigest(cases) {
+    const normalized = cases.map((item, index) => ({
+        id: item.id ?? `case-${index + 1}`,
+        input: item.input,
+        expected: item.expected ?? null,
+        metadata: item.metadata ?? {},
+        tags: item.tags ?? [],
+    }));
+    if (new Set(normalized.map((item) => item.id)).size !== cases.length) {
+        throw new Error("Eval case IDs must be unique");
+    }
+    const hash = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(canonicalJson(normalized)));
+    return "sha256:" + [...new Uint8Array(hash)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+async function datasetCases(resources, datasetId) {
+    const records = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.datasets.records.list({ datasetId, limit: 500, offset });
+        const items = field(page, "items");
+        const batch = Array.isArray(items) ? items : [];
+        for (const item of batch) {
+            const recordId = field(item, "id");
+            if (typeof recordId !== "string" || recordId.length === 0) {
+                throw new Error("dataset record response did not contain an ID");
+            }
+            const rawMetadata = field(item, "extra");
+            if (rawMetadata !== undefined && rawMetadata !== null && !isRecord(rawMetadata)) {
+                throw new Error("dataset record response contained invalid metadata");
+            }
+            const rawTags = field(item, "categories");
+            if (rawTags !== undefined &&
+                rawTags !== null &&
+                (!Array.isArray(rawTags) || rawTags.some((tag) => typeof tag !== "string"))) {
+                throw new Error("dataset record response contained invalid categories");
+            }
+            records.push({
+                id: recordId,
+                input: field(item, "input"),
+                expected: field(item, "expected"),
+                metadata: (rawMetadata ?? {}),
+                tags: (rawTags ?? []),
+            });
+        }
+        offset += batch.length;
+        const total = field(field(page, "page"), "total");
+        if (batch.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET) {
+            throw new Error("dataset record resolution exceeded 10000 rows");
+        }
+    }
+    return records;
+}
+function snapshotCases(experiment) {
+    const snapshot = field(experiment, "datasetSnapshot");
+    const raw = field(snapshot, "records");
+    if (!Array.isArray(raw))
+        return undefined;
+    return raw.map((record) => {
+        const id = field(record, "id");
+        if (typeof id !== "string" || id.length === 0) {
+            throw new Error("experiment dataset snapshot contains a record without an ID");
+        }
+        const extra = field(record, "extra");
+        const categories = field(record, "categories");
+        return {
+            id,
+            input: field(record, "input"),
+            expected: field(record, "expected"),
+            metadata: (isRecord(extra) ? extra : {}),
+            tags: (Array.isArray(categories) ? categories : []),
+        };
+    });
+}
+async function experimentResults(resources, experimentId) {
+    const results = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.experiments.results.list({
+            experimentId,
+            limit: 500,
+            offset,
+        });
+        const raw = field(page, "items");
+        const items = Array.isArray(raw) ? raw : [];
+        results.push(...items);
+        offset += items.length;
+        const total = field(field(page, "page"), "total");
+        if (items.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET) {
+            throw new Error("experiment result resolution exceeded 10000 rows");
+        }
+    }
+    return results;
+}
+async function findScorer(resources, name, workspaceId) {
+    const matches = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.scorers.list({ limit: 500, offset });
+        const items = field(page, "items");
+        const batch = Array.isArray(items) ? items : [];
+        matches.push(...batch.filter((item) => field(item, "name") === name && field(item, "workspaceId") === workspaceId));
+        if (matches.length > 1)
+            break;
+        offset += batch.length;
+        const total = field(field(page, "page"), "total");
+        if (batch.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET)
+            throw new Error("scorer resolution exceeded 10000 rows");
+    }
+    if (matches.length > 1)
+        throw new Error(`multiple scorers are named '${name}'`);
+    return matches[0];
+}
+function scorerIdentity(value, workspaceId) {
+    const id = field(value, "id");
+    const version = field(value, "versionOrdinal");
+    if (field(value, "workspaceId") !== workspaceId) {
+        throw new Error("scorer resolved outside the evaluation workspace");
+    }
+    if (typeof id !== "string" || !id.startsWith("scorer_")) {
+        throw new Error("scorer response did not contain a valid ID");
+    }
+    if (typeof version !== "number" || !Number.isInteger(version) || version < 1) {
+        throw new Error("scorer response did not contain a valid version");
+    }
+    return { id, version };
+}
+async function scorerDefinition(definition) {
+    const outputs = definition.outputs ?? { [definition.name]: { type: "number", minimum: 0, maximum: 1 } };
+    if (definition.spec !== undefined)
+        return { spec: definition.spec, outputs };
+    if (!globalThis.crypto?.subtle)
+        throw new Error("Web Crypto is required to identify a scorer");
+    const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(String(definition.scorer)));
+    const codeDigest = "sha256:" +
+        [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+    return {
+        outputs,
+        spec: {
+            executor: "external",
+            sdk: "@atlanai/sdk",
+            entrypoint: definition.scorer.name || "anonymous",
+            code_digest: codeDigest,
+        },
+    };
+}
+async function reconcileScorer(resources, existing, workspaceId, definition) {
+    const identity = scorerIdentity(existing, workspaceId);
+    const scorerKind = definition.scorerKind ?? "code";
+    const scope = definition.scope ?? "result";
+    if (field(existing, "scorerKind") !== scorerKind || field(existing, "scope") !== scope) {
+        throw new Error(`scorer '${definition.name}' reuses a Registry name with a different identity`);
+    }
+    const desired = await scorerDefinition(definition);
+    if (canonicalJson(field(existing, "spec")) === canonicalJson(desired.spec) &&
+        canonicalJson(field(existing, "outputs")) === canonicalJson(desired.outputs)) {
+        return identity;
+    }
+    return scorerIdentity(await resources.scorers.update(identity.id, desired), workspaceId);
+}
+async function resolveScorer(resources, workspaceId, evalName, definition) {
+    if (!definition.name || /[\r\n]/.test(definition.name)) {
+        throw new Error("each scorer needs a non-empty single-line name");
+    }
+    if (definition.scorerId !== undefined) {
+        if (!definition.scorerId.startsWith("scorer_")) {
+            throw new Error(`scorer '${definition.name}' has an invalid scorerId`);
+        }
+        const scorer = await resources.scorers.get(definition.scorerId);
+        const identity = scorerIdentity(scorer, workspaceId);
+        if (definition.scorerVersion !== undefined) {
+            if (!Number.isInteger(definition.scorerVersion) || definition.scorerVersion < 1) {
+                throw new Error(`scorer '${definition.name}' has an invalid scorerVersion`);
+            }
+            return scorerIdentity(await resources.scorers.versions.get(identity.id, definition.scorerVersion), workspaceId);
+        }
+        return identity;
+    }
+    if (definition.scorerVersion !== undefined) {
+        throw new Error(`scorer '${definition.name}' has scorerVersion without scorerId`);
+    }
+    const artifactName = definition.registryName ?? `${slug(evalName, "eval")}-${slug(definition.name, "score")}`;
+    const existing = await findScorer(resources, artifactName, workspaceId);
+    if (existing !== undefined) {
+        return reconcileScorer(resources, existing, workspaceId, definition);
+    }
+    const desired = await scorerDefinition(definition);
+    try {
+        return scorerIdentity(await resources.scorers.create({
+            workspaceId,
+            name: artifactName,
+            displayName: definition.name,
+            sourceType: "local",
+            sourceId: `atlan-eval:${slug(evalName, "eval")}:${slug(definition.name, "score")}`,
+            scorerKind: definition.scorerKind ?? "code",
+            scope: definition.scope ?? "result",
+            outputs: desired.outputs,
+            spec: desired.spec,
+        }), workspaceId);
+    }
+    catch (error) {
+        if (!(error instanceof errors_1.AtlanAPIError) || error.status !== 409)
+            throw error;
+        const raced = await findScorer(resources, artifactName, workspaceId);
+        if (raced === undefined)
+            throw error;
+        return reconcileScorer(resources, raced, workspaceId, definition);
+    }
+}
+function normalizedScores(value, defaultName) {
+    if (value === null)
+        return [{ name: defaultName, score: null }];
+    if (typeof value === "number" || typeof value === "boolean") {
+        return [{ name: defaultName, score: value }];
+    }
+    const values = Array.isArray(value) ? value : [value];
+    return values.map((item) => {
+        if (!isRecord(item) || !("score" in item)) {
+            throw new Error(`scorer '${defaultName}' returned an unsupported value`);
+        }
+        const score = item.score;
+        if (score !== null &&
+            typeof score !== "boolean" &&
+            !(typeof score === "number" && Number.isFinite(score))) {
+            throw new Error(`scorer '${defaultName}' returned a non-numeric score`);
+        }
+        return {
+            name: typeof item.name === "string" && item.name ? item.name : defaultName,
+            score,
+            ...(isRecord(item.metadata) ? { metadata: item.metadata } : {}),
+        };
+    });
+}
+function numericScore(value) {
+    return typeof value === "boolean" ? (value ? 1 : 0) : value;
+}
+function localSummary(results, durationMs) {
+    const values = new Map();
+    for (const result of results) {
+        const scores = field(result, "scores");
+        if (scores !== undefined && !isRecord(scores)) {
+            throw new Error("experiment result response contained invalid scores");
+        }
+        for (const [name, score] of Object.entries(scores ?? {})) {
+            if (score === null)
+                continue;
+            if (typeof score !== "number" || !Number.isFinite(score)) {
+                throw new Error("experiment result response contained an invalid score value");
+            }
+            const observed = values.get(name) ?? [];
+            observed.push(score);
+            values.set(name, observed);
+        }
+    }
+    return {
+        scores: Object.fromEntries([...values].map(([name, scores]) => [
+            name,
+            {
+                mean: scores.length === 0 ? null : scores.reduce((sum, value) => sum + value, 0) / scores.length,
+                count: scores.length,
+            },
+        ])),
+        metrics: {
+            cases: results.length,
+            successfulCases: results.filter((result) => field(result, "error") == null).length,
+            failedCases: results.filter((result) => field(result, "error") != null).length,
+            durationMs,
+        },
+    };
+}
+async function traceSpans(resources, experimentId, traceId) {
+    const spans = [];
+    const seenCursors = new Set();
+    let cursor;
+    while (true) {
+        const page = await resources.experiments.traces.listSpans({
+            experimentId,
+            traceId,
+            limit: 500,
+            fields: "io",
+            ...(cursor === undefined ? {} : { cursor }),
+        });
+        const items = field(page, "items");
+        if (Array.isArray(items))
+            spans.push(...items);
+        const nextCursor = field(field(page, "page"), "nextCursor");
+        if (typeof nextCursor !== "string" || nextCursor.length === 0)
+            return spans;
+        if (seenCursors.has(nextCursor)) {
+            throw new Error("experiment trace span pagination repeated a cursor");
+        }
+        seenCursors.add(nextCursor);
+        cursor = nextCursor;
+    }
+}
+async function listedTraceIds(resources, experimentId) {
+    const traceIds = new Set();
+    const seenCursors = new Set();
+    let cursor;
+    const windowEnd = Math.floor(Date.now() / 1_000);
+    while (true) {
+        const page = await resources.experiments.traces.list({
+            experimentId,
+            startTimeUnixSeconds: windowEnd - 30 * 24 * 60 * 60,
+            endTimeUnixSeconds: windowEnd,
+            limit: 500,
+            ...(cursor === undefined ? {} : { cursor }),
+        });
+        const items = field(page, "items");
+        if (Array.isArray(items)) {
+            for (const item of items) {
+                const traceId = field(item, "traceId");
+                if (typeof traceId === "string")
+                    traceIds.add(traceId);
+            }
+        }
+        const nextCursor = field(field(page, "page"), "nextCursor");
+        if (typeof nextCursor !== "string" || nextCursor.length === 0)
+            return traceIds;
+        if (seenCursors.has(nextCursor)) {
+            throw new Error("experiment trace pagination repeated a cursor");
+        }
+        seenCursors.add(nextCursor);
+        cursor = nextCursor;
+    }
+}
+function spanAttributes(span) {
+    const attributes = field(span, "attributes");
+    return isRecord(attributes) ? attributes : {};
+}
+function missingTraceEvidence(spans, expected) {
+    const missing = [];
+    const byId = new Map();
+    for (const span of spans) {
+        const spanId = field(span, "spanId");
+        if (typeof spanId === "string")
+            byId.set(spanId, span);
+    }
+    const root = expected.rootSpanId === undefined
+        ? spans.find((span) => spanAttributes(span)["atlan.eval.case_id"] === expected.caseId &&
+            spanAttributes(span)["atlan.eval.evidence_complete"] === true)
+        : byId.get(expected.rootSpanId);
+    if (root === undefined) {
+        missing.push("case root");
+    }
+    else {
+        const attributes = spanAttributes(root);
+        if (attributes["atlan.eval.evidence_complete"] !== true) {
+            missing.push("producer completion marker");
+        }
+        if (attributes["atlan.eval.case_id"] !== expected.caseId)
+            missing.push("case identity");
+        if (expected.datasetRecordId !== undefined &&
+            attributes["atlan.eval.dataset_record_id"] !== expected.datasetRecordId) {
+            missing.push("dataset record identity");
+        }
+    }
+    const absentIds = [...expected.requiredSpanIds].filter((spanId) => !byId.has(spanId));
+    if (absentIds.length > 0)
+        missing.push(`${absentIds.length} SDK-owned spans`);
+    if (spans.some((span) => spanAttributes(span)["atlan.eval.evidence_truncated"] === true)) {
+        missing.push("untruncated payloads");
+    }
+    for (const [scoreName, scorer] of expected.scores) {
+        const present = spans.some((span) => field(span, "scoreName") === scoreName &&
+            field(span, "scorerId") === scorer.id &&
+            field(span, "scorerVersion") === scorer.version);
+        if (!present)
+            missing.push(`score '${scoreName}'`);
+    }
+    return missing;
+}
+async function verifyTraceEvidence(resources, experimentId, expected, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    let missing = new Map();
+    let lastError;
+    while (true) {
+        try {
+            missing = new Map();
+            if (typeof resources.experiments.traces.listSpans === "function") {
+                for (const [traceId, expectation] of expected) {
+                    const gaps = missingTraceEvidence(await traceSpans(resources, experimentId, traceId), expectation);
+                    if (gaps.length > 0)
+                        missing.set(traceId, gaps);
+                }
+            }
+            else {
+                const observed = await listedTraceIds(resources, experimentId);
+                for (const traceId of expected.keys()) {
+                    if (!observed.has(traceId))
+                        missing.set(traceId, ["trace read"]);
+                }
+            }
+            lastError = undefined;
+        }
+        catch (error) {
+            if (!(error instanceof errors_1.AtlanAPIError) ||
+                ![404, 429, 500, 502, 503, 504].includes(error.status)) {
+                throw error;
+            }
+            missing = new Map([...expected.keys()].map((traceId) => [traceId, ["trace read"]]));
+            lastError = error;
+        }
+        if (missing.size === 0)
+            return;
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+            const failure = new Error(`Registry did not expose complete trace evidence for ${missing.size} of ` +
+                `${expected.size} case traces before the verification deadline: ` +
+                [...missing].map(([traceId, gaps]) => `${traceId} (${gaps.join(", ")})`).join("; "));
+            if (lastError !== undefined)
+                failure.cause = lastError;
+            throw failure;
+        }
+        await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining)));
+    }
+}
+function defaultConnections(name, options) {
+    const apiKey = options.apiKey ?? environment("ATLAN_API_KEY");
+    const workspaceId = nonEmpty(options.workspaceId ?? options.client?.workspace ?? environment("ATLAN_WORKSPACE_ID"), "Eval needs workspaceId, client.workspace, or ATLAN_WORKSPACE_ID");
+    const gatewayOrigin = options.gatewayOrigin ?? environment("ATLAN_BASE_URL") ?? DEFAULT_GATEWAY_ORIGIN;
+    const client = options.client ??
+        new client_1.AtlanClient({
+            gatewayOrigin,
+            bearerToken: nonEmpty(apiKey, "Eval needs apiKey, client, or ATLAN_API_KEY"),
+            workspace: workspaceId,
+        });
+    const tracing = tracingModule();
+    const traceLogger = options.logger ??
+        tracing.initLogger({
+            projectName: name,
+            apiKey: nonEmpty(apiKey, "Eval needs apiKey, logger, or ATLAN_API_KEY for traces"),
+            baseUrl: gatewayOrigin,
+            workspaceId,
+            asyncFlush: true,
+            sampleRate: 1,
+        });
+    if (!traceLogger.client.isEnabled) {
+        throw new Error("Eval tracing is disabled; every case must produce a Registry trace");
+    }
+    const sampleRate = traceLogger.client.config
+        ?.sampleRate;
+    if (sampleRate !== undefined && sampleRate !== 1) {
+        throw new Error("Eval tracing requires sampleRate=1 so no case trace is sampled out");
+    }
+    const traceContent = traceLogger.client.config
+        ?.traceContent;
+    if (traceContent === false) {
+        throw new Error("Eval tracing requires traceContent=true so case evidence is readable");
+    }
+    return { client, logger: traceLogger, workspaceId, tracing };
+}
+/**
+ * Run a complete Registry evaluation from a compact Braintrust-shaped definition.
+ *
+ * Every case is a root OTel trace. Task/provider spans and scorer spans nest
+ * beneath it, verdicts cite immutable Registry scorer versions, result rows
+ * carry the root trace ID, traces flush before summary, and the experiment is
+ * explicitly finalized.
+ */
+async function Eval(name, evaluator, options = {}) {
+    if (!name || /[\r\n]/.test(name))
+        throw new Error("Eval name must be non-empty and single-line");
+    if ((evaluator.data === undefined) === (evaluator.dataset === undefined)) {
+        throw new Error("Eval needs exactly one of data or dataset");
+    }
+    if (evaluator.scores.length === 0)
+        throw new Error("Eval needs at least one scorer");
+    if (options.verifyTraces !== undefined && typeof options.verifyTraces !== "boolean") {
+        throw new Error("verifyTraces must be a boolean");
+    }
+    const traceVerificationTimeoutMs = options.traceVerificationTimeoutMs ?? 30_000;
+    if (!Number.isFinite(traceVerificationTimeoutMs) || traceVerificationTimeoutMs < 0) {
+        throw new Error("traceVerificationTimeoutMs must be a non-negative number");
+    }
+    const { client, logger: traceLogger, workspaceId, tracing } = defaultConnections(name, options);
+    const resources = asResources(client);
+    const displayName = evaluator.experimentName ?? name;
+    const experimentBody = {
+        workspaceId,
+        name: runName(displayName),
+        displayName,
+        description: evaluator.description,
+        projectId: options.projectId,
+        subjectKind: options.subjectKind,
+        subjectId: options.subjectId,
+        baselineExperimentId: options.baselineExperimentId,
+        experimentStatus: "running",
+        config: {
+            runner: "@atlanai/sdk.Eval",
+            ...(options.config ?? {}),
+        },
+    };
+    for (const [key, value] of Object.entries(experimentBody)) {
+        if (value === undefined)
+            delete experimentBody[key];
+    }
+    let run;
+    let cases = [];
+    if (options.resumeExperimentId !== undefined) {
+        const experiment = await resources.experiments.get(options.resumeExperimentId);
+        if (field(experiment, "workspaceId") !== workspaceId) {
+            throw new Error("resume experiment resolved outside the requested workspace");
+        }
+        if (field(experiment, "experimentStatus") !== "running") {
+            throw new Error("resumeExperimentId must name a running experiment");
+        }
+        const storedConfig = field(experiment, "config") ?? {};
+        if (field(storedConfig, "context_manifest_digest") !== options.contextManifest?.digest) {
+            throw new Error("resume experiment uses a different context manifest");
+        }
+        const storedUserConfig = Object.fromEntries(Object.entries(storedConfig).filter(([key]) => !["runner", "registry_dataset", "context_manifest", "context_manifest_digest", "eval_cases_digest"].includes(key)));
+        if (canonicalJson(storedUserConfig) !== canonicalJson(options.config ?? {})) {
+            throw new Error("resume experiment uses a different config");
+        }
+        let resolvedDataset;
+        if (evaluator.dataset !== undefined) {
+            resolvedDataset = await resolveDataset(client, evaluator.dataset, workspaceId);
+            if (field(experiment, "datasetId") !== field(resolvedDataset, "id")) {
+                throw new Error("resume experiment uses a different dataset");
+            }
+        }
+        else if (field(experiment, "datasetId") !== undefined) {
+            throw new Error("resume experiment expects a Registry dataset");
+        }
+        else {
+            cases = await collectData(evaluator.data);
+            if (field(storedConfig, "eval_cases_digest") !== await casesDigest(cases)) {
+                throw new Error("resume inline data differs or has no saved digest; start a new experiment");
+            }
+        }
+        run = new EvalRun(experiment, resolvedDataset, options.contextManifest);
+    }
+    else if (evaluator.dataset !== undefined) {
+        run = await startExperiment(client, evaluator.dataset, experimentBody, {
+            contextManifest: options.contextManifest,
+        });
+    }
+    else {
+        cases = await collectData(evaluator.data);
+        const config = { ...experimentBody.config };
+        config.eval_cases_digest = await casesDigest(cases);
+        if (options.contextManifest !== undefined) {
+            config.context_manifest = options.contextManifest;
+            config.context_manifest_digest = options.contextManifest.digest;
+        }
+        const experiment = await resources.experiments.create({ ...experimentBody, config });
+        run = new EvalRun(experiment, undefined, options.contextManifest);
+        void run.id;
+        if (field(experiment, "workspaceId") !== workspaceId) {
+            throw new Error("experiment resolved outside the requested workspace");
+        }
+    }
+    const scorerCache = new Map();
+    const traceEvidence = new Map();
+    const started = Date.now();
+    const results = [];
+    try {
+        options.onStart?.({ experimentId: run.id, experiment: run.experiment });
+        if (evaluator.dataset !== undefined) {
+            const snapshot = snapshotCases(run.experiment);
+            if (options.resumeExperimentId !== undefined && snapshot === undefined) {
+                throw new Error("resume requires the experiment's immutable dataset snapshot");
+            }
+            cases =
+                snapshot ??
+                    (await datasetCases(resources, String(field(run.dataset, "id"))));
+        }
+        const caseIds = new Set(cases.map((item, index) => item.id ?? `case-${index + 1}`));
+        if (caseIds.size !== cases.length)
+            throw new Error("Eval case IDs must be unique");
+        const persisted = options.resumeExperimentId === undefined ? [] : await experimentResults(resources, run.id);
+        const completedCaseIds = new Set(persisted.map((item) => field(item, "caseId")));
+        if (completedCaseIds.size !== persisted.length || [...completedCaseIds].some((id) => typeof id !== "string" || !caseIds.has(id))) {
+            throw new Error("resume experiment contains unexpected or duplicate case IDs");
+        }
+        for (const definition of evaluator.scores) {
+            scorerCache.set(definition.name, await resolveScorer(resources, workspaceId, name, definition));
+        }
+        for (const [index, item] of cases.entries()) {
+            const caseId = item.id ?? `case-${index + 1}`;
+            const datasetRecordId = evaluator.dataset === undefined ? undefined : item.id;
+            if (completedCaseIds.has(caseId))
+                continue;
+            const caseStarted = Date.now();
+            const scores = {};
+            const failures = [];
+            let output;
+            let traceId = "";
+            let rootSpanId;
+            const requiredSpanIds = new Set();
+            const scoreEvidence = new Map();
+            const metadata = (item.metadata ?? {});
+            await tracing.propagateAttributes({
+                ...run.traceOptions,
+                caseId,
+                ...(datasetRecordId === undefined ? {} : { datasetRecordId }),
+            }, () => traceLogger.client.startAsCurrentSpan(`eval.case.${index + 1}`, {
+                asType: "task",
+                fields: {
+                    input: item.input,
+                    ...(item.expected === undefined ? {} : { metadata: { expected: item.expected } }),
+                    ...(item.tags === undefined ? {} : { tags: item.tags }),
+                },
+            }, async (rootSpan) => {
+                traceId = rootSpan.traceId;
+                rootSpanId = rootSpan.spanId;
+                if (rootSpan.spanId !== undefined)
+                    requiredSpanIds.add(rootSpan.spanId);
+                try {
+                    output = await traceLogger.client.startAsCurrentSpan("eval.task", { asType: "task", fields: { input: item.input } }, async (taskSpan) => {
+                        if (taskSpan.spanId !== undefined)
+                            requiredSpanIds.add(taskSpan.spanId);
+                        const value = await evaluator.task(item.input, {
+                            expected: item.expected,
+                            metadata,
+                            span: taskSpan,
+                            tags: item.tags,
+                            trialIndex: 0,
+                        });
+                        taskSpan.update({ output: value });
+                        return value;
+                    });
+                }
+                catch (error) {
+                    failures.push(safeError(error));
+                    rootSpan.recordFailure(error);
+                    rootSpan.markEvidenceComplete?.();
+                    return;
+                }
+                for (const definition of evaluator.scores) {
+                    try {
+                        const returned = await traceLogger.client.startAsCurrentSpan(`scorer.${definition.name}`, {
+                            asType: "function",
+                            fields: {
+                                input: {
+                                    input: item.input,
+                                    output,
+                                    expected: item.expected,
+                                },
+                            },
+                        }, async (scoreSpan) => {
+                            if (scoreSpan.spanId !== undefined)
+                                requiredSpanIds.add(scoreSpan.spanId);
+                            const value = await definition.scorer({ ...item, output: output });
+                            scoreSpan.update({ output: value });
+                            return value;
+                        });
+                        for (const scored of normalizedScores(returned, definition.name)) {
+                            const scoreName = scored.name ?? definition.name;
+                            if (scored.score === null) {
+                                scores[scoreName] = null;
+                                continue;
+                            }
+                            let registered = scorerCache.get(scoreName);
+                            if (registered === undefined) {
+                                registered = await resolveScorer(resources, workspaceId, name, {
+                                    ...definition,
+                                    name: scoreName,
+                                });
+                                scorerCache.set(scoreName, registered);
+                            }
+                            const numeric = numericScore(scored.score);
+                            scores[scoreName] = numeric;
+                            scoreEvidence.set(scoreName, registered);
+                            rootSpan.score(scoreName, numeric, {
+                                scorerId: registered.id,
+                                scorerVersion: registered.version,
+                                ...(scored.metadata === undefined
+                                    ? {}
+                                    : { comment: JSON.stringify(scored.metadata).slice(0, 2_000) }),
+                            });
+                        }
+                    }
+                    catch (error) {
+                        failures.push(`${definition.name}: ${safeError(error)}`);
+                        rootSpan.recordFailure(error);
+                    }
+                }
+                rootSpan.update({ output, metadata });
+                rootSpan.markEvidenceComplete?.();
+            }));
+            if (!TRACE_ID.test(traceId) || /^0+$/.test(traceId)) {
+                throw new Error(`case ${index + 1} did not produce a trace ID`);
+            }
+            results.push({
+                ...item,
+                caseId,
+                ...(output === undefined ? {} : { output }),
+                ...(failures.length === 0 ? {} : { error: failures.join("; ").slice(0, 2_000) }),
+                scores,
+                traceId,
+                durationMs: Date.now() - caseStarted,
+            });
+            traceEvidence.set(traceId, {
+                caseId,
+                ...(datasetRecordId === undefined ? {} : { datasetRecordId }),
+                ...(rootSpanId === undefined ? {} : { rootSpanId }),
+                requiredSpanIds,
+                scores: scoreEvidence,
+            });
+            // Persist each completed case before starting another costly task.
+            // Keep the source index so a resumed subset cannot reuse earlier names.
+            const result = results[results.length - 1];
+            await traceLogger.flush();
+            if (options.verifyTraces !== false) {
+                await verifyTraceEvidence(resources, run.id, new Map([[result.traceId, traceEvidence.get(result.traceId)]]), traceVerificationTimeoutMs);
+            }
+            const resultBody = {
+                workspaceId,
+                name: `${slug(run.id, "experiment")}-case-${index + 1}`,
+                caseId: result.caseId,
+                datasetRecordId,
+                input: registryObject(result.input),
+                expected: registryObject(result.expected),
+                output: registryObject(result.output),
+                error: result.error,
+                scores: result.scores,
+                traceId: result.traceId,
+                durationMs: result.durationMs,
+                sourceType: "local",
+            };
+            const response = await resources.experiments.results.createBulk(run.id, {
+                items: [resultBody],
+            });
+            const slots = field(response, "items");
+            const status = Array.isArray(slots) && slots.length === 1 ? field(slots[0], "statusCode") : undefined;
+            if (typeof status !== "number" || status < 200 || status >= 300) {
+                throw new Error("1 Eval results were rejected or not acknowledged");
+            }
+        }
+        const persistedResults = await experimentResults(resources, run.id);
+        const persistedCaseIds = new Set(persistedResults.map((item) => field(item, "caseId")));
+        if (persistedResults.length !== caseIds.size || persistedCaseIds.size !== caseIds.size || [...caseIds].some((id) => !persistedCaseIds.has(id))) {
+            throw new Error("Registry result coverage does not match the evaluation cases");
+        }
+        if (options.verifyTraces !== false && persistedResults.length > 0) {
+            const traceIds = new Set();
+            for (const item of persistedResults) {
+                const traceId = field(item, "traceId");
+                if (typeof traceId !== "string" || !TRACE_ID.test(traceId) || /^0+$/.test(traceId)) {
+                    throw new Error("Registry result is missing a valid trace ID");
+                }
+                traceIds.add(traceId);
+            }
+            const expectations = new Map();
+            for (const item of persistedResults) {
+                const traceId = field(item, "traceId");
+                const current = traceEvidence.get(traceId);
+                if (current !== undefined) {
+                    expectations.set(traceId, current);
+                    continue;
+                }
+                const persistedScores = field(item, "scores");
+                const scorePins = new Map();
+                if (isRecord(persistedScores)) {
+                    for (const [scoreName, value] of Object.entries(persistedScores)) {
+                        const scorer = scorerCache.get(scoreName);
+                        if (value !== null && scorer !== undefined)
+                            scorePins.set(scoreName, scorer);
+                    }
+                }
+                const datasetRecordId = field(item, "datasetRecordId");
+                expectations.set(traceId, {
+                    caseId: String(field(item, "caseId")),
+                    ...(typeof datasetRecordId === "string" ? { datasetRecordId } : {}),
+                    requiredSpanIds: new Set(),
+                    scores: scorePins,
+                });
+            }
+            await verifyTraceEvidence(resources, run.id, expectations, traceVerificationTimeoutMs);
+        }
+        const local = localSummary(persistedResults, Date.now() - started);
+        const experiment = await resources.experiments.update(run.id, {
+            experimentStatus: "completed",
+            summary: local,
+        });
+        const remote = field(experiment, "summary");
+        const summary = isRecord(remote)
+            ? remote
+            : local;
+        return {
+            experimentId: run.id,
+            experiment,
+            dataset: run.dataset,
+            summary,
+            results: options.returnResults === false ? [] : results,
+        };
+    }
+    catch (error) {
+        // A custom exporter's failure must not replace the task, scorer, or
+        // Registry error that caused the evaluation to fail.
+        try {
+            await traceLogger.flush();
+        }
+        catch {
+            // Preserve the original runner/upload failure.
+        }
+        // Leave interrupted infrastructure work running: terminal experiments
+        // cannot accept more results. Resume reconciles Registry case IDs.
+        throw error;
+    }
+}