@atlanai/sdk 0.2.3 → 0.3.0

package/dist/evals.js

@@ -4,10 +4,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EvalRun = exports.CONTEXT_MANIFEST_SCHEMA = void 0;
 exports.createContextManifest = createContextManifest;
 exports.resolveDataset = resolveDataset;
+exports.recordName = recordName;
+exports.pushRecords = pushRecords;
+exports.pushDataset = pushDataset;
 exports.startExperiment = startExperiment;
+exports.experimentTraces = experimentTraces;
+exports.recordExperimentSession = recordExperimentSession;
 exports.Eval = Eval;
+exports.verifyExperiment = verifyExperiment;
 const client_1 = require("./client");
 const errors_1 = require("./errors");
+const AgentSubjectKind_1 = require("./raw/agent/models/AgentSubjectKind");
 exports.CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
 const SHA256 = /^sha256:[0-9a-f]{64}$/;
 const TRACE_ID = /^[0-9a-f]{32}$/;
@@ -107,6 +114,8 @@ async function createContextManifest(input) {
     const digest = "sha256:" + [...new Uint8Array(hash)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
     return Object.freeze({ ...body, items: Object.freeze(items), digest });
 }
+class DatasetNotFoundError extends Error {
+}
 /** Resolve one dataset by artifact ID or exact name, never fuzzy matching. */
 async function resolveDataset(client, idOrExactName, workspaceId) {
     if (!idOrExactName || idOrExactName.trim() !== idOrExactName) {
@@ -141,17 +150,263 @@ async function resolveDataset(client, idOrExactName, workspaceId) {
     }
     if (matches.length === 0) {
         const scope = workspace === undefined ? "" : ` in workspace '${workspace}'`;
-        throw new Error(`no dataset named '${idOrExactName}'${scope}`);
+        throw new DatasetNotFoundError(`no dataset named '${idOrExactName}'${scope}`);
     }
     if (matches.length > 1) {
         throw new Error(`multiple datasets are named '${idOrExactName}'; pass the dataset ID`);
     }
     return matches[0];
 }
+// --- Pushing a suite -------------------------------------------------------
+//
+// A record name is unique per `(workspace, kind)`, not per dataset, and
+// archiving never frees it, so a developer's name is spendable once per
+// workspace. `name` is therefore a derived per-dataset handle and
+// `displayName` carries the developer's key. Nothing reads a record by name.
+const RECORD_NAME_DIGEST = 12;
+const BULK_RECORD_LIMIT = 100;
+// `name` is excluded: the gateway accepts it in a patch and ignores it
+// (measured: the old name came back with the ordinal bumped).
+const PUSHED_RECORD_FIELDS = [
+    "input",
+    "expected",
+    "label",
+    "categories",
+    "description",
+    "displayName",
+    "extra",
+];
+/**
+ * The Registry handle for the case called `key` inside `datasetId`.
+ *
+ * Deterministic, so pushing the same suite twice lands on the same row, and
+ * dataset-scoped, so the same suite can also be pushed into a new dataset.
+ */
+async function recordName(datasetId, key) {
+    nonEmpty(datasetId, "datasetId must be a non-empty string");
+    nonEmpty(key, "a pushed record needs a non-empty name");
+    if (!globalThis.crypto?.subtle) {
+        throw new Error("Web Crypto is required to derive a dataset record name");
+    }
+    const hash = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${datasetId}\u0000${key}`));
+    const digest = [...new Uint8Array(hash)]
+        .map((byte) => byte.toString(16).padStart(2, "0"))
+        .join("")
+        .slice(0, RECORD_NAME_DIGEST);
+    // The digest over the whole key is what keeps two cases apart; the prefix
+    // is only readability. The Python SDK derives the same handle.
+    return `${slug(key, "record")}-${digest}`;
+}
+function pushKey(record) {
+    const key = record.name;
+    if (typeof key !== "string" || key.trim().length === 0) {
+        throw new Error("every pushed record needs a non-empty 'name' to key it by");
+    }
+    return key;
+}
+/**
+ * The subset of the caller's fields the live row does not already carry.
+ *
+ * The gateway mints a new `versionOrdinal` for every PATCH it accepts,
+ * including one that changes nothing, so an unconditional re-push would pile
+ * up empty versions on rows nobody edited.
+ */
+function recordChanges(existing, record) {
+    const changes = {};
+    for (const name of PUSHED_RECORD_FIELDS) {
+        if (!(name in record))
+            continue;
+        if (canonicalJson(field(existing, name) ?? null) !== canonicalJson(record[name] ?? null)) {
+            changes[name] = record[name];
+        }
+    }
+    return changes;
+}
+async function listDatasetRecords(resources, datasetId) {
+    const items = [];
+    let offset = 0;
+    while (true) {
+        const page = await resources.datasets.records.list({ datasetId, limit: 500, offset });
+        const raw = field(page, "items");
+        const batch = Array.isArray(raw) ? raw : [];
+        items.push(...batch);
+        offset += batch.length;
+        const total = field(field(page, "page"), "total");
+        if (batch.length === 0 || (typeof total === "number" && offset >= total))
+            break;
+        if (offset > MAX_LIST_OFFSET)
+            throw new Error("dataset record listing exceeded 10000 rows");
+    }
+    return items;
+}
+/**
+ * Create or update each case, so the same suite can be pushed repeatedly.
+ *
+ * Each record's `name` is the developer's key for the case: it becomes the
+ * row's `displayName` and keys the Registry handle. A case already in the
+ * dataset is patched when its content moved and left alone when it did not.
+ */
+async function pushRecords(client, datasetId, records, workspaceId) {
+    const resources = asResources(client);
+    const workspace = workspaceId ?? resources.workspace;
+    if (typeof workspace !== "string" || workspace.length === 0) {
+        throw new Error("pushRecords needs a workspace: pass workspaceId or set one on the client");
+    }
+    const wanted = new Map();
+    for (const record of records) {
+        const key = pushKey(record);
+        if (record.input === undefined)
+            throw new Error(`pushed record '${key}' has no 'input'`);
+        const name = await recordName(datasetId, key);
+        if (wanted.has(name))
+            throw new Error(`the pushed suite has two records named '${key}'`);
+        wanted.set(name, { key, record });
+    }
+    if (wanted.size === 0)
+        return [];
+    const existing = new Map((await listDatasetRecords(resources, datasetId)).map((item) => [field(item, "name"), item]));
+    const pushed = new Map();
+    const toCreate = [];
+    for (const [name, { key, record }] of wanted) {
+        const live = existing.get(name);
+        if (live === undefined) {
+            const { name: _key, ...rest } = record;
+            toCreate.push({
+                key,
+                name,
+                body: {
+                    ...rest,
+                    workspaceId: workspace,
+                    name,
+                    displayName: record.displayName ?? key,
+                },
+            });
+            continue;
+        }
+        const id = String(field(live, "id"));
+        const changes = recordChanges(live, record);
+        if (Object.keys(changes).length === 0) {
+            pushed.set(name, { key, name, id, action: "unchanged", record: live });
+            continue;
+        }
+        const updated = await resources.datasets.records.update(datasetId, id, changes);
+        pushed.set(name, { key, name, id, action: "updated", record: updated });
+    }
+    for (let start = 0; start < toCreate.length; start += BULK_RECORD_LIMIT) {
+        const chunk = toCreate.slice(start, start + BULK_RECORD_LIMIT);
+        let response;
+        try {
+            response = await resources.datasets.records.createBulk(datasetId, {
+                items: chunk.map((item) => item.body),
+            });
+        }
+        catch (error) {
+            // The transport refuses an all-rejected batch before the slots are
+            // visible, and can only say "the batch". Every case in this chunk was
+            // rejected, so they can be named without the slots.
+            if (!(error instanceof errors_1.AtlanAPIError) || error.code !== "bulk_all_rejected")
+                throw error;
+            throw rejectedRecords(error.status, chunk);
+        }
+        const raw = field(response, "items");
+        const slots = Array.isArray(raw) ? raw : [];
+        if (slots.length !== chunk.length) {
+            throw new Error(`bulk record create returned ${slots.length} slots for ${chunk.length} items`);
+        }
+        for (const [index, slot] of slots.entries()) {
+            const item = chunk[index];
+            const created = field(slot, "record");
+            const id = field(created, "id");
+            if (typeof id !== "string" || id.length === 0) {
+                const status = field(slot, "statusCode") ?? field(slot, "status_code");
+                throw rejectedRecords(Number(status), [item]);
+            }
+            pushed.set(item.name, { key: item.key, name: item.name, id, action: "created", record: created });
+        }
+    }
+    return [...wanted.keys()].map((name) => pushed.get(name));
+}
+/**
+ * Name the cases a bulk create refused, rather than blaming "the batch".
+ *
+ * The batch-level code is all the transport can know; the keys are only known
+ * here, and they are what a developer has to go and edit.
+ */
+function rejectedRecords(status, cases) {
+    const named = cases.map((item) => `'${item.key}' (registry handle '${item.name}')`).join(", ");
+    const spent = status === 409
+        ? cases.length === 1
+            ? "; that handle is already spent in this workspace, which a record in another dataset can do"
+            : "; those handles are already spent in this workspace, which records in another dataset can do"
+        : "";
+    return new Error(cases.length === 1
+        ? `case ${named} was refused with status ${status}${spent}`
+        : `${cases.length} cases were refused with status ${status}: ${named}${spent}`);
+}
+/**
+ * Push a suite under `name`, creating the dataset the first time only.
+ *
+ * Idempotent: run it again after correcting an expected value and the
+ * correction lands on the same rows, in the same dataset, without a 409.
+ */
+async function pushDataset(client, name, records, options = {}) {
+    const resources = asResources(client);
+    const workspace = options.workspaceId ?? resources.workspace;
+    if (typeof workspace !== "string" || workspace.length === 0) {
+        throw new Error("pushDataset needs a workspace: pass workspaceId or set one on the client");
+    }
+    let dataset;
+    let created = false;
+    try {
+        dataset = await resolveDataset(client, name, workspace);
+    }
+    catch (error) {
+        if (!(error instanceof DatasetNotFoundError))
+            throw error;
+        const body = { workspaceId: workspace, name };
+        if (options.displayName !== undefined)
+            body.displayName = options.displayName;
+        if (options.description !== undefined)
+            body.description = options.description;
+        try {
+            dataset = await resources.datasets.create(body);
+            created = true;
+        }
+        catch (createError) {
+            if (!(createError instanceof errors_1.AtlanAPIError) || createError.status !== 409)
+                throw createError;
+            try {
+                dataset = await resolveDataset(client, name, workspace);
+            }
+            catch (resolveError) {
+                if (resolveError instanceof DatasetNotFoundError)
+                    throw createError;
+                throw resolveError;
+            }
+        }
+    }
+    const datasetId = field(dataset, "id");
+    if (typeof datasetId !== "string" || datasetId.length === 0) {
+        throw new Error("dataset response did not contain an ID");
+    }
+    return {
+        dataset,
+        id: datasetId,
+        created,
+        records: await pushRecords(client, datasetId, records, workspace),
+    };
+}
 class EvalRun {
     experiment;
     dataset;
     contextManifest;
+    /**
+     * The session id stamped on every span this run emits.
+     *
+     * It is the only join key the gateway promotes off an eval span that can
+     * reach the run's subject: `recordExperimentSession` registers this value
+     * against the experiment's subject.
+     */
     constructor(experiment, dataset, contextManifest) {
         this.experiment = experiment;
         this.dataset = dataset;
@@ -167,10 +422,20 @@ class EvalRun {
     get experimentId() {
         return this.id;
     }
-    /** Options for `propagateAttributes` from `@atlanai/sdk/tracing`. */
+    /** Stable across retries because the experiment is the resumable unit. */
+    get sessionId() {
+        return this.id;
+    }
+    /**
+     * Options for `propagateAttributes` from `@atlanai/sdk/tracing`.
+     *
+     * Carries the experiment join and the subject join. A nested
+     * `propagateAttributes({ sessionId })` still wins for the spans inside it.
+     */
     get traceOptions() {
         return {
             experimentId: this.id,
+            sessionId: this.sessionId,
             ...(this.contextManifest === undefined
                 ? {}
                 : { metadata: { context_manifest_digest: this.contextManifest.digest } }),
@@ -245,6 +510,124 @@ async function startExperiment(client, dataset, body, options = {}) {
     }
     return run;
 }
+/**
+ * Every trace an experiment recorded, newest first.
+ *
+ * **The experiment is the scope eval traces are filed under.** A subject's own
+ * Traces tab (`GET /agent/v1/agents/{id}/traces`) resolves on the trace's
+ * *creator identity*, so it lists a run only when the run's spans were exported
+ * with that agent's own credential. An eval exported with your user key or a
+ * service-account key is filed under that identity instead, and the agent's
+ * Traces tab reads empty even though every trace exists. That is not the traces
+ * going missing; it is a different scope. Read them here, and call
+ * `recordExperimentSession` so the run is also reachable from the subject's
+ * Sessions tab.
+ *
+ * To fill the agent's own Traces tab, export the spans as the agent: pass
+ * `logger: initLogger({ apiKey: <the agent's credential> })` while the
+ * management `client` keeps your own, since the agent's identity is not
+ * entitled to create experiments.
+ */
+async function experimentTraces(client, experimentId, options = {}) {
+    if (experimentId.trim().length === 0)
+        throw new Error("experimentId is required");
+    const limit = options.limit ?? 500;
+    if (limit <= 0)
+        throw new Error("limit must be greater than zero");
+    const resources = asResources(client);
+    const traces = [];
+    let cursor;
+    while (traces.length < limit) {
+        const page = await resources.experiments.traces.list({
+            experimentId,
+            limit: Math.min(500, limit - traces.length),
+            ...(cursor === undefined ? {} : { cursor }),
+        });
+        const items = field(page, "items");
+        const batch = Array.isArray(items) ? items : [];
+        traces.push(...batch);
+        const next = field(field(page, "page"), "nextCursor");
+        cursor = typeof next === "string" && next.length > 0 ? next : undefined;
+        if (batch.length === 0 || cursor === undefined)
+            break;
+    }
+    return traces.slice(0, limit);
+}
+/**
+ * Bind this run's traces to the experiment's subject, and return the record.
+ *
+ * The gateway promotes exactly one span attribute that can reach a subject: the
+ * session id. This registers a `session` artifact whose `subjectId` is the
+ * experiment's subject and whose `externalSessionId` is `run.sessionId`, the
+ * value `run.traceOptions` stamps on every span of the run. After it, the run
+ * appears on the subject's Sessions tab and `client.sessions.traces.list(id)`
+ * returns the run's traces.
+ *
+ * Returns `undefined` when the experiment names no subject, because then there
+ * is nothing to bind the run to. A prior accepted create is reconciled on
+ * `409`, so a resume after a lost response does not strand the experiment.
+ */
+// Which subject kinds a `session` artifact can name. Read off the generated
+// enum, so a gateway that later adds one is picked up by a contract sync
+// rather than needing this list edited.
+const SESSION_SUBJECT_KINDS = new Set(Object.values(AgentSubjectKind_1.AgentSubjectKind));
+async function recordExperimentSession(client, run, options = {}) {
+    const subjectKind = field(run.experiment, "subjectKind");
+    const subjectId = field(run.experiment, "subjectId");
+    if (typeof subjectKind !== "string" || typeof subjectId !== "string")
+        return undefined;
+    if (subjectKind.length === 0 || subjectId.length === 0)
+        return undefined;
+    // EvalSubjectKind has `skill`, AgentSubjectKind does not, so a skill eval
+    // has no session to bind to. Throwing would fail a run whose results are
+    // already persisted, with no remedy that could succeed.
+    if (!SESSION_SUBJECT_KINDS.has(subjectKind))
+        return undefined;
+    const resources = asResources(client);
+    const workspaceId = field(run.experiment, "workspaceId") ?? resources.workspace;
+    if (typeof workspaceId !== "string" || workspaceId.length === 0) {
+        throw new Error("experiment has no workspaceId and the client has no default");
+    }
+    const displayName = field(run.experiment, "displayName") ?? field(run.experiment, "name");
+    const body = {
+        workspaceId,
+        name: `${slug(run.id, "experiment")}-traces`,
+        subjectKind,
+        subjectId,
+        sessionStatus: options.sessionStatus ?? "completed",
+        externalSessionId: run.sessionId,
+        sourceType: "local",
+    };
+    if (typeof displayName === "string" && displayName.length > 0)
+        body.displayName = displayName;
+    if (options.title !== undefined)
+        body.title = options.title;
+    try {
+        return await resources.sessions.createRecord(body);
+    }
+    catch (error) {
+        if (!(error instanceof errors_1.AtlanAPIError) || error.status !== 409)
+            throw error;
+        const page = await resources.sessions.list({
+            limit: 500,
+            offset: 0,
+            subjectKind,
+            subjectId,
+            externalSessionId: run.sessionId,
+        });
+        const items = field(page, "items");
+        const matches = (Array.isArray(items) ? items : []).filter((item) => field(item, "workspaceId") === workspaceId &&
+            field(item, "subjectKind") === subjectKind &&
+            field(item, "subjectId") === subjectId &&
+            field(item, "externalSessionId") === run.sessionId);
+        if (matches.length === 1)
+            return matches[0];
+        if (matches.length > 1) {
+            throw new Error(`multiple sessions bind experiment ${run.id} to subject ${subjectId}`, { cause: error });
+        }
+        throw error;
+    }
+}
 // ---------------------------------------------------------------------------
 // Declarative evaluator
 const DEFAULT_GATEWAY_ORIGIN = "https://api.atlan.com";
@@ -605,7 +988,11 @@ async function traceSpans(resources, experimentId, traceId) {
             experimentId,
             traceId,
             limit: 500,
-            fields: "io",
+            // Every check in `missingTraceEvidence` reads a span attribute, and
+            // `attributes` is its own projection group: asking for `io` alone
+            // returns `attributes: null` on every span, so the gate can only ever
+            // fail. `core` carries `spanId`, which the same checks match on.
+            fields: "core,io,attributes",
             ...(cursor === undefined ? {} : { cursor }),
         });
         const items = field(page, "items");
@@ -621,16 +1008,34 @@ async function traceSpans(resources, experimentId, traceId) {
         cursor = nextCursor;
     }
 }
-async function listedTraceIds(resources, experimentId) {
+function artifactUnixSeconds(value) {
+    if (typeof value !== "string" || value.length === 0)
+        return undefined;
+    const milliseconds = Date.parse(value);
+    return Number.isFinite(milliseconds) ? Math.floor(milliseconds / 1_000) : undefined;
+}
+function experimentTraceWindow(experiment) {
+    const created = artifactUnixSeconds(field(experiment, "createdAt"));
+    const updated = artifactUnixSeconds(field(experiment, "updatedAt"));
+    const startTimeUnixSeconds = Math.max(0, (created ?? 0) - 60 * 60);
+    const endTimeUnixSeconds = field(experiment, "experimentStatus") === "running"
+        ? Math.floor(Date.now() / 1_000)
+        : (updated ?? created ?? Math.floor(Date.now() / 1_000)) + 24 * 60 * 60;
+    return { startTimeUnixSeconds, endTimeUnixSeconds };
+}
+async function listedTraceIds(resources, experimentId, window) {
     const traceIds = new Set();
     const seenCursors = new Set();
     let cursor;
     const windowEnd = Math.floor(Date.now() / 1_000);
+    const bounds = window ?? {
+        startTimeUnixSeconds: windowEnd - 30 * 24 * 60 * 60,
+        endTimeUnixSeconds: windowEnd,
+    };
     while (true) {
         const page = await resources.experiments.traces.list({
             experimentId,
-            startTimeUnixSeconds: windowEnd - 30 * 24 * 60 * 60,
-            endTimeUnixSeconds: windowEnd,
+            ...bounds,
             limit: 500,
             ...(cursor === undefined ? {} : { cursor }),
         });
@@ -652,9 +1057,42 @@ async function listedTraceIds(resources, experimentId) {
         cursor = nextCursor;
     }
 }
+/** Rejoin one exploded attribute subtree into dotted keys. */
+function flattenAttributes(value, prefix, into) {
+    if (isRecord(value)) {
+        for (const [key, child] of Object.entries(value)) {
+            flattenAttributes(child, prefix === "" ? key : `${prefix}.${key}`, into);
+        }
+    }
+    else if (prefix !== "") {
+        into[prefix] = value;
+    }
+}
+/**
+ * The span's attributes, keyed the way this module writes them.
+ *
+ * An OTel attribute key is a flat dotted string, and that is what the tracing
+ * side sets. The gateway does not return it that way: it splits the key on
+ * every dot into a nested object and sorts the result into `resource`, `scope`
+ * and `span` envelopes, so `atlan.eval.case_id` arrives as
+ * `{span: {atlan: {eval: {case_id: ...}}}}`. `SpanViewDto.attributes` is
+ * untyped in the contract, so neither shape is wrong; this is the side that has
+ * to adapt. An already-flat mapping is passed through unchanged.
+ */
 function spanAttributes(span) {
     const attributes = field(span, "attributes");
-    return isRecord(attributes) ? attributes : {};
+    if (!isRecord(attributes))
+        return {};
+    const envelopes = ["resource", "scope", "span"].map((name) => attributes[name]);
+    if (!envelopes.some((envelope) => isRecord(envelope)))
+        return attributes;
+    // Span last: a span attribute outranks the resource-wide one it shadows.
+    const flat = {};
+    for (const envelope of envelopes) {
+        if (isRecord(envelope))
+            flattenAttributes(envelope, "", flat);
+    }
+    return flat;
 }
 function missingTraceEvidence(spans, expected) {
     const missing = [];
@@ -786,6 +1224,73 @@ function defaultConnections(name, options) {
  * carry the root trace ID, traces flush before summary, and the experiment is
  * explicitly finalized.
  */
+// The Registry resource each subject kind lives in.
+const SUBJECT_RESOURCES = {
+    agent: "agents",
+    harness: "harnesses",
+    skill: "skills",
+};
+/**
+ * Refuse an experiment whose subject lives in another workspace.
+ *
+ * The dataset is already checked ("dataset ID resolved outside the requested
+ * workspace") and so is every scorer ("scorer resolved outside the evaluation
+ * workspace"). The subject was not, and it is the field most likely to be
+ * wrong: the listings people resolve a subject's name through are not
+ * workspace-scoped, so `agents.list()` answers with agents from every workspace
+ * the caller can see and "reuse the agent called X" quietly picks a stranger's.
+ * The gateway accepts the row, and the run is then attributed to an agent its
+ * own workspace has never heard of.
+ *
+ * A read that fails is not proof of a mismatch, so only a subject that reads
+ * back with a different workspace fails the run. This is a wrong-binding check,
+ * not an authorization gate.
+ */
+async function checkSubjectWorkspace(client, subjectKind, subjectId, workspaceId) {
+    const resourceName = SUBJECT_RESOURCES[subjectKind ?? ""];
+    if (resourceName === undefined || subjectId === undefined || subjectId.length === 0)
+        return;
+    const resource = field(client, resourceName);
+    const getter = field(resource, "get");
+    if (typeof getter !== "function")
+        return;
+    let subject;
+    try {
+        subject = await getter.call(resource, subjectId);
+    }
+    catch {
+        // See the doc comment: a read that fails is not proof of a mismatch.
+        return;
+    }
+    const actual = field(subject, "workspaceId");
+    if (actual !== undefined && actual !== workspaceId) {
+        throw new Error(`subject ${subjectId} lives in workspace ${String(actual)}, not ${workspaceId}; ` +
+            `an experiment cannot be attributed to a subject outside its own workspace`);
+    }
+}
+function nameOpenExperiment(error, experimentId) {
+    if (!(error instanceof Error))
+        return error;
+    const note = `experiment ${experimentId} was left running; pass ` +
+        `resumeExperimentId: ${JSON.stringify(experimentId)} to continue it`;
+    try {
+        Object.defineProperty(error, "experimentId", {
+            configurable: true,
+            enumerable: true,
+            value: experimentId,
+        });
+    }
+    catch {
+        // Preserve the original error even when a caller froze it.
+    }
+    try {
+        error.message = `${error.message} (${note})`;
+    }
+    catch {
+        // Preserve the original error even when its message is read-only.
+    }
+    return error;
+}
 async function Eval(name, evaluator, options = {}) {
     if (!name || /[\r\n]/.test(name))
         throw new Error("Eval name must be non-empty and single-line");
@@ -823,6 +1328,7 @@ async function Eval(name, evaluator, options = {}) {
         if (value === undefined)
             delete experimentBody[key];
     }
+    await checkSubjectWorkspace(client, options.subjectKind, options.subjectId, workspaceId);
     let run;
     let cases = [];
     if (options.resumeExperimentId !== undefined) {
@@ -894,6 +1400,14 @@ async function Eval(name, evaluator, options = {}) {
                 snapshot ??
                     (await datasetCases(resources, String(field(run.dataset, "id"))));
         }
+        if (evaluator.dataset !== undefined && cases.length === 0) {
+            // A named dataset that yields nothing is a mistake upstream, not an eval
+            // with no work in it. Finishing here would write a `completed`
+            // experiment carrying no results and no scores, which reads downstream
+            // as a run that happened and found nothing.
+            throw new Error(`dataset ${JSON.stringify(evaluator.dataset)} resolved to ` +
+                `${String(field(run.dataset, "id"))} and it has no records, so there is nothing to evaluate`);
+        }
         const caseIds = new Set(cases.map((item, index) => item.id ?? `case-${index + 1}`));
         if (caseIds.size !== cases.length)
             throw new Error("Eval case IDs must be unique");
@@ -1098,6 +1612,21 @@ async function Eval(name, evaluator, options = {}) {
             }
             await verifyTraceEvidence(resources, run.id, expectations, traceVerificationTimeoutMs);
         }
+        // Without this the traces are readable by experiment but nothing on the
+        // subject leads to them. Bind before the terminal update so a failed
+        // create leaves a resumable experiment instead of a completed one with no
+        // route from its subject to the traces.
+        let session;
+        try {
+            session = await recordExperimentSession(client, run);
+        }
+        catch (error) {
+            throw new Error(`experiment ${run.id} has every result persisted, but recording ` +
+                `the session that binds its traces to subject ` +
+                `${String(field(run.experiment, "subjectId"))} failed: ${safeError(error)}. ` +
+                `The traces are readable with experimentTraces(client, "${run.id}"); the ` +
+                `experiment remains running and can be resumed safely.`, { cause: error });
+        }
         const local = localSummary(persistedResults, Date.now() - started);
         const experiment = await resources.experiments.update(run.id, {
             experimentStatus: "completed",
@@ -1113,6 +1642,8 @@ async function Eval(name, evaluator, options = {}) {
             dataset: run.dataset,
             summary,
             results: options.returnResults === false ? [] : results,
+            sessionId: run.sessionId,
+            ...(session === undefined ? {} : { session }),
         };
     }
     catch (error) {
@@ -1126,6 +1657,191 @@ async function Eval(name, evaluator, options = {}) {
         }
         // Leave interrupted infrastructure work running: terminal experiments
         // cannot accept more results. Resume reconciles Registry case IDs.
-        throw error;
+        throw nameOpenExperiment(error, run.id);
     }
 }
+/**
+ * A span's score column, from the typed projection or the raw attribute.
+ *
+ * The gateway promotes `atlan.score.*` onto typed columns, but only when it was
+ * asked for the projection that carries them. Falling back to the attribute
+ * keeps this working on spans read any other way.
+ */
+function promoted(span, name, attribute) {
+    const value = field(span, name);
+    return value === undefined || value === null
+        ? spanAttributes(span)[`atlan.score.${attribute}`]
+        : value;
+}
+/**
+ * True only for a span that actually carries a score verdict.
+ *
+ * Span type is not the test. `startAsCurrentSpan({asType: "score"})` produces a
+ * `score`-typed wrapper around the real verdict, and the gateway answers for
+ * that wrapper with `scoreName: ""`, `scoreValue: 0`, `scorerId: ""`,
+ * `scorerVersion: 0` rather than omitting the columns. Read it as evidence and
+ * an unscored wrapper becomes "scored 0 by nobody". Only a non-empty
+ * `scoreName` marks a span a scorer actually wrote.
+ */
+function isScoreVerdict(span) {
+    const name = promoted(span, "scoreName", "score_name");
+    return typeof name === "string" && name.trim().length > 0;
+}
+/**
+ * Run the documented seven-point evidence gate over a finished experiment.
+ *
+ * Reads the experiment, its result rows, its trace list and the spans of up to
+ * `maxTraces` case traces, and reports whether the evidence chain actually
+ * holds. It only reads; nothing is written.
+ *
+ * Gate 7, the content and masking policy, cannot be decided from the data and
+ * is reported as a manual check rather than silently passed.
+ */
+async function verifyExperiment(client, experimentId, options = {}) {
+    if (experimentId.length === 0 || experimentId !== experimentId.trim()) {
+        throw new Error("experimentId must be a non-empty ID with no outer whitespace");
+    }
+    const resources = client;
+    const maxTraces = options.maxTraces === undefined ? 3 : options.maxTraces;
+    const experiment = await resources.experiments.get(experimentId);
+    const results = await experimentResults(resources, experimentId);
+    const checks = [];
+    const record = (gate, name, ok, detail = "") => {
+        checks.push({ gate, name, ok, detail });
+    };
+    // Gate 1 -------------------------------------------------------------------
+    const status = field(experiment, "experimentStatus");
+    record(1, "the experiment reached a terminal status", status === "completed" || status === "failed", `experimentStatus=${JSON.stringify(status)}; 'running' after a finished run ` +
+        `is a lifecycle failure, not a slow write`);
+    // Gate 2 -------------------------------------------------------------------
+    record(2, "at least one result row was written", results.length > 0, `${results.length} rows`);
+    if (options.expectedCaseCount !== undefined) {
+        record(2, "the result count equals the case count", results.length === options.expectedCaseCount, `${results.length} rows for ${options.expectedCaseCount} cases`);
+    }
+    const caseIds = results.map((row) => field(row, "caseId") ?? field(row, "name"));
+    const duplicates = [
+        ...new Set(caseIds.filter((key) => caseIds.filter((other) => other === key).length > 1)),
+    ];
+    record(2, "no duplicate result rows", duplicates.length === 0, duplicates.length === 0 ? "" : `repeated case ids: ${duplicates.join(", ")}`);
+    // Gate 3 -------------------------------------------------------------------
+    const rawTraceIds = results.map((row) => field(row, "traceId"));
+    const malformed = rawTraceIds.filter((value) => typeof value !== "string" || !TRACE_ID.test(value));
+    record(3, "every result carries a W3C trace id", malformed.length === 0, malformed.length === 0 ? "" : `${malformed.length} malformed trace ids`);
+    const zeroed = rawTraceIds.filter((value) => typeof value === "string" && /^0+$/.test(value));
+    record(3, "no result carries the all-zero trace id", zeroed.length === 0, zeroed.length === 0
+        ? ""
+        : `${zeroed.length} rows read as 'a trace that exists and is empty' rather ` +
+            `than 'this case was not traced'`);
+    record(3, "trace ids are distinct per case", new Set(rawTraceIds).size === rawTraceIds.length, `${rawTraceIds.length - new Set(rawTraceIds).size} rows share a trace with another row`);
+    // Gate 4 -------------------------------------------------------------------
+    let listed = new Set();
+    let traceListReadable = true;
+    try {
+        listed = await listedTraceIds(resources, experimentId, experimentTraceWindow(experiment));
+    }
+    catch (error) {
+        traceListReadable = false;
+        record(4, "the experiment trace filter is readable", false, String(error));
+    }
+    if (traceListReadable) {
+        // An empty listing is a real answer here, not a missing one: it means every
+        // result row points at a trace the experiment does not own.
+        const missing = rawTraceIds.filter((value) => typeof value === "string" && !listed.has(value));
+        record(4, "every result trace id appears in the experiment trace list", missing.length === 0, missing.length === 0
+            ? `${listed.size} traces listed`
+            : `${missing.length} results point at a trace the experiment filter does not return`);
+    }
+    // Gates 5 and 6 ------------------------------------------------------------
+    const wellFormed = rawTraceIds.filter((value) => typeof value === "string" && TRACE_ID.test(value));
+    const sampled = maxTraces === null ? wellFormed : wellFormed.slice(0, maxTraces);
+    const verified = [];
+    for (const traceId of sampled) {
+        const spans = await traceSpans(resources, experimentId, traceId);
+        verified.push(traceId);
+        const label = `trace ${traceId}`;
+        if (spans.length === 0) {
+            record(5, `${label} has spans`, false, "0 spans returned");
+            continue;
+        }
+        const verdicts = spans.filter(isScoreVerdict);
+        const typedScore = spans.filter((span) => field(span, "spanType") === "score" ||
+            spanAttributes(span)["atlan.span.type"] === "score");
+        record(5, `${label} carries a score verdict`, verdicts.length > 0, verdicts.length > 0
+            ? `${verdicts.length} of ${spans.length} spans carry a verdict`
+            : typedScore.length > 0
+                ? `${typedScore.length} spans are typed \`score\` but none carries a ` +
+                    `scoreName, so none of them is evidence`
+                : "no scorer wrote a span on this trace");
+        const hollow = typedScore.filter((span) => !verdicts.includes(span));
+        record(5, `${label} has no hollow \`score\` spans`, hollow.length === 0, hollow.length === 0
+            ? ""
+            : `${hollow.length} spans are typed \`score\` with an empty scoreName; the ` +
+                `gateway answers for those with scoreValue 0, which reads as a real ` +
+                `zero to anything that does not filter them out`);
+        const unpinned = verdicts.filter((span) => {
+            const scorerId = promoted(span, "scorerId", "scorer_id");
+            return typeof scorerId !== "string" || scorerId.length === 0;
+        });
+        record(5, `${label}: every verdict cites a scorer id`, unpinned.length === 0, unpinned.length === 0
+            ? ""
+            : `${unpinned.length} verdict spans carry an empty scorerId; a score ` +
+                `without an identity is a number, not evidence`);
+        const unversioned = verdicts.filter((span) => {
+            const version = promoted(span, "scorerVersion", "scorer_version");
+            return typeof version !== "number" || version < 1;
+        });
+        record(5, `${label}: every verdict cites a scorer version`, unversioned.length === 0, unversioned.length === 0
+            ? ""
+            : `${unversioned.length} verdict spans carry no usable scorerVersion`);
+        const byId = new Set();
+        for (const span of spans)
+            byId.add(field(span, "spanId"));
+        const roots = spans.filter((span) => {
+            const parent = field(span, "parentSpanId");
+            return parent === undefined || parent === null || parent === "";
+        });
+        record(6, `${label} has exactly one root span`, roots.length === 1, `${roots.length} spans have no parent`);
+        const orphans = spans.filter((span) => {
+            const parent = field(span, "parentSpanId");
+            return typeof parent === "string" && parent.length > 0 && !byId.has(parent);
+        });
+        record(6, `${label} has no orphaned spans`, orphans.length === 0, orphans.length === 0
+            ? ""
+            : `${orphans.length} spans name a parent that is not in this trace`);
+        record(6, `${label} is nested, not flat`, spans.length > 1 && roots.length === 1, `${spans.length} spans, ${roots.length} roots`);
+    }
+    // Gate 7 -------------------------------------------------------------------
+    record(7, "inputs and outputs follow the approved content policy", undefined, "re-run with traceContent=false and confirm the span payloads are gone; " +
+        "no read-back can decide this for you");
+    const failures = checks.filter((check) => check.ok === false);
+    const manual = checks.filter((check) => check.ok === undefined);
+    return {
+        experimentId,
+        experiment,
+        checks,
+        results,
+        traceIds: rawTraceIds.filter((value) => typeof value === "string"),
+        verifiedTraceIds: verified,
+        failures,
+        manual,
+        ok: failures.length === 0,
+        raiseForStatus() {
+            if (failures.length === 0)
+                return;
+            throw new Error(`experiment ${experimentId} failed ${failures.length} of ${checks.length} ` +
+                `evidence checks: ` +
+                failures.map((check) => `${check.name} (${check.detail})`).join("; "));
+        },
+        toString() {
+            const passed = checks.filter((check) => check.ok === true).length;
+            const header = `experiment ${experimentId}: ${passed} passed, ${failures.length} failed, ` +
+                `${manual.length} manual`;
+            const lines = checks.map((check) => {
+                const mark = check.ok === true ? "PASS" : check.ok === false ? "FAIL" : "MANUAL";
+                const first = `  [${mark}] gate ${check.gate}: ${check.name}`;
+                return check.detail ? `${first}\n        ${check.detail}` : first;
+            });
+            return [header, ...lines].join("\n");
+        },
+    };
+}