@hue-run/sdk 0.1.2

package/dist/evals/client.js

@@ -0,0 +1,195 @@
+import { validateOptions } from "../config.js";
+import { json, uuid } from "./json.js";
+export class HueApiError extends Error {
+    status;
+    constructor(status) {
+        super(status ? `Hue API request failed (HTTP ${status})` : "Hue API connection or response failed");
+        this.status = status;
+        this.name = "HueApiError";
+    }
+}
+/** No implicit mutation retry: callers retain stable idempotency keys for experiments/results. */
+export class EvaluationClient {
+    baseUrl;
+    apiKey;
+    timeoutMillis;
+    constructor(options) {
+        const validated = validateOptions({
+            ...options,
+            serviceName: "hue-evaluations",
+            captureContent: false,
+        });
+        this.baseUrl = validated.baseUrl;
+        this.apiKey = validated.apiKey;
+        this.timeoutMillis = validated.timeoutMillis;
+    }
+    async request(method, path, body) {
+        // Optional top-level fields are omitted intentionally; nested undefined remains invalid.
+        const payload = body === undefined
+            ? undefined
+            : JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), 1024 * 1024));
+        let response;
+        try {
+            response = await fetch(`${this.baseUrl}/api/v1${path}`, {
+                method,
+                headers: {
+                    Authorization: `Bearer ${this.apiKey}`,
+                    ...(payload ? { "Content-Type": "application/json" } : {}),
+                },
+                body: payload,
+                redirect: "error",
+                signal: AbortSignal.timeout(this.timeoutMillis),
+            });
+        }
+        catch {
+            throw new HueApiError();
+        }
+        if (!response.ok) {
+            await response.body?.cancel();
+            throw new HueApiError(response.status);
+        }
+        try {
+            const reader = response.body?.getReader();
+            if (!reader)
+                throw new Error("Missing response");
+            const chunks = [];
+            let size = 0;
+            try {
+                for (;;) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    size += value.byteLength;
+                    if (size > 4 * 1024 * 1024)
+                        throw new Error("Oversized response");
+                    chunks.push(value);
+                }
+            }
+            finally {
+                await reader.cancel();
+            }
+            return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        }
+        catch {
+            throw new HueApiError();
+        }
+    }
+    page(options = {}) {
+        const query = new URLSearchParams();
+        if (options.after)
+            query.set("after", uuid(options.after));
+        if (options.limit !== undefined) {
+            if (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 100)
+                throw new RangeError("Page limit must be 1–100");
+            query.set("limit", String(options.limit));
+        }
+        return query.size ? `?${query}` : "";
+    }
+    checkConnection() {
+        return this.request("GET", "/projects/current");
+    }
+    createDataset(input) {
+        return this.request("POST", "/datasets", input);
+    }
+    getDataset(id) {
+        return this.request("GET", `/datasets/${uuid(id)}`);
+    }
+    listDatasets(page) {
+        return this.request("GET", `/datasets${this.page(page)}`);
+    }
+    createDatasetVersion(id, input = {}) {
+        return this.request("POST", `/datasets/${uuid(id)}/versions`, input);
+    }
+    getDatasetVersion(id) {
+        return this.request("GET", `/dataset-versions/${uuid(id)}`);
+    }
+    listCases(id, page) {
+        return this.request("GET", `/dataset-versions/${uuid(id)}/cases${this.page(page)}`);
+    }
+    addCase(id, input) {
+        return this.request("POST", `/dataset-versions/${uuid(id)}/cases`, input);
+    }
+    freezeDatasetVersion(id, expectedRevision) {
+        return this.request("POST", `/dataset-versions/${uuid(id)}/freeze`, {
+            expectedRevision,
+        });
+    }
+    createScorer(input) {
+        return this.request("POST", "/scorers", input);
+    }
+    getScorer(id) {
+        return this.request("GET", `/scorers/${uuid(id)}`);
+    }
+    listScorers(page) {
+        return this.request("GET", `/scorers${this.page(page)}`);
+    }
+    publishScorerVersion(id, definition) {
+        return this.request("POST", `/scorers/${uuid(id)}/versions`, { definition });
+    }
+    getScorerVersion(id) {
+        return this.request("GET", `/scorer-versions/${uuid(id)}`);
+    }
+    createExperiment(input) {
+        return this.request("POST", "/experiments", input);
+    }
+    getExperiment(id) {
+        return this.request("GET", `/experiments/${uuid(id)}`);
+    }
+    listExperimentItems(id, page) {
+        return this.request("GET", `/experiments/${uuid(id)}/items${this.page(page)}`);
+    }
+    getExperimentCase(id, caseId) {
+        return this.request("GET", `/experiments/${uuid(id)}/items/${uuid(caseId)}`);
+    }
+    startExecution(id, caseId, input) {
+        return this.request("POST", `/experiments/${uuid(id)}/items/${uuid(caseId)}/start`, input);
+    }
+    getExecution(id) {
+        return this.request("GET", `/experiment-executions/${uuid(id)}`);
+    }
+    completeExecution(id, input) {
+        return this.request("POST", `/experiment-executions/${uuid(id)}/complete`, input);
+    }
+    finishExperiment(id, idempotencyKey) {
+        return this.request("POST", `/experiments/${uuid(id)}/finish`, { idempotencyKey });
+    }
+    createEvaluationRun(input) {
+        return this.request("POST", "/evaluation-runs", input);
+    }
+    getEvaluationRun(id) {
+        return this.request("GET", `/evaluation-runs/${uuid(id)}`);
+    }
+    listEvaluationItems(id, page) {
+        return this.request("GET", `/evaluation-runs/${uuid(id)}/items${this.page(page)}`);
+    }
+    getSubject(id) {
+        return this.request("GET", `/evaluation-subjects/${uuid(id)}`);
+    }
+    submitResults(id, input) {
+        return this.request("POST", `/evaluation-runs/${uuid(id)}/results`, input);
+    }
+    listResults(id, page) {
+        return this.request("GET", `/evaluation-runs/${uuid(id)}/results${this.page(page)}`);
+    }
+    getResult(id) {
+        return this.request("GET", `/evaluation-results/${uuid(id)}`);
+    }
+    createJudgeJobs(id, input) {
+        return this.request("POST", `/evaluation-runs/${uuid(id)}/judge-jobs`, input);
+    }
+    listJudgeJobs(id, page) {
+        return this.request("GET", `/evaluation-runs/${uuid(id)}/judge-jobs${this.page(page)}`);
+    }
+    getJudgeJob(id) {
+        return this.request("GET", `/judge-jobs/${uuid(id)}`);
+    }
+    cancelJudgeJob(id, reason) {
+        return this.request("POST", `/judge-jobs/${uuid(id)}/cancel`, { reason });
+    }
+    getJudgeBudget() {
+        return this.request("GET", "/judge-budget");
+    }
+}
+export function createEvaluationClient(options) {
+    return new EvaluationClient(options);
+}

package/dist/evals/json.d.ts

@@ -0,0 +1,6 @@
+import type { JsonValue } from "../types.js";
+/** Reject lossy JSON serialization before creating requests/checkpoints. */
+export declare function json(value: unknown, maxBytes?: number): JsonValue;
+export declare function digest(value: unknown): string;
+export declare function sourceDigest(source: string | Uint8Array): string;
+export declare function uuid(value: string): string;

package/dist/evals/json.js

@@ -0,0 +1,61 @@
+import { createHash } from "node:crypto";
+/** Reject lossy JSON serialization before creating requests/checkpoints. */
+export function json(value, maxBytes = 200_000) {
+    const ancestors = new Set();
+    let nodes = 0;
+    const visit = (item, depth) => {
+        if (++nodes > 20_000 || depth > 32)
+            throw new RangeError("JSON exceeds depth/node limits");
+        if (item === null || typeof item === "boolean")
+            return item;
+        if (typeof item === "number" && Number.isFinite(item))
+            return item;
+        if (typeof item === "string" && item.isWellFormed() && !item.includes("\u0000"))
+            return item;
+        if (!item || typeof item !== "object" || ancestors.has(item))
+            throw new TypeError("Expected finite JSON without cycles or invalid Unicode");
+        ancestors.add(item);
+        try {
+            if (Array.isArray(item)) {
+                if (Object.keys(item).length !== item.length)
+                    throw new TypeError("Sparse/extended arrays are not JSON");
+                return item.map((entry) => visit(entry, depth + 1));
+            }
+            if (Object.getPrototypeOf(item) !== Object.prototype && Object.getPrototypeOf(item) !== null)
+                throw new TypeError("JSON objects must be plain objects");
+            if (Object.getOwnPropertySymbols(item).length)
+                throw new TypeError("JSON cannot contain symbol properties");
+            const result = Object.create(null);
+            for (const key of Object.keys(item).sort()) {
+                visit(key, depth + 1);
+                const descriptor = Object.getOwnPropertyDescriptor(item, key);
+                if (!("value" in descriptor))
+                    throw new TypeError("JSON cannot contain accessors");
+                result[key] = visit(descriptor.value, depth + 1);
+            }
+            return result;
+        }
+        finally {
+            // Repeated siblings serialize independently; only an active ancestor is a cycle.
+            ancestors.delete(item);
+        }
+    };
+    const result = visit(value, 0);
+    if (Buffer.byteLength(JSON.stringify(result)) > maxBytes)
+        throw new RangeError("JSON exceeds byte limit");
+    return result;
+}
+export function digest(value) {
+    return createHash("sha256")
+        .update(JSON.stringify(json(value, 8 * 1024 * 1024)))
+        .digest("hex");
+}
+export function sourceDigest(source) {
+    return createHash("sha256").update(source).digest("hex");
+}
+export function uuid(value) {
+    if (typeof value !== "string" ||
+        !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value))
+        throw new TypeError("Expected a UUID");
+    return value;
+}

package/dist/evals/runner.d.ts

@@ -0,0 +1,49 @@
+import type { HueClient } from "../client.js";
+import type { HueSpan } from "../types.js";
+import { EvaluationClient } from "./client.js";
+import type { ExperimentCase, JsonValue, LocalScorer } from "./types.js";
+export declare class UncertainExecutionError extends Error {
+    readonly caseId: string;
+    readonly executionId?: string | undefined;
+    constructor(caseId: string, executionId?: string | undefined);
+}
+export declare class OutcomeSerializationError extends Error {
+    readonly executionId: string;
+    constructor(executionId: string);
+}
+interface RunnerOptions {
+    client: EvaluationClient;
+    checkpointDirectory: string;
+    persistResultContent: boolean;
+    scorers?: LocalScorer[];
+    concurrency?: number;
+    schemaTimeoutMillis?: number;
+}
+export interface RunExperimentOptions extends RunnerOptions {
+    hue: HueClient;
+    experimentId: string;
+    traceEvidence: {
+        mode: "required";
+    } | {
+        mode: "omit";
+        reason: string;
+    };
+    target(inputs: JsonValue, context: {
+        config: JsonValue;
+        item: ExperimentCase;
+        span: HueSpan;
+    }): JsonValue | undefined | Promise<JsonValue | undefined>;
+}
+export interface RescoreOptions extends RunnerOptions {
+    runId: string;
+}
+export interface RunnerReport {
+    runId: string;
+    subjectIds: string[];
+    resultIds: string[];
+    deferredScorerVersionIds: string[];
+}
+export declare function runExperiment(options: RunExperimentOptions): Promise<RunnerReport>;
+/** Scores existing immutable subjects; this API has no target callback. */
+export declare function rescore(options: RescoreOptions): Promise<RunnerReport>;
+export {};

package/dist/evals/runner.js

@@ -0,0 +1,369 @@
+import { randomUUID } from "node:crypto";
+import { ROOT_CONTEXT } from "@opentelemetry/api";
+import { HueExportError } from "../transport.js";
+import { CheckpointStore } from "./checkpoint.js";
+import { json, uuid } from "./json.js";
+import { persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
+export class UncertainExecutionError extends Error {
+    caseId;
+    executionId;
+    constructor(caseId, executionId) {
+        super("Execution has no saved outcome. The target will not run again. Inspect it and explicitly authorize a new attempt through startExecution if required.");
+        this.caseId = caseId;
+        this.executionId = executionId;
+        this.name = "UncertainExecutionError";
+    }
+}
+export class OutcomeSerializationError extends Error {
+    executionId;
+    constructor(executionId) {
+        super("Target completed, but its output could not be serialized. Resolve completion explicitly; the runner will not invoke the target again.");
+        this.executionId = executionId;
+        this.name = "OutcomeSerializationError";
+    }
+}
+function settings(options) {
+    if (typeof options.persistResultContent !== "boolean")
+        throw new TypeError("Choose persistResultContent explicitly: true or false");
+    const concurrency = options.concurrency ?? 1;
+    if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16)
+        throw new RangeError("concurrency must be 1–16");
+    const timeout = options.schemaTimeoutMillis ?? 2000;
+    if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
+        throw new RangeError("schemaTimeoutMillis must be 100–60000");
+    return concurrency;
+}
+async function allPages(page) {
+    const items = [];
+    const cursors = new Set();
+    let after;
+    do {
+        const response = await page(after);
+        items.push(...response.items);
+        if (items.length > 5000)
+            throw new RangeError("Runner supports at most 5000 items");
+        if (response.nextCursor === null)
+            break;
+        after = uuid(response.nextCursor);
+        if (cursors.has(after))
+            throw new Error("API pagination repeated a cursor");
+        cursors.add(after);
+    } while (after !== undefined);
+    return items;
+}
+async function pool(items, concurrency, execute) {
+    let position = 0;
+    const failures = [];
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+        while (position < items.length && !failures.length) {
+            const item = items[position++];
+            try {
+                await execute(item);
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+    }));
+    if (failures.length === 1)
+        throw failures[0];
+    if (failures.length)
+        throw new AggregateError(failures, "Multiple case operations failed; resume uses saved outcomes");
+}
+async function scoresFor(versions, context, options) {
+    const scores = [];
+    for (const version of versions) {
+        // Hosted/manual pins remain pending for their authorized executor.
+        if (version.definition.kind === "llm_judge" || version.definition.kind === "manual")
+            continue;
+        const score = persistedScore(await scoreLocally(version, context, options), options.persistResultContent);
+        scores.push({
+            key: randomUUID(),
+            payload: {
+                ...score,
+                scorerVersionId: version.id,
+                ...(version.definition.kind === "local_code"
+                    ? { sourceDigest: version.definition.sourceDigest }
+                    : {}),
+            },
+        });
+    }
+    return scores;
+}
+async function uploadScores(options, runId, scores, save) {
+    for (const score of scores) {
+        if (score.receipt)
+            continue;
+        if (!score.payload.evaluationItemId)
+            throw new Error("Scoring requires the acknowledged evaluation item identity");
+        const result = await options.client.submitResults(runId, {
+            idempotencyKey: score.key,
+            results: [score.payload],
+        });
+        score.receipt = result.ids;
+        await save();
+    }
+}
+export async function runExperiment(options) {
+    const concurrency = settings(options);
+    if (!options.traceEvidence || !["required", "omit"].includes(options.traceEvidence.mode))
+        throw new TypeError("Choose a trace evidence policy explicitly");
+    if (options.traceEvidence.mode === "omit" &&
+        (!options.traceEvidence.reason?.trim() || options.traceEvidence.reason.length > 4000))
+        throw new TypeError("Omitting evidence requires a bounded reason");
+    if (options.hue.transport.options.baseUrl !== options.client.baseUrl)
+        throw new Error("Telemetry and evaluations must use the same Hue origin");
+    const [project, telemetryProject, experiment] = await Promise.all([
+        options.client.checkConnection(),
+        options.hue.checkConnection(),
+        options.client.getExperiment(options.experimentId),
+    ]);
+    if (project.id !== telemetryProject.id)
+        throw new Error("Telemetry and evaluations must use the same Hue project");
+    const version = await options.client.getDatasetVersion(experiment.datasetVersionId);
+    if (!version.frozenAt || !version.contentDigest)
+        throw new Error("Experiment dataset must be frozen");
+    const versions = experiment.evaluation.scorerVersions;
+    validateScorerBindings(versions, options.scorers);
+    const items = await allPages((after) => options.client.listExperimentItems(experiment.id, { after }));
+    if (items.length !== experiment.caseCount)
+        throw new Error("Frozen experiment case count differs from API items");
+    const store = await CheckpointStore.acquire(options.checkpointDirectory, {
+        kind: "experiment",
+        projectId: project.id,
+        baseUrl: options.client.baseUrl,
+        experimentId: experiment.id,
+        datasetVersionId: version.id,
+        datasetDigest: version.contentDigest,
+        configDigest: experiment.configDigest,
+        pins: versions
+            .map(({ id, contentDigest }) => ({ id, contentDigest }))
+            .sort((a, b) => a.id.localeCompare(b.id)),
+        persistResultContent: options.persistResultContent,
+        captureContent: options.hue.captureContent,
+        traceEvidence: options.traceEvidence,
+    });
+    const report = {
+        runId: experiment.evaluation.id,
+        subjectIds: [],
+        resultIds: [],
+        deferredScorerVersionIds: versions
+            .filter((version) => ["llm_judge", "manual"].includes(version.definition.kind))
+            .map((version) => version.id),
+    };
+    try {
+        await pool(items, concurrency, async (item) => {
+            const file = `case-${uuid(item.id)}`;
+            let checkpoint = await store.read(file);
+            if (checkpoint && checkpoint.stage !== "prepared") {
+                if (checkpoint.stage === "serialization_failed")
+                    throw new OutcomeSerializationError(checkpoint.executionId);
+                const execution = checkpoint.stage === "starting"
+                    ? await options.client.startExecution(experiment.id, item.id, {
+                        idempotencyKey: checkpoint.startKey,
+                        traceExternalId: checkpoint.traceExternalId,
+                    })
+                    : await options.client.getExecution(checkpoint.executionId);
+                throw new UncertainExecutionError(item.id, execution.id);
+            }
+            if (!checkpoint) {
+                if (item.execution)
+                    throw new UncertainExecutionError(item.id, item.execution.id);
+                const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
+                if (frozenCase.datasetVersionId !== version.id)
+                    throw new Error("Case is not from the pinned dataset version");
+                const failureSequenceBefore = options.hue.transport.getFailureSequence();
+                checkpoint = await options.hue.withSpan("hue.experiment.case", async (span) => {
+                    const start = {
+                        stage: "starting",
+                        startKey: randomUUID(),
+                        traceExternalId: span.traceId,
+                    };
+                    await store.write(file, start);
+                    const execution = await options.client.startExecution(experiment.id, item.id, {
+                        idempotencyKey: start.startKey,
+                        traceExternalId: span.traceId,
+                    });
+                    await store.write(file, { stage: "running", executionId: execution.id });
+                    let state = "succeeded";
+                    let output;
+                    let targetError;
+                    try {
+                        output = await options.target(json(frozenCase.inputs), {
+                            config: json(experiment.config),
+                            item: structuredClone(frozenCase),
+                            span,
+                        });
+                    }
+                    catch (error) {
+                        state = "error";
+                        targetError = error;
+                        options.hue.recordError(span.span, error);
+                    }
+                    // Output validation is outside the target catch: a successful target never becomes an agent error because upload serialization failed.
+                    if (output !== undefined) {
+                        try {
+                            output = json(output);
+                        }
+                        catch {
+                            await store.write(file, {
+                                stage: "serialization_failed",
+                                executionId: execution.id,
+                            });
+                            throw new OutcomeSerializationError(execution.id);
+                        }
+                    }
+                    if (output !== undefined)
+                        span.setOutput(output);
+                    const scores = await scoresFor(versions, {
+                        inputs: frozenCase.inputs,
+                        hasExpected: frozenCase.hasExpected,
+                        ...(frozenCase.hasExpected ? { expected: frozenCase.expected } : {}),
+                        metadata: frozenCase.metadata,
+                        hasOutput: output !== undefined,
+                        ...(output !== undefined ? { output } : {}),
+                        executionState: state,
+                    }, options);
+                    const complete = {
+                        idempotencyKey: randomUUID(),
+                        state,
+                        ...(options.persistResultContent && output !== undefined ? { output } : {}),
+                        ...(state === "error"
+                            ? {
+                                error: {
+                                    type: "TargetError",
+                                    ...(options.persistResultContent && targetError instanceof Error
+                                        ? {
+                                            message: targetError.message
+                                                .slice(0, 4000)
+                                                .toWellFormed()
+                                                .replaceAll("\u0000", ""),
+                                        }
+                                        : {}),
+                                },
+                            }
+                            : {}),
+                        traceEvidence: options.traceEvidence.mode,
+                        ...(options.traceEvidence.mode === "omit"
+                            ? { omissionReason: options.traceEvidence.reason }
+                            : {}),
+                    };
+                    const prepared = {
+                        stage: "prepared",
+                        executionId: execution.id,
+                        complete,
+                        scores,
+                        exportState: "pending",
+                    };
+                    await store.write(file, prepared);
+                    return prepared;
+                }, {
+                    parentContext: ROOT_CONTEXT,
+                    input: frozenCase.inputs,
+                    attributes: { "hue.experiment.id": experiment.id, "hue.dataset.case.id": item.id },
+                });
+                // End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
+                try {
+                    await options.hue.flush();
+                    // Another concurrent flush may already have surfaced this failure. OTLP
+                    // partial acknowledgements do not identify individual rejected records.
+                    if (options.hue.transport.getFailureSequence() !== failureSequenceBefore)
+                        throw new HueExportError(options.hue.transport
+                            .getIssues()
+                            .filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning"), options.hue.transport.getReport());
+                    checkpoint.exportState = "accepted";
+                    await store.write(file, checkpoint);
+                }
+                catch (error) {
+                    // The explicitly chosen omission policy can complete without acknowledged telemetry.
+                    if (options.traceEvidence.mode !== "omit")
+                        throw error;
+                }
+            }
+            const prepared = checkpoint;
+            await options.client.getExecution(prepared.executionId);
+            if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
+                throw new Error("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
+            const save = () => store.write(file, prepared);
+            if (!prepared.completion) {
+                prepared.completion = await options.client.completeExecution(prepared.executionId, prepared.complete);
+                for (const score of prepared.scores)
+                    score.payload.evaluationItemId = prepared.completion.evaluationItemId;
+                await save();
+            }
+            await uploadScores(options, experiment.evaluation.id, prepared.scores, save);
+            report.subjectIds.push(prepared.completion.subjectId);
+            report.resultIds.push(...prepared.scores.flatMap((score) => score.receipt ?? []));
+        });
+        let finish = await store.read("finish");
+        if (!finish) {
+            finish = { key: randomUUID() };
+            await store.write("finish", finish);
+        }
+        await options.client.finishExperiment(experiment.id, finish.key);
+        return report;
+    }
+    finally {
+        await store.release();
+    }
+}
+/** Scores existing immutable subjects; this API has no target callback. */
+export async function rescore(options) {
+    const concurrency = settings(options);
+    const [project, run] = await Promise.all([
+        options.client.checkConnection(),
+        options.client.getEvaluationRun(options.runId),
+    ]);
+    validateScorerBindings(run.scorerVersions, options.scorers);
+    const items = await allPages((after) => options.client.listEvaluationItems(run.id, { after }));
+    if (items.length !== run.itemCount)
+        throw new Error("Frozen evaluation run item count differs from API items");
+    const store = await CheckpointStore.acquire(options.checkpointDirectory, {
+        kind: "rescore",
+        projectId: project.id,
+        baseUrl: options.client.baseUrl,
+        runId: run.id,
+        pins: run.scorerVersions
+            .map(({ id, contentDigest }) => ({ id, contentDigest }))
+            .sort((a, b) => a.id.localeCompare(b.id)),
+        persistResultContent: options.persistResultContent,
+    });
+    const report = {
+        runId: run.id,
+        subjectIds: [],
+        resultIds: [],
+        deferredScorerVersionIds: run.scorerVersions
+            .filter((version) => ["llm_judge", "manual"].includes(version.definition.kind))
+            .map((version) => version.id),
+    };
+    try {
+        await pool(items, concurrency, async (item) => {
+            const file = `item-${uuid(item.id)}`;
+            let saved = await store.read(file);
+            if (!saved) {
+                const subject = await options.client.getSubject(item.subjectId);
+                const scores = await scoresFor(run.scorerVersions, {
+                    inputs: subject.inputs,
+                    hasOutput: subject.hasOutput,
+                    hasExpected: subject.hasExpected,
+                    ...(subject.hasOutput ? { output: subject.output } : {}),
+                    ...(subject.hasExpected ? { expected: subject.expected } : {}),
+                    metadata: subject.metadata,
+                    executionState: subject.executionState,
+                }, options);
+                for (const score of scores)
+                    score.payload.evaluationItemId = item.id;
+                saved = { scores };
+                await store.write(file, saved);
+            }
+            const current = saved;
+            await uploadScores(options, run.id, current.scores, () => store.write(file, current));
+            report.subjectIds.push(item.subjectId);
+            report.resultIds.push(...current.scores.flatMap((score) => score.receipt ?? []));
+        });
+        return report;
+    }
+    finally {
+        await store.release();
+    }
+}

package/dist/evals/schema-worker.d.ts

@@ -0,0 +1 @@
+export {};