@arizeai/phoenix-client 2.3.2 → 2.3.4

package/dist/src/experiments/runExperiment.js

@@ -1,13 +1,4 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -59,138 +50,141 @@ const urlUtils_1 = require("../utils/urlUtils");
  * });
  * ```
  */
-function runExperiment(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ experimentName, experimentDescription, experimentMetadata = {}, client: _client, dataset: DatasetSelector, task, evaluators, logger = console, record = true, concurrency = 5, dryRun = false, }) {
-        var _b, _c, _d, _e, _f;
-        let provider;
-        const isDryRun = typeof dryRun === "number" || dryRun === true;
-        const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
-        const dataset = yield (0, getDataset_1.getDataset)({ dataset: DatasetSelector, client });
-        (0, tiny_invariant_1.default)(dataset, `Dataset not found`);
-        (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset has no examples`);
-        const nExamples = typeof dryRun === "number"
-            ? Math.min(dryRun, dataset.examples.length)
-            : dataset.examples.length;
-        let projectName = `${dataset.name}-exp-${new Date().toISOString()}`;
-        // initialize the tracer into scope
-        let taskTracer;
-        let experiment;
-        if (isDryRun) {
-            experiment = {
-                id: localId(),
-                datasetId: dataset.id,
-                datasetVersionId: dataset.versionId,
-                projectName,
-                metadata: experimentMetadata,
-            };
-            taskTracer = (0, instrumention_1.createNoOpProvider)().getTracer("no-op");
-        }
-        else {
-            const experimentResponse = yield client
-                .POST("/v1/datasets/{dataset_id}/experiments", {
-                params: {
-                    path: {
-                        dataset_id: dataset.id,
-                    },
-                },
-                body: {
-                    name: experimentName,
-                    description: experimentDescription,
-                    metadata: experimentMetadata,
-                    project_name: projectName,
+async function runExperiment({ experimentName, experimentDescription, experimentMetadata = {}, client: _client, dataset: DatasetSelector, task, evaluators, logger = console, record = true, concurrency = 5, dryRun = false, setGlobalTracerProvider = true, }) {
+    var _a, _b, _c, _d, _e;
+    let provider;
+    const isDryRun = typeof dryRun === "number" || dryRun === true;
+    const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
+    const dataset = await (0, getDataset_1.getDataset)({ dataset: DatasetSelector, client });
+    (0, tiny_invariant_1.default)(dataset, `Dataset not found`);
+    (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset has no examples`);
+    const nExamples = typeof dryRun === "number"
+        ? Math.min(dryRun, dataset.examples.length)
+        : dataset.examples.length;
+    let projectName = `${dataset.name}-exp-${new Date().toISOString()}`;
+    // initialize the tracer into scope
+    let taskTracer;
+    let experiment;
+    if (isDryRun) {
+        experiment = {
+            id: localId(),
+            datasetId: dataset.id,
+            datasetVersionId: dataset.versionId,
+            projectName,
+            metadata: experimentMetadata,
+        };
+        taskTracer = (0, instrumention_1.createNoOpProvider)().getTracer("no-op");
+    }
+    else {
+        const experimentResponse = await client
+            .POST("/v1/datasets/{dataset_id}/experiments", {
+            params: {
+                path: {
+                    dataset_id: dataset.id,
                 },
-            })
-                .then((res) => { var _a; return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data; });
-            (0, tiny_invariant_1.default)(experimentResponse, `Failed to create experiment`);
-            projectName = (_b = experimentResponse.project_name) !== null && _b !== void 0 ? _b : projectName;
-            experiment = {
-                id: experimentResponse.id,
-                datasetId: experimentResponse.dataset_id,
-                datasetVersionId: experimentResponse.dataset_version_id,
-                projectName,
-                metadata: experimentResponse.metadata,
-            };
-            // Initialize the tracer, now that we have a project name
-            const baseUrl = client.config.baseUrl;
-            (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
-            provider = (0, instrumention_1.createProvider)({
-                projectName,
-                baseUrl,
-                headers: (_c = client.config.headers) !== null && _c !== void 0 ? _c : {},
-            });
-            taskTracer = provider.getTracer(projectName);
-        }
-        if (!record) {
-            logger.info(`🔧 Running experiment in readonly mode. Results will not be recorded.`);
-        }
-        if (!isDryRun && client.config.baseUrl) {
-            const datasetUrl = (0, urlUtils_1.getDatasetUrl)({
-                baseUrl: client.config.baseUrl,
-                datasetId: dataset.id,
-            });
-            const datasetExperimentsUrl = (0, urlUtils_1.getDatasetExperimentsUrl)({
-                baseUrl: client.config.baseUrl,
-                datasetId: dataset.id,
-            });
-            const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
-                baseUrl: client.config.baseUrl,
-                datasetId: dataset.id,
-                experimentId: experiment.id,
-            });
-            logger.info(`📊 View dataset: ${datasetUrl}`);
-            logger.info(`📺 View dataset experiments: ${datasetExperimentsUrl}`);
-            logger.info(`🔗 View this experiment: ${experimentUrl}`);
-        }
-        logger.info(`🧪 Starting experiment "${experimentName || `<unnamed>`}" on dataset "${dataset.id}" with task "${task.name}" and ${(_d = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _d !== void 0 ? _d : 0} ${(0, pluralize_1.pluralize)("evaluator", (_e = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _e !== void 0 ? _e : 0)} and ${concurrency} concurrent runs`);
-        const runs = {};
-        yield runTaskWithExamples({
-            client,
-            experimentId: experiment.id,
-            task,
-            dataset,
-            logger,
-            onComplete: (run) => {
-                runs[run.id] = run;
             },
-            concurrency,
-            isDryRun,
-            nExamples,
-            tracer: taskTracer,
+            body: {
+                name: experimentName,
+                description: experimentDescription,
+                metadata: experimentMetadata,
+                project_name: projectName,
+            },
+        })
+            .then((res) => { var _a; return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data; });
+        (0, tiny_invariant_1.default)(experimentResponse, `Failed to create experiment`);
+        projectName = (_a = experimentResponse.project_name) !== null && _a !== void 0 ? _a : projectName;
+        experiment = {
+            id: experimentResponse.id,
+            datasetId: experimentResponse.dataset_id,
+            datasetVersionId: experimentResponse.dataset_version_id,
+            projectName,
+            metadata: experimentResponse.metadata,
+        };
+        // Initialize the tracer, now that we have a project name
+        const baseUrl = client.config.baseUrl;
+        (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
+        provider = (0, instrumention_1.createProvider)({
+            projectName,
+            baseUrl,
+            headers: (_b = client.config.headers) !== null && _b !== void 0 ? _b : {},
         });
-        logger.info(`✅ Task runs completed`);
-        const ranExperiment = Object.assign(Object.assign({}, experiment), { runs });
-        // Shut down the provider so that the experiments run
-        if (provider) {
-            yield ((_f = provider.shutdown) === null || _f === void 0 ? void 0 : _f.call(provider));
+        // Register the provider
+        if (setGlobalTracerProvider) {
+            provider.register();
         }
-        const { evaluationRuns } = yield evaluateExperiment({
-            experiment: ranExperiment,
-            evaluators: evaluators !== null && evaluators !== void 0 ? evaluators : [],
-            client,
-            logger,
-            concurrency,
-            dryRun,
+        taskTracer = provider.getTracer(projectName);
+    }
+    if (!record) {
+        logger.info(`🔧 Running experiment in readonly mode. Results will not be recorded.`);
+    }
+    if (!isDryRun && client.config.baseUrl) {
+        const datasetUrl = (0, urlUtils_1.getDatasetUrl)({
+            baseUrl: client.config.baseUrl,
+            datasetId: dataset.id,
         });
-        ranExperiment.evaluationRuns = evaluationRuns;
-        logger.info(`✅ Experiment ${experiment.id} completed`);
-        if (!isDryRun && client.config.baseUrl) {
-            const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
-                baseUrl: client.config.baseUrl,
-                datasetId: dataset.id,
-                experimentId: experiment.id,
-            });
-            logger.info(`🔍 View results: ${experimentUrl}`);
-        }
-        return ranExperiment;
+        const datasetExperimentsUrl = (0, urlUtils_1.getDatasetExperimentsUrl)({
+            baseUrl: client.config.baseUrl,
+            datasetId: dataset.id,
+        });
+        const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
+            baseUrl: client.config.baseUrl,
+            datasetId: dataset.id,
+            experimentId: experiment.id,
+        });
+        logger.info(`📊 View dataset: ${datasetUrl}`);
+        logger.info(`📺 View dataset experiments: ${datasetExperimentsUrl}`);
+        logger.info(`🔗 View this experiment: ${experimentUrl}`);
+    }
+    logger.info(`🧪 Starting experiment "${experimentName || `<unnamed>`}" on dataset "${dataset.id}" with task "${task.name}" and ${(_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0} ${(0, pluralize_1.pluralize)("evaluator", (_d = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _d !== void 0 ? _d : 0)} and ${concurrency} concurrent runs`);
+    const runs = {};
+    await runTaskWithExamples({
+        client,
+        experimentId: experiment.id,
+        task,
+        dataset,
+        logger,
+        onComplete: (run) => {
+            runs[run.id] = run;
+        },
+        concurrency,
+        isDryRun,
+        nExamples,
+        tracer: taskTracer,
     });
+    logger.info(`✅ Task runs completed`);
+    const ranExperiment = Object.assign(Object.assign({}, experiment), { runs });
+    // Shut down the provider so that the experiments run
+    if (provider) {
+        await ((_e = provider.shutdown) === null || _e === void 0 ? void 0 : _e.call(provider));
+    }
+    const { evaluationRuns } = await evaluateExperiment({
+        experiment: ranExperiment,
+        evaluators: evaluators !== null && evaluators !== void 0 ? evaluators : [],
+        client,
+        logger,
+        concurrency,
+        dryRun,
+        setGlobalTracerProvider,
+    });
+    ranExperiment.evaluationRuns = evaluationRuns;
+    logger.info(`✅ Experiment ${experiment.id} completed`);
+    if (!isDryRun && client.config.baseUrl) {
+        const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
+            baseUrl: client.config.baseUrl,
+            datasetId: dataset.id,
+            experimentId: experiment.id,
+        });
+        logger.info(`🔍 View results: ${experimentUrl}`);
+    }
+    return ranExperiment;
 }
 /**
  * Run a task against n examples in a dataset.
  */
 function runTaskWithExamples({ client, experimentId, task, dataset, onComplete, logger, concurrency = 5, isDryRun, nExamples, tracer, }) {
     logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
-    const run = (example) => __awaiter(this, void 0, void 0, function* () {
-        return tracer.startActiveSpan(`Task: ${task.name}`, (span) => __awaiter(this, void 0, void 0, function* () {
+    const run = async (example) => {
+        return tracer.startActiveSpan(`Task: ${task.name}`, async (span) => {
             var _a, _b;
             logger.info(`🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`);
             const traceId = span.spanContext().traceId;
@@ -205,7 +199,7 @@ function runTaskWithExamples({ client, experimentId, task, dataset, onComplete,
                 error: null,
             };
             try {
-                const taskOutput = yield (0, promisifyResult_1.promisifyResult)(task(example));
+                const taskOutput = await (0, promisifyResult_1.promisifyResult)(task(example));
                 thisRun.output = taskOutput;
             }
             catch (error) {
@@ -216,7 +210,7 @@ function runTaskWithExamples({ client, experimentId, task, dataset, onComplete,
             thisRun.endTime = new Date();
             if (!isDryRun) {
                 // Log the run to the server
-                const res = yield client.POST("/v1/experiments/{experiment_id}/runs", {
+                const res = await client.POST("/v1/experiments/{experiment_id}/runs", {
                     params: {
                         path: {
                             experiment_id: experimentId,
@@ -248,8 +242,8 @@ function runTaskWithExamples({ client, experimentId, task, dataset, onComplete,
             span === null || span === void 0 ? void 0 : span.end();
             onComplete(thisRun);
             return thisRun;
-        }));
-    });
+        });
+    };
     const q = (0, async_1.queue)(run, concurrency);
     const examplesToUse = dataset.examples.slice(0, nExamples);
     examplesToUse.forEach((example) => q.push(example, (err) => {
@@ -264,179 +258,178 @@ function runTaskWithExamples({ client, experimentId, task, dataset, onComplete,
  *
  * @experimental This feature is not complete, and will change in the future.
  */
-function evaluateExperiment(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ experiment, evaluators, client: _client, logger = console, concurrency = 5, dryRun = false, }) {
-        var _b, _c, _d, _e;
-        const isDryRun = typeof dryRun === "number" || dryRun === true;
-        const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
-        const baseUrl = client.config.baseUrl;
-        (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
-        let provider;
-        if (!isDryRun) {
-            provider = (0, instrumention_1.createProvider)({
-                projectName: "evaluators",
-                baseUrl,
-                headers: (_b = client.config.headers) !== null && _b !== void 0 ? _b : {},
-            });
-        }
-        else {
-            provider = (0, instrumention_1.createNoOpProvider)();
-        }
-        const tracer = isDryRun
-            ? provider.getTracer("no-op")
-            : provider.getTracer("evaluators");
-        const nRuns = typeof dryRun === "number"
-            ? Math.max(dryRun, Object.keys(experiment.runs).length)
-            : Object.keys(experiment.runs).length;
-        const dataset = yield (0, getDataset_1.getDataset)({
-            dataset: { datasetId: experiment.datasetId },
-            client,
+async function evaluateExperiment({ experiment, evaluators, client: _client, logger = console, concurrency = 5, dryRun = false, setGlobalTracerProvider = true, }) {
+    var _a, _b, _c, _d;
+    const isDryRun = typeof dryRun === "number" || dryRun === true;
+    const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
+    const baseUrl = client.config.baseUrl;
+    (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
+    let provider;
+    if (!isDryRun) {
+        provider = (0, instrumention_1.createProvider)({
+            projectName: "evaluators",
+            baseUrl,
+            headers: (_a = client.config.headers) !== null && _a !== void 0 ? _a : {},
         });
-        (0, tiny_invariant_1.default)(dataset, `Dataset "${experiment.datasetId}" not found`);
-        (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset "${experiment.datasetId}" has no examples`);
-        (0, tiny_invariant_1.default)(experiment.runs, `Experiment "${experiment.id}" has no runs`);
-        const runsToEvaluate = Object.values(experiment.runs).slice(0, nRuns);
-        if ((evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) === 0) {
-            return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
+        if (setGlobalTracerProvider) {
+            provider.register();
         }
-        logger.info(`🧠 Evaluating experiment "${experiment.id}" with ${(_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0} ${(0, pluralize_1.pluralize)("evaluator", (_d = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _d !== void 0 ? _d : 0)}`);
-        if (!isDryRun && client.config.baseUrl) {
-            const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
-                baseUrl: client.config.baseUrl,
-                datasetId: experiment.datasetId,
-                experimentId: experiment.id,
+    }
+    else {
+        provider = (0, instrumention_1.createNoOpProvider)();
+    }
+    const tracer = isDryRun
+        ? provider.getTracer("no-op")
+        : provider.getTracer("evaluators");
+    const nRuns = typeof dryRun === "number"
+        ? Math.min(dryRun, Object.keys(experiment.runs).length)
+        : Object.keys(experiment.runs).length;
+    const dataset = await (0, getDataset_1.getDataset)({
+        dataset: { datasetId: experiment.datasetId },
+        client,
+    });
+    (0, tiny_invariant_1.default)(dataset, `Dataset "${experiment.datasetId}" not found`);
+    (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset "${experiment.datasetId}" has no examples`);
+    (0, tiny_invariant_1.default)(experiment.runs, `Experiment "${experiment.id}" has no runs`);
+    const runsToEvaluate = Object.values(experiment.runs).slice(0, nRuns);
+    if ((evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) === 0) {
+        return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
+    }
+    logger.info(`🧠 Evaluating experiment "${experiment.id}" with ${(_b = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _b !== void 0 ? _b : 0} ${(0, pluralize_1.pluralize)("evaluator", (_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0)}`);
+    if (!isDryRun && client.config.baseUrl) {
+        const experimentUrl = (0, urlUtils_1.getExperimentUrl)({
+            baseUrl: client.config.baseUrl,
+            datasetId: experiment.datasetId,
+            experimentId: experiment.id,
+        });
+        logger.info(`🔗 View experiment evaluation: ${experimentUrl}`);
+    }
+    const evaluationRuns = {};
+    const examplesById = {};
+    for (const example of dataset.examples) {
+        examplesById[example.id] = example;
+    }
+    const onEvaluationComplete = (run) => {
+        evaluationRuns[run.id] = run;
+    };
+    // Run evaluators against all runs
+    // Flat list of evaluator + run tuples
+    const evaluatorsAndRuns = evaluators.flatMap((evaluator) => runsToEvaluate.map((run) => ({
+        evaluator,
+        run,
+    })));
+    const evaluatorsQueue = (0, async_1.queue)(async (evaluatorAndRun) => {
+        return tracer.startActiveSpan(`Evaluation: ${evaluatorAndRun.evaluator.name}`, async (span) => {
+            var _a, _b, _c;
+            const evalResult = await runEvaluator({
+                evaluator: evaluatorAndRun.evaluator,
+                run: evaluatorAndRun.run,
+                exampleCache: examplesById,
+                onComplete: onEvaluationComplete,
+                logger,
             });
-            logger.info(`🔗 View experiment evaluation: ${experimentUrl}`);
-        }
-        const evaluationRuns = {};
-        const examplesById = {};
-        for (const example of dataset.examples) {
-            examplesById[example.id] = example;
-        }
-        const onEvaluationComplete = (run) => {
-            evaluationRuns[run.id] = run;
-        };
-        // Run evaluators against all runs
-        // Flat list of evaluator + run tuples
-        const evaluatorsAndRuns = evaluators.flatMap((evaluator) => runsToEvaluate.map((run) => ({
-            evaluator,
-            run,
-        })));
-        const evaluatorsQueue = (0, async_1.queue)((evaluatorAndRun) => __awaiter(this, void 0, void 0, function* () {
-            return tracer.startActiveSpan(`Evaluation: ${evaluatorAndRun.evaluator.name}`, (span) => __awaiter(this, void 0, void 0, function* () {
-                var _a, _b, _c;
-                const evalResult = yield runEvaluator({
-                    evaluator: evaluatorAndRun.evaluator,
-                    run: evaluatorAndRun.run,
-                    exampleCache: examplesById,
-                    onComplete: onEvaluationComplete,
-                    logger,
+            span.setAttributes({
+                [openinference_semantic_conventions_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: openinference_semantic_conventions_1.OpenInferenceSpanKind.EVALUATOR,
+                [openinference_semantic_conventions_1.SemanticConventions.INPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
+                [openinference_semantic_conventions_1.SemanticConventions.INPUT_VALUE]: JSON.stringify({
+                    input: (_a = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _a === void 0 ? void 0 : _a.input,
+                    output: evaluatorAndRun.run.output,
+                    expected: (_b = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _b === void 0 ? void 0 : _b.output,
+                    metadata: (_c = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _c === void 0 ? void 0 : _c.metadata,
+                }),
+                [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
+                [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(evalResult.result),
+            });
+            if (evalResult.error) {
+                span.setStatus({
+                    code: api_1.SpanStatusCode.ERROR,
+                    message: evalResult.error,
                 });
-                span.setAttributes({
-                    [openinference_semantic_conventions_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: openinference_semantic_conventions_1.OpenInferenceSpanKind.EVALUATOR,
-                    [openinference_semantic_conventions_1.SemanticConventions.INPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
-                    [openinference_semantic_conventions_1.SemanticConventions.INPUT_VALUE]: JSON.stringify({
-                        input: (_a = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _a === void 0 ? void 0 : _a.input,
-                        output: evaluatorAndRun.run.output,
-                        expected: (_b = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _b === void 0 ? void 0 : _b.output,
-                        metadata: (_c = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _c === void 0 ? void 0 : _c.metadata,
-                    }),
-                    [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
-                    [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(evalResult.result),
+            }
+            else {
+                span.setStatus({ code: api_1.SpanStatusCode.OK });
+            }
+            if (evalResult.result) {
+                span.setAttributes((0, objectAsAttributes_1.objectAsAttributes)(evalResult.result));
+            }
+            evalResult.traceId = span.spanContext().traceId;
+            if (!isDryRun) {
+                // Log the evaluation to the server
+                // We log this without awaiting (e.g. best effort)
+                client.POST("/v1/experiment_evaluations", {
+                    body: {
+                        experiment_run_id: evaluatorAndRun.run.id,
+                        name: evaluatorAndRun.evaluator.name,
+                        annotator_kind: evaluatorAndRun.evaluator.kind,
+                        start_time: evalResult.startTime.toISOString(),
+                        end_time: evalResult.endTime.toISOString(),
+                        result: Object.assign({}, evalResult.result),
+                        error: evalResult.error,
+                        trace_id: evalResult.traceId,
+                    },
                 });
-                if (evalResult.error) {
-                    span.setStatus({
-                        code: api_1.SpanStatusCode.ERROR,
-                        message: evalResult.error,
-                    });
-                }
-                else {
-                    span.setStatus({ code: api_1.SpanStatusCode.OK });
-                }
-                if (evalResult.result) {
-                    span.setAttributes((0, objectAsAttributes_1.objectAsAttributes)(evalResult.result));
-                }
-                evalResult.traceId = span.spanContext().traceId;
-                if (!isDryRun) {
-                    // Log the evaluation to the server
-                    // We log this without awaiting (e.g. best effort)
-                    client.POST("/v1/experiment_evaluations", {
-                        body: {
-                            experiment_run_id: evaluatorAndRun.run.id,
-                            name: evaluatorAndRun.evaluator.name,
-                            annotator_kind: evaluatorAndRun.evaluator.kind,
-                            start_time: evalResult.startTime.toISOString(),
-                            end_time: evalResult.endTime.toISOString(),
-                            result: Object.assign({}, evalResult.result),
-                            error: evalResult.error,
-                            trace_id: evalResult.traceId,
-                        },
-                    });
-                }
-                span.end();
-                return evalResult;
-            }));
-        }), concurrency);
-        if (!evaluatorsAndRuns.length) {
-            logger.info(`⛔ No evaluators to run`);
-            return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
-        }
-        evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun, (err) => {
-            if (err) {
-                logger.error(`❌ Error running evaluator "${evaluatorAndRun.evaluator.name}" on run "${evaluatorAndRun.run.id}": ${err}`);
             }
-        }));
-        yield evaluatorsQueue.drain();
-        logger.info(`✅ Evaluation runs completed`);
-        if (provider) {
-            yield ((_e = provider.shutdown) === null || _e === void 0 ? void 0 : _e.call(provider));
+            span.end();
+            return evalResult;
+        });
+    }, concurrency);
+    if (!evaluatorsAndRuns.length) {
+        logger.info(`⛔ No evaluators to run`);
+        return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
+    }
+    evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun, (err) => {
+        if (err) {
+            logger.error(`❌ Error running evaluator "${evaluatorAndRun.evaluator.name}" on run "${evaluatorAndRun.run.id}": ${err}`);
         }
-        return Object.assign(Object.assign({}, experiment), { evaluationRuns: Object.values(evaluationRuns) });
-    });
+    }));
+    await evaluatorsQueue.drain();
+    logger.info(`✅ Evaluation runs completed`);
+    if (provider) {
+        await ((_d = provider.shutdown) === null || _d === void 0 ? void 0 : _d.call(provider));
+    }
+    return Object.assign(Object.assign({}, experiment), { evaluationRuns: Object.values(evaluationRuns) });
 }
 /**
  * Run an evaluator against a run.
  *
  * @experimental This feature is not complete, and will change in the future.
  */
-function runEvaluator(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ evaluator, run, exampleCache, onComplete, logger, }) {
-        const example = exampleCache[run.datasetExampleId];
-        (0, tiny_invariant_1.default)(example, `Example "${run.datasetExampleId}" not found`);
-        const evaluate = () => __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            logger.info(`🧠 Evaluating run "${run.id}" with evaluator "${evaluator.name}"`);
-            const thisEval = {
-                id: localId(),
-                traceId: null,
-                experimentRunId: run.id,
-                startTime: new Date(),
-                endTime: new Date(), // will get replaced with actual end time
-                name: evaluator.name,
-                result: null,
-                error: null,
-                annotatorKind: evaluator.kind,
-            };
-            try {
-                const result = yield evaluator.evaluate({
-                    input: example.input,
-                    output: (_a = run.output) !== null && _a !== void 0 ? _a : null,
-                    expected: example.output,
-                    metadata: example.metadata,
-                });
-                thisEval.result = result;
-                logger.info(`✅ Evaluator "${evaluator.name}" on run "${run.id}" completed`);
-            }
-            catch (error) {
-                thisEval.error = error instanceof Error ? error.message : "Unknown error";
-                logger.error(`❌ Evaluator "${evaluator.name}" on run "${run.id}" failed: ${thisEval.error}`);
-            }
-            thisEval.endTime = new Date();
-            onComplete(thisEval);
-            return thisEval;
-        });
-        return evaluate();
-    });
+async function runEvaluator({ evaluator, run, exampleCache, onComplete, logger, }) {
+    const example = exampleCache[run.datasetExampleId];
+    (0, tiny_invariant_1.default)(example, `Example "${run.datasetExampleId}" not found`);
+    const evaluate = async () => {
+        var _a;
+        logger.info(`🧠 Evaluating run "${run.id}" with evaluator "${evaluator.name}"`);
+        const thisEval = {
+            id: localId(),
+            traceId: null,
+            experimentRunId: run.id,
+            startTime: new Date(),
+            endTime: new Date(), // will get replaced with actual end time
+            name: evaluator.name,
+            result: null,
+            error: null,
+            annotatorKind: evaluator.kind,
+        };
+        try {
+            const result = await evaluator.evaluate({
+                input: example.input,
+                output: (_a = run.output) !== null && _a !== void 0 ? _a : null,
+                expected: example.output,
+                metadata: example.metadata,
+            });
+            thisEval.result = result;
+            logger.info(`✅ Evaluator "${evaluator.name}" on run "${run.id}" completed`);
+        }
+        catch (error) {
+            thisEval.error = error instanceof Error ? error.message : "Unknown error";
+            logger.error(`❌ Evaluator "${evaluator.name}" on run "${run.id}" failed: ${thisEval.error}`);
+        }
+        thisEval.endTime = new Date();
+        onComplete(thisEval);
+        return thisEval;
+    };
+    return evaluate();
 }
 /**
  * Wrap an evaluator function in an object with a name property.