@arizeai/phoenix-client 1.0.2 → 1.2.0

package/dist/esm/experiments/runExperiment.js

@@ -1,40 +1,80 @@
-import { createClient } from "../client.js";
-import { promisifyResult } from "../utils/promisifyResult.js";
+import { queue } from "async";
 import invariant from "tiny-invariant";
-import { pluralize } from "../utils/pluralize.js";
+import { createClient } from "../client.js";
 import { getDatasetBySelector } from "../utils/getDatasetBySelector.js";
+import { pluralize } from "../utils/pluralize.js";
+import { promisifyResult } from "../utils/promisifyResult.js";
 /**
  * Run an experiment.
  *
+ * @example
+ * ```ts
+ * import { asEvaluator, runExperiment } from "@phoenix/client/experiments";
+ *
+ * const experiment = await runExperiment({
+ *   dataset: "my-dataset",
+ *   task: async (example) => example.input,
+ *   evaluators: [
+ *     asEvaluator("my-evaluator", "CODE", async (params) => params.output),
+ *   ],
+ * });
+ * ```
+ *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
-export async function runExperiment({ experimentName: _experimentName, client: _client, dataset: _dataset, task, evaluators, repetitions = 1, projectName = "default", logger = console, record = true, }) {
+export async function runExperiment({ experimentName: _experimentName, experimentDescription, experimentMetadata, client: _client, dataset: _dataset, task, evaluators, projectName = "default", logger = console, record = true, concurrency = 5, dryRun = false, }) {
+    const isDryRun = typeof dryRun === "number" || dryRun === true;
     const client = _client ?? createClient();
     const dataset = await getDatasetBySelector({ dataset: _dataset, client });
     invariant(dataset, `Dataset not found`);
     invariant(dataset.examples.length > 0, `Dataset has no examples`);
+    const nExamples = typeof dryRun === "number"
+        ? Math.max(dryRun, dataset.examples.length)
+        : dataset.examples.length;
     const experimentName = _experimentName ?? `${dataset.name}-${new Date().toISOString()}`;
     const experimentParams = {
-        nRepetitions: repetitions,
-        // TODO: Make configurable?
-        nExamples: dataset.examples.length,
-    };
-    const experiment = {
-        id: id(),
-        datasetId: dataset.id,
-        datasetVersionId: dataset.versionId,
-        repetitions,
-        projectName,
+        nExamples,
     };
+    let experiment;
+    if (isDryRun) {
+        experiment = {
+            id: id(),
+            datasetId: dataset.id,
+            datasetVersionId: dataset.versionId,
+            projectName,
+        };
+    }
+    else {
+        const experimentResponse = await client
+            .POST("/v1/datasets/{dataset_id}/experiments", {
+            params: {
+                path: {
+                    dataset_id: dataset.id,
+                },
+            },
+            body: {
+                name: experimentName,
+                description: experimentDescription,
+                metadata: experimentMetadata,
+                project_name: projectName,
+            },
+        })
+            .then((res) => res.data?.data);
+        invariant(experimentResponse, `Failed to create experiment`);
+        experiment = {
+            id: experimentResponse.id,
+            datasetId: dataset.id,
+            datasetVersionId: dataset.versionId,
+            projectName,
+        };
+    }
     if (!record) {
         logger.info(`🔧 Running experiment in readonly mode. Results will not be recorded.`);
     }
-    logger.info(`🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${evaluators?.length ?? 0} ${pluralize("evaluator", evaluators?.length ?? 0)}`);
-    logger.info(`🔁 Running ${repetitions} ${pluralize("repetition", repetitions)} of task "${task.name}"`);
+    logger.info(`🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${evaluators?.length ?? 0} ${pluralize("evaluator", evaluators?.length ?? 0)} and ${concurrency} concurrent runs`);
     const runs = {};
-    await Promise.all(Array.from({ length: repetitions }, (_, i) => runTask({
-        repetition: i + 1,
+    await runTask({
+        client,
         experimentId: experiment.id,
         task,
         dataset,
@@ -42,7 +82,10 @@ export async function runExperiment({ experimentName: _experimentName, client: _
         onComplete: (run) => {
             runs[run.id] = run;
         },
-    })));
+        concurrency,
+        isDryRun,
+        nExamples,
+    });
     logger.info(`✅ Task runs completed`);
     const ranExperiment = {
         ...experiment,
@@ -54,24 +97,26 @@ export async function runExperiment({ experimentName: _experimentName, client: _
         evaluators: evaluators ?? [],
         client,
         logger,
+        concurrency,
+        dryRun,
     });
     ranExperiment.evaluationRuns = evaluationRuns;
     logger.info(`✅ Experiment ${experiment.id} completed`);
     return ranExperiment;
 }
 /**
- * Run a task against all examples in a dataset.
+ * Run a task against n examples in a dataset.
  */
-function runTask({ experimentId, task, dataset, repetition, onComplete, logger, }) {
-    logger.info(`🔧 (${repetition}) Running task "${task.name}" on dataset "${dataset.id}"`);
+function runTask({ client, experimentId, task, dataset, onComplete, logger, concurrency = 5, isDryRun, nExamples, }) {
+    logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
     const run = async (example) => {
+        logger.info(`🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`);
         const thisRun = {
             id: id(),
-            traceId: id(),
+            traceId: null, // TODO: fill this in once we trace experiments
             experimentId,
             datasetExampleId: example.id,
             startTime: new Date(),
-            repetitionNumber: repetition,
             endTime: new Date(), // will get replaced with actual end time
             output: null,
             error: null,
@@ -88,17 +133,46 @@ function runTask({ experimentId, task, dataset, repetition, onComplete, logger,
             thisRun.error = error instanceof Error ? error.message : "Unknown error";
         }
         thisRun.endTime = new Date();
+        if (!isDryRun) {
+            // Log the run to the server
+            // We log this without awaiting (e.g. best effort)
+            const res = await client.POST("/v1/experiments/{experiment_id}/runs", {
+                params: {
+                    path: {
+                        experiment_id: experimentId,
+                    },
+                },
+                body: {
+                    dataset_example_id: example.id,
+                    output: thisRun.output,
+                    repetition_number: 0,
+                    start_time: thisRun.startTime.toISOString(),
+                    end_time: thisRun.endTime.toISOString(),
+                    trace_id: thisRun.traceId,
+                    error: thisRun.error,
+                },
+            });
+            // replace the local run id with the server-assigned id
+            thisRun.id = res.data?.data.id ?? thisRun.id;
+        }
         onComplete(thisRun);
+        return thisRun;
     };
-    return Promise.all(dataset.examples.map(run));
+    const q = queue(run, concurrency);
+    const examplesToUse = dataset.examples.slice(0, nExamples);
+    examplesToUse.forEach((example) => q.push(example));
+    return q.drain();
 }
 /**
  * Evaluate an experiment.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
-export async function evaluateExperiment({ experiment, evaluators, client: _client, logger, }) {
+export async function evaluateExperiment({ experiment, evaluators, client: _client, logger, concurrency = 5, dryRun = false, }) {
+    const isDryRun = typeof dryRun === "number" || dryRun === true;
+    const nRuns = typeof dryRun === "number"
+        ? Math.max(dryRun, Object.keys(experiment.runs).length)
+        : Object.keys(experiment.runs).length;
     const client = _client ?? createClient();
     const dataset = await getDatasetBySelector({
         dataset: experiment.datasetId,
@@ -107,6 +181,7 @@ export async function evaluateExperiment({ experiment, evaluators, client: _clie
     invariant(dataset, `Dataset "${experiment.datasetId}" not found`);
     invariant(dataset.examples.length > 0, `Dataset "${experiment.datasetId}" has no examples`);
     invariant(experiment.runs, `Experiment "${experiment.id}" has no runs`);
+    const runsToEvaluate = Object.values(experiment.runs).slice(0, nRuns);
     if (evaluators?.length === 0) {
         return {
             ...experiment,
@@ -123,12 +198,40 @@ export async function evaluateExperiment({ experiment, evaluators, client: _clie
         evaluationRuns[run.id] = run;
     };
     // Run evaluators against all runs
-    await Promise.all(evaluators.map((evaluator) => Promise.all(Object.values(experiment.runs).map((run) => runEvaluator({
+    // Flat list of evaluator + run tuples
+    const evaluatorsAndRuns = evaluators.flatMap((evaluator) => runsToEvaluate.map((run) => ({
         evaluator,
         run,
-        exampleCache: examplesById,
-        onComplete: onEvaluationComplete,
-    })))));
+    })));
+    const evaluatorsQueue = queue(async (evaluatorAndRun) => {
+        const evalResult = await runEvaluator({
+            evaluator: evaluatorAndRun.evaluator,
+            run: evaluatorAndRun.run,
+            exampleCache: examplesById,
+            onComplete: onEvaluationComplete,
+        });
+        if (!isDryRun) {
+            logger.info(`📝 Logging evaluation ${evalResult.id}`);
+            // Log the evaluation to the server
+            // We log this without awaiting (e.g. best effort)
+            client.POST("/v1/experiment_evaluations", {
+                body: {
+                    experiment_run_id: evaluatorAndRun.run.id,
+                    name: evaluatorAndRun.evaluator.name,
+                    annotator_kind: evaluatorAndRun.evaluator.kind,
+                    start_time: evalResult.startTime.toISOString(),
+                    end_time: evalResult.endTime.toISOString(),
+                    result: {
+                        ...evalResult.result,
+                    },
+                    error: evalResult.error,
+                    trace_id: evalResult.traceId,
+                },
+            });
+        }
+    }, concurrency);
+    evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun));
+    await evaluatorsQueue.drain();
     logger.info(`✅ Evaluation runs completed`);
     return {
         ...experiment,
@@ -139,7 +242,6 @@ export async function evaluateExperiment({ experiment, evaluators, client: _clie
  * Run an evaluator against a run.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 async function runEvaluator({ evaluator, run, exampleCache, onComplete, }) {
     const example = exampleCache[run.datasetExampleId];
@@ -147,7 +249,7 @@ async function runEvaluator({ evaluator, run, exampleCache, onComplete, }) {
     const evaluate = async () => {
         const thisEval = {
             id: id(),
-            traceId: id(),
+            traceId: null, // TODO: fill this in once we trace experiments
             experimentRunId: run.id,
             startTime: new Date(),
             endTime: new Date(), // will get replaced with actual end time
@@ -170,6 +272,7 @@ async function runEvaluator({ evaluator, run, exampleCache, onComplete, }) {
         }
         thisEval.endTime = new Date();
         onComplete(thisEval);
+        return thisEval;
     };
     return evaluate();
 }
@@ -177,15 +280,15 @@ async function runEvaluator({ evaluator, run, exampleCache, onComplete, }) {
  * Wrap an evaluator function in an object with a name property.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  *
  * @param name - The name of the evaluator.
  * @param evaluate - The evaluator function.
  * @returns The evaluator object.
  */
-export function asEvaluator(name, evaluate) {
+export function asEvaluator({ name, kind, evaluate, }) {
     return {
         name,
+        kind,
         evaluate,
     };
 }

package/dist/esm/experiments/runExperiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAsB,MAAM,WAAW,CAAC;AAU7D,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AA6CrE;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,CAAC,EACf,WAAW,GAAG,SAAS,EACvB,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,GACO;IACpB,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC1E,SAAS,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAClE,MAAM,cAAc,GAClB,eAAe,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACnE,MAAM,gBAAgB,GAAyB;QAC7C,YAAY,EAAE,WAAW;QACzB,2BAA2B;QAC3B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;KACnC,CAAC;IACF,MAAM,UAAU,GAAe;QAC7B,EAAE,EAAE,EAAE,EAAE;QACR,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;QACnC,WAAW;QACX,WAAW;KACZ,CAAC;IAEF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,SAAS,CACxI,WAAW,EACX,UAAU,EAAE,MAAM,IAAI,CAAC,CACxB,EAAE,CACJ,CAAC;IAEF,MAAM,CAAC,IAAI,CACT,cAAc,WAAW,IAAI,SAAS,CAAC,YAAY,EAAE,WAAW,CAAC,aAAa,IAAI,CAAC,IAAI,GAAG,CAC3F,CAAC;IAIF,MAAM,IAAI,GAA2C,EAAE,CAAC;IACxD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3C,OAAO,CAAC;QACN,UAAU,EAAE,CAAC,GAAG,CAAC;QACjB,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,IAAI;QACJ,OAAO;QACP,MAAM;QACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QACrB,CAAC;KACF,CAAC,CACH,CACF,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAErC,MAAM,aAAa,GAAkB;QACnC,GAAG,UAAU;QACb,MAAM,EAAE,gBAAgB;QACxB,IAAI;KACL,CAAC;IAEF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;QAClD,UAAU,EAAE,aAAa;QACzB,UAAU,EAAE,UAAU,IAAI,EAAE;QAC5B,MAAM;QACN,MAAM;KACP,CAAC,CAAC;IACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;IAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;IAEvD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,EACf,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,EACV,MAAM,GAcP;IACC,MAAM,CAAC,IAAI,CACT,OAAO,UAAU,mBAAmB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAC5E,CAAC;IACF,MAAM,GAAG,GAAG,KAAK,EAAE,OAAgB,EAAE,EAAE;QACrC,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,EAAE,EAAE;YACb,YAAY;YACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,gBAAgB,EAAE,UAAU;YAC5B,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,4DAA4D;YAC5D,OAAO,CAAC,MAAM;gBACZ,OAAO,UAAU,KAAK,QAAQ;oBAC5B,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,UAAU,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAC;IACF,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,GAaP;IACC,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC;QACzC,OAAO,EAAE,UAAU,CAAC,SAAS;QAC7B,MAAM;KACP,CAAC,CAAC;IACH,SAAS,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;IAClE,SAAS,CACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;IACF,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;IAExE,IAAI,UAAU,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,GAAG,UAAU;YACb,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,SAAS,CACtF,WAAW,EACX,UAAU,EAAE,MAAM,IAAI,CAAC,CACxB,EAAE,CACJ,CAAC;IAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;IAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;QAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;IAC/B,CAAC,CAAC;IAEF,kCAAkC;IAClC,MAAM,OAAO,CAAC,GAAG,CACf,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAC3B,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACzC,YAAY,CAAC;QACX,SAAS;QACT,GAAG;QACH,YAAY,EAAE,YAAY;QAC1B,UAAU,EAAE,oBAAoB;KACjC,CAAC,CACH,CACF,CACF,CACF,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,UAAU;QACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,GAMX;IACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnD,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,MAAM,QAAQ,GAA4B;YACxC,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,EAAE,EAAE;YACb,eAAe,EAAE,GAAG,CAAC,EAAE;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;YACX,aAAa,EAAE,KAAK,EAAE,4CAA4C;SACnE,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;gBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;gBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;gBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,CAAC;YACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC5E,CAAC;QACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,OAAO,QAAQ,EAAE,CAAC;AACpB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,QAA+B;IAE/B,OAAO;QACL,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,GAAG,GAAG,IAAI,CAAC;AAEf;;;;;GAKG;AACH,MAAM,UAAU,EAAE;IAChB,OAAO,CAAC,GAAG,EAAE;QACX,GAAG,EAAE,CAAC;QACN,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;AACP,CAAC"}
+{"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,YAAY,EAAsB,MAAM,WAAW,CAAC;AAa7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAyD3D;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,qBAAqB,EACrB,kBAAkB,EAClB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,SAAS,EACvB,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,EACb,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GACM;IACpB,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC1E,SAAS,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAClE,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ;QACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC3C,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAE9B,MAAM,cAAc,GAClB,eAAe,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACnE,MAAM,gBAAgB,GAAyB;QAC7C,SAAS;KACV,CAAC;IACF,IAAI,UAAsB,CAAC;IAC3B,IAAI,QAAQ,EAAE,CAAC;QACb,UAAU,GAAG;YACX,EAAE,EAAE,EAAE,EAAE;YACR,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,WAAW;SACZ,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,kBAAkB,GAAG,MAAM,MAAM;aACpC,IAAI,CAAC,uCAAuC,EAAE;YAC7C,MAAM,EAAE;gBACN,IAAI,EAAE;oBACJ,UAAU,EAAE,OAAO,CAAC,EAAE;iBACvB;aACF;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,qBAAqB;gBAClC,QAAQ,EAAE,kBAAkB;gBAC5B,YAAY,EAAE,WAAW;aAC1B;SACF,CAAC;aACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACjC,SAAS,CAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC;QAC7D,UAAU,GAAG;YACX,EAAE,EAAE,kBAAkB,CAAC,EAAE;YACzB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,WAAW;SACZ,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,SAAS,CACxI,WAAW,EACX,UAAU,EAAE,MAAM,IAAI,CAAC,CACxB,QAAQ,WAAW,kBAAkB,CACvC,CAAC;IAIF,MAAM,IAAI,GAA2C,EAAE,CAAC;IACxD,MAAM,OAAO,CAAC;QACZ,MAAM;QACN,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,IAAI;QACJ,OAAO;QACP,MAAM;QACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QACrB,CAAC;QACD,WAAW;QACX,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAErC,MAAM,aAAa,GAAkB;QACnC,GAAG,UAAU;QACb,MAAM,EAAE,gBAAgB;QACxB,IAAI;KACL,CAAC;IAEF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;QAClD,UAAU,EAAE,aAAa;QACzB,UAAU,EAAE,UAAU,IAAI,EAAE;QAC5B,MAAM;QACN,MAAM;QACN,WAAW;QACX,MAAM;KACP,CAAC,CAAC;IACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;IAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;IAEvD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,EACf,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,WAAW,GAAG,CAAC,EACf,QAAQ,EACR,SAAS,GAoBV;IACC,MAAM,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,KAAK,EAAE,OAAgB,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,OAAO,CAAC,EAAE,GAAG,CACtF,CAAC;QACF,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;YAC9D,YAAY;YACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,4DAA4D;YAC5D,OAAO,CAAC,MAAM;gBACZ,OAAO,UAAU,KAAK,QAAQ;oBAC5B,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,4BAA4B;YAC5B,kDAAkD;YAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;gBACpE,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,aAAa,EAAE,YAAY;qBAC5B;iBACF;gBACD,IAAI,EAAE;oBACJ,kBAAkB,EAAE,OAAO,CAAC,EAAE;oBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,iBAAiB,EAAE,CAAC;oBACpB,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;oBACvC,QAAQ,EAAE,OAAO,CAAC,OAAO;oBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB;aACF,CAAC,CAAC;YACH,uDAAuD;YACvD,OAAO,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,CAAC;QAC/C,CAAC;QACD,UAAU,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IACF,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3D,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,EACN,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GAqBf;IACC,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;IAC/D,MAAM,KAAK,GACT,OAAO,MAAM,KAAK,QAAQ;QACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACvD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC1C,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC;QACzC,OAAO,EAAE,UAAU,CAAC,SAAS;QAC7B,MAAM;KACP,CAAC,CAAC;IACH,SAAS,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;IAClE,SAAS,CACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;IACF,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;IAExE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEtE,IAAI,UAAU,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,GAAG,UAAU;YACb,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,SAAS,CACtF,WAAW,EACX,UAAU,EAAE,MAAM,IAAI,CAAC,CACxB,EAAE,CACJ,CAAC;IAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;IAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;QAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;IAC/B,CAAC,CAAC;IAEF,kCAAkC;IAClC,sCAAsC;IACtC,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3B,SAAS;QACT,GAAG;KACJ,CAAC,CAAC,CACJ,CAAC;IACF,MAAM,eAAe,GAAG,KAAK,CAC3B,KAAK,EAAE,eAA6D,EAAE,EAAE;QACtE,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;YACpC,SAAS,EAAE,eAAe,CAAC,SAAS;YACpC,GAAG,EAAE,eAAe,CAAC,GAAG;YACxB,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,oBAAoB;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YACtD,mCAAmC;YACnC,kDAAkD;YAClD,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBACxC,IAAI,EAAE;oBACJ,iBAAiB,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE;oBACzC,IAAI,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;oBACpC,cAAc,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;oBAC9C,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC9C,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE;oBAC1C,MAAM,EAAE;wBACN,GAAG,UAAU,CAAC,MAAM;qBACrB;oBACD,KAAK,EAAE,UAAU,CAAC,KAAK;oBACvB,QAAQ,EAAE,UAAU,CAAC,OAAO;iBAC7B;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC,EACD,WAAW,CACZ,CAAC;IACF,iBAAiB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CACtC,CAAC;IACF,MAAM,eAAe,CAAC,KAAK,EAAE,CAAC;IAC9B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,UAAU;QACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,YAAY,CAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,GAMX;IACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnD,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,MAAM,QAAQ,GAA4B;YACxC,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;YAC9D,eAAe,EAAE,GAAG,CAAC,EAAE;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;YACX,aAAa,EAAE,KAAK,EAAE,4CAA4C;SACnE,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;gBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;gBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;gBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,CAAC;YACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC5E,CAAC;QACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;QACrB,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,OAAO,QAAQ,EAAE,CAAC;AACpB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GAKT;IACC,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,GAAG,GAAG,IAAI,CAAC;AAEf;;;;;GAKG;AACH,MAAM,UAAU,EAAE;IAChB,OAAO,CAAC,GAAG,EAAE;QACX,GAAG,EAAE,CAAC;QACN,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;AACP,CAAC"}

package/dist/esm/schemas/llm/constants.d.ts

@@ -6681,7 +6681,7 @@ export declare const SDKProviderConverterMap: {
             type: "tool_call";
             tool_call_id: string;
         } | {
-            tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+            tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
             type: "tool_result";
             tool_call_id: string;
         })[];

package/dist/esm/schemas/llm/converters.d.ts

@@ -6682,7 +6682,7 @@ export declare const safelyConvertMessageToProvider: <TargetProviderSDK extends
             type: "tool_call";
             tool_call_id: string;
         } | {
-            tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+            tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
             type: "tool_result";
             tool_call_id: string;
         })[];
@@ -15783,7 +15783,7 @@ export declare const safelyConvertToolCallToProvider: <TargetProviderSDK extends
             type: "tool_call";
             tool_call_id: string;
         } | {
-            tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+            tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
             type: "tool_result";
             tool_call_id: string;
         })[];
@@ -24884,7 +24884,7 @@ export declare const safelyConvertToolDefinitionToProvider: <TargetProviderSDK e
             type: "tool_call";
             tool_call_id: string;
         } | {
-            tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+            tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
             type: "tool_result";
             tool_call_id: string;
         })[];
@@ -33985,7 +33985,7 @@ export declare const safelyConvertToolChoiceToProvider: <TargetProviderSDK exten
             type: "tool_call";
             tool_call_id: string;
         } | {
-            tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+            tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
             type: "tool_result";
             tool_call_id: string;
         })[];

package/dist/esm/schemas/llm/openai/converters.d.ts

@@ -1043,7 +1043,7 @@ export declare const openAIMessageToPhoenixPrompt: import("zod").ZodEffects<impo
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];

package/dist/esm/schemas/llm/phoenixPrompt/converters.d.ts

@@ -44,11 +44,11 @@ export declare const phoenixMessagePartToOpenAI: import("zod").ZodEffects<import
     tool_call_id: import("zod").ZodString;
     tool_result: import("zod").ZodUnion<[import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodBoolean, import("zod").ZodNull, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodUnknown>, import("zod").ZodArray<import("zod").ZodUnknown, "many">]>;
 }, "strip", import("zod").ZodTypeAny, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }>]>, {
@@ -66,7 +66,7 @@ export declare const phoenixMessagePartToOpenAI: import("zod").ZodEffects<import
     type: "tool_call";
     tool_call_id: string;
 } | {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }>;
@@ -121,11 +121,11 @@ export declare const phoenixMessageToOpenAI: import("zod").ZodEffects<import("zo
         tool_call_id: import("zod").ZodString;
         tool_result: import("zod").ZodUnion<[import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodBoolean, import("zod").ZodNull, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodUnknown>, import("zod").ZodArray<import("zod").ZodUnknown, "many">]>;
     }, "strip", import("zod").ZodTypeAny, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }>]>, "many">]>;
@@ -142,7 +142,7 @@ export declare const phoenixMessageToOpenAI: import("zod").ZodEffects<import("zo
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -160,7 +160,7 @@ export declare const phoenixMessageToOpenAI: import("zod").ZodEffects<import("zo
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -216,7 +216,7 @@ export declare const phoenixMessageToOpenAI: import("zod").ZodEffects<import("zo
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];

package/dist/esm/schemas/llm/phoenixPrompt/messagePartSchemas.d.ts

@@ -49,11 +49,11 @@ export declare const toolResultPartSchema: z.ZodObject<{
     tool_call_id: z.ZodString;
     tool_result: z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodUnknown, "many">]>;
 }, "strip", z.ZodTypeAny, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }>;
@@ -104,11 +104,11 @@ export declare const phoenixContentPartSchema: z.ZodDiscriminatedUnion<"type", [
     tool_call_id: z.ZodString;
     tool_result: z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodUnknown, "many">]>;
 }, "strip", z.ZodTypeAny, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }, {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 }>]>;
@@ -121,7 +121,7 @@ export declare const makeTextPart: (text?: string | null) => {
 export declare const asToolCallPart: (maybePart: unknown) => ToolCallPart | null;
 export declare const asToolResultPart: (maybePart: unknown) => ToolResultPart | null;
 export declare const makeToolResultPart: (toolCallId?: string | null, result?: unknown) => {
-    tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+    tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
     type: "tool_result";
     tool_call_id: string;
 } | null;

package/dist/esm/schemas/llm/phoenixPrompt/messageSchemas.d.ts

@@ -54,11 +54,11 @@ export declare const phoenixMessageSchema: z.ZodObject<{
         tool_call_id: z.ZodString;
         tool_result: z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodUnknown, "many">]>;
     }, "strip", z.ZodTypeAny, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }>]>, "many">]>;
@@ -75,7 +75,7 @@ export declare const phoenixMessageSchema: z.ZodObject<{
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -93,7 +93,7 @@ export declare const phoenixMessageSchema: z.ZodObject<{
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -148,11 +148,11 @@ export declare const phoenixMessagesSchema: z.ZodArray<z.ZodObject<{
         tool_call_id: z.ZodString;
         tool_result: z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodUnknown, "many">]>;
     }, "strip", z.ZodTypeAny, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }>]>, "many">]>;
@@ -169,7 +169,7 @@ export declare const phoenixMessagesSchema: z.ZodArray<z.ZodObject<{
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -187,7 +187,7 @@ export declare const phoenixMessagesSchema: z.ZodArray<z.ZodObject<{
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];

package/dist/esm/schemas/llm/schemas.d.ts

@@ -800,11 +800,11 @@ export declare const llmProviderMessageSchema: z.ZodUnion<[z.ZodDiscriminatedUni
         tool_call_id: z.ZodString;
         tool_result: z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodUnknown, "many">]>;
     }, "strip", z.ZodTypeAny, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }, {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     }>]>, "many">]>;
@@ -821,7 +821,7 @@ export declare const llmProviderMessageSchema: z.ZodUnion<[z.ZodDiscriminatedUni
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];
@@ -839,7 +839,7 @@ export declare const llmProviderMessageSchema: z.ZodUnion<[z.ZodDiscriminatedUni
         type: "tool_call";
         tool_call_id: string;
     } | {
-        tool_result: string | number | boolean | Record<string, unknown> | unknown[] | null;
+        tool_result: string | number | boolean | unknown[] | Record<string, unknown> | null;
         type: "tool_result";
         tool_call_id: string;
     })[];

package/dist/esm/spans/addSpanAnnotation.d.ts

@@ -0,0 +1,39 @@
+import { ClientFn } from "../types/core.js";
+import { SpanAnnotation } from "./types.js";
+/**
+ * Parameters to add a span annotation
+ */
+interface AddSpanAnnotationParams extends ClientFn {
+    spanAnnotation: SpanAnnotation;
+}
+/**
+ * Add an annotation to a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to add a span annotation
+ * @returns The ID of the created or updated annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addSpanAnnotation({
+ *   spanAnnotation: {
+ *     spanId: "123abc",
+ *     name: "quality_score",
+ *     label: "good",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     identifier: "custom_id_123",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export declare function addSpanAnnotation({ client: _client, spanAnnotation, }: AddSpanAnnotationParams): Promise<{
+    id: string;
+}>;
+export {};
+//# sourceMappingURL=addSpanAnnotation.d.ts.map

package/dist/esm/spans/addSpanAnnotation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"addSpanAnnotation.d.ts","sourceRoot":"","sources":["../../../src/spans/addSpanAnnotation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,cAAc,EAAwB,MAAM,SAAS,CAAC;AAE/D;;GAEG;AACH,UAAU,uBAAwB,SAAQ,QAAQ;IAChD,cAAc,EAAE,cAAc,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,cAAc,GACf,EAAE,uBAAuB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAkBnD"}