@arizeai/phoenix-client 1.1.0 → 1.2.0

package/dist/src/experiments/runExperiment.js

@@ -16,45 +16,85 @@ exports.runExperiment = runExperiment;
 exports.evaluateExperiment = evaluateExperiment;
 exports.asEvaluator = asEvaluator;
 exports.id = id;
-const client_1 = require("../client");
-const promisifyResult_1 = require("../utils/promisifyResult");
+const async_1 = require("async");
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
-const pluralize_1 = require("../utils/pluralize");
+const client_1 = require("../client");
 const getDatasetBySelector_1 = require("../utils/getDatasetBySelector");
+const pluralize_1 = require("../utils/pluralize");
+const promisifyResult_1 = require("../utils/promisifyResult");
 /**
  * Run an experiment.
  *
+ * @example
+ * ```ts
+ * import { asEvaluator, runExperiment } from "@phoenix/client/experiments";
+ *
+ * const experiment = await runExperiment({
+ *   dataset: "my-dataset",
+ *   task: async (example) => example.input,
+ *   evaluators: [
+ *     asEvaluator("my-evaluator", "CODE", async (params) => params.output),
+ *   ],
+ * });
+ * ```
+ *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 function runExperiment(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ experimentName: _experimentName, client: _client, dataset: _dataset, task, evaluators, repetitions = 1, projectName = "default", logger = console, record = true, }) {
+    return __awaiter(this, arguments, void 0, function* ({ experimentName: _experimentName, experimentDescription, experimentMetadata, client: _client, dataset: _dataset, task, evaluators, projectName = "default", logger = console, record = true, concurrency = 5, dryRun = false, }) {
         var _b, _c;
+        const isDryRun = typeof dryRun === "number" || dryRun === true;
         const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
         const dataset = yield (0, getDatasetBySelector_1.getDatasetBySelector)({ dataset: _dataset, client });
         (0, tiny_invariant_1.default)(dataset, `Dataset not found`);
         (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset has no examples`);
+        const nExamples = typeof dryRun === "number"
+            ? Math.max(dryRun, dataset.examples.length)
+            : dataset.examples.length;
         const experimentName = _experimentName !== null && _experimentName !== void 0 ? _experimentName : `${dataset.name}-${new Date().toISOString()}`;
         const experimentParams = {
-            nRepetitions: repetitions,
-            // TODO: Make configurable?
-            nExamples: dataset.examples.length,
-        };
-        const experiment = {
-            id: id(),
-            datasetId: dataset.id,
-            datasetVersionId: dataset.versionId,
-            repetitions,
-            projectName,
+            nExamples,
         };
+        let experiment;
+        if (isDryRun) {
+            experiment = {
+                id: id(),
+                datasetId: dataset.id,
+                datasetVersionId: dataset.versionId,
+                projectName,
+            };
+        }
+        else {
+            const experimentResponse = yield client
+                .POST("/v1/datasets/{dataset_id}/experiments", {
+                params: {
+                    path: {
+                        dataset_id: dataset.id,
+                    },
+                },
+                body: {
+                    name: experimentName,
+                    description: experimentDescription,
+                    metadata: experimentMetadata,
+                    project_name: projectName,
+                },
+            })
+                .then((res) => { var _a; return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data; });
+            (0, tiny_invariant_1.default)(experimentResponse, `Failed to create experiment`);
+            experiment = {
+                id: experimentResponse.id,
+                datasetId: dataset.id,
+                datasetVersionId: dataset.versionId,
+                projectName,
+            };
+        }
         if (!record) {
             logger.info(`🔧 Running experiment in readonly mode. Results will not be recorded.`);
         }
-        logger.info(`🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${(_b = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _b !== void 0 ? _b : 0} ${(0, pluralize_1.pluralize)("evaluator", (_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0)}`);
-        logger.info(`🔁 Running ${repetitions} ${(0, pluralize_1.pluralize)("repetition", repetitions)} of task "${task.name}"`);
+        logger.info(`🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${(_b = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _b !== void 0 ? _b : 0} ${(0, pluralize_1.pluralize)("evaluator", (_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0)} and ${concurrency} concurrent runs`);
         const runs = {};
-        yield Promise.all(Array.from({ length: repetitions }, (_, i) => runTask({
-            repetition: i + 1,
+        yield runTask({
+            client,
             experimentId: experiment.id,
             task,
             dataset,
@@ -62,7 +102,10 @@ function runExperiment(_a) {
             onComplete: (run) => {
                 runs[run.id] = run;
             },
-        })));
+            concurrency,
+            isDryRun,
+            nExamples,
+        });
         logger.info(`✅ Task runs completed`);
         const ranExperiment = Object.assign(Object.assign({}, experiment), { params: experimentParams, runs });
         const { evaluationRuns } = yield evaluateExperiment({
@@ -70,6 +113,8 @@ function runExperiment(_a) {
             evaluators: evaluators !== null && evaluators !== void 0 ? evaluators : [],
             client,
             logger,
+            concurrency,
+            dryRun,
         });
         ranExperiment.evaluationRuns = evaluationRuns;
         logger.info(`✅ Experiment ${experiment.id} completed`);
@@ -77,18 +122,19 @@ function runExperiment(_a) {
     });
 }
 /**
- * Run a task against all examples in a dataset.
+ * Run a task against n examples in a dataset.
  */
-function runTask({ experimentId, task, dataset, repetition, onComplete, logger, }) {
-    logger.info(`🔧 (${repetition}) Running task "${task.name}" on dataset "${dataset.id}"`);
+function runTask({ client, experimentId, task, dataset, onComplete, logger, concurrency = 5, isDryRun, nExamples, }) {
+    logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
     const run = (example) => __awaiter(this, void 0, void 0, function* () {
+        var _a, _b;
+        logger.info(`🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`);
         const thisRun = {
             id: id(),
-            traceId: id(),
+            traceId: null, // TODO: fill this in once we trace experiments
             experimentId,
             datasetExampleId: example.id,
             startTime: new Date(),
-            repetitionNumber: repetition,
             endTime: new Date(), // will get replaced with actual end time
             output: null,
             error: null,
@@ -105,19 +151,48 @@ function runTask({ experimentId, task, dataset, repetition, onComplete, logger,
             thisRun.error = error instanceof Error ? error.message : "Unknown error";
         }
         thisRun.endTime = new Date();
+        if (!isDryRun) {
+            // Log the run to the server
+            // We log this without awaiting (e.g. best effort)
+            const res = yield client.POST("/v1/experiments/{experiment_id}/runs", {
+                params: {
+                    path: {
+                        experiment_id: experimentId,
+                    },
+                },
+                body: {
+                    dataset_example_id: example.id,
+                    output: thisRun.output,
+                    repetition_number: 0,
+                    start_time: thisRun.startTime.toISOString(),
+                    end_time: thisRun.endTime.toISOString(),
+                    trace_id: thisRun.traceId,
+                    error: thisRun.error,
+                },
+            });
+            // replace the local run id with the server-assigned id
+            thisRun.id = (_b = (_a = res.data) === null || _a === void 0 ? void 0 : _a.data.id) !== null && _b !== void 0 ? _b : thisRun.id;
+        }
         onComplete(thisRun);
+        return thisRun;
     });
-    return Promise.all(dataset.examples.map(run));
+    const q = (0, async_1.queue)(run, concurrency);
+    const examplesToUse = dataset.examples.slice(0, nExamples);
+    examplesToUse.forEach((example) => q.push(example));
+    return q.drain();
 }
 /**
  * Evaluate an experiment.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 function evaluateExperiment(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ experiment, evaluators, client: _client, logger, }) {
+    return __awaiter(this, arguments, void 0, function* ({ experiment, evaluators, client: _client, logger, concurrency = 5, dryRun = false, }) {
         var _b, _c;
+        const isDryRun = typeof dryRun === "number" || dryRun === true;
+        const nRuns = typeof dryRun === "number"
+            ? Math.max(dryRun, Object.keys(experiment.runs).length)
+            : Object.keys(experiment.runs).length;
         const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
         const dataset = yield (0, getDatasetBySelector_1.getDatasetBySelector)({
             dataset: experiment.datasetId,
@@ -126,6 +201,7 @@ function evaluateExperiment(_a) {
         (0, tiny_invariant_1.default)(dataset, `Dataset "${experiment.datasetId}" not found`);
         (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset "${experiment.datasetId}" has no examples`);
         (0, tiny_invariant_1.default)(experiment.runs, `Experiment "${experiment.id}" has no runs`);
+        const runsToEvaluate = Object.values(experiment.runs).slice(0, nRuns);
         if ((evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) === 0) {
             return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
         }
@@ -139,12 +215,38 @@ function evaluateExperiment(_a) {
             evaluationRuns[run.id] = run;
         };
         // Run evaluators against all runs
-        yield Promise.all(evaluators.map((evaluator) => Promise.all(Object.values(experiment.runs).map((run) => runEvaluator({
+        // Flat list of evaluator + run tuples
+        const evaluatorsAndRuns = evaluators.flatMap((evaluator) => runsToEvaluate.map((run) => ({
             evaluator,
             run,
-            exampleCache: examplesById,
-            onComplete: onEvaluationComplete,
-        })))));
+        })));
+        const evaluatorsQueue = (0, async_1.queue)((evaluatorAndRun) => __awaiter(this, void 0, void 0, function* () {
+            const evalResult = yield runEvaluator({
+                evaluator: evaluatorAndRun.evaluator,
+                run: evaluatorAndRun.run,
+                exampleCache: examplesById,
+                onComplete: onEvaluationComplete,
+            });
+            if (!isDryRun) {
+                logger.info(`📝 Logging evaluation ${evalResult.id}`);
+                // Log the evaluation to the server
+                // We log this without awaiting (e.g. best effort)
+                client.POST("/v1/experiment_evaluations", {
+                    body: {
+                        experiment_run_id: evaluatorAndRun.run.id,
+                        name: evaluatorAndRun.evaluator.name,
+                        annotator_kind: evaluatorAndRun.evaluator.kind,
+                        start_time: evalResult.startTime.toISOString(),
+                        end_time: evalResult.endTime.toISOString(),
+                        result: Object.assign({}, evalResult.result),
+                        error: evalResult.error,
+                        trace_id: evalResult.traceId,
+                    },
+                });
+            }
+        }), concurrency);
+        evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun));
+        yield evaluatorsQueue.drain();
         logger.info(`✅ Evaluation runs completed`);
         return Object.assign(Object.assign({}, experiment), { evaluationRuns: Object.values(evaluationRuns) });
     });
@@ -153,7 +255,6 @@ function evaluateExperiment(_a) {
  * Run an evaluator against a run.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 function runEvaluator(_a) {
     return __awaiter(this, arguments, void 0, function* ({ evaluator, run, exampleCache, onComplete, }) {
@@ -163,7 +264,7 @@ function runEvaluator(_a) {
             var _a;
             const thisEval = {
                 id: id(),
-                traceId: id(),
+                traceId: null, // TODO: fill this in once we trace experiments
                 experimentRunId: run.id,
                 startTime: new Date(),
                 endTime: new Date(), // will get replaced with actual end time
@@ -186,6 +287,7 @@ function runEvaluator(_a) {
             }
             thisEval.endTime = new Date();
             onComplete(thisEval);
+            return thisEval;
         });
         return evaluate();
     });
@@ -194,15 +296,15 @@ function runEvaluator(_a) {
  * Wrap an evaluator function in an object with a name property.
  *
  * @experimental This feature is not complete, and will change in the future.
- * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  *
  * @param name - The name of the evaluator.
  * @param evaluate - The evaluator function.
  * @returns The evaluator object.
  */
-function asEvaluator(name, evaluate) {
+function asEvaluator({ name, kind, evaluate, }) {
     return {
         name,
+        kind,
         evaluate,
     };
 }

package/dist/src/experiments/runExperiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAkEA,sCAmFC;AA+DD,gDA6EC;AA6DD,kCAQC;AAUD,gBAKC;AApXD,sCAA6D;AAU7D,8DAA2D;AAC3D,oEAAuC;AACvC,kDAA+C;AAE/C,wEAAqE;AA6CrE;;;;;GAKG;AACH,SAAsB,aAAa;yDAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,CAAC,EACf,WAAW,GAAG,SAAS,EACvB,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,GACO;;QACpB,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,IAAA,wBAAS,EAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACxC,IAAA,wBAAS,EAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAClE,MAAM,cAAc,GAClB,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QACnE,MAAM,gBAAgB,GAAyB;YAC7C,YAAY,EAAE,WAAW;YACzB,2BAA2B;YAC3B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;SACnC,CAAC;QACF,MAAM,UAAU,GAAe;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,WAAW;YACX,WAAW;SACZ,CAAC;QAEF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACxI,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,EAAE,CACJ,CAAC;QAEF,MAAM,CAAC,IAAI,CACT,cAAc,WAAW,IAAI,IAAA,qBAAS,EAAC,YAAY,EAAE,WAAW,CAAC,aAAa,IAAI,CAAC,IAAI,GAAG,CAC3F,CAAC;QAIF,MAAM,IAAI,GAA2C,EAAE,CAAC;QACxD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3C,OAAO,CAAC;YACN,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,IAAI;YACJ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;SACF,CAAC,CACH,CACF,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAErC,MAAM,aAAa,mCACd,UAAU,KACb,MAAM,EAAE,gBAAgB,EACxB,IAAI,GACL,CAAC;QAEF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;YAClD,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,EAAE;YAC5B,MAAM;YACN,MAAM;SACP,CAAC,CAAC;QACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QAEvD,OAAO,aAAa,CAAC;IACvB,CAAC;CAAA;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,EACf,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,EACV,MAAM,GAcP;IACC,MAAM,CAAC,IAAI,CACT,OAAO,UAAU,mBAAmB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAC5E,CAAC;IACF,MAAM,GAAG,GAAG,CAAO,OAAgB,EAAE,EAAE;QACrC,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,EAAE,EAAE;YACb,YAAY;YACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,gBAAgB,EAAE,UAAU;YAC5B,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,4DAA4D;YAC5D,OAAO,CAAC,MAAM;gBACZ,OAAO,UAAU,KAAK,QAAQ;oBAC5B,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,UAAU,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAA,CAAC;IACF,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,GAaP;;QACC,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC;YACzC,OAAO,EAAE,UAAU,CAAC,SAAS;YAC7B,MAAM;SACP,CAAC,CAAC;QACH,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;QAClE,IAAA,wBAAS,EACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;QACF,IAAA,wBAAS,EAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;QAExE,IAAI,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YAC7B,uCACK,UAAU,KACb,cAAc,EAAE,EAAE,IAClB;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACtF,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,EAAE,CACJ,CAAC;QAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;QAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;YAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QAC/B,CAAC,CAAC;QAEF,kCAAkC;QAClC,MAAM,OAAO,CAAC,GAAG,CACf,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAC3B,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACzC,YAAY,CAAC;YACX,SAAS;YACT,GAAG;YACH,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,oBAAoB;SACjC,CAAC,CACH,CACF,CACF,CACF,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAE3C,uCACK,UAAU,KACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAC7C;IACJ,CAAC;CAAA;AAED;;;;;GAKG;AACH,SAAe,YAAY;yDAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,GAMX;QACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,GAAS,EAAE;;YAC1B,MAAM,QAAQ,GAA4B;gBACxC,EAAE,EAAE,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE,EAAE;gBACb,eAAe,EAAE,GAAG,CAAC,EAAE;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;gBAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,KAAK,EAAE,4CAA4C;aACnE,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;oBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,MAAA,GAAG,CAAC,MAAM,mCAAI,IAAI;oBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;oBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;gBACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC5E,CAAC;YACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC,CAAA,CAAC;QAEF,OAAO,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;;;;;GASG;AACH,SAAgB,WAAW,CACzB,IAAY,EACZ,QAA+B;IAE/B,OAAO;QACL,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,GAAG,GAAG,IAAI,CAAC;AAEf;;;;;GAKG;AACH,SAAgB,EAAE;IAChB,OAAO,CAAC,GAAG,EAAE;QACX,GAAG,EAAE,CAAC;QACN,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;AACP,CAAC"}
+{"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AA4FA,sCAgHC;AAiGD,gDAwHC;AA4DD,kCAcC;AAUD,gBAKC;AA9fD,iCAA8B;AAC9B,oEAAuC;AACvC,sCAA6D;AAa7D,wEAAqE;AACrE,kDAA+C;AAC/C,8DAA2D;AAyD3D;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAsB,aAAa;yDAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,qBAAqB,EACrB,kBAAkB,EAClB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,SAAS,EACvB,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,EACb,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GACM;;QACpB,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,IAAA,wBAAS,EAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACxC,IAAA,wBAAS,EAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAClE,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9B,MAAM,cAAc,GAClB,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QACnE,MAAM,gBAAgB,GAAyB;YAC7C,SAAS;SACV,CAAC;QACF,IAAI,UAAsB,CAAC;QAC3B,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,GAAG;gBACX,EAAE,EAAE,EAAE,EAAE;gBACR,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,WAAW;aACZ,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,kBAAkB,GAAG,MAAM,MAAM;iBACpC,IAAI,CAAC,uCAAuC,EAAE;gBAC7C,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,UAAU,EAAE,OAAO,CAAC,EAAE;qBACvB;iBACF;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,cAAc;oBACpB,WAAW,EAAE,qBAAqB;oBAClC,QAAQ,EAAE,kBAAkB;oBAC5B,YAAY,EAAE,WAAW;iBAC1B;aACF,CAAC;iBACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAA,EAAA,CAAC,CAAC;YACjC,IAAA,wBAAS,EAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC;YAC7D,UAAU,GAAG;gBACX,EAAE,EAAE,kBAAkB,CAAC,EAAE;gBACzB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACxI,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,QAAQ,WAAW,kBAAkB,CACvC,CAAC;QAIF,MAAM,IAAI,GAA2C,EAAE,CAAC;QACxD,MAAM,OAAO,CAAC;YACZ,MAAM;YACN,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,IAAI;YACJ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;YACD,WAAW;YACX,QAAQ;YACR,SAAS;SACV,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAErC,MAAM,aAAa,mCACd,UAAU,KACb,MAAM,EAAE,gBAAgB,EACxB,IAAI,GACL,CAAC;QAEF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;YAClD,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,EAAE;YAC5B,MAAM;YACN,MAAM;YACN,WAAW;YACX,MAAM;SACP,CAAC,CAAC;QACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QAEvD,OAAO,aAAa,CAAC;IACvB,CAAC;CAAA;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,EACf,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,WAAW,GAAG,CAAC,EACf,QAAQ,EACR,SAAS,GAoBV;IACC,MAAM,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,CAAO,OAAgB,EAAE,EAAE;;QACrC,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,OAAO,CAAC,EAAE,GAAG,CACtF,CAAC;QACF,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;YAC9D,YAAY;YACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,4DAA4D;YAC5D,OAAO,CAAC,MAAM;gBACZ,OAAO,UAAU,KAAK,QAAQ;oBAC5B,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,4BAA4B;YAC5B,kDAAkD;YAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;gBACpE,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,aAAa,EAAE,YAAY;qBAC5B;iBACF;gBACD,IAAI,EAAE;oBACJ,kBAAkB,EAAE,OAAO,CAAC,EAAE;oBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,iBAAiB,EAAE,CAAC;oBACpB,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;oBACvC,QAAQ,EAAE,OAAO,CAAC,OAAO;oBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB;aACF,CAAC,CAAC;YACH,uDAAuD;YACvD,OAAO,CAAC,EAAE,GAAG,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAC,EAAE,mCAAI,OAAO,CAAC,EAAE,CAAC;QAC/C,CAAC;QACD,UAAU,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC,CAAA,CAAC;IACF,MAAM,CAAC,GAAG,IAAA,aAAK,EAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3D,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,EACN,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GAqBf;;QACC,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,KAAK,GACT,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACvD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC;YACzC,OAAO,EAAE,UAAU,CAAC,SAAS;YAC7B,MAAM;SACP,CAAC,CAAC;QACH,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;QAClE,IAAA,wBAAS,EACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;QACF,IAAA,wBAAS,EAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;QAExE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEtE,IAAI,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YAC7B,uCACK,UAAU,KACb,cAAc,EAAE,EAAE,IAClB;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACtF,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,EAAE,CACJ,CAAC;QAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;QAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;YAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QAC/B,CAAC,CAAC;QAEF,kCAAkC;QAClC,sCAAsC;QACtC,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3B,SAAS;YACT,GAAG;SACJ,CAAC,CAAC,CACJ,CAAC;QACF,MAAM,eAAe,GAAG,IAAA,aAAK,EAC3B,CAAO,eAA6D,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;gBACpC,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,GAAG,EAAE,eAAe,CAAC,GAAG;gBACxB,YAAY,EAAE,YAAY;gBAC1B,UAAU,EAAE,oBAAoB;aACjC,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtD,mCAAmC;gBACnC,kDAAkD;gBAClD,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;oBACxC,IAAI,EAAE;wBACJ,iBAAiB,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE;wBACzC,IAAI,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;wBACpC,cAAc,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;wBAC9C,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,WAAW,EAAE;wBAC9C,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE;wBAC1C,MAAM,oBACD,UAAU,CAAC,MAAM,CACrB;wBACD,KAAK,EAAE,UAAU,CAAC,KAAK;wBACvB,QAAQ,EAAE,UAAU,CAAC,OAAO;qBAC7B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAA,EACD,WAAW,CACZ,CAAC;QACF,iBAAiB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CACtC,CAAC;QACF,MAAM,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAE3C,uCACK,UAAU,KACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAC7C;IACJ,CAAC;CAAA;AAED;;;;GAIG;AACH,SAAe,YAAY;yDAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,GAMX;QACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,GAAS,EAAE;;YAC1B,MAAM,QAAQ,GAA4B;gBACxC,EAAE,EAAE,EAAE,EAAE;gBACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;gBAC9D,eAAe,EAAE,GAAG,CAAC,EAAE;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;gBAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,KAAK,EAAE,4CAA4C;aACnE,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;oBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,MAAA,GAAG,CAAC,MAAM,mCAAI,IAAI;oBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;oBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;gBACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC5E,CAAC;YACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAA,CAAC;QAEF,OAAO,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GAKT;IACC,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,GAAG,GAAG,IAAI,CAAC;AAEf;;;;;GAKG;AACH,SAAgB,EAAE;IAChB,OAAO,CAAC,GAAG,EAAE;QACX,GAAG,EAAE,CAAC;QACN,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;AACP,CAAC"}

package/dist/src/spans/addSpanAnnotation.d.ts

@@ -0,0 +1,39 @@
+import { ClientFn } from "../types/core";
+import { SpanAnnotation } from "./types";
+/**
+ * Parameters to add a span annotation
+ */
+interface AddSpanAnnotationParams extends ClientFn {
+    spanAnnotation: SpanAnnotation;
+}
+/**
+ * Add an annotation to a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to add a span annotation
+ * @returns The ID of the created or updated annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addSpanAnnotation({
+ *   spanAnnotation: {
+ *     spanId: "123abc",
+ *     name: "quality_score",
+ *     label: "good",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     identifier: "custom_id_123",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export declare function addSpanAnnotation({ client: _client, spanAnnotation, }: AddSpanAnnotationParams): Promise<{
+    id: string;
+}>;
+export {};
+//# sourceMappingURL=addSpanAnnotation.d.ts.map

package/dist/src/spans/addSpanAnnotation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"addSpanAnnotation.d.ts","sourceRoot":"","sources":["../../../src/spans/addSpanAnnotation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,cAAc,EAAwB,MAAM,SAAS,CAAC;AAE/D;;GAEG;AACH,UAAU,uBAAwB,SAAQ,QAAQ;IAChD,cAAc,EAAE,cAAc,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,cAAc,GACf,EAAE,uBAAuB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAkBnD"}

package/dist/src/spans/addSpanAnnotation.js

@@ -0,0 +1,59 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addSpanAnnotation = addSpanAnnotation;
+const client_1 = require("../client");
+const types_1 = require("./types");
+/**
+ * Add an annotation to a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to add a span annotation
+ * @returns The ID of the created or updated annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addSpanAnnotation({
+ *   spanAnnotation: {
+ *     spanId: "123abc",
+ *     name: "quality_score",
+ *     label: "good",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     identifier: "custom_id_123",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+function addSpanAnnotation(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ client: _client, spanAnnotation, }) {
+        var _b, _c;
+        const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
+        const { data, error } = yield client.POST("/v1/span_annotations", {
+            body: {
+                data: [(0, types_1.toSpanAnnotationData)(spanAnnotation)],
+            },
+        });
+        if (error) {
+            throw new Error(`Failed to add span annotation: ${error}`);
+        }
+        if (!((_c = (_b = data === null || data === void 0 ? void 0 : data.data) === null || _b === void 0 ? void 0 : _b[0]) === null || _c === void 0 ? void 0 : _c.id)) {
+            throw new Error("No annotation ID returned from server");
+        }
+        return data.data[0];
+    });
+}
+//# sourceMappingURL=addSpanAnnotation.js.map

package/dist/src/spans/addSpanAnnotation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"addSpanAnnotation.js","sourceRoot":"","sources":["../../../src/spans/addSpanAnnotation.ts"],"names":[],"mappings":";;;;;;;;;;;AAqCA,8CAqBC;AA1DD,sCAAyC;AAEzC,mCAA+D;AAS/D;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAsB,iBAAiB;yDAAC,EACtC,MAAM,EAAE,OAAO,EACf,cAAc,GACU;;QACxB,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QAEzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAChE,IAAI,EAAE;gBACJ,IAAI,EAAE,CAAC,IAAA,4BAAoB,EAAC,cAAc,CAAC,CAAC;aAC7C;SACF,CAAC,CAAC;QAEH,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,CAAA,MAAA,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,0CAAG,CAAC,CAAC,0CAAE,EAAE,CAAA,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;CAAA"}

package/dist/src/spans/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./addSpanAnnotation";
+export * from "./logSpanAnnotations";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/spans/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/spans/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC"}

package/dist/src/spans/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./addSpanAnnotation"), exports);
+__exportStar(require("./logSpanAnnotations"), exports);
+//# sourceMappingURL=index.js.map

package/dist/src/spans/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/spans/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,uDAAqC"}

package/dist/src/spans/logSpanAnnotations.d.ts

@@ -0,0 +1,51 @@
+import { ClientFn } from "../types/core";
+import { SpanAnnotation } from "./types";
+/**
+ * Parameters to log multiple span annotations
+ */
+interface LogSpanAnnotationsParams extends ClientFn {
+    /**
+     * The span annotations to log
+     */
+    spanAnnotations: SpanAnnotation[];
+}
+/**
+ * Log multiple span annotations in a single request.
+ *
+ * Each annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to log span annotations
+ * @returns The IDs of the created or updated annotations
+ *
+ * @example
+ * ```ts
+ * const results = await logSpanAnnotations({
+ *   spanAnnotations: [
+ *     {
+ *       spanId: "123abc",
+ *       name: "quality_score",
+ *       label: "good",
+ *       score: 0.95,
+ *       annotatorKind: "LLM",
+ *       identifier: "custom_id_123",
+ *       metadata: {
+ *         model: "gpt-4"
+ *       }
+ *     },
+ *     {
+ *       spanId: "456def",
+ *       name: "sentiment",
+ *       label: "positive",
+ *       score: 0.8,
+ *       annotatorKind: "CODE"
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export declare function logSpanAnnotations({ client: _client, spanAnnotations, }: LogSpanAnnotationsParams): Promise<{
+    id: string;
+}[]>;
+export {};
+//# sourceMappingURL=logSpanAnnotations.d.ts.map

package/dist/src/spans/logSpanAnnotations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logSpanAnnotations.d.ts","sourceRoot":"","sources":["../../../src/spans/logSpanAnnotations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,cAAc,EAAwB,MAAM,SAAS,CAAC;AAE/D;;GAEG;AACH,UAAU,wBAAyB,SAAQ,QAAQ;IACjD;;OAEG;IACH,eAAe,EAAE,cAAc,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,MAAM,EAAE,OAAO,EACf,eAAe,GAChB,EAAE,wBAAwB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkBtD"}

package/dist/src/spans/logSpanAnnotations.js

@@ -0,0 +1,68 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logSpanAnnotations = logSpanAnnotations;
+const client_1 = require("../client");
+const types_1 = require("./types");
+/**
+ * Log multiple span annotations in a single request.
+ *
+ * Each annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to log span annotations
+ * @returns The IDs of the created or updated annotations
+ *
+ * @example
+ * ```ts
+ * const results = await logSpanAnnotations({
+ *   spanAnnotations: [
+ *     {
+ *       spanId: "123abc",
+ *       name: "quality_score",
+ *       label: "good",
+ *       score: 0.95,
+ *       annotatorKind: "LLM",
+ *       identifier: "custom_id_123",
+ *       metadata: {
+ *         model: "gpt-4"
+ *       }
+ *     },
+ *     {
+ *       spanId: "456def",
+ *       name: "sentiment",
+ *       label: "positive",
+ *       score: 0.8,
+ *       annotatorKind: "CODE"
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+function logSpanAnnotations(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ client: _client, spanAnnotations, }) {
+        var _b;
+        const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
+        const { data, error } = yield client.POST("/v1/span_annotations", {
+            body: {
+                data: spanAnnotations.map(types_1.toSpanAnnotationData),
+            },
+        });
+        if (error) {
+            throw new Error(`Failed to log span annotations: ${error}`);
+        }
+        if (!((_b = data === null || data === void 0 ? void 0 : data.data) === null || _b === void 0 ? void 0 : _b.length)) {
+            throw new Error("No annotation IDs returned from server");
+        }
+        return data.data;
+    });
+}
+//# sourceMappingURL=logSpanAnnotations.js.map

package/dist/src/spans/logSpanAnnotations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logSpanAnnotations.js","sourceRoot":"","sources":["../../../src/spans/logSpanAnnotations.ts"],"names":[],"mappings":";;;;;;;;;;;AAiDA,gDAqBC;AAtED,sCAAyC;AAEzC,mCAA+D;AAY/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,MAAM,EAAE,OAAO,EACf,eAAe,GACU;;QACzB,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QAEzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAChE,IAAI,EAAE;gBACJ,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,4BAAoB,CAAC;aAChD;SACF,CAAC,CAAC;QAEH,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,CAAA,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,0CAAE,MAAM,CAAA,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;CAAA"}

package/dist/src/spans/types.d.ts

@@ -0,0 +1,43 @@
+import { paths } from "../__generated__/api/v1";
+type SpanAnnotationData = paths["/v1/span_annotations"]["post"]["requestBody"]["content"]["application/json"]["data"][0];
+/**
+ * Parameters for a single span annotation
+ */
+export interface SpanAnnotation {
+    /**
+     * The OpenTelemetry Span ID (hex format without 0x prefix)
+     */
+    spanId: string;
+    /**
+     * The name of the annotation
+     */
+    name: string;
+    /**
+     * The label assigned by the annotation
+     */
+    label?: string;
+    /**
+     * The score assigned by the annotation
+     */
+    score?: number;
+    /**
+     * The identifier of the annotation. If provided, the annotation will be updated if it already exists.
+     */
+    identifier?: string;
+    /**
+     * Metadata for the annotation
+     */
+    metadata?: Record<string, unknown>;
+    /**
+     * The kind of annotator used for the annotation
+     * Can be "HUMAN", "LLM", or "CODE"
+     * @default "HUMAN"
+     */
+    annotatorKind?: SpanAnnotationData["annotator_kind"];
+}
+/**
+ * Convert a SpanAnnotation to the API format
+ */
+export declare function toSpanAnnotationData(annotation: SpanAnnotation): SpanAnnotationData;
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/src/spans/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/spans/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,KAAK,kBAAkB,GACrB,KAAK,CAAC,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjG;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;;OAIG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;CACtD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,cAAc,GACzB,kBAAkB,CAapB"}