@arizeai/phoenix-client 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/README.md +118 -0
  2. package/dist/esm/client.d.ts +13 -1
  3. package/dist/esm/client.d.ts.map +1 -1
  4. package/dist/esm/client.js +4 -1
  5. package/dist/esm/client.js.map +1 -1
  6. package/dist/esm/datasets/appendDatasetExamples.d.ts +21 -0
  7. package/dist/esm/datasets/appendDatasetExamples.d.ts.map +1 -0
  8. package/dist/esm/datasets/appendDatasetExamples.js +32 -0
  9. package/dist/esm/datasets/appendDatasetExamples.js.map +1 -0
  10. package/dist/esm/datasets/createDataset.d.ts +25 -0
  11. package/dist/esm/datasets/createDataset.d.ts.map +1 -0
  12. package/dist/esm/datasets/createDataset.js +34 -0
  13. package/dist/esm/datasets/createDataset.js.map +1 -0
  14. package/dist/esm/datasets/getDataset.d.ts +10 -0
  15. package/dist/esm/datasets/getDataset.d.ts.map +1 -0
  16. package/dist/esm/datasets/getDataset.js +18 -0
  17. package/dist/esm/datasets/getDataset.js.map +1 -0
  18. package/dist/esm/datasets/getDatasetExamples.d.ts +10 -0
  19. package/dist/esm/datasets/getDatasetExamples.d.ts.map +1 -0
  20. package/dist/esm/datasets/getDatasetExamples.js +25 -0
  21. package/dist/esm/datasets/getDatasetExamples.js.map +1 -0
  22. package/dist/esm/datasets/getDatasetInfo.d.ts +11 -0
  23. package/dist/esm/datasets/getDatasetInfo.d.ts.map +1 -0
  24. package/dist/esm/datasets/getDatasetInfo.js +25 -0
  25. package/dist/esm/datasets/getDatasetInfo.js.map +1 -0
  26. package/dist/esm/datasets/index.d.ts +7 -0
  27. package/dist/esm/datasets/index.d.ts.map +1 -0
  28. package/dist/esm/datasets/index.js +7 -0
  29. package/dist/esm/datasets/index.js.map +1 -0
  30. package/dist/esm/datasets/listDatasets.d.ts +23 -0
  31. package/dist/esm/datasets/listDatasets.d.ts.map +1 -0
  32. package/dist/esm/datasets/listDatasets.js +26 -0
  33. package/dist/esm/datasets/listDatasets.js.map +1 -0
  34. package/dist/esm/experiments/getExperiment.d.ts +14 -0
  35. package/dist/esm/experiments/getExperiment.d.ts.map +1 -0
  36. package/dist/esm/experiments/getExperiment.js +25 -0
  37. package/dist/esm/experiments/getExperiment.js.map +1 -0
  38. package/dist/esm/experiments/getExperimentInfo.d.ts +13 -0
  39. package/dist/esm/experiments/getExperimentInfo.d.ts.map +1 -0
  40. package/dist/esm/experiments/getExperimentInfo.js +24 -0
  41. package/dist/esm/experiments/getExperimentInfo.js.map +1 -0
  42. package/dist/esm/experiments/getExperimentRuns.d.ts +15 -0
  43. package/dist/esm/experiments/getExperimentRuns.d.ts.map +1 -0
  44. package/dist/esm/experiments/getExperimentRuns.js +33 -0
  45. package/dist/esm/experiments/getExperimentRuns.js.map +1 -0
  46. package/dist/esm/experiments/index.d.ts +3 -0
  47. package/dist/esm/experiments/index.d.ts.map +1 -1
  48. package/dist/esm/experiments/index.js +3 -0
  49. package/dist/esm/experiments/index.js.map +1 -1
  50. package/dist/esm/experiments/instrumention.d.ts +18 -0
  51. package/dist/esm/experiments/instrumention.d.ts.map +1 -0
  52. package/dist/esm/experiments/instrumention.js +34 -0
  53. package/dist/esm/experiments/instrumention.js.map +1 -0
  54. package/dist/esm/experiments/runExperiment.d.ts +24 -21
  55. package/dist/esm/experiments/runExperiment.d.ts.map +1 -1
  56. package/dist/esm/experiments/runExperiment.js +221 -108
  57. package/dist/esm/experiments/runExperiment.js.map +1 -1
  58. package/dist/esm/schemas/llm/anthropic/converters.d.ts +28 -28
  59. package/dist/esm/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
  60. package/dist/esm/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
  61. package/dist/esm/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
  62. package/dist/esm/schemas/llm/constants.d.ts +6 -6
  63. package/dist/esm/schemas/llm/converters.d.ts +24 -24
  64. package/dist/esm/schemas/llm/openai/converters.d.ts +6 -6
  65. package/dist/esm/schemas/llm/schemas.d.ts +22 -22
  66. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  67. package/dist/esm/types/datasets.d.ts +33 -8
  68. package/dist/esm/types/datasets.d.ts.map +1 -1
  69. package/dist/esm/types/experiments.d.ts +17 -4
  70. package/dist/esm/types/experiments.d.ts.map +1 -1
  71. package/dist/esm/utils/ensureString.d.ts +8 -0
  72. package/dist/esm/utils/ensureString.d.ts.map +1 -0
  73. package/dist/esm/utils/ensureString.js +14 -0
  74. package/dist/esm/utils/ensureString.js.map +1 -0
  75. package/dist/esm/utils/objectAsAttributes.d.ts +3 -0
  76. package/dist/esm/utils/objectAsAttributes.d.ts.map +1 -0
  77. package/dist/esm/utils/objectAsAttributes.js +4 -0
  78. package/dist/esm/utils/objectAsAttributes.js.map +1 -0
  79. package/dist/src/client.d.ts +13 -1
  80. package/dist/src/client.d.ts.map +1 -1
  81. package/dist/src/client.js +1 -1
  82. package/dist/src/client.js.map +1 -1
  83. package/dist/src/datasets/appendDatasetExamples.d.ts +21 -0
  84. package/dist/src/datasets/appendDatasetExamples.d.ts.map +1 -0
  85. package/dist/src/datasets/appendDatasetExamples.js +50 -0
  86. package/dist/src/datasets/appendDatasetExamples.js.map +1 -0
  87. package/dist/src/datasets/createDataset.d.ts +25 -0
  88. package/dist/src/datasets/createDataset.d.ts.map +1 -0
  89. package/dist/src/datasets/createDataset.js +52 -0
  90. package/dist/src/datasets/createDataset.js.map +1 -0
  91. package/dist/src/datasets/getDataset.d.ts +10 -0
  92. package/dist/src/datasets/getDataset.d.ts.map +1 -0
  93. package/dist/src/datasets/getDataset.js +29 -0
  94. package/dist/src/datasets/getDataset.js.map +1 -0
  95. package/dist/src/datasets/getDatasetExamples.d.ts +10 -0
  96. package/dist/src/datasets/getDatasetExamples.d.ts.map +1 -0
  97. package/dist/src/datasets/getDatasetExamples.js +40 -0
  98. package/dist/src/datasets/getDatasetExamples.js.map +1 -0
  99. package/dist/src/datasets/getDatasetInfo.d.ts +11 -0
  100. package/dist/src/datasets/getDatasetInfo.d.ts.map +1 -0
  101. package/dist/src/datasets/getDatasetInfo.js +43 -0
  102. package/dist/src/datasets/getDatasetInfo.js.map +1 -0
  103. package/dist/src/datasets/index.d.ts +7 -0
  104. package/dist/src/datasets/index.d.ts.map +1 -0
  105. package/dist/src/datasets/index.js +23 -0
  106. package/dist/src/datasets/index.js.map +1 -0
  107. package/dist/src/datasets/listDatasets.d.ts +23 -0
  108. package/dist/src/datasets/listDatasets.d.ts.map +1 -0
  109. package/dist/src/datasets/listDatasets.js +40 -0
  110. package/dist/src/datasets/listDatasets.js.map +1 -0
  111. package/dist/src/experiments/getExperiment.d.ts +14 -0
  112. package/dist/src/experiments/getExperiment.d.ts.map +1 -0
  113. package/dist/src/experiments/getExperiment.js +36 -0
  114. package/dist/src/experiments/getExperiment.js.map +1 -0
  115. package/dist/src/experiments/getExperimentInfo.d.ts +13 -0
  116. package/dist/src/experiments/getExperimentInfo.d.ts.map +1 -0
  117. package/dist/src/experiments/getExperimentInfo.js +41 -0
  118. package/dist/src/experiments/getExperimentInfo.js.map +1 -0
  119. package/dist/src/experiments/getExperimentRuns.d.ts +15 -0
  120. package/dist/src/experiments/getExperimentRuns.d.ts.map +1 -0
  121. package/dist/src/experiments/getExperimentRuns.js +50 -0
  122. package/dist/src/experiments/getExperimentRuns.js.map +1 -0
  123. package/dist/src/experiments/index.d.ts +3 -0
  124. package/dist/src/experiments/index.d.ts.map +1 -1
  125. package/dist/src/experiments/index.js +3 -0
  126. package/dist/src/experiments/index.js.map +1 -1
  127. package/dist/src/experiments/instrumention.d.ts +18 -0
  128. package/dist/src/experiments/instrumention.d.ts.map +1 -0
  129. package/dist/src/experiments/instrumention.js +38 -0
  130. package/dist/src/experiments/instrumention.js.map +1 -0
  131. package/dist/src/experiments/runExperiment.d.ts +24 -21
  132. package/dist/src/experiments/runExperiment.d.ts.map +1 -1
  133. package/dist/src/experiments/runExperiment.js +222 -111
  134. package/dist/src/experiments/runExperiment.js.map +1 -1
  135. package/dist/src/schemas/llm/anthropic/converters.d.ts +28 -28
  136. package/dist/src/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
  137. package/dist/src/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
  138. package/dist/src/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
  139. package/dist/src/schemas/llm/constants.d.ts +6 -6
  140. package/dist/src/schemas/llm/converters.d.ts +24 -24
  141. package/dist/src/schemas/llm/openai/converters.d.ts +6 -6
  142. package/dist/src/schemas/llm/schemas.d.ts +22 -22
  143. package/dist/src/types/datasets.d.ts +33 -8
  144. package/dist/src/types/datasets.d.ts.map +1 -1
  145. package/dist/src/types/experiments.d.ts +17 -4
  146. package/dist/src/types/experiments.d.ts.map +1 -1
  147. package/dist/src/utils/ensureString.d.ts +8 -0
  148. package/dist/src/utils/ensureString.d.ts.map +1 -0
  149. package/dist/src/utils/ensureString.js +18 -0
  150. package/dist/src/utils/ensureString.js.map +1 -0
  151. package/dist/src/utils/objectAsAttributes.d.ts +3 -0
  152. package/dist/src/utils/objectAsAttributes.d.ts.map +1 -0
  153. package/dist/src/utils/objectAsAttributes.js +7 -0
  154. package/dist/src/utils/objectAsAttributes.js.map +1 -0
  155. package/dist/tsconfig.tsbuildinfo +1 -1
  156. package/package.json +13 -1
  157. package/src/client.ts +4 -1
  158. package/src/datasets/appendDatasetExamples.ts +55 -0
  159. package/src/datasets/createDataset.ts +60 -0
  160. package/src/datasets/getDataset.ts +27 -0
  161. package/src/datasets/getDatasetExamples.ts +34 -0
  162. package/src/datasets/getDatasetInfo.ts +34 -0
  163. package/src/datasets/index.ts +6 -0
  164. package/src/datasets/listDatasets.ts +37 -0
  165. package/src/experiments/getExperiment.ts +40 -0
  166. package/src/experiments/getExperimentInfo.ts +39 -0
  167. package/src/experiments/getExperimentRuns.ts +45 -0
  168. package/src/experiments/index.ts +3 -0
  169. package/src/experiments/instrumention.ts +52 -0
  170. package/src/experiments/runExperiment.ts +277 -133
  171. package/src/types/datasets.ts +35 -9
  172. package/src/types/experiments.ts +19 -4
  173. package/src/utils/ensureString.ts +14 -0
  174. package/src/utils/objectAsAttributes.ts +9 -0
  175. package/dist/esm/utils/getDatasetBySelector.d.ts +0 -25
  176. package/dist/esm/utils/getDatasetBySelector.d.ts.map +0 -1
  177. package/dist/esm/utils/getDatasetBySelector.js +0 -37
  178. package/dist/esm/utils/getDatasetBySelector.js.map +0 -1
  179. package/dist/src/utils/getDatasetBySelector.d.ts +0 -25
  180. package/dist/src/utils/getDatasetBySelector.d.ts.map +0 -1
  181. package/dist/src/utils/getDatasetBySelector.js +0 -47
  182. package/dist/src/utils/getDatasetBySelector.js.map +0 -1
  183. package/src/utils/getDatasetBySelector.ts +0 -55
@@ -15,15 +15,35 @@ Object.defineProperty(exports, "__esModule", { value: true });
15
15
  exports.runExperiment = runExperiment;
16
16
  exports.evaluateExperiment = evaluateExperiment;
17
17
  exports.asEvaluator = asEvaluator;
18
- exports.id = id;
19
18
  const async_1 = require("async");
20
19
  const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
21
20
  const client_1 = require("../client");
22
- const getDatasetBySelector_1 = require("../utils/getDatasetBySelector");
21
+ const getDataset_1 = require("../datasets/getDataset");
23
22
  const pluralize_1 = require("../utils/pluralize");
24
23
  const promisifyResult_1 = require("../utils/promisifyResult");
24
+ const instrumention_1 = require("./instrumention");
25
+ const api_1 = require("@opentelemetry/api");
26
+ const openinference_semantic_conventions_1 = require("@arizeai/openinference-semantic-conventions");
27
+ const ensureString_1 = require("../utils/ensureString");
28
+ const objectAsAttributes_1 = require("../utils/objectAsAttributes");
25
29
  /**
26
- * Run an experiment.
30
+ * Runs an experiment using a given set of dataset of examples.
31
+ *
32
+ * An experiment is a user-defined task that runs on each example in a dataset. The results from
33
+ * each experiment can be evaluated using any number of evaluators to measure the behavior of the
34
+ * task. The experiment and evaluation results are stored in the Phoenix database for comparison
35
+ * and analysis.
36
+ *
37
+ * A `task` is either a sync or async function that returns a JSON serializable
38
+ * output. If the `task` is a function of one argument then that argument will be bound to the
39
+ * `input` field of the dataset example. Alternatively, the `task` can be a function of any
40
+ * combination of specific argument names that will be bound to special values:
41
+ *
42
+ * - `input`: The input field of the dataset example
43
+ * - `expected`: The expected or reference output of the dataset example
44
+ * - `reference`: An alias for `expected`
45
+ * - `metadata`: Metadata associated with the dataset example
46
+ * - `example`: The dataset `Example` object with all associated fields
27
47
  *
28
48
  * @example
29
49
  * ```ts
@@ -33,36 +53,36 @@ const promisifyResult_1 = require("../utils/promisifyResult");
33
53
  * dataset: "my-dataset",
34
54
  * task: async (example) => example.input,
35
55
  * evaluators: [
36
- * asEvaluator("my-evaluator", "CODE", async (params) => params.output),
56
+ * asEvaluator({ name: "my-evaluator", kind: "CODE", evaluate: async (params) => params.output }),
37
57
  * ],
38
58
  * });
39
59
  * ```
40
- *
41
- * @experimental This feature is not complete, and will change in the future.
42
60
  */
43
61
  function runExperiment(_a) {
44
- return __awaiter(this, arguments, void 0, function* ({ experimentName: _experimentName, experimentDescription, experimentMetadata, client: _client, dataset: _dataset, task, evaluators, projectName = "default", logger = console, record = true, concurrency = 5, dryRun = false, }) {
45
- var _b, _c;
62
+ return __awaiter(this, arguments, void 0, function* ({ experimentName, experimentDescription, experimentMetadata = {}, client: _client, dataset: DatasetSelector, task, evaluators, logger = console, record = true, concurrency = 5, dryRun = false, }) {
63
+ var _b, _c, _d, _e, _f;
64
+ let provider;
46
65
  const isDryRun = typeof dryRun === "number" || dryRun === true;
47
66
  const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
48
- const dataset = yield (0, getDatasetBySelector_1.getDatasetBySelector)({ dataset: _dataset, client });
67
+ const dataset = yield (0, getDataset_1.getDataset)({ dataset: DatasetSelector, client });
49
68
  (0, tiny_invariant_1.default)(dataset, `Dataset not found`);
50
69
  (0, tiny_invariant_1.default)(dataset.examples.length > 0, `Dataset has no examples`);
51
70
  const nExamples = typeof dryRun === "number"
52
- ? Math.max(dryRun, dataset.examples.length)
71
+ ? Math.min(dryRun, dataset.examples.length)
53
72
  : dataset.examples.length;
54
- const experimentName = _experimentName !== null && _experimentName !== void 0 ? _experimentName : `${dataset.name}-${new Date().toISOString()}`;
55
- const experimentParams = {
56
- nExamples,
57
- };
73
+ let projectName = `${dataset.name}-exp-${new Date().toISOString()}`;
74
+ // initialize the tracer into scope
75
+ let taskTracer;
58
76
  let experiment;
59
77
  if (isDryRun) {
60
78
  experiment = {
61
- id: id(),
79
+ id: localId(),
62
80
  datasetId: dataset.id,
63
81
  datasetVersionId: dataset.versionId,
64
82
  projectName,
83
+ metadata: experimentMetadata,
65
84
  };
85
+ taskTracer = (0, instrumention_1.createNoOpProvider)().getTracer("no-op");
66
86
  }
67
87
  else {
68
88
  const experimentResponse = yield client
@@ -81,19 +101,30 @@ function runExperiment(_a) {
81
101
  })
82
102
  .then((res) => { var _a; return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data; });
83
103
  (0, tiny_invariant_1.default)(experimentResponse, `Failed to create experiment`);
104
+ projectName = (_b = experimentResponse.project_name) !== null && _b !== void 0 ? _b : projectName;
84
105
  experiment = {
85
106
  id: experimentResponse.id,
86
- datasetId: dataset.id,
87
- datasetVersionId: dataset.versionId,
107
+ datasetId: experimentResponse.dataset_id,
108
+ datasetVersionId: experimentResponse.dataset_version_id,
88
109
  projectName,
110
+ metadata: experimentResponse.metadata,
89
111
  };
112
+ // Initialize the tracer, now that we have a project name
113
+ const baseUrl = client.config.baseUrl;
114
+ (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
115
+ provider = (0, instrumention_1.createProvider)({
116
+ projectName,
117
+ baseUrl,
118
+ headers: (_c = client.config.headers) !== null && _c !== void 0 ? _c : {},
119
+ });
120
+ taskTracer = provider.getTracer(projectName);
90
121
  }
91
122
  if (!record) {
92
123
  logger.info(`🔧 Running experiment in readonly mode. Results will not be recorded.`);
93
124
  }
94
- logger.info(`🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${(_b = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _b !== void 0 ? _b : 0} ${(0, pluralize_1.pluralize)("evaluator", (_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0)} and ${concurrency} concurrent runs`);
125
+ logger.info(`🧪 Starting experiment "${experimentName || `<unnamed>`}" on dataset "${dataset.id}" with task "${task.name}" and ${(_d = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _d !== void 0 ? _d : 0} ${(0, pluralize_1.pluralize)("evaluator", (_e = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _e !== void 0 ? _e : 0)} and ${concurrency} concurrent runs`);
95
126
  const runs = {};
96
- yield runTask({
127
+ yield runTaskWithExamples({
97
128
  client,
98
129
  experimentId: experiment.id,
99
130
  task,
@@ -105,9 +136,14 @@ function runExperiment(_a) {
105
136
  concurrency,
106
137
  isDryRun,
107
138
  nExamples,
139
+ tracer: taskTracer,
108
140
  });
109
141
  logger.info(`✅ Task runs completed`);
110
- const ranExperiment = Object.assign(Object.assign({}, experiment), { params: experimentParams, runs });
142
+ const ranExperiment = Object.assign(Object.assign({}, experiment), { runs });
143
+ // Shut down the provider so that the experiments run
144
+ if (provider) {
145
+ yield ((_f = provider.shutdown) === null || _f === void 0 ? void 0 : _f.call(provider));
146
+ }
111
147
  const { evaluationRuns } = yield evaluateExperiment({
112
148
  experiment: ranExperiment,
113
149
  evaluators: evaluators !== null && evaluators !== void 0 ? evaluators : [],
@@ -124,61 +160,79 @@ function runExperiment(_a) {
124
160
  /**
125
161
  * Run a task against n examples in a dataset.
126
162
  */
127
- function runTask({ client, experimentId, task, dataset, onComplete, logger, concurrency = 5, isDryRun, nExamples, }) {
163
+ function runTaskWithExamples({ client, experimentId, task, dataset, onComplete, logger, concurrency = 5, isDryRun, nExamples, tracer, }) {
128
164
  logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
129
165
  const run = (example) => __awaiter(this, void 0, void 0, function* () {
130
- var _a, _b;
131
- logger.info(`🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`);
132
- const thisRun = {
133
- id: id(),
134
- traceId: null, // TODO: fill this in once we trace experiments
135
- experimentId,
136
- datasetExampleId: example.id,
137
- startTime: new Date(),
138
- endTime: new Date(), // will get replaced with actual end time
139
- output: null,
140
- error: null,
141
- };
142
- try {
143
- const taskOutput = yield (0, promisifyResult_1.promisifyResult)(task(example));
144
- // TODO: why doesn't run output type match task output type?
145
- thisRun.output =
146
- typeof taskOutput === "string"
147
- ? taskOutput
148
- : JSON.stringify(taskOutput);
149
- }
150
- catch (error) {
151
- thisRun.error = error instanceof Error ? error.message : "Unknown error";
152
- }
153
- thisRun.endTime = new Date();
154
- if (!isDryRun) {
155
- // Log the run to the server
156
- // We log this without awaiting (e.g. best effort)
157
- const res = yield client.POST("/v1/experiments/{experiment_id}/runs", {
158
- params: {
159
- path: {
160
- experiment_id: experimentId,
166
+ return tracer.startActiveSpan(`Task: ${task.name}`, (span) => __awaiter(this, void 0, void 0, function* () {
167
+ var _a, _b;
168
+ logger.info(`🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`);
169
+ const traceId = span.spanContext().traceId;
170
+ const thisRun = {
171
+ id: localId(), // initialized with local id, will be replaced with server-assigned id when dry run is false
172
+ traceId,
173
+ experimentId,
174
+ datasetExampleId: example.id,
175
+ startTime: new Date(),
176
+ endTime: new Date(), // will get replaced with actual end time
177
+ output: null,
178
+ error: null,
179
+ };
180
+ try {
181
+ const taskOutput = yield (0, promisifyResult_1.promisifyResult)(task(example));
182
+ thisRun.output =
183
+ typeof taskOutput === "string"
184
+ ? taskOutput
185
+ : JSON.stringify(taskOutput);
186
+ }
187
+ catch (error) {
188
+ thisRun.error =
189
+ error instanceof Error ? error.message : "Unknown error";
190
+ span.setStatus({ code: api_1.SpanStatusCode.ERROR });
191
+ }
192
+ thisRun.endTime = new Date();
193
+ if (!isDryRun) {
194
+ // Log the run to the server
195
+ const res = yield client.POST("/v1/experiments/{experiment_id}/runs", {
196
+ params: {
197
+ path: {
198
+ experiment_id: experimentId,
199
+ },
161
200
  },
162
- },
163
- body: {
164
- dataset_example_id: example.id,
165
- output: thisRun.output,
166
- repetition_number: 0,
167
- start_time: thisRun.startTime.toISOString(),
168
- end_time: thisRun.endTime.toISOString(),
169
- trace_id: thisRun.traceId,
170
- error: thisRun.error,
171
- },
172
- });
173
- // replace the local run id with the server-assigned id
174
- thisRun.id = (_b = (_a = res.data) === null || _a === void 0 ? void 0 : _a.data.id) !== null && _b !== void 0 ? _b : thisRun.id;
175
- }
176
- onComplete(thisRun);
177
- return thisRun;
201
+ body: {
202
+ dataset_example_id: example.id,
203
+ output: thisRun.output,
204
+ repetition_number: 0,
205
+ start_time: thisRun.startTime.toISOString(),
206
+ end_time: thisRun.endTime.toISOString(),
207
+ trace_id: thisRun.traceId,
208
+ error: thisRun.error,
209
+ },
210
+ });
211
+ // replace the local run id with the server-assigned id
212
+ thisRun.id = (_b = (_a = res.data) === null || _a === void 0 ? void 0 : _a.data.id) !== null && _b !== void 0 ? _b : thisRun.id;
213
+ const inputMimeType = typeof example.input === "string" ? openinference_semantic_conventions_1.MimeType.TEXT : openinference_semantic_conventions_1.MimeType.JSON;
214
+ const outputMimeType = typeof thisRun.output === "string" ? openinference_semantic_conventions_1.MimeType.TEXT : openinference_semantic_conventions_1.MimeType.JSON;
215
+ span.setStatus({ code: api_1.SpanStatusCode.OK });
216
+ span.setAttributes({
217
+ [openinference_semantic_conventions_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: openinference_semantic_conventions_1.OpenInferenceSpanKind.CHAIN,
218
+ [openinference_semantic_conventions_1.SemanticConventions.INPUT_MIME_TYPE]: inputMimeType,
219
+ [openinference_semantic_conventions_1.SemanticConventions.INPUT_VALUE]: (0, ensureString_1.ensureString)(example.input),
220
+ [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_MIME_TYPE]: outputMimeType,
221
+ [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(thisRun.output),
222
+ });
223
+ }
224
+ span === null || span === void 0 ? void 0 : span.end();
225
+ onComplete(thisRun);
226
+ return thisRun;
227
+ }));
178
228
  });
179
229
  const q = (0, async_1.queue)(run, concurrency);
180
230
  const examplesToUse = dataset.examples.slice(0, nExamples);
181
- examplesToUse.forEach((example) => q.push(example));
231
+ examplesToUse.forEach((example) => q.push(example, (err) => {
232
+ if (err) {
233
+ logger.error(`Error running task "${task.name}" on example "${example.id}": ${err}`);
234
+ }
235
+ }));
182
236
  return q.drain();
183
237
  }
184
238
  /**
@@ -187,15 +241,31 @@ function runTask({ client, experimentId, task, dataset, onComplete, logger, conc
187
241
  * @experimental This feature is not complete, and will change in the future.
188
242
  */
189
243
  function evaluateExperiment(_a) {
190
- return __awaiter(this, arguments, void 0, function* ({ experiment, evaluators, client: _client, logger, concurrency = 5, dryRun = false, }) {
191
- var _b, _c;
244
+ return __awaiter(this, arguments, void 0, function* ({ experiment, evaluators, client: _client, logger = console, concurrency = 5, dryRun = false, }) {
245
+ var _b, _c, _d, _e;
192
246
  const isDryRun = typeof dryRun === "number" || dryRun === true;
247
+ const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
248
+ const baseUrl = client.config.baseUrl;
249
+ (0, tiny_invariant_1.default)(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
250
+ let provider;
251
+ if (!isDryRun) {
252
+ provider = (0, instrumention_1.createProvider)({
253
+ projectName: "evaluators",
254
+ baseUrl,
255
+ headers: (_b = client.config.headers) !== null && _b !== void 0 ? _b : {},
256
+ });
257
+ }
258
+ else {
259
+ provider = (0, instrumention_1.createNoOpProvider)();
260
+ }
261
+ const tracer = isDryRun
262
+ ? provider.getTracer("no-op")
263
+ : provider.getTracer("evaluators");
193
264
  const nRuns = typeof dryRun === "number"
194
265
  ? Math.max(dryRun, Object.keys(experiment.runs).length)
195
266
  : Object.keys(experiment.runs).length;
196
- const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
197
- const dataset = yield (0, getDatasetBySelector_1.getDatasetBySelector)({
198
- dataset: experiment.datasetId,
267
+ const dataset = yield (0, getDataset_1.getDataset)({
268
+ dataset: { datasetId: experiment.datasetId },
199
269
  client,
200
270
  });
201
271
  (0, tiny_invariant_1.default)(dataset, `Dataset "${experiment.datasetId}" not found`);
@@ -205,7 +275,7 @@ function evaluateExperiment(_a) {
205
275
  if ((evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) === 0) {
206
276
  return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
207
277
  }
208
- logger.info(`🧠 Evaluating experiment "${experiment.id}" with ${(_b = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _b !== void 0 ? _b : 0} ${(0, pluralize_1.pluralize)("evaluator", (_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0)}`);
278
+ logger.info(`🧠 Evaluating experiment "${experiment.id}" with ${(_c = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _c !== void 0 ? _c : 0} ${(0, pluralize_1.pluralize)("evaluator", (_d = evaluators === null || evaluators === void 0 ? void 0 : evaluators.length) !== null && _d !== void 0 ? _d : 0)}`);
209
279
  const evaluationRuns = {};
210
280
  const examplesById = {};
211
281
  for (const example of dataset.examples) {
@@ -221,33 +291,74 @@ function evaluateExperiment(_a) {
221
291
  run,
222
292
  })));
223
293
  const evaluatorsQueue = (0, async_1.queue)((evaluatorAndRun) => __awaiter(this, void 0, void 0, function* () {
224
- const evalResult = yield runEvaluator({
225
- evaluator: evaluatorAndRun.evaluator,
226
- run: evaluatorAndRun.run,
227
- exampleCache: examplesById,
228
- onComplete: onEvaluationComplete,
229
- });
230
- if (!isDryRun) {
231
- logger.info(`📝 Logging evaluation ${evalResult.id}`);
232
- // Log the evaluation to the server
233
- // We log this without awaiting (e.g. best effort)
234
- client.POST("/v1/experiment_evaluations", {
235
- body: {
236
- experiment_run_id: evaluatorAndRun.run.id,
237
- name: evaluatorAndRun.evaluator.name,
238
- annotator_kind: evaluatorAndRun.evaluator.kind,
239
- start_time: evalResult.startTime.toISOString(),
240
- end_time: evalResult.endTime.toISOString(),
241
- result: Object.assign({}, evalResult.result),
242
- error: evalResult.error,
243
- trace_id: evalResult.traceId,
244
- },
294
+ return tracer.startActiveSpan(`Evaluation: ${evaluatorAndRun.evaluator.name}`, (span) => __awaiter(this, void 0, void 0, function* () {
295
+ var _a, _b, _c;
296
+ const evalResult = yield runEvaluator({
297
+ evaluator: evaluatorAndRun.evaluator,
298
+ run: evaluatorAndRun.run,
299
+ exampleCache: examplesById,
300
+ onComplete: onEvaluationComplete,
301
+ logger,
245
302
  });
246
- }
303
+ span.setAttributes({
304
+ [openinference_semantic_conventions_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: openinference_semantic_conventions_1.OpenInferenceSpanKind.EVALUATOR,
305
+ [openinference_semantic_conventions_1.SemanticConventions.INPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
306
+ [openinference_semantic_conventions_1.SemanticConventions.INPUT_VALUE]: JSON.stringify({
307
+ input: (_a = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _a === void 0 ? void 0 : _a.input,
308
+ output: evaluatorAndRun.run.output,
309
+ expected: (_b = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _b === void 0 ? void 0 : _b.output,
310
+ metadata: (_c = examplesById[evaluatorAndRun.run.datasetExampleId]) === null || _c === void 0 ? void 0 : _c.metadata,
311
+ }),
312
+ [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_MIME_TYPE]: openinference_semantic_conventions_1.MimeType.JSON,
313
+ [openinference_semantic_conventions_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(evalResult.result),
314
+ });
315
+ if (evalResult.error) {
316
+ span.setStatus({
317
+ code: api_1.SpanStatusCode.ERROR,
318
+ message: evalResult.error,
319
+ });
320
+ }
321
+ else {
322
+ span.setStatus({ code: api_1.SpanStatusCode.OK });
323
+ }
324
+ if (evalResult.result) {
325
+ span.setAttributes((0, objectAsAttributes_1.objectAsAttributes)(evalResult.result));
326
+ }
327
+ evalResult.traceId = span.spanContext().traceId;
328
+ if (!isDryRun) {
329
+ // Log the evaluation to the server
330
+ // We log this without awaiting (e.g. best effort)
331
+ client.POST("/v1/experiment_evaluations", {
332
+ body: {
333
+ experiment_run_id: evaluatorAndRun.run.id,
334
+ name: evaluatorAndRun.evaluator.name,
335
+ annotator_kind: evaluatorAndRun.evaluator.kind,
336
+ start_time: evalResult.startTime.toISOString(),
337
+ end_time: evalResult.endTime.toISOString(),
338
+ result: Object.assign({}, evalResult.result),
339
+ error: evalResult.error,
340
+ trace_id: evalResult.traceId,
341
+ },
342
+ });
343
+ }
344
+ span.end();
345
+ return evalResult;
346
+ }));
247
347
  }), concurrency);
248
- evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun));
348
+ if (!evaluatorsAndRuns.length) {
349
+ logger.info(`⛔ No evaluators to run`);
350
+ return Object.assign(Object.assign({}, experiment), { evaluationRuns: [] });
351
+ }
352
+ evaluatorsAndRuns.forEach((evaluatorAndRun) => evaluatorsQueue.push(evaluatorAndRun, (err) => {
353
+ if (err) {
354
+ logger.error(`❌ Error running evaluator "${evaluatorAndRun.evaluator.name}" on run "${evaluatorAndRun.run.id}": ${err}`);
355
+ }
356
+ }));
249
357
  yield evaluatorsQueue.drain();
250
358
  logger.info(`✅ Evaluation runs completed`);
359
+ if (provider) {
360
+ yield ((_e = provider.shutdown) === null || _e === void 0 ? void 0 : _e.call(provider));
361
+ }
251
362
  return Object.assign(Object.assign({}, experiment), { evaluationRuns: Object.values(evaluationRuns) });
252
363
  });
253
364
  }
@@ -257,21 +368,22 @@ function evaluateExperiment(_a) {
257
368
  * @experimental This feature is not complete, and will change in the future.
258
369
  */
259
370
  function runEvaluator(_a) {
260
- return __awaiter(this, arguments, void 0, function* ({ evaluator, run, exampleCache, onComplete, }) {
371
+ return __awaiter(this, arguments, void 0, function* ({ evaluator, run, exampleCache, onComplete, logger, }) {
261
372
  const example = exampleCache[run.datasetExampleId];
262
373
  (0, tiny_invariant_1.default)(example, `Example "${run.datasetExampleId}" not found`);
263
374
  const evaluate = () => __awaiter(this, void 0, void 0, function* () {
264
375
  var _a;
376
+ logger.info(`🧠 Evaluating run "${run.id}" with evaluator "${evaluator.name}"`);
265
377
  const thisEval = {
266
- id: id(),
267
- traceId: null, // TODO: fill this in once we trace experiments
378
+ id: localId(),
379
+ traceId: null,
268
380
  experimentRunId: run.id,
269
381
  startTime: new Date(),
270
382
  endTime: new Date(), // will get replaced with actual end time
271
383
  name: evaluator.name,
272
384
  result: null,
273
385
  error: null,
274
- annotatorKind: "LLM", // TODO: make configurable via evaluator def
386
+ annotatorKind: evaluator.kind,
275
387
  };
276
388
  try {
277
389
  const result = yield evaluator.evaluate({
@@ -281,9 +393,11 @@ function runEvaluator(_a) {
281
393
  metadata: example.metadata,
282
394
  });
283
395
  thisEval.result = result;
396
+ logger.info(`✅ Evaluator "${evaluator.name}" on run "${run.id}" completed`);
284
397
  }
285
398
  catch (error) {
286
399
  thisEval.error = error instanceof Error ? error.message : "Unknown error";
400
+ logger.error(`❌ Evaluator "${evaluator.name}" on run "${run.id}" failed: ${thisEval.error}`);
287
401
  }
288
402
  thisEval.endTime = new Date();
289
403
  onComplete(thisEval);
@@ -308,17 +422,14 @@ function asEvaluator({ name, kind, evaluate, }) {
308
422
  evaluate,
309
423
  };
310
424
  }
311
- let _id = 1000;
425
+ let _localIdIndex = 1000;
312
426
  /**
313
- * Generate a unique id.
427
+ * Generate a local id.
314
428
  *
315
- * @deprecated Use id generated by phoenix instead.
316
- * @returns A unique id.
429
+ * @returns A semi-unique id.
317
430
  */
318
- function id() {
319
- return (() => {
320
- _id++;
321
- return _id.toString();
322
- })();
431
+ function localId() {
432
+ _localIdIndex++;
433
+ return `local_${_localIdIndex}`;
323
434
  }
324
435
  //# sourceMappingURL=runExperiment.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AA4FA,sCAgHC;AAiGD,gDAwHC;AA4DD,kCAcC;AAUD,gBAKC;AA9fD,iCAA8B;AAC9B,oEAAuC;AACvC,sCAA6D;AAa7D,wEAAqE;AACrE,kDAA+C;AAC/C,8DAA2D;AAyD3D;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAsB,aAAa;yDAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,qBAAqB,EACrB,kBAAkB,EAClB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,SAAS,EACvB,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,EACb,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GACM;;QACpB,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,IAAA,wBAAS,EAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACxC,IAAA,wBAAS,EAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAClE,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9B,MAAM,cAAc,GAClB,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QACnE,MAAM,gBAAgB,GAAyB;YAC7C,SAAS;SACV,CAAC;QACF,IAAI,UAAsB,CAAC;QAC3B,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,GAAG;gBACX,EAAE,EAAE,EAAE,EAAE;gBACR,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,WAAW;aACZ,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,kBAAkB,GAAG,MAAM,MAAM;iBACpC,IAAI,CAAC,uCAAuC,EAAE;gBAC7C,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,UAAU,EAAE,OAAO,CAAC,EAAE;qBACvB;iBACF;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,cAAc;oBACpB,WAAW,EAAE,qBAAqB;oBAClC,QAAQ,EAAE,kBAAkB;oBAC5B,YAAY,EAAE,WAAW;iBAC1B;aACF,CAAC;iBACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAA,EAAA,CAAC,CAAC;YACjC,IAAA,wBAAS,EAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC;YAC7D,UAAU,GAAG;gBACX,EAAE,EAAE,kBAAkB,CAAC,EAAE;gBACzB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACxI,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,QAAQ,WAAW,kBAAkB,CACvC,CAAC;QAIF,MAAM,IAAI,GAA2C,EAAE,CAAC;QACxD,MAAM,OAAO,CAAC;YACZ,MAAM;YACN,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,IAAI;YACJ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;YACD,WAAW;YACX,QAAQ;YACR,SAAS;SACV,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAErC,MAAM,aAAa,mCACd,UAAU,KACb,MAAM,EAAE,gBAAgB,EACxB,IAAI,GACL,CAAC;QAEF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;YAClD,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,EAAE;YAC5B,MAAM;YACN,MAAM;YACN,WAAW;YACX,MAAM;SACP,CAAC,CAAC;QACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QAEvD,OAAO,aAAa,CAAC;IACvB,CAAC;CAAA;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,EACf,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,WAAW,GAAG,CAAC,EACf,QAAQ,EACR,SAAS,GAoBV;IACC,MAAM,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,CAAO,OAAgB,EAAE,EAAE;;QACrC,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,OAAO,CAAC,EAAE,GAAG,CACtF,CAAC;QACF,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,EAAE,EAAE;YACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;YAC9D,YAAY;YACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;YAC9D,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,4DAA4D;YAC5D,OAAO,CAAC,MAAM;gBACZ,OAAO,UAAU,KAAK,QAAQ;oBAC5B,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,4BAA4B;YAC5B,kDAAkD;YAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;gBACpE,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,aAAa,EAAE,YAAY;qBAC5B;iBACF;gBACD,IAAI,EAAE;oBACJ,kBAAkB,EAAE,OAAO,CAAC,EAAE;oBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,iBAAiB,EAAE,CAAC;oBACpB,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;oBACvC,QAAQ,EAAE,OAAO,CAAC,OAAO;oBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB;aACF,CAAC,CAAC;YACH,uDAAuD;YACvD,OAAO,CAAC,EAAE,GAAG,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAC,EAAE,mCAAI,OAAO,CAAC,EAAE,CAAC;QAC/C,CAAC;QACD,UAAU,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC,CAAA,CAAC;IACF,MAAM,CAAC,GAAG,IAAA,aAAK,EAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3D,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,EACN,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GAqBf;;QACC,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,KAAK,GACT,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACvD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,2CAAoB,EAAC;YACzC,OAAO,EAAE,UAAU,CAAC,SAAS;YAC7B,MAAM;SACP,CAAC,CAAC;QACH,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;QAClE,IAAA,wBAAS,EACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;QACF,IAAA,wBAAS,EAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;QAExE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEtE,IAAI,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YAC7B,uCACK,UAAU,KACb,cAAc,EAAE,EAAE,IAClB;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACtF,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,EAAE,CACJ,CAAC;QAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;QAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;YAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QAC/B,CAAC,CAAC;QAEF,kCAAkC;QAClC,sCAAsC;QACtC,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3B,SAAS;YACT,GAAG;SACJ,CAAC,CAAC,CACJ,CAAC;QACF,MAAM,eAAe,GAAG,IAAA,aAAK,EAC3B,CAAO,eAA6D,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;gBACpC,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,GAAG,EAAE,eAAe,CAAC,GAAG;gBACxB,YAAY,EAAE,YAAY;gBAC1B,UAAU,EAAE,oBAAoB;aACjC,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtD,mCAAmC;gBACnC,kDAAkD;gBAClD,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;oBACxC,IAAI,EAAE;wBACJ,iBAAiB,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE;wBACzC,IAAI,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;wBACpC,cAAc,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;wBAC9C,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,WAAW,EAAE;wBAC9C,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE;wBAC1C,MAAM,oBACD,UAAU,CAAC,MAAM,CACrB;wBACD,KAAK,EAAE,UAAU,CAAC,KAAK;wBACvB,QAAQ,EAAE,UAAU,CAAC,OAAO;qBAC7B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAA,EACD,WAAW,CACZ,CAAC;QACF,iBAAiB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CACtC,CAAC;QACF,MAAM,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAE3C,uCACK,UAAU,KACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAC7C;IACJ,CAAC;CAAA;AAED;;;;GAIG;AACH,SAAe,YAAY;yDAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,GAMX;QACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,GAAS,EAAE;;YAC1B,MAAM,QAAQ,GAA4B;gBACxC,EAAE,EAAE,EAAE,EAAE;gBACR,OAAO,EAAE,IAAI,EAAE,+CAA+C;gBAC9D,eAAe,EAAE,GAAG,CAAC,EAAE;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;gBAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,KAAK,EAAE,4CAA4C;aACnE,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;oBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,MAAA,GAAG,CAAC,MAAM,mCAAI,IAAI;oBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;oBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;gBACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC5E,CAAC;YACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAA,CAAC;QAEF,OAAO,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GAKT;IACC,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,GAAG,GAAG,IAAI,CAAC;AAEf;;;;;GAKG;AACH,SAAgB,EAAE;IAChB,OAAO,CAAC,GAAG,EAAE;QACX,GAAG,EAAE,CAAC;QACN,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;AACP,CAAC"}
1
+ {"version":3,"file":"runExperiment.js","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAsHA,sCAgIC;AA6HD,gDA0LC;AAuED,kCAcC;AAloBD,iCAA8B;AAC9B,oEAAuC;AACvC,sCAA6D;AAkB7D,uDAAoD;AACpD,kDAA+C;AAC/C,8DAA2D;AAE3D,mDAAqE;AACrE,4CAA4D;AAC5D,oGAIqD;AACrD,wDAAqD;AAErD,oEAAiE;AAqDjE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,SAAsB,aAAa;yDAAC,EAClC,cAAc,EACd,qBAAqB,EACrB,kBAAkB,GAAG,EAAE,EACvB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,eAAe,EACxB,IAAI,EACJ,UAAU,EACV,MAAM,GAAG,OAAO,EAChB,MAAM,GAAG,IAAI,EACb,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GACM;;QACpB,IAAI,QAAwC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAU,EAAC,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC;QACvE,IAAA,wBAAS,EAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACxC,IAAA,wBAAS,EAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAClE,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9B,IAAI,WAAW,GAAG,GAAG,OAAO,CAAC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QACpE,mCAAmC;QACnC,IAAI,UAAkB,CAAC;QACvB,IAAI,UAA0B,CAAC;QAC/B,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,GAAG;gBACX,EAAE,EAAE,OAAO,EAAE;gBACb,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,WAAW;gBACX,QAAQ,EAAE,kBAAkB;aAC7B,CAAC;YACF,UAAU,GAAG,IAAA,kCAAkB,GAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,kBAAkB,GAAG,MAAM,MAAM;iBACpC,IAAI,CAAC,uCAAuC,EAAE;gBAC7C,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,UAAU,EAAE,OAAO,CAAC,EAAE;qBACvB;iBACF;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,cAAc;oBACpB,WAAW,EAAE,qBAAqB;oBAClC,QAAQ,EAAE,kBAAkB;oBAC5B,YAAY,EAAE,WAAW;iBAC1B;aACF,CAAC;iBACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAA,EAAA,CAAC,CAAC;YACjC,IAAA,wBAAS,EAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC;YAC7D,WAAW,GAAG,MAAA,kBAAkB,CAAC,YAAY,mCAAI,WAAW,CAAC;YAC7D,UAAU,GAAG;gBACX,EAAE,EAAE,kBAAkB,CAAC,EAAE;gBACzB,SAAS,EAAE,kBAAkB,CAAC,UAAU;gBACxC,gBAAgB,EAAE,kBAAkB,CAAC,kBAAkB;gBACvD,WAAW;gBACX,QAAQ,EAAE,kBAAkB,CAAC,QAAQ;aACtC,CAAC;YACF,yDAAyD;YACzD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;YACtC,IAAA,wBAAS,EACP,OAAO,EACP,mFAAmF,CACpF,CAAC;YACF,QAAQ,GAAG,IAAA,8BAAc,EAAC;gBACxB,WAAW;gBACX,OAAO;gBACP,OAAO,EAAE,MAAA,MAAM,CAAC,MAAM,CAAC,OAAO,mCAAI,EAAE;aACrC,CAAC,CAAC;YACH,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CACT,uEAAuE,CACxE,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,2BAA2B,cAAc,IAAI,WAAW,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,IAAI,SAAS,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACvJ,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,QAAQ,WAAW,kBAAkB,CACvC,CAAC;QAEF,MAAM,IAAI,GAA2C,EAAE,CAAC;QACxD,MAAM,mBAAmB,CAAC;YACxB,MAAM;YACN,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,IAAI;YACJ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;YACD,WAAW;YACX,QAAQ;YACR,SAAS;YACT,MAAM,EAAE,UAAU;SACnB,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAErC,MAAM,aAAa,mCACd,UAAU,KACb,IAAI,GACL,CAAC;QAEF,qDAAqD;QACrD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAA,MAAA,QAAQ,CAAC,QAAQ,wDAAI,CAAA,CAAC;QAC9B,CAAC;QAED,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAkB,CAAC;YAClD,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,EAAE;YAC5B,MAAM;YACN,MAAM;YACN,WAAW;YACX,MAAM;SACP,CAAC,CAAC;QACH,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QAEvD,OAAO,aAAa,CAAC;IACvB,CAAC;CAAA;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,EAC3B,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,WAAW,GAAG,CAAC,EACf,QAAQ,EACR,SAAS,EACT,MAAM,GAsBP;IACC,MAAM,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,CAAO,OAAsB,EAAE,EAAE;QAC3C,OAAO,MAAM,CAAC,eAAe,CAAC,SAAS,IAAI,CAAC,IAAI,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;;YACjE,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,gBAAgB,OAAO,CAAC,EAAE,GAAG,CACtF,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC;YAC3C,MAAM,OAAO,GAAkB;gBAC7B,EAAE,EAAE,OAAO,EAAE,EAAE,4FAA4F;gBAC3G,OAAO;gBACP,YAAY;gBACZ,gBAAgB,EAAE,OAAO,CAAC,EAAE;gBAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;gBAC9D,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;aACZ,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxD,OAAO,CAAC,MAAM;oBACZ,OAAO,UAAU,KAAK,QAAQ;wBAC5B,CAAC,CAAC,UAAU;wBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK;oBACX,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBAC3D,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAc,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,4BAA4B;gBAC5B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;oBACpE,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,aAAa,EAAE,YAAY;yBAC5B;qBACF;oBACD,IAAI,EAAE;wBACJ,kBAAkB,EAAE,OAAO,CAAC,EAAE;wBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,iBAAiB,EAAE,CAAC;wBACpB,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;wBAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;wBACvC,QAAQ,EAAE,OAAO,CAAC,OAAO;wBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;qBACrB;iBACF,CAAC,CAAC;gBACH,uDAAuD;gBACvD,OAAO,CAAC,EAAE,GAAG,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,IAAI,CAAC,EAAE,mCAAI,OAAO,CAAC,EAAE,CAAC;gBAC7C,MAAM,aAAa,GACjB,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,6CAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,6CAAQ,CAAC,IAAI,CAAC;gBACpE,MAAM,cAAc,GAClB,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,6CAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,6CAAQ,CAAC,IAAI,CAAC;gBACrE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAc,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,CAAC,aAAa,CAAC;oBACjB,CAAC,wDAAmB,CAAC,uBAAuB,CAAC,EAC3C,0DAAqB,CAAC,KAAK;oBAC7B,CAAC,wDAAmB,CAAC,eAAe,CAAC,EAAE,aAAa;oBACpD,CAAC,wDAAmB,CAAC,WAAW,CAAC,EAAE,IAAA,2BAAY,EAAC,OAAO,CAAC,KAAK,CAAC;oBAC9D,CAAC,wDAAmB,CAAC,gBAAgB,CAAC,EAAE,cAAc;oBACtD,CAAC,wDAAmB,CAAC,YAAY,CAAC,EAAE,IAAA,2BAAY,EAAC,OAAO,CAAC,MAAM,CAAC;iBACjE,CAAC,CAAC;YACL,CAAC;YACD,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,GAAG,EAAE,CAAC;YACZ,UAAU,CAAC,OAAO,CAAC,CAAC;YACpB,OAAO,OAAO,CAAC;QACjB,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAA,CAAC;IACF,MAAM,CAAC,GAAG,IAAA,aAAK,EAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3D,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAChC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACtB,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,CAAC,KAAK,CACV,uBAAuB,IAAI,CAAC,IAAI,iBAAiB,OAAO,CAAC,EAAE,MAAM,GAAG,EAAE,CACvE,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,GAAG,OAAO,EAChB,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,KAAK,GAoBf;;QACC,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC;QAC/D,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QACtC,IAAA,wBAAS,EACP,OAAO,EACP,mFAAmF,CACpF,CAAC;QACF,IAAI,QAA4B,CAAC;QACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAA,8BAAc,EAAC;gBACxB,WAAW,EAAE,YAAY;gBACzB,OAAO;gBACP,OAAO,EAAE,MAAA,MAAM,CAAC,MAAM,CAAC,OAAO,mCAAI,EAAE;aACrC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAA,kCAAkB,GAAE,CAAC;QAClC,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ;YACrB,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACrC,MAAM,KAAK,GACT,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACvD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAU,EAAC;YAC/B,OAAO,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE;YAC5C,MAAM;SACP,CAAC,CAAC;QACH,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,UAAU,CAAC,SAAS,aAAa,CAAC,CAAC;QAClE,IAAA,wBAAS,EACP,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAC3B,YAAY,UAAU,CAAC,SAAS,mBAAmB,CACpD,CAAC;QACF,IAAA,wBAAS,EAAC,UAAU,CAAC,IAAI,EAAE,eAAe,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;QAExE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtE,IAAI,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YAC7B,uCACK,UAAU,KACb,cAAc,EAAE,EAAE,IAClB;QACJ,CAAC;QACD,MAAM,CAAC,IAAI,CACT,6BAA6B,UAAU,CAAC,EAAE,UAAU,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,IAAI,IAAA,qBAAS,EACtF,WAAW,EACX,MAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,MAAM,mCAAI,CAAC,CACxB,EAAE,CACJ,CAAC;QAEF,MAAM,cAAc,GAAkD,EAAE,CAAC;QAEzE,MAAM,YAAY,GAA4B,EAAE,CAAC;QACjD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,MAAM,oBAAoB,GAAG,CAAC,GAA4B,EAAE,EAAE;YAC5D,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;QAC/B,CAAC,CAAC;QAEF,kCAAkC;QAClC,sCAAsC;QACtC,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3B,SAAS;YACT,GAAG;SACJ,CAAC,CAAC,CACJ,CAAC;QACF,MAAM,eAAe,GAAG,IAAA,aAAK,EAC3B,CAAO,eAA6D,EAAE,EAAE;YACtE,OAAO,MAAM,CAAC,eAAe,CAC3B,eAAe,eAAe,CAAC,SAAS,CAAC,IAAI,EAAE,EAC/C,CAAO,IAAI,EAAE,EAAE;;gBACb,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;oBACpC,SAAS,EAAE,eAAe,CAAC,SAAS;oBACpC,GAAG,EAAE,eAAe,CAAC,GAAG;oBACxB,YAAY,EAAE,YAAY;oBAC1B,UAAU,EAAE,oBAAoB;oBAChC,MAAM;iBACP,CAAC,CAAC;gBACH,IAAI,CAAC,aAAa,CAAC;oBACjB,CAAC,wDAAmB,CAAC,uBAAuB,CAAC,EAC3C,0DAAqB,CAAC,SAAS;oBACjC,CAAC,wDAAmB,CAAC,eAAe,CAAC,EAAE,6CAAQ,CAAC,IAAI;oBACpD,CAAC,wDAAmB,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC;wBAChD,KAAK,EAAE,MAAA,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,0CAAE,KAAK;wBAChE,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM;wBAClC,QAAQ,EACN,MAAA,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,0CAAE,MAAM;wBAC5D,QAAQ,EACN,MAAA,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,0CAAE,QAAQ;qBAC/D,CAAC;oBACF,CAAC,wDAAmB,CAAC,gBAAgB,CAAC,EAAE,6CAAQ,CAAC,IAAI;oBACrD,CAAC,wDAAmB,CAAC,YAAY,CAAC,EAAE,IAAA,2BAAY,EAAC,UAAU,CAAC,MAAM,CAAC;iBACpE,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,IAAI,CAAC,SAAS,CAAC;wBACb,IAAI,EAAE,oBAAc,CAAC,KAAK;wBAC1B,OAAO,EAAE,UAAU,CAAC,KAAK;qBAC1B,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAc,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9C,CAAC;gBACD,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACtB,IAAI,CAAC,aAAa,CAAC,IAAA,uCAAkB,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5D,CAAC;gBACD,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC;gBAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,mCAAmC;oBACnC,kDAAkD;oBAClD,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;wBACxC,IAAI,EAAE;4BACJ,iBAAiB,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE;4BACzC,IAAI,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;4BACpC,cAAc,EAAE,eAAe,CAAC,SAAS,CAAC,IAAI;4BAC9C,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,WAAW,EAAE;4BAC9C,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE;4BAC1C,MAAM,oBACD,UAAU,CAAC,MAAM,CACrB;4BACD,KAAK,EAAE,UAAU,CAAC,KAAK;4BACvB,QAAQ,EAAE,UAAU,CAAC,OAAO;yBAC7B;qBACF,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACX,OAAO,UAAU,CAAC;YACpB,CAAC,CAAA,CACF,CAAC;QACJ,CAAC,CAAA,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACtC,uCACK,UAAU,KACb,cAAc,EAAE,EAAE,IAClB;QACJ,CAAC;QACD,iBAAiB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,GAAG,EAAE,EAAE;YAC5C,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,KAAK,CACV,8BAA8B,eAAe,CAAC,SAAS,CAAC,IAAI,aAAa,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,EAAE,CAC3G,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,MAAM,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAE3C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAA,MAAA,QAAQ,CAAC,QAAQ,wDAAI,CAAA,CAAC;QAC9B,CAAC;QAED,uCACK,UAAU,KACb,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAC7C;IACJ,CAAC;CAAA;AAED;;;;GAIG;AACH,SAAe,YAAY;yDAAC,EAC1B,SAAS,EACT,GAAG,EACH,YAAY,EACZ,UAAU,EACV,MAAM,GAOP;QACC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAA,wBAAS,EAAC,OAAO,EAAE,YAAY,GAAG,CAAC,gBAAgB,aAAa,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,GAAS,EAAE;;YAC1B,MAAM,CAAC,IAAI,CACT,sBAAsB,GAAG,CAAC,EAAE,qBAAqB,SAAS,CAAC,IAAI,GAAG,CACnE,CAAC;YACF,MAAM,QAAQ,GAA4B;gBACxC,EAAE,EAAE,OAAO,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,GAAG,CAAC,EAAE;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,yCAAyC;gBAC9D,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,SAAS,CAAC,IAAI;aAC9B,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC;oBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,MAAA,GAAG,CAAC,MAAM,mCAAI,IAAI;oBAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM;oBACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;gBACH,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;gBACzB,MAAM,CAAC,IAAI,CACT,gBAAgB,SAAS,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,CAC/D,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBAC1E,MAAM,CAAC,KAAK,CACV,gBAAgB,SAAS,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,QAAQ,CAAC,KAAK,EAAE,CAC/E,CAAC;YACJ,CAAC;YACD,QAAQ,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAA,CAAC;QAEF,OAAO,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GAKT;IACC,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,IAAI,aAAa,GAAG,IAAI,CAAC;AAEzB;;;;GAIG;AACH,SAAS,OAAO;IACd,aAAa,EAAE,CAAC;IAChB,OAAO,SAAS,aAAa,EAAE,CAAC;AAClC,CAAC"}