@arizeai/phoenix-client 5.2.1 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +3 -3
  2. package/dist/esm/__generated__/api/v1.d.ts +321 -9
  3. package/dist/esm/__generated__/api/v1.d.ts.map +1 -1
  4. package/dist/esm/experiments/createExperiment.d.ts +39 -0
  5. package/dist/esm/experiments/createExperiment.d.ts.map +1 -0
  6. package/dist/esm/experiments/createExperiment.js +43 -0
  7. package/dist/esm/experiments/createExperiment.js.map +1 -0
  8. package/dist/esm/experiments/deleteExperiment.d.ts +36 -0
  9. package/dist/esm/experiments/deleteExperiment.d.ts.map +1 -0
  10. package/dist/esm/experiments/deleteExperiment.js +49 -0
  11. package/dist/esm/experiments/deleteExperiment.js.map +1 -0
  12. package/dist/esm/experiments/getExperimentInfo.d.ts.map +1 -1
  13. package/dist/esm/experiments/getExperimentInfo.js +9 -2
  14. package/dist/esm/experiments/getExperimentInfo.js.map +1 -1
  15. package/dist/esm/experiments/helpers/asExperimentEvaluator.d.ts +19 -0
  16. package/dist/esm/experiments/helpers/asExperimentEvaluator.d.ts.map +1 -0
  17. package/dist/esm/experiments/helpers/asExperimentEvaluator.js +19 -0
  18. package/dist/esm/experiments/helpers/asExperimentEvaluator.js.map +1 -0
  19. package/dist/esm/experiments/helpers/fromPhoenixLLMEvaluator.d.ts +9 -0
  20. package/dist/esm/experiments/helpers/fromPhoenixLLMEvaluator.d.ts.map +1 -0
  21. package/dist/esm/experiments/helpers/fromPhoenixLLMEvaluator.js +18 -0
  22. package/dist/esm/experiments/helpers/fromPhoenixLLMEvaluator.js.map +1 -0
  23. package/dist/esm/experiments/helpers/getExperimentEvaluators.d.ts +6 -0
  24. package/dist/esm/experiments/helpers/getExperimentEvaluators.d.ts.map +1 -0
  25. package/dist/esm/experiments/helpers/getExperimentEvaluators.js +58 -0
  26. package/dist/esm/experiments/helpers/getExperimentEvaluators.js.map +1 -0
  27. package/dist/esm/experiments/helpers/index.d.ts +4 -0
  28. package/dist/esm/experiments/helpers/index.d.ts.map +1 -0
  29. package/dist/esm/experiments/helpers/index.js +4 -0
  30. package/dist/esm/experiments/helpers/index.js.map +1 -0
  31. package/dist/esm/experiments/index.d.ts +6 -0
  32. package/dist/esm/experiments/index.d.ts.map +1 -1
  33. package/dist/esm/experiments/index.js +6 -0
  34. package/dist/esm/experiments/index.js.map +1 -1
  35. package/dist/esm/experiments/listExperiments.d.ts +29 -0
  36. package/dist/esm/experiments/listExperiments.d.ts.map +1 -0
  37. package/dist/esm/experiments/listExperiments.js +59 -0
  38. package/dist/esm/experiments/listExperiments.js.map +1 -0
  39. package/dist/esm/experiments/resumeEvaluation.d.ts +105 -0
  40. package/dist/esm/experiments/resumeEvaluation.d.ts.map +1 -0
  41. package/dist/esm/experiments/resumeEvaluation.js +559 -0
  42. package/dist/esm/experiments/resumeEvaluation.js.map +1 -0
  43. package/dist/esm/experiments/resumeExperiment.d.ts +102 -0
  44. package/dist/esm/experiments/resumeExperiment.d.ts.map +1 -0
  45. package/dist/esm/experiments/resumeExperiment.js +517 -0
  46. package/dist/esm/experiments/resumeExperiment.js.map +1 -0
  47. package/dist/esm/experiments/runExperiment.d.ts +4 -3
  48. package/dist/esm/experiments/runExperiment.d.ts.map +1 -1
  49. package/dist/esm/experiments/runExperiment.js +32 -3
  50. package/dist/esm/experiments/runExperiment.js.map +1 -1
  51. package/dist/esm/prompts/createPrompt.d.ts +19 -1
  52. package/dist/esm/prompts/createPrompt.d.ts.map +1 -1
  53. package/dist/esm/prompts/createPrompt.js +14 -1
  54. package/dist/esm/prompts/createPrompt.js.map +1 -1
  55. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  56. package/dist/esm/types/experiments.d.ts +66 -3
  57. package/dist/esm/types/experiments.d.ts.map +1 -1
  58. package/dist/esm/utils/channel.d.ts +229 -0
  59. package/dist/esm/utils/channel.d.ts.map +1 -0
  60. package/dist/esm/utils/channel.js +352 -0
  61. package/dist/esm/utils/channel.js.map +1 -0
  62. package/dist/esm/utils/formatPromptMessages.d.ts.map +1 -1
  63. package/dist/esm/utils/getPromptBySelector.d.ts.map +1 -1
  64. package/dist/esm/utils/isHttpError.d.ts +21 -0
  65. package/dist/esm/utils/isHttpError.d.ts.map +1 -0
  66. package/dist/esm/utils/isHttpError.js +33 -0
  67. package/dist/esm/utils/isHttpError.js.map +1 -0
  68. package/dist/src/__generated__/api/v1.d.ts +321 -9
  69. package/dist/src/__generated__/api/v1.d.ts.map +1 -1
  70. package/dist/src/experiments/createExperiment.d.ts +39 -0
  71. package/dist/src/experiments/createExperiment.d.ts.map +1 -0
  72. package/dist/src/experiments/createExperiment.js +43 -0
  73. package/dist/src/experiments/createExperiment.js.map +1 -0
  74. package/dist/src/experiments/deleteExperiment.d.ts +36 -0
  75. package/dist/src/experiments/deleteExperiment.d.ts.map +1 -0
  76. package/dist/src/experiments/deleteExperiment.js +52 -0
  77. package/dist/src/experiments/deleteExperiment.js.map +1 -0
  78. package/dist/src/experiments/getExperimentInfo.d.ts.map +1 -1
  79. package/dist/src/experiments/getExperimentInfo.js +9 -2
  80. package/dist/src/experiments/getExperimentInfo.js.map +1 -1
  81. package/dist/src/experiments/helpers/asExperimentEvaluator.d.ts +19 -0
  82. package/dist/src/experiments/helpers/asExperimentEvaluator.d.ts.map +1 -0
  83. package/dist/src/experiments/helpers/asExperimentEvaluator.js +22 -0
  84. package/dist/src/experiments/helpers/asExperimentEvaluator.js.map +1 -0
  85. package/dist/src/experiments/helpers/fromPhoenixLLMEvaluator.d.ts +9 -0
  86. package/dist/src/experiments/helpers/fromPhoenixLLMEvaluator.d.ts.map +1 -0
  87. package/dist/src/experiments/helpers/fromPhoenixLLMEvaluator.js +21 -0
  88. package/dist/src/experiments/helpers/fromPhoenixLLMEvaluator.js.map +1 -0
  89. package/dist/src/experiments/helpers/getExperimentEvaluators.d.ts +6 -0
  90. package/dist/src/experiments/helpers/getExperimentEvaluators.d.ts.map +1 -0
  91. package/dist/src/experiments/helpers/getExperimentEvaluators.js +61 -0
  92. package/dist/src/experiments/helpers/getExperimentEvaluators.js.map +1 -0
  93. package/dist/src/experiments/helpers/index.d.ts +4 -0
  94. package/dist/src/experiments/helpers/index.d.ts.map +1 -0
  95. package/dist/src/experiments/helpers/index.js +20 -0
  96. package/dist/src/experiments/helpers/index.js.map +1 -0
  97. package/dist/src/experiments/index.d.ts +6 -0
  98. package/dist/src/experiments/index.d.ts.map +1 -1
  99. package/dist/src/experiments/index.js +6 -0
  100. package/dist/src/experiments/index.js.map +1 -1
  101. package/dist/src/experiments/listExperiments.d.ts +29 -0
  102. package/dist/src/experiments/listExperiments.d.ts.map +1 -0
  103. package/dist/src/experiments/listExperiments.js +66 -0
  104. package/dist/src/experiments/listExperiments.js.map +1 -0
  105. package/dist/src/experiments/resumeEvaluation.d.ts +105 -0
  106. package/dist/src/experiments/resumeEvaluation.d.ts.map +1 -0
  107. package/dist/src/experiments/resumeEvaluation.js +585 -0
  108. package/dist/src/experiments/resumeEvaluation.js.map +1 -0
  109. package/dist/src/experiments/resumeExperiment.d.ts +102 -0
  110. package/dist/src/experiments/resumeExperiment.d.ts.map +1 -0
  111. package/dist/src/experiments/resumeExperiment.js +540 -0
  112. package/dist/src/experiments/resumeExperiment.js.map +1 -0
  113. package/dist/src/experiments/runExperiment.d.ts +4 -3
  114. package/dist/src/experiments/runExperiment.d.ts.map +1 -1
  115. package/dist/src/experiments/runExperiment.js +32 -3
  116. package/dist/src/experiments/runExperiment.js.map +1 -1
  117. package/dist/src/prompts/createPrompt.d.ts +19 -1
  118. package/dist/src/prompts/createPrompt.d.ts.map +1 -1
  119. package/dist/src/prompts/createPrompt.js +14 -1
  120. package/dist/src/prompts/createPrompt.js.map +1 -1
  121. package/dist/src/types/experiments.d.ts +66 -3
  122. package/dist/src/types/experiments.d.ts.map +1 -1
  123. package/dist/src/utils/channel.d.ts +229 -0
  124. package/dist/src/utils/channel.d.ts.map +1 -0
  125. package/dist/src/utils/channel.js +385 -0
  126. package/dist/src/utils/channel.js.map +1 -0
  127. package/dist/src/utils/formatPromptMessages.d.ts.map +1 -1
  128. package/dist/src/utils/getPromptBySelector.d.ts.map +1 -1
  129. package/dist/src/utils/isHttpError.d.ts +21 -0
  130. package/dist/src/utils/isHttpError.d.ts.map +1 -0
  131. package/dist/src/utils/isHttpError.js +37 -0
  132. package/dist/src/utils/isHttpError.js.map +1 -0
  133. package/dist/tsconfig.tsbuildinfo +1 -1
  134. package/package.json +6 -5
  135. package/src/__generated__/api/v1.ts +321 -9
  136. package/src/experiments/createExperiment.ts +90 -0
  137. package/src/experiments/deleteExperiment.ts +67 -0
  138. package/src/experiments/getExperimentInfo.ts +9 -2
  139. package/src/experiments/helpers/asExperimentEvaluator.ts +29 -0
  140. package/src/experiments/helpers/fromPhoenixLLMEvaluator.ts +24 -0
  141. package/src/experiments/helpers/getExperimentEvaluators.ts +74 -0
  142. package/src/experiments/helpers/index.ts +3 -0
  143. package/src/experiments/index.ts +6 -0
  144. package/src/experiments/listExperiments.ts +83 -0
  145. package/src/experiments/resumeEvaluation.ts +804 -0
  146. package/src/experiments/resumeExperiment.ts +745 -0
  147. package/src/experiments/runExperiment.ts +37 -5
  148. package/src/prompts/createPrompt.ts +19 -1
  149. package/src/types/experiments.ts +72 -3
  150. package/src/utils/channel.ts +397 -0
  151. package/src/utils/isHttpError.ts +45 -0
@@ -0,0 +1,559 @@
1
+ import { MimeType, OpenInferenceSpanKind, SemanticConventions, } from "@arizeai/openinference-semantic-conventions";
2
+ import { objectAsAttributes, register, SpanStatusCode, } from "@arizeai/phoenix-otel";
3
+ import { createClient } from "../client.js";
4
+ import { Channel, ChannelError } from "../utils/channel.js";
5
+ import { ensureString } from "../utils/ensureString.js";
6
+ import { toObjectHeaders } from "../utils/toObjectHeaders.js";
7
+ import { getExperimentInfo } from "./getExperimentInfo.js";
8
+ import { getExperimentEvaluators } from "./helpers/index.js";
9
+ import invariant from "tiny-invariant";
10
+ /**
11
+ * Error thrown when evaluation is aborted due to a failure in stopOnFirstError mode.
12
+ * This provides semantic context that the abort was intentional, not an infrastructure failure.
13
+ * @internal - Not exported to minimize API surface area
14
+ */
15
+ class EvaluationAbortedError extends Error {
16
+ constructor(message, cause) {
17
+ super(message);
18
+ this.name = "EvaluationAbortedError";
19
+ this.cause = cause;
20
+ }
21
+ }
22
+ /**
23
+ * Error thrown when the producer fails to fetch incomplete evaluations from the server.
24
+ * This is a critical error that should always be surfaced, even in stopOnFirstError=false mode.
25
+ * @internal - Not exported to minimize API surface area
26
+ */
27
+ class EvaluationFetchError extends Error {
28
+ constructor(message, cause) {
29
+ super(message);
30
+ this.name = "EvaluationFetchError";
31
+ this.cause = cause;
32
+ }
33
+ }
34
+ const DEFAULT_PAGE_SIZE = 50;
35
+ /**
36
+ * Channel capacity multiplier for producer-consumer buffering.
37
+ * A value of 2 enables pipeline efficiency: workers process batch N while
38
+ * the producer fetches batch N+1, eliminating idle time without excessive
39
+ * memory usage. The channel blocks when full, providing natural backpressure.
40
+ */
41
+ const CHANNEL_CAPACITY_MULTIPLIER = 2;
42
+ /**
43
+ * Transforms API incomplete evaluation response to IncompleteEvaluation
44
+ */
45
+ function buildIncompleteEvaluation(apiResponse) {
46
+ return {
47
+ experimentRun: {
48
+ id: apiResponse.experiment_run.id,
49
+ experimentId: apiResponse.experiment_run.experiment_id,
50
+ datasetExampleId: apiResponse.experiment_run.dataset_example_id,
51
+ output: apiResponse.experiment_run.output ?? null,
52
+ startTime: new Date(apiResponse.experiment_run.start_time),
53
+ endTime: new Date(apiResponse.experiment_run.end_time),
54
+ error: apiResponse.experiment_run.error ?? null,
55
+ traceId: apiResponse.experiment_run.trace_id ?? null,
56
+ },
57
+ datasetExample: {
58
+ id: apiResponse.dataset_example.id,
59
+ input: apiResponse.dataset_example.input,
60
+ output: apiResponse.dataset_example.output ?? null,
61
+ metadata: apiResponse.dataset_example.metadata || {},
62
+ updatedAt: new Date(apiResponse.dataset_example.updated_at),
63
+ },
64
+ evaluationNames: apiResponse.evaluation_names,
65
+ };
66
+ }
67
+ /**
68
+ * Determines if an evaluator should run for an incomplete evaluation
69
+ */
70
+ function shouldRunEvaluator(evaluator, incompleteEval) {
71
+ // Match evaluator name directly
72
+ return incompleteEval.evaluationNames.includes(evaluator.name);
73
+ }
74
+ /**
75
+ * Handles fetch errors with helpful version information for unsupported features
76
+ */
77
+ async function handleEvaluationFetchError(error, client, featureName) {
78
+ // Check if this is a JSON parse error (likely 404 HTML response from old server)
79
+ const isJsonError = error instanceof SyntaxError &&
80
+ error.message.toLowerCase().includes("json");
81
+ if (isJsonError) {
82
+ // Fetch server version to provide helpful context
83
+ let versionInfo = "";
84
+ try {
85
+ const baseUrl = client.config.baseUrl || "";
86
+ const versionRes = await fetch(`${baseUrl}/arize_phoenix_version`);
87
+ if (versionRes.ok) {
88
+ const version = await versionRes.text();
89
+ versionInfo = ` Your current server version is ${version}.`;
90
+ }
91
+ }
92
+ catch {
93
+ // Ignore errors fetching version
94
+ }
95
+ throw new Error(`The ${featureName} feature is not available on this Phoenix server. ` +
96
+ "Please upgrade your Phoenix server to use this feature." +
97
+ versionInfo);
98
+ }
99
+ throw error;
100
+ }
101
+ /**
102
+ * Sets up OpenTelemetry tracer for evaluation tracing
103
+ */
104
+ function setupEvaluationTracer({ projectName, baseUrl, headers, useBatchSpanProcessor, diagLogLevel, setGlobalTracerProvider, }) {
105
+ if (!projectName) {
106
+ return null;
107
+ }
108
+ const provider = register({
109
+ projectName,
110
+ url: baseUrl,
111
+ headers,
112
+ batch: useBatchSpanProcessor,
113
+ diagLogLevel,
114
+ global: setGlobalTracerProvider,
115
+ });
116
+ const tracer = provider.getTracer(projectName);
117
+ return { provider, tracer };
118
+ }
119
+ /**
120
+ * Prints evaluation summary to logger
121
+ */
122
+ function printEvaluationSummary({ logger, experimentId, totalProcessed, totalCompleted, }) {
123
+ logger.info("\n" + "=".repeat(70));
124
+ logger.info("📊 Evaluation Resume Summary");
125
+ logger.info("=".repeat(70));
126
+ logger.info(`Experiment ID: ${experimentId}`);
127
+ logger.info(`Runs processed: ${totalProcessed}`);
128
+ logger.info(`Evaluations completed: ${totalCompleted}`);
129
+ logger.info("=".repeat(70));
130
+ }
131
+ /**
132
+ * Resume incomplete evaluations for an experiment.
133
+ *
134
+ * This function identifies which evaluations have not been completed (either missing or failed)
135
+ * and runs the evaluators only for those runs. This is useful for:
136
+ * - Recovering from transient evaluator failures
137
+ * - Adding new evaluators to completed experiments
138
+ * - Completing partially evaluated experiments
139
+ *
140
+ * The function processes incomplete evaluations in batches using pagination to minimize memory usage.
141
+ *
142
+ * Evaluation names are matched to evaluator names. For example, if you pass
143
+ * an evaluator with name "accuracy", it will check for and resume any runs missing the "accuracy" evaluation.
144
+ *
145
+ * **Note:** Multi-output evaluators (evaluators that return an array of results) are not
146
+ * supported for resume operations. Each evaluator should produce a single evaluation
147
+ * result with a name matching the evaluator's name.
148
+ *
149
+ * @throws {Error} Throws different error types based on failure:
150
+ * - "EvaluationFetchError": Unable to fetch incomplete evaluations from the server.
151
+ * Always thrown regardless of stopOnFirstError, as it indicates critical infrastructure failure.
152
+ * - "EvaluationAbortedError": stopOnFirstError=true and an evaluator failed.
153
+ * Original error preserved in `cause` property.
154
+ * - Generic Error: Other evaluator execution errors or unexpected failures.
155
+ *
156
+ * @example
157
+ * ```ts
158
+ * import { resumeEvaluation } from "@arizeai/phoenix-client/experiments";
159
+ *
160
+ * // Standard usage: evaluation name matches evaluator name
161
+ * try {
162
+ * await resumeEvaluation({
163
+ * experimentId: "exp_123",
164
+ * evaluators: [{
165
+ * name: "correctness",
166
+ * kind: "CODE",
167
+ * evaluate: async ({ output, expected }) => ({
168
+ * score: output === expected ? 1 : 0
169
+ * })
170
+ * }],
171
+ * });
172
+ * } catch (error) {
173
+ * // Handle by error name (no instanceof needed)
174
+ * if (error.name === "EvaluationFetchError") {
175
+ * console.error("Failed to connect to server:", error.cause);
176
+ * } else if (error.name === "EvaluationAbortedError") {
177
+ * console.error("Evaluation stopped due to error:", error.cause);
178
+ * } else {
179
+ * console.error("Unexpected error:", error);
180
+ * }
181
+ * }
182
+ *
183
+ * // Stop on first error (useful for debugging)
184
+ * await resumeEvaluation({
185
+ * experimentId: "exp_123",
186
+ * evaluators: [myEvaluator],
187
+ * stopOnFirstError: true, // Exit immediately on first failure
188
+ * });
189
+ * ```
190
+ */
191
+ export async function resumeEvaluation({ client: _client, experimentId, evaluators: _evaluators, logger = console, concurrency = 5, setGlobalTracerProvider = true, useBatchSpanProcessor = true, diagLogLevel, stopOnFirstError = false, }) {
192
+ const client = _client ?? createClient();
193
+ const pageSize = DEFAULT_PAGE_SIZE;
194
+ // Normalize evaluators to array
195
+ const evaluators = getExperimentEvaluators(Array.isArray(_evaluators) ? _evaluators : [_evaluators]);
196
+ // Validate inputs
197
+ invariant(evaluators.length > 0, "Must specify at least one evaluator");
198
+ // Get experiment info
199
+ logger.info(`🔍 Checking for incomplete evaluations...`);
200
+ const experiment = await getExperimentInfo({ client, experimentId });
201
+ // Initialize tracer (only if experiment has a project_name)
202
+ const baseUrl = client.config.baseUrl;
203
+ invariant(baseUrl, "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client.");
204
+ const tracerSetup = setupEvaluationTracer({
205
+ projectName: experiment.projectName,
206
+ baseUrl,
207
+ headers: client.config.headers
208
+ ? toObjectHeaders(client.config.headers)
209
+ : undefined,
210
+ useBatchSpanProcessor,
211
+ diagLogLevel,
212
+ setGlobalTracerProvider,
213
+ });
214
+ const provider = tracerSetup?.provider ?? null;
215
+ const evalTracer = tracerSetup?.tracer ?? null;
216
+ // Build evaluation names list for query - derive from evaluator names
217
+ const evaluationNamesList = evaluators.map((e) => e.name);
218
+ // Create a CSP-style bounded buffer for evaluation distribution
219
+ const evalChannel = new Channel(pageSize * CHANNEL_CAPACITY_MULTIPLIER);
220
+ // Abort controller for stopOnFirstError coordination
221
+ const abortController = new AbortController();
222
+ const { signal } = abortController;
223
+ let totalProcessed = 0;
224
+ let totalCompleted = 0;
225
+ let totalFailed = 0;
226
+ // Producer: Fetch incomplete evaluations and send to channel
227
+ async function fetchIncompleteEvaluations() {
228
+ let cursor = null;
229
+ try {
230
+ do {
231
+ // Stop fetching if abort signal received
232
+ if (signal.aborted) {
233
+ logger.info("🛑 Stopping fetch due to error in evaluation");
234
+ break;
235
+ }
236
+ let res;
237
+ try {
238
+ res = await client.GET("/v1/experiments/{experiment_id}/incomplete-evaluations", {
239
+ params: {
240
+ path: {
241
+ experiment_id: experimentId,
242
+ },
243
+ query: {
244
+ cursor,
245
+ limit: pageSize,
246
+ evaluation_name: evaluationNamesList,
247
+ },
248
+ },
249
+ });
250
+ }
251
+ catch (error) {
252
+ // Check for version compatibility issues and throw helpful error
253
+ try {
254
+ await handleEvaluationFetchError(error, client, "resume_evaluation");
255
+ // TypeScript: handleEvaluationFetchError never returns, but add throw for safety
256
+ throw new Error("handleEvaluationFetchError should never return");
257
+ }
258
+ catch (handledError) {
259
+ // Wrap the error (from handleEvaluationFetchError or original) in semantic error type
260
+ throw new EvaluationFetchError("Failed to fetch incomplete evaluations from server", handledError instanceof Error ? handledError : undefined);
261
+ }
262
+ }
263
+ // Check for API errors
264
+ if (res.error) {
265
+ throw new EvaluationFetchError(`Failed to fetch incomplete evaluations: ${ensureString(res.error)}`);
266
+ }
267
+ cursor = res.data?.next_cursor ?? null;
268
+ const batchIncomplete = res.data?.data;
269
+ invariant(batchIncomplete, "Failed to fetch incomplete evaluations");
270
+ if (batchIncomplete.length === 0) {
271
+ if (totalProcessed === 0) {
272
+ logger.info("✅ No incomplete evaluations found. All evaluations are complete.");
273
+ }
274
+ break;
275
+ }
276
+ if (totalProcessed === 0) {
277
+ logger.info("🧠 Resuming evaluations...");
278
+ }
279
+ // Build evaluation tasks and send to channel
280
+ let batchCount = 0;
281
+ for (const incomplete of batchIncomplete) {
282
+ // Stop sending items if abort signal received
283
+ if (signal.aborted) {
284
+ break;
285
+ }
286
+ const incompleteEval = buildIncompleteEvaluation(incomplete);
287
+ const evaluatorsToRun = evaluators.filter((evaluator) => shouldRunEvaluator(evaluator, incompleteEval));
288
+ // Flatten: Send one channel item per evaluator
289
+ for (const evaluator of evaluatorsToRun) {
290
+ // Stop sending items if abort signal received
291
+ if (signal.aborted) {
292
+ break;
293
+ }
294
+ await evalChannel.send({ incompleteEval, evaluator });
295
+ batchCount++;
296
+ totalProcessed++;
297
+ }
298
+ }
299
+ logger.info(`Fetched batch of ${batchCount} evaluation tasks (channel buffer: ${evalChannel.length})`);
300
+ } while (cursor !== null && !signal.aborted);
301
+ }
302
+ catch (error) {
303
+ // Re-throw with context preservation
304
+ if (error instanceof EvaluationFetchError) {
305
+ throw error;
306
+ }
307
+ // ChannelError from blocked send() should bubble up naturally
308
+ // (happens when channel closes while producer is blocked)
309
+ if (error instanceof ChannelError) {
310
+ throw error;
311
+ }
312
+ // Wrap any unexpected errors from channel operations
313
+ throw new EvaluationFetchError("Unexpected error during evaluation fetch", error instanceof Error ? error : undefined);
314
+ }
315
+ finally {
316
+ evalChannel.close(); // Signal workers we're done
317
+ }
318
+ }
319
+ // Worker: Process evaluations from channel
320
+ async function processEvaluationsFromChannel() {
321
+ for await (const item of evalChannel) {
322
+ // Stop processing if abort signal received
323
+ if (signal.aborted) {
324
+ break;
325
+ }
326
+ try {
327
+ await runSingleEvaluation({
328
+ client,
329
+ experimentId,
330
+ evaluator: item.evaluator,
331
+ experimentRun: item.incompleteEval.experimentRun,
332
+ datasetExample: item.incompleteEval.datasetExample,
333
+ tracer: evalTracer,
334
+ });
335
+ totalCompleted++;
336
+ }
337
+ catch (error) {
338
+ totalFailed++;
339
+ logger.error(`Failed to run evaluator "${item.evaluator.name}" for run ${item.incompleteEval.experimentRun.id}: ${error}`);
340
+ // If stopOnFirstError is enabled, abort and re-throw
341
+ if (stopOnFirstError) {
342
+ logger.error("🛑 Stopping on first error");
343
+ abortController.abort();
344
+ throw error;
345
+ }
346
+ }
347
+ }
348
+ }
349
+ // Start concurrent execution
350
+ // Wrap in try-finally to ensure channel is always closed, even if Promise.all throws
351
+ let executionError = null;
352
+ try {
353
+ const producerTask = fetchIncompleteEvaluations();
354
+ const workerTasks = Array.from({ length: concurrency }, () => processEvaluationsFromChannel());
355
+ // Wait for producer and all workers to finish
356
+ await Promise.all([producerTask, ...workerTasks]);
357
+ }
358
+ catch (error) {
359
+ // Classify and handle errors based on their nature
360
+ const err = error instanceof Error ? error : new Error(String(error));
361
+ // Always surface producer/infrastructure errors
362
+ if (error instanceof EvaluationFetchError) {
363
+ // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
364
+ logger.error(`❌ Critical: Failed to fetch evaluations from server`);
365
+ executionError = err;
366
+ }
367
+ else if (error instanceof ChannelError && signal.aborted) {
368
+ // Channel closed due to intentional abort - wrap in semantic error
369
+ executionError = new EvaluationAbortedError("Evaluation stopped due to error in concurrent evaluator", err);
370
+ }
371
+ else if (stopOnFirstError) {
372
+ // Worker error in stopOnFirstError mode - already logged by worker
373
+ executionError = err;
374
+ }
375
+ else {
376
+ // Unexpected error (not from worker, not from producer fetch)
377
+ // This could be a bug in our code or infrastructure failure
378
+ logger.error(`❌ Unexpected error during evaluation: ${err.message}`);
379
+ executionError = err;
380
+ }
381
+ }
382
+ finally {
383
+ // Ensure channel is closed even if there are unexpected errors
384
+ // This is a safety net in case producer's finally block didn't execute
385
+ if (!evalChannel.isClosed) {
386
+ evalChannel.close();
387
+ }
388
+ }
389
+ // Only show completion message if we didn't stop on error
390
+ if (!executionError) {
391
+ logger.info(`✅ Evaluations completed.`);
392
+ }
393
+ if (totalFailed > 0 && !executionError) {
394
+ logger.info(`⚠️ Warning: ${totalFailed} out of ${totalProcessed} evaluations failed.`);
395
+ }
396
+ // Print summary
397
+ printEvaluationSummary({
398
+ logger,
399
+ experimentId: experiment.id,
400
+ totalProcessed,
401
+ totalCompleted,
402
+ });
403
+ // Flush spans (if tracer was initialized)
404
+ if (provider) {
405
+ await provider.forceFlush();
406
+ }
407
+ // Re-throw error if evaluation failed
408
+ if (executionError) {
409
+ throw executionError;
410
+ }
411
+ }
412
+ /**
413
+ * Record evaluation results to API.
414
+ */
415
+ async function recordEvaluationResults({ client, evaluator, experimentRun, results, error, startTime, endTime, traceId = null, }) {
416
+ if (results) {
417
+ // Success case: record each evaluation result
418
+ for (const singleResult of results) {
419
+ await client.POST("/v1/experiment_evaluations", {
420
+ body: {
421
+ experiment_run_id: experimentRun.id,
422
+ name: evaluator.name,
423
+ annotator_kind: evaluator.kind,
424
+ result: {
425
+ score: singleResult.score ?? null,
426
+ label: singleResult.label ?? null,
427
+ explanation: singleResult.explanation ?? null,
428
+ metadata: singleResult.metadata ?? {},
429
+ },
430
+ start_time: startTime.toISOString(),
431
+ end_time: endTime.toISOString(),
432
+ error: null,
433
+ trace_id: traceId,
434
+ },
435
+ });
436
+ }
437
+ }
438
+ else if (error) {
439
+ // Error case: record failed evaluation with evaluator name
440
+ await client.POST("/v1/experiment_evaluations", {
441
+ body: {
442
+ experiment_run_id: experimentRun.id,
443
+ name: evaluator.name,
444
+ annotator_kind: evaluator.kind,
445
+ result: null,
446
+ start_time: startTime.toISOString(),
447
+ end_time: endTime.toISOString(),
448
+ error,
449
+ trace_id: traceId,
450
+ },
451
+ });
452
+ }
453
+ }
454
+ /**
455
+ * Run a single evaluation and record the result.
456
+ */
457
+ async function runSingleEvaluation({ client, experimentId, evaluator, experimentRun, datasetExample, tracer, }) {
458
+ const startTime = new Date();
459
+ // Prepare evaluator inputs
460
+ const taskOutput = experimentRun.output ?? null;
461
+ const expectedOutput = datasetExample.output ?? undefined;
462
+ // If no tracer (no project_name), execute without tracing
463
+ if (!tracer) {
464
+ let results;
465
+ let error;
466
+ try {
467
+ const result = await Promise.resolve(evaluator.evaluate({
468
+ input: datasetExample.input,
469
+ output: taskOutput,
470
+ expected: expectedOutput,
471
+ metadata: datasetExample.metadata,
472
+ }));
473
+ results = Array.isArray(result) ? result : [result];
474
+ }
475
+ catch (err) {
476
+ error = err instanceof Error ? err.message : String(err);
477
+ throw err;
478
+ }
479
+ finally {
480
+ const endTime = new Date();
481
+ await recordEvaluationResults({
482
+ client,
483
+ evaluator,
484
+ experimentRun,
485
+ results,
486
+ error,
487
+ startTime,
488
+ endTime,
489
+ });
490
+ }
491
+ return;
492
+ }
493
+ // With tracer: wrap execution in a span for observability
494
+ return tracer.startActiveSpan(`Evaluation: ${evaluator.name}`, async (span) => {
495
+ // Set span attributes for input
496
+ span.setAttributes({
497
+ [SemanticConventions.OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.EVALUATOR,
498
+ [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON,
499
+ [SemanticConventions.INPUT_VALUE]: ensureString({
500
+ input: datasetExample.input,
501
+ output: experimentRun.output,
502
+ expected: datasetExample.output,
503
+ metadata: datasetExample.metadata,
504
+ }),
505
+ ...objectAsAttributes({
506
+ experiment_id: experimentId,
507
+ experiment_run_id: experimentRun.id,
508
+ dataset_example_id: datasetExample.id,
509
+ }),
510
+ });
511
+ let results;
512
+ let error;
513
+ try {
514
+ // Execute the evaluator (only once!)
515
+ const result = await Promise.resolve(evaluator.evaluate({
516
+ input: datasetExample.input,
517
+ output: taskOutput,
518
+ expected: expectedOutput,
519
+ metadata: datasetExample.metadata,
520
+ }));
521
+ results = Array.isArray(result) ? result : [result];
522
+ // Set output span attributes
523
+ span.setAttributes({
524
+ [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON,
525
+ [SemanticConventions.OUTPUT_VALUE]: ensureString(result),
526
+ });
527
+ // Set attributes from first result for span metadata
528
+ if (results[0]) {
529
+ span.setAttributes(objectAsAttributes(results[0]));
530
+ }
531
+ span.setStatus({ code: SpanStatusCode.OK });
532
+ }
533
+ catch (err) {
534
+ error = err instanceof Error ? err.message : String(err);
535
+ span.setStatus({
536
+ code: SpanStatusCode.ERROR,
537
+ message: error,
538
+ });
539
+ span.recordException(err);
540
+ throw err;
541
+ }
542
+ finally {
543
+ const endTime = new Date();
544
+ span.end();
545
+ // Record results to API
546
+ await recordEvaluationResults({
547
+ client,
548
+ evaluator,
549
+ experimentRun,
550
+ results,
551
+ error,
552
+ startTime,
553
+ endTime,
554
+ traceId: span.spanContext().traceId,
555
+ });
556
+ }
557
+ });
558
+ }
559
+ //# sourceMappingURL=resumeEvaluation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"resumeEvaluation.js","sourceRoot":"","sources":["../../../src/experiments/resumeEvaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAGL,kBAAkB,EAClB,QAAQ,EACR,cAAc,GAEf,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,YAAY,EAAsB,MAAM,WAAW,CAAC;AAU7D,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AAEpD,OAAO,SAAS,MAAM,gBAAgB,CAAC;AAEvC;;;;GAIG;AACH,MAAM,sBAAuB,SAAQ,KAAK;IACxC,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,oBAAqB,SAAQ,KAAK;IACtC,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AA4CD,MAAM,iBAAiB,GAAG,EAAW,CAAC;AACtC;;;;;GAKG;AACH,MAAM,2BAA2B,GAAG,CAAU,CAAC;AAU/C;;GAEG;AACH,SAAS,yBAAyB,CAChC,WAAoE;IAEpE,OAAO;QACL,aAAa,EAAE;YACb,EAAE,EAAE,WAAW,CAAC,cAAc,CAAC,EAAE;YACjC,YAAY,EAAE,WAAW,CAAC,cAAc,CAAC,aAAa;YACtD,gBAAgB,EAAE,WAAW,CAAC,cAAc,CAAC,kBAAkB;YAC/D,MAAM,EAAE,WAAW,CAAC,cAAc,CAAC,MAAM,IAAI,IAAI;YACjD,SAAS,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,UAAU,CAAC;YAC1D,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC;YACtD,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,KAAK,IAAI,IAAI;YAC/C,OAAO,EAAE,WAAW,CAAC,cAAc,CAAC,QAAQ,IAAI,IAAI;SACrD;QACD,cAAc,EAAE;YACd,EAAE,EAAE,WAAW,CAAC,eAAe,CAAC,EAAE;YAClC,KAAK,EAAE,WAAW,CAAC,eAAe,CAAC,KAAK;YACxC,MAAM,EAAE,WAAW,CAAC,eAAe,CAAC,MAAM,IAAI,IAAI;YAClD,QAAQ,EAAE,WAAW,CAAC,eAAe,CAAC,QAAQ,IAAI,EAAE;YACpD,SAAS,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,UAAU,CAAC;SAC5D;QACD,eAAe,EAAE,WAAW,CAAC,gBAAgB;KAC9C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,SAAoB,EACpB,cAAoC;IAEpC,gCAAgC;IAChC,OAAO,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,0BAA0B,CACvC,KAAc,EACd,MAAqB,EACrB,WAAmB;IAEnB,iFAAiF;IACjF,MAAM,WAAW,GACf,KAAK,YAAY,WAAW;QAC5B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE/C,IAAI,WAAW,EAAE,CAAC;QAChB,kDAAkD;QAClD,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,wBAAwB,CAAC,CAAC;YACnE,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;gBACxC,WAAW,GAAG,mCAAmC,OAAO,GAAG,CAAC;YAC9D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;QACnC,CAAC;QAED,MAAM,IAAI,KAAK,CACb,OAAO,WAAW,oDAAoD;YACpE,yDAAyD;YACzD,WAAW,CACd,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAAqB,EACrB,YAAY,EACZ,uBAAuB,GAQxB;IACC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QACxB,WAAW;QACX,GAAG,EAAE,OAAO;QACZ,OAAO;QACP,KAAK,EAAE,qBAAqB;QAC5B,YAAY;QACZ,MAAM,EAAE,uBAAuB;KAChC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,EAC9B,MAAM,EACN,YAAY,EACZ,cAAc,EACd,cAAc,GAMf;IACC,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,MAAM,CAAC,IAAI,CAAC,kBAAkB,YAAY,EAAE,CAAC,CAAC;IAC9C,MAAM,CAAC,IAAI,CAAC,mBAAmB,cAAc,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,IAAI,CAAC,0BAA0B,cAAc,EAAE,CAAC,CAAC;IACxD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EACrC,MAAM,EAAE,OAAO,EACf,YAAY,EACZ,UAAU,EAAE,WAAW,EACvB,MAAM,GAAG,OAAO,EAChB,WAAW,GAAG,CAAC,EACf,uBAAuB,GAAG,IAAI,EAC9B,qBAAqB,GAAG,IAAI,EAC5B,YAAY,EACZ,gBAAgB,GAAG,KAAK,GACD;IACvB,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,iBAAiB,CAAC;IAEnC,gCAAgC;IAChC,MAAM,UAAU,GAAG,uBAAuB,CACxC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CACzD,CAAC;IACF,kBAAkB;IAClB,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,qCAAqC,CAAC,CAAC;IAExE,sBAAsB;IACtB,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAErE,4DAA4D;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;IACtC,SAAS,CACP,OAAO,EACP,mFAAmF,CACpF,CAAC;IAEF,MAAM,WAAW,GAAG,qBAAqB,CAAC;QACxC,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,OAAO;QACP,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC5B,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;YACxC,CAAC,CAAC,SAAS;QACb,qBAAqB;QACrB,YAAY;QACZ,uBAAuB;KACxB,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,WAAW,EAAE,QAAQ,IAAI,IAAI,CAAC;IAC/C,MAAM,UAAU,GAAG,WAAW,EAAE,MAAM,IAAI,IAAI,CAAC;IAE/C,sEAAsE;IACtE,MAAM,mBAAmB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE1D,gEAAgE;IAChE,MAAM,WAAW,GAAG,IAAI,OAAO,CAC7B,QAAQ,GAAG,2BAA2B,CACvC,CAAC;IAEF,qDAAqD;IACrD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;IAEnC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,6DAA6D;IAC7D,KAAK,UAAU,0BAA0B;QACvC,IAAI,MAAM,GAAkB,IAAI,CAAC;QAEjC,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,yCAAyC;gBACzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;oBAC5D,MAAM;gBACR,CAAC;gBAED,IAAI,GAGH,CAAC;gBAEF,IAAI,CAAC;oBACH,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CACpB,wDAAwD,EACxD;wBACE,MAAM,EAAE;4BACN,IAAI,EAAE;gCACJ,aAAa,EAAE,YAAY;6BAC5B;4BACD,KAAK,EAAE;gCACL,MAAM;gCACN,KAAK,EAAE,QAAQ;gCACf,eAAe,EAAE,mBAAmB;6BACrC;yBACF;qBACF,CACF,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAc,EAAE,CAAC;oBACxB,iEAAiE;oBACjE,IAAI,CAAC;wBACH,MAAM,0BAA0B,CAC9B,KAAK,EACL,MAAM,EACN,mBAAmB,CACpB,CAAC;wBACF,iFAAiF;wBACjF,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;oBACpE,CAAC;oBAAC,OAAO,YAAY,EAAE,CAAC;wBACtB,sFAAsF;wBACtF,MAAM,IAAI,oBAAoB,CAC5B,oDAAoD,EACpD,YAAY,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CACzD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,uBAAuB;gBACvB,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACd,MAAM,IAAI,oBAAoB,CAC5B,2CAA2C,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACrE,CAAC;gBACJ,CAAC;gBAED,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,IAAI,CAAC;gBACvC,MAAM,eAAe,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBACvC,SAAS,CAAC,eAAe,EAAE,wCAAwC,CAAC,CAAC;gBAErE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;wBACzB,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACJ,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;gBAC5C,CAAC;gBAED,6CAA6C;gBAC7C,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,KAAK,MAAM,UAAU,IAAI,eAAe,EAAE,CAAC;oBACzC,8CAA8C;oBAC9C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,MAAM;oBACR,CAAC;oBAED,MAAM,cAAc,GAAG,yBAAyB,CAAC,UAAU,CAAC,CAAC;oBAE7D,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CACtD,kBAAkB,CAAC,SAAS,EAAE,cAAc,CAAC,CAC9C,CAAC;oBAEF,+CAA+C;oBAC/C,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;wBACxC,8CAA8C;wBAC9C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;4BACnB,MAAM;wBACR,CAAC;wBAED,MAAM,WAAW,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,CAAC;wBACtD,UAAU,EAAE,CAAC;wBACb,cAAc,EAAE,CAAC;oBACnB,CAAC;gBACH,CAAC;gBAED,MAAM,CAAC,IAAI,CACT,oBAAoB,UAAU,sCAAsC,WAAW,CAAC,MAAM,GAAG,CAC1F,CAAC;YACJ,CAAC,QAAQ,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,qCAAqC;YACrC,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;gBAC1C,MAAM,KAAK,CAAC;YACd,CAAC;YACD,8DAA8D;YAC9D,0DAA0D;YAC1D,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,qDAAqD;YACrD,MAAM,IAAI,oBAAoB,CAC5B,0CAA0C,EAC1C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,4BAA4B;QACnD,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,KAAK,UAAU,6BAA6B;QAC1C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YACrC,2CAA2C;YAC3C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,mBAAmB,CAAC;oBACxB,MAAM;oBACN,YAAY;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,cAAc,CAAC,aAAa;oBAChD,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,cAAc;oBAClD,MAAM,EAAE,UAAU;iBACnB,CAAC,CAAC;gBACH,cAAc,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,WAAW,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CACV,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,aAAa,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,KAAK,KAAK,EAAE,CAC7G,CAAC;gBAEF,qDAAqD;gBACrD,IAAI,gBAAgB,EAAE,CAAC;oBACrB,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;oBAC3C,eAAe,CAAC,KAAK,EAAE,CAAC;oBACxB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,qFAAqF;IACrF,IAAI,cAAc,GAAiB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,0BAA0B,EAAE,CAAC;QAClD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAC3D,6BAA6B,EAAE,CAChC,CAAC;QAEF,8CAA8C;QAC9C,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mDAAmD;QACnD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAEtE,gDAAgD;QAChD,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;YAC1C,2EAA2E;YAC3E,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACpE,cAAc,GAAG,GAAG,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,YAAY,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC3D,mEAAmE;YACnE,cAAc,GAAG,IAAI,sBAAsB,CACzC,yDAAyD,EACzD,GAAG,CACJ,CAAC;QACJ,CAAC;aAAM,IAAI,gBAAgB,EAAE,CAAC;YAC5B,mEAAmE;YACnE,cAAc,GAAG,GAAG,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,8DAA8D;YAC9D,4DAA4D;YAC5D,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACrE,cAAc,GAAG,GAAG,CAAC;QACvB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,+DAA+D;QAC/D,uEAAuE;QACvE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC1B,WAAW,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CACT,gBAAgB,WAAW,WAAW,cAAc,sBAAsB,CAC3E,CAAC;IACJ,CAAC;IAED,gBAAgB;IAChB,sBAAsB,CAAC;QACrB,MAAM;QACN,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,cAAc;QACd,cAAc;KACf,CAAC,CAAC;IAEH,0CAA0C;IAC1C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;IAC9B,CAAC;IAED,sCAAsC;IACtC,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,uBAAuB,CAAC,EACrC,MAAM,EACN,SAAS,EACT,aAAa,EACb,OAAO,EACP,KAAK,EACL,SAAS,EACT,OAAO,EACP,OAAO,GAAG,IAAI,GAUf;IACC,IAAI,OAAO,EAAE,CAAC;QACZ,8CAA8C;QAC9C,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBAC9C,IAAI,EAAE;oBACJ,iBAAiB,EAAE,aAAa,CAAC,EAAE;oBACnC,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,cAAc,EAAE,SAAS,CAAC,IAAI;oBAC9B,MAAM,EAAE;wBACN,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,IAAI;wBACjC,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,IAAI;wBACjC,WAAW,EAAE,YAAY,CAAC,WAAW,IAAI,IAAI;wBAC7C,QAAQ,EAAE,YAAY,CAAC,QAAQ,IAAI,EAAE;qBACtC;oBACD,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;oBACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;oBAC/B,KAAK,EAAE,IAAI;oBACX,QAAQ,EAAE,OAAO;iBAClB;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,IAAI,KAAK,EAAE,CAAC;QACjB,2DAA2D;QAC3D,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;YAC9C,IAAI,EAAE;gBACJ,iBAAiB,EAAE,aAAa,CAAC,EAAE;gBACnC,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,cAAc,EAAE,SAAS,CAAC,IAAI;gBAC9B,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;gBACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;gBAC/B,KAAK;gBACL,QAAQ,EAAE,OAAO;aAClB;SACF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,mBAAmB,CAAC,EACjC,MAAM,EACN,YAAY,EACZ,SAAS,EACT,aAAa,EACb,cAAc,EACd,MAAM,GAQP;IACC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,2BAA2B;IAC3B,MAAM,UAAU,GAAe,aAAa,CAAC,MAAM,IAAI,IAAI,CAAC;IAC5D,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,IAAI,SAAS,CAAC;IAE1D,0DAA0D;IAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,IAAI,OAAgD,CAAC;QACrD,IAAI,KAAyB,CAAC;QAE9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAClC,SAAS,CAAC,QAAQ,CAAC;gBACjB,KAAK,EAAE,cAAc,CAAC,KAAK;gBAC3B,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,cAAc;gBACxB,QAAQ,EAAE,cAAc,CAAC,QAAQ;aAClC,CAAC,CACH,CAAC;YACF,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzD,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC3B,MAAM,uBAAuB,CAAC;gBAC5B,MAAM;gBACN,SAAS;gBACT,aAAa;gBACb,OAAO;gBACP,KAAK;gBACL,SAAS;gBACT,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QACD,OAAO;IACT,CAAC;IAED,0DAA0D;IAC1D,OAAO,MAAM,CAAC,eAAe,CAC3B,eAAe,SAAS,CAAC,IAAI,EAAE,EAC/B,KAAK,EAAE,IAAI,EAAE,EAAE;QACb,gCAAgC;QAChC,IAAI,CAAC,aAAa,CAAC;YACjB,CAAC,mBAAmB,CAAC,uBAAuB,CAAC,EAC3C,qBAAqB,CAAC,SAAS;YACjC,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,IAAI;YACpD,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,YAAY,CAAC;gBAC9C,KAAK,EAAE,cAAc,CAAC,KAAK;gBAC3B,MAAM,EAAE,aAAa,CAAC,MAAM;gBAC5B,QAAQ,EAAE,cAAc,CAAC,MAAM;gBAC/B,QAAQ,EAAE,cAAc,CAAC,QAAQ;aAClC,CAAC;YACF,GAAG,kBAAkB,CAAC;gBACpB,aAAa,EAAE,YAAY;gBAC3B,iBAAiB,EAAE,aAAa,CAAC,EAAE;gBACnC,kBAAkB,EAAE,cAAc,CAAC,EAAE;aACtC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,OAAgD,CAAC;QACrD,IAAI,KAAyB,CAAC;QAE9B,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAClC,SAAS,CAAC,QAAQ,CAAC;gBACjB,KAAK,EAAE,cAAc,CAAC,KAAK;gBAC3B,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,cAAc;gBACxB,QAAQ,EAAE,cAAc,CAAC,QAAQ;aAClC,CAAC,CACH,CAAC;YAEF,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAEpD,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC;gBACjB,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,IAAI;gBACrD,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC;aACzD,CAAC,CAAC;YAEH,qDAAqD;YACrD,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAEzD,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,GAAY,CAAC,CAAC;YAEnC,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;YAEX,wBAAwB;YACxB,MAAM,uBAAuB,CAAC;gBAC5B,MAAM;gBACN,SAAS;gBACT,aAAa;gBACb,OAAO;gBACP,KAAK;gBACL,SAAS;gBACT,OAAO;gBACP,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO;aACpC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CACF,CAAC;AACJ,CAAC"}