@uipath/eval-tool 1.203.0-preview.203

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1593 @@
1
+ import {
2
+ catchError,
3
+ extractErrorMessage,
4
+ RESULTS,
5
+ Pagination,
6
+ OutputFormatter,
7
+ processContext,
8
+ getAuthContext
9
+ } from "./tool-pa6awsya.js";
10
+ // package.json
11
+ var package_default = {
12
+ name: "@uipath/eval-tool",
13
+ author: "UiPath",
14
+ license: "SEE LICENSE IN LICENSE.txt",
15
+ version: "1.203.0-preview.203",
16
+ description: "Manage runtime evaluations, evaluators, eval sets, evaluations, and schedules.",
17
+ private: false,
18
+ repository: {
19
+ type: "git",
20
+ url: "https://github.com/UiPath/cli.git",
21
+ directory: "packages/eval-tool"
22
+ },
23
+ publishConfig: {
24
+ registry: "https://npm.pkg.github.com/@uipath"
25
+ },
26
+ keywords: [
27
+ "cli-tool"
28
+ ],
29
+ type: "module",
30
+ main: "./dist/tool.js",
31
+ exports: {
32
+ ".": "./dist/tool.js"
33
+ },
34
+ bin: {
35
+ "eval-tool": "./dist/index.js"
36
+ },
37
+ files: [
38
+ "dist"
39
+ ],
40
+ scripts: {
41
+ build: "bun ../../tools/build-tool.ts",
42
+ package: "bun run build && bun pm pack",
43
+ test: "vitest run",
44
+ "test:coverage": "vitest run --coverage",
45
+ lint: "biome check .",
46
+ "lint:fix": "biome check --write ."
47
+ },
48
+ devDependencies: {
49
+ "@types/node": "^25.5.2",
50
+ "@uipath/auth": "workspace:*",
51
+ "@uipath/common": "workspace:*",
52
+ "@uipath/orchestrator-sdk": "workspace:*",
53
+ commander: "^14.0.3",
54
+ typescript: "^7.0.2"
55
+ }
56
+ };
57
+
58
+ // src/utils/eval-api.ts
59
+ async function resolveEvalAuth(tenant) {
60
+ const ctx = await getAuthContext({
61
+ tenant,
62
+ ensureTokenValidityMinutes: 10,
63
+ requireOrganizationId: true,
64
+ requireTenantName: true,
65
+ requireTenantId: true
66
+ });
67
+ return {
68
+ agentsApiBase: `${ctx.baseUrl}/${ctx.organizationId}/${ctx.tenantName}/agentsruntime_/api`,
69
+ authHeaders: {
70
+ Authorization: `Bearer ${ctx.accessToken}`,
71
+ "Content-Type": "application/json",
72
+ Accept: "application/json",
73
+ "X-UiPath-Internal-TenantId": ctx.tenantId ?? "",
74
+ "X-UiPath-Internal-AccountId": ctx.organizationId ?? ""
75
+ },
76
+ tenantId: ctx.tenantId ?? ""
77
+ };
78
+ }
79
+ async function evalApiRequest(url, headers, method, body) {
80
+ const response = await fetch(url, {
81
+ method,
82
+ headers,
83
+ body: body !== undefined ? JSON.stringify(body) : undefined
84
+ });
85
+ const text = await response.text();
86
+ if (!response.ok) {
87
+ const hint = response.status === 401 ? " Token may have expired — run 'uip login' and retry." : "";
88
+ throw new Error(`${method} ${url.split("?")[0]} returned ${response.status}: ${text}${hint}`);
89
+ }
90
+ if (!text)
91
+ return {};
92
+ try {
93
+ return JSON.parse(text);
94
+ } catch {
95
+ throw new Error(`API returned non-JSON response (${response.status}): ${text.slice(0, 200)}`);
96
+ }
97
+ }
98
+ async function resolveEvalAuthOrFail(tenant) {
99
+ const [authErr, auth] = await catchError(resolveEvalAuth(tenant));
100
+ if (authErr) {
101
+ OutputFormatter.error({
102
+ Result: RESULTS.Failure,
103
+ Message: `Authentication failed: ${authErr.message}`,
104
+ Instructions: "Run 'uip login' first."
105
+ });
106
+ processContext.exit(1);
107
+ return null;
108
+ }
109
+ return auth;
110
+ }
111
+ async function evalApiRequestOrFail(url, headers, method, instructions, body) {
112
+ const [err, data] = await catchError(evalApiRequest(url, headers, method, body));
113
+ if (err) {
114
+ OutputFormatter.error({
115
+ Result: RESULTS.Failure,
116
+ Message: err.message,
117
+ Instructions: instructions
118
+ });
119
+ processContext.exit(1);
120
+ return null;
121
+ }
122
+ return data;
123
+ }
124
+ function parseIntOrFail(value, flag, min = 0) {
125
+ const n = Number.parseInt(value, 10);
126
+ if (Number.isNaN(n) || n < min) {
127
+ const constraint = min > 0 ? "a positive integer" : "a non-negative integer";
128
+ OutputFormatter.error({
129
+ Result: RESULTS.Failure,
130
+ Message: `Invalid ${flag} "${value}": must be ${constraint}.`,
131
+ Instructions: `Example: ${flag} 10`
132
+ });
133
+ processContext.exit(1);
134
+ return null;
135
+ }
136
+ return n;
137
+ }
138
+ function parseJsonOrFail(value, flag, example) {
139
+ const [parseErr, parsed] = catchError(() => JSON.parse(value));
140
+ if (parseErr || parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
141
+ OutputFormatter.error({
142
+ Result: RESULTS.Failure,
143
+ Message: `${flag} is not valid JSON: ${parseErr?.message ?? "expected an object"}`,
144
+ Instructions: `Example: ${flag} '${example ?? '{"key":"value"}'}'`
145
+ });
146
+ processContext.exit(1);
147
+ return null;
148
+ }
149
+ return parsed;
150
+ }
151
+ async function fetchSimulatableComponents(auth, processKey) {
152
+ const params = new URLSearchParams({ workloadType: "Agents" });
153
+ const [err, response] = await catchError(evalApiRequest(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(processKey)}/simulatableComponents?${params.toString()}`, auth.authHeaders, "GET"));
154
+ if (err || !response)
155
+ return [];
156
+ const components = Array.isArray(response.components) ? response.components : [];
157
+ return components.filter((c) => c.isEnabled);
158
+ }
159
+
160
+ // src/commands/eval-evaluations.ts
161
+ var SIMULATION_STRATEGY_LABELS = {
162
+ 0: "Llm",
163
+ 1: "Mockito",
164
+ 2: "Static"
165
+ };
166
+ function formatSimulation(sim) {
167
+ return {
168
+ ComponentId: sim.componentId,
169
+ ComponentType: sim.componentType,
170
+ ...sim.componentDescription ? { ComponentDescription: sim.componentDescription } : {},
171
+ SimulationStrategy: SIMULATION_STRATEGY_LABELS[sim.simulationStrategy ?? 0] ?? String(sim.simulationStrategy),
172
+ ...sim.simulationInstruction ? { SimulationInstruction: sim.simulationInstruction } : {},
173
+ ...sim.outputSchema ? { OutputSchema: sim.outputSchema } : {},
174
+ ...sim.mockValue !== undefined ? { MockValue: sim.mockValue } : {},
175
+ ...sim.behaviors !== undefined ? { Behaviors: sim.behaviors } : {},
176
+ ...sim.childSimulations && sim.childSimulations.length > 0 ? {
177
+ ChildSimulations: sim.childSimulations.map(formatSimulation)
178
+ } : {}
179
+ };
180
+ }
181
+ function formatEvaluation(ev) {
182
+ return {
183
+ EvaluationId: ev.id,
184
+ EvalSetId: ev.unifiedEvalSetId,
185
+ Name: ev.name,
186
+ Inputs: ev.inputs,
187
+ ExpectedOutput: ev.expectedOutput ?? {},
188
+ ExpectedBehavior: ev.expectedBehavior ?? "",
189
+ ...ev.simulations && ev.simulations.length > 0 ? { Simulations: ev.simulations.map(formatSimulation) } : {},
190
+ CreatedAt: ev.createdAt,
191
+ ...ev.updatedAt ? { UpdatedAt: ev.updatedAt } : {}
192
+ };
193
+ }
194
+ async function resolveSimulationSchemas(simulations, auth, processKey) {
195
+ const needsResolve = simulations.some((s) => (s.simulationStrategy === undefined || s.simulationStrategy === 0) && !s.outputSchema);
196
+ if (!needsResolve)
197
+ return simulations;
198
+ const componentsList = await fetchSimulatableComponents(auth, processKey);
199
+ if (componentsList.length === 0)
200
+ return simulations;
201
+ const componentsById = new Map(componentsList.map((c) => [c.id, c]));
202
+ return simulations.map((sim) => {
203
+ if ((sim.simulationStrategy === undefined || sim.simulationStrategy === 0) && !sim.outputSchema) {
204
+ const comp = componentsById.get(sim.componentId);
205
+ if (comp?.outputSchema) {
206
+ return { ...sim, outputSchema: comp.outputSchema };
207
+ }
208
+ }
209
+ return sim;
210
+ });
211
+ }
212
+ function parseSimulationsOrFail(value) {
213
+ const [parseErr, parsed] = catchError(() => JSON.parse(value));
214
+ if (parseErr || !Array.isArray(parsed)) {
215
+ OutputFormatter.error({
216
+ Result: RESULTS.Failure,
217
+ Message: `--simulations is not a valid JSON array: ${parseErr?.message ?? "expected an array"}`,
218
+ Instructions: `Example: --simulations '[{"componentId":"web-search","componentType":"tool"}]'`
219
+ });
220
+ processContext.exit(1);
221
+ return null;
222
+ }
223
+ for (let i = 0;i < parsed.length; i++) {
224
+ const item = parsed[i];
225
+ if (!item || typeof item !== "object" || Array.isArray(item)) {
226
+ OutputFormatter.error({
227
+ Result: RESULTS.Failure,
228
+ Message: `Simulation at index ${i} is not an object.`,
229
+ Instructions: 'Each simulation must have at least "componentId" and "componentType".'
230
+ });
231
+ processContext.exit(1);
232
+ return null;
233
+ }
234
+ if (typeof item.componentId !== "string" || item.componentId.length === 0) {
235
+ OutputFormatter.error({
236
+ Result: RESULTS.Failure,
237
+ Message: `Simulation at index ${i} is missing a valid "componentId".`,
238
+ Instructions: "Use 'uip eval simulatable-components list --process-key ...' to find component IDs."
239
+ });
240
+ processContext.exit(1);
241
+ return null;
242
+ }
243
+ if (typeof item.componentType !== "string" || item.componentType.length === 0) {
244
+ OutputFormatter.error({
245
+ Result: RESULTS.Failure,
246
+ Message: `Simulation at index ${i} is missing a valid "componentType".`,
247
+ Instructions: 'componentType is typically "tool", "escalation", or "mcpTool".'
248
+ });
249
+ processContext.exit(1);
250
+ return null;
251
+ }
252
+ }
253
+ return parsed;
254
+ }
255
+ var INPUTS_EXAMPLE = '{"input":"hello"}';
256
+ var LIST_EXAMPLES = [
257
+ {
258
+ Description: "List evaluations (data points) in an eval set",
259
+ Command: "uip eval evaluation list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --eval-set-id f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
260
+ Output: {
261
+ Code: "EvaluationList",
262
+ Data: [
263
+ {
264
+ EvaluationId: "c1d2e3f4-0000-0000-0000-000000000001",
265
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
266
+ Name: "Greeting test",
267
+ Inputs: { input: "hello" },
268
+ ExpectedOutput: { content: "Hi there!" },
269
+ ExpectedBehavior: "",
270
+ CreatedAt: "2026-08-01T10:00:00Z"
271
+ }
272
+ ]
273
+ }
274
+ }
275
+ ];
276
+ var UPDATE_EXAMPLES = [
277
+ {
278
+ Description: "Update an evaluation's inputs and expected output",
279
+ Command: `uip eval evaluation update c1d2e3f4-0000-0000-0000-000000000001 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --eval-set-id f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04 --inputs '{"input":"updated"}' --expected-output '{"content":"new response"}'`,
280
+ Output: {
281
+ Code: "EvaluationUpdated",
282
+ Data: {
283
+ EvaluationId: "c1d2e3f4-0000-0000-0000-000000000001",
284
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
285
+ Name: "Greeting test"
286
+ }
287
+ }
288
+ }
289
+ ];
290
+ var CREATE_EXAMPLES = [
291
+ {
292
+ Description: "Create an evaluation (data point) in an eval set",
293
+ Command: `uip eval evaluation create --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --eval-set-id f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04 --folder-key f1f2f3f4-0000-0000-0000-000000000001 --name "Greeting test" --inputs '{"input":"hello"}' --expected-output '{"content":"Hi there!"}'`,
294
+ Output: {
295
+ Code: "EvaluationCreated",
296
+ Data: {
297
+ EvaluationId: "c1d2e3f4-0000-0000-0000-000000000001",
298
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
299
+ Name: "Greeting test"
300
+ }
301
+ }
302
+ }
303
+ ];
304
+ function registerEvalEvaluationsCommand(evalCmd) {
305
+ const evaluationCmd = evalCmd.command("evaluation").description("Manage evaluations (data points) within eval sets by process key.");
306
+ evaluationCmd.command("list").description("List evaluations in an eval set.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID)").option("--limit <number>", "Maximum number of items to return", "100").option("--offset <number>", "Number of items to skip", "0").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_EXAMPLES).trackedAction(processContext, async (options) => {
307
+ const limit = parseIntOrFail(options.limit, "--limit");
308
+ if (limit === null)
309
+ return;
310
+ const offset = parseIntOrFail(options.offset, "--offset");
311
+ if (offset === null)
312
+ return;
313
+ const auth = await resolveEvalAuthOrFail(options.tenant);
314
+ if (!auth)
315
+ return;
316
+ const evaluations = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}/evaluations?skip=${offset}&take=${limit}`, auth.authHeaders, "GET", "Check the process key and eval set ID.");
317
+ if (!evaluations)
318
+ return;
319
+ OutputFormatter.success({
320
+ Result: RESULTS.Success,
321
+ Code: "EvaluationList",
322
+ Data: evaluations.map(formatEvaluation),
323
+ Pagination: new Pagination({
324
+ returned: evaluations.length,
325
+ limit,
326
+ offset
327
+ })
328
+ });
329
+ });
330
+ evaluationCmd.command("get").description("Get details of a specific evaluation.").argument("<evaluationId>", "Evaluation ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID)").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evaluationId, options) => {
331
+ const auth = await resolveEvalAuthOrFail(options.tenant);
332
+ if (!auth)
333
+ return;
334
+ const evaluation = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}/evaluations/${encodeURIComponent(evaluationId)}`, auth.authHeaders, "GET", "Check the evaluation ID, eval set ID, and process key.");
335
+ if (!evaluation)
336
+ return;
337
+ OutputFormatter.success({
338
+ Result: RESULTS.Success,
339
+ Code: "EvaluationDetails",
340
+ Data: formatEvaluation(evaluation)
341
+ });
342
+ });
343
+ evaluationCmd.command("create").description("Create an evaluation (data point) in an eval set.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID)").requiredOption("--folder-key <guid>", "Folder key (GUID)").requiredOption("--name <name>", "Evaluation name").requiredOption("--inputs <json>", `Input values as JSON (e.g. '{"input":"hello"}')`).option("--expected-output <json>", "Expected output as JSON").option("--expected-behavior <text>", "Expected agent behavior description (for trajectory evaluators)").option("--evaluation-criterias <json>", "Per-evaluator criteria overrides as JSON").option("--simulations <json>", 'JSON array of simulations. Each object needs "componentId" and "componentType". ' + "For LLM strategy (default), outputSchema is auto-resolved from the deployed agent. " + "Use 'uip eval simulatable-components list' to see available components.").option("--tenant <tenant>", "UiPath tenant name").examples(CREATE_EXAMPLES).trackedAction(processContext, async (options) => {
344
+ const inputs = parseJsonOrFail(options.inputs, "--inputs", INPUTS_EXAMPLE);
345
+ if (inputs === null)
346
+ return;
347
+ let expectedOutput;
348
+ if (options.expectedOutput !== undefined) {
349
+ const parsed = parseJsonOrFail(options.expectedOutput, "--expected-output");
350
+ if (parsed === null)
351
+ return;
352
+ expectedOutput = parsed;
353
+ }
354
+ let evaluationCriterias;
355
+ if (options.evaluationCriterias !== undefined) {
356
+ const parsed = parseJsonOrFail(options.evaluationCriterias, "--evaluation-criterias");
357
+ if (parsed === null)
358
+ return;
359
+ evaluationCriterias = parsed;
360
+ }
361
+ const auth = await resolveEvalAuthOrFail(options.tenant);
362
+ if (!auth)
363
+ return;
364
+ let simulations;
365
+ if (options.simulations !== undefined) {
366
+ const parsed = parseSimulationsOrFail(options.simulations);
367
+ if (parsed === null)
368
+ return;
369
+ simulations = await resolveSimulationSchemas(parsed, auth, options.processKey);
370
+ }
371
+ const body = {
372
+ folderKey: options.folderKey,
373
+ externalId: crypto.randomUUID(),
374
+ name: options.name,
375
+ inputs,
376
+ expectedOutput: expectedOutput ?? {},
377
+ expectedBehavior: options.expectedBehavior ?? "",
378
+ evaluationCriterias: evaluationCriterias ?? {},
379
+ ...simulations ? { simulations } : {}
380
+ };
381
+ const evaluation = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}/evaluations`, auth.authHeaders, "POST", "Check the process key, eval set ID, and folder key.", body);
382
+ if (!evaluation)
383
+ return;
384
+ OutputFormatter.success({
385
+ Result: RESULTS.Success,
386
+ Code: "EvaluationCreated",
387
+ Data: formatEvaluation(evaluation)
388
+ });
389
+ });
390
+ evaluationCmd.command("update").description("Update an existing evaluation. Provide at least one field to change.").argument("<evaluationId>", "Evaluation ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID)").option("--name <name>", "New evaluation name").option("--inputs <json>", "New input values as JSON").option("--expected-output <json>", "New expected output as JSON").option("--expected-behavior <text>", "New expected behavior description").option("--evaluation-criterias <json>", "New per-evaluator criteria overrides as JSON").option("--simulations <json>", 'New simulations as JSON array. Each object needs "componentId" and "componentType". ' + "For LLM strategy, outputSchema is auto-resolved from the deployed agent.").option("--tenant <tenant>", "UiPath tenant name").examples(UPDATE_EXAMPLES).trackedAction(processContext, async (evaluationId, options) => {
391
+ if (options.name === undefined && options.inputs === undefined && options.expectedOutput === undefined && options.expectedBehavior === undefined && options.evaluationCriterias === undefined && options.simulations === undefined) {
392
+ OutputFormatter.error({
393
+ Result: RESULTS.Failure,
394
+ Message: "At least one of --name, --inputs, --expected-output, --expected-behavior, --evaluation-criterias, or --simulations must be provided.",
395
+ Instructions: "Provide the field(s) you want to update."
396
+ });
397
+ processContext.exit(1);
398
+ return;
399
+ }
400
+ let parsedInputs;
401
+ if (options.inputs !== undefined) {
402
+ const parsed = parseJsonOrFail(options.inputs, "--inputs", INPUTS_EXAMPLE);
403
+ if (parsed === null)
404
+ return;
405
+ parsedInputs = parsed;
406
+ }
407
+ let parsedExpectedOutput;
408
+ if (options.expectedOutput !== undefined) {
409
+ const parsed = parseJsonOrFail(options.expectedOutput, "--expected-output");
410
+ if (parsed === null)
411
+ return;
412
+ parsedExpectedOutput = parsed;
413
+ }
414
+ let parsedCriterias;
415
+ if (options.evaluationCriterias !== undefined) {
416
+ const parsed = parseJsonOrFail(options.evaluationCriterias, "--evaluation-criterias");
417
+ if (parsed === null)
418
+ return;
419
+ parsedCriterias = parsed;
420
+ }
421
+ const auth = await resolveEvalAuthOrFail(options.tenant);
422
+ if (!auth)
423
+ return;
424
+ let simulations;
425
+ if (options.simulations !== undefined) {
426
+ const parsed = parseSimulationsOrFail(options.simulations);
427
+ if (parsed === null)
428
+ return;
429
+ simulations = await resolveSimulationSchemas(parsed, auth, options.processKey);
430
+ }
431
+ const url = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}/evaluations/${encodeURIComponent(evaluationId)}`;
432
+ const existing = await evalApiRequestOrFail(url, auth.authHeaders, "GET", "Check the evaluation ID, eval set ID, and process key.");
433
+ if (!existing)
434
+ return;
435
+ const body = {
436
+ folderKey: existing.folderKey,
437
+ externalId: existing.externalId,
438
+ name: options.name ?? existing.name,
439
+ inputs: parsedInputs ?? existing.inputs,
440
+ expectedOutput: parsedExpectedOutput ?? existing.expectedOutput ?? {},
441
+ expectedBehavior: options.expectedBehavior ?? existing.expectedBehavior ?? "",
442
+ evaluationCriterias: parsedCriterias ?? existing.evaluationCriterias ?? {},
443
+ simulations: simulations ?? existing.simulations
444
+ };
445
+ const evaluation = await evalApiRequestOrFail(url, auth.authHeaders, "PUT", "Check the evaluation ID, eval set ID, process key, and provided values.", body);
446
+ if (!evaluation)
447
+ return;
448
+ OutputFormatter.success({
449
+ Result: RESULTS.Success,
450
+ Code: "EvaluationUpdated",
451
+ Data: formatEvaluation(evaluation)
452
+ });
453
+ });
454
+ evaluationCmd.command("delete").description("Delete an evaluation from an eval set.").argument("<evaluationId>", "Evaluation ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID)").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evaluationId, options) => {
455
+ const auth = await resolveEvalAuthOrFail(options.tenant);
456
+ if (!auth)
457
+ return;
458
+ const result = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}/evaluations/${encodeURIComponent(evaluationId)}`, auth.authHeaders, "DELETE", "Check the evaluation ID, eval set ID, and process key.");
459
+ if (result === null)
460
+ return;
461
+ OutputFormatter.success({
462
+ Result: RESULTS.Success,
463
+ Code: "EvaluationDeleted",
464
+ Data: { EvaluationId: evaluationId }
465
+ });
466
+ });
467
+ }
468
+
469
+ // src/commands/eval-evaluators.ts
470
+ function formatEvaluator(ev) {
471
+ return {
472
+ EvaluatorId: ev.id,
473
+ ExternalId: ev.externalId,
474
+ Name: ev.name,
475
+ Description: ev.description,
476
+ EvaluatorTypeId: ev.evaluatorTypeId,
477
+ Version: ev.version,
478
+ CreatedAt: ev.createdAt,
479
+ ...ev.updatedAt ? { UpdatedAt: ev.updatedAt } : {}
480
+ };
481
+ }
482
+ var LIST_EXAMPLES2 = [
483
+ {
484
+ Description: "List evaluators for a process",
485
+ Command: "uip eval evaluator list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
486
+ Output: {
487
+ Code: "EvaluatorList",
488
+ Data: [
489
+ {
490
+ EvaluatorId: "1d4cb04c-2ee5-4efd-908c-b8c4159893d5",
491
+ ExternalId: "llm-judge-output",
492
+ Name: "LLM Judge Output",
493
+ Description: "Use LLM to judge semantic similarity of outputs",
494
+ EvaluatorTypeId: "uipath-llm-judge-output-semantic-similarity",
495
+ Version: "1.0",
496
+ CreatedAt: "2026-09-18T18:17:48.7037826Z",
497
+ UpdatedAt: "2026-09-18T18:17:48.7037826Z"
498
+ }
499
+ ]
500
+ }
501
+ }
502
+ ];
503
+ var UPDATE_EXAMPLES2 = [
504
+ {
505
+ Description: "Update an evaluator's name and config",
506
+ Command: `uip eval evaluator update a1b2c3d4-0000-0000-0000-000000000001 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --name "Updated Evaluator" --evaluator-config '{"name":"Updated","prompt":"New prompt"}'`,
507
+ Output: {
508
+ Code: "EvaluatorUpdated",
509
+ Data: {
510
+ EvaluatorId: "a1b2c3d4-0000-0000-0000-000000000001",
511
+ Name: "Updated Evaluator",
512
+ EvaluatorTypeId: "uipath-llm-judge-output-semantic-similarity"
513
+ }
514
+ }
515
+ }
516
+ ];
517
+ var CREATE_EXAMPLES2 = [
518
+ {
519
+ Description: "Create a semantic similarity evaluator",
520
+ Command: `uip eval evaluator create --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --workload-id a1b2c3d4-0000-0000-0000-000000000001 --folder-key f1f2f3f4-0000-0000-0000-000000000001 --name "Semantic Similarity" --description "LLM-based output comparison" --evaluator-type-id uipath-llm-judge-output-semantic-similarity --evaluator-config '{"name":"Semantic","prompt":"Score 0-100...","model":"gpt-4.1-2025-04-14","targetOutputKey":"*"}' --version 1.0`,
521
+ Output: {
522
+ Code: "EvaluatorCreated",
523
+ Data: {
524
+ EvaluatorId: "a1b2c3d4-0000-0000-0000-000000000001",
525
+ Name: "Semantic Similarity",
526
+ EvaluatorTypeId: "uipath-llm-judge-output-semantic-similarity"
527
+ }
528
+ }
529
+ }
530
+ ];
531
+ function registerEvalEvaluatorsCommand(evalCmd) {
532
+ const evaluatorCmd = evalCmd.command("evaluator").description("Manage evaluators by process key.");
533
+ evaluatorCmd.command("list").description("List evaluators for a process.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--limit <number>", "Maximum number of items to return", "100").option("--offset <number>", "Number of items to skip", "0").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_EXAMPLES2).trackedAction(processContext, async (options) => {
534
+ const limit = parseIntOrFail(options.limit, "--limit");
535
+ if (limit === null)
536
+ return;
537
+ const offset = parseIntOrFail(options.offset, "--offset");
538
+ if (offset === null)
539
+ return;
540
+ const auth = await resolveEvalAuthOrFail(options.tenant);
541
+ if (!auth)
542
+ return;
543
+ const evaluators = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evaluators?skip=${offset}&take=${limit}`, auth.authHeaders, "GET", "Check that the process key is correct and you have access.");
544
+ if (!evaluators)
545
+ return;
546
+ OutputFormatter.success({
547
+ Result: RESULTS.Success,
548
+ Code: "EvaluatorList",
549
+ Data: evaluators.map(formatEvaluator),
550
+ Pagination: new Pagination({
551
+ returned: evaluators.length,
552
+ limit,
553
+ offset
554
+ })
555
+ });
556
+ });
557
+ evaluatorCmd.command("get").description("Get details of a specific evaluator.").argument("<evaluatorId>", "Evaluator ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evaluatorId, options) => {
558
+ const auth = await resolveEvalAuthOrFail(options.tenant);
559
+ if (!auth)
560
+ return;
561
+ const evaluator = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evaluators/${encodeURIComponent(evaluatorId)}`, auth.authHeaders, "GET", "Check the evaluator ID and process key.");
562
+ if (!evaluator)
563
+ return;
564
+ OutputFormatter.success({
565
+ Result: RESULTS.Success,
566
+ Code: "EvaluatorDetails",
567
+ Data: formatEvaluator(evaluator)
568
+ });
569
+ });
570
+ evaluatorCmd.command("create").description("Create an evaluator for a process.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--workload-id <guid>", "Workload ID (GUID)").requiredOption("--folder-key <guid>", "Folder key (GUID)").requiredOption("--name <name>", "Evaluator name").requiredOption("--description <text>", "Evaluator description").requiredOption("--evaluator-type-id <id>", "Evaluator type ID (e.g. uipath-exact-match, uipath-llm-judge-output-semantic-similarity, uipath-llm-judge-trajectory-similarity)").requiredOption("--evaluator-config <json>", "Evaluator config as JSON object").option("--version <version>", "Evaluator version", "1.0").option("--tenant <tenant>", "UiPath tenant name").examples(CREATE_EXAMPLES2).trackedAction(processContext, async (options) => {
571
+ const config = parseJsonOrFail(options.evaluatorConfig, "--evaluator-config");
572
+ if (!config)
573
+ return;
574
+ const auth = await resolveEvalAuthOrFail(options.tenant);
575
+ if (!auth)
576
+ return;
577
+ const body = {
578
+ workloadId: options.workloadId,
579
+ folderKey: options.folderKey,
580
+ externalId: crypto.randomUUID(),
581
+ name: options.name,
582
+ description: options.description,
583
+ evaluatorTypeId: options.evaluatorTypeId,
584
+ evaluatorConfig: config,
585
+ version: options.version
586
+ };
587
+ const evaluator = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evaluators`, auth.authHeaders, "POST", "Check the process key, workload ID, and evaluator config.", body);
588
+ if (!evaluator)
589
+ return;
590
+ OutputFormatter.success({
591
+ Result: RESULTS.Success,
592
+ Code: "EvaluatorCreated",
593
+ Data: formatEvaluator(evaluator)
594
+ });
595
+ });
596
+ evaluatorCmd.command("update").description("Update an existing evaluator. Provide at least one field to change.").argument("<evaluatorId>", "Evaluator ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--name <name>", "New evaluator name").option("--description <text>", "New evaluator description").option("--evaluator-type-id <id>", "Evaluator type ID (e.g. uipath-exact-match, uipath-llm-judge-output-semantic-similarity, uipath-llm-judge-trajectory-similarity)").option("--evaluator-config <json>", "Evaluator config as JSON object").option("--version <version>", "Evaluator version").option("--tenant <tenant>", "UiPath tenant name").examples(UPDATE_EXAMPLES2).trackedAction(processContext, async (evaluatorId, options) => {
597
+ if (options.name === undefined && options.description === undefined && options.evaluatorTypeId === undefined && options.evaluatorConfig === undefined && options.version === undefined) {
598
+ OutputFormatter.error({
599
+ Result: RESULTS.Failure,
600
+ Message: "At least one of --name, --description, --evaluator-type-id, --evaluator-config, or --version must be provided.",
601
+ Instructions: "Provide the field(s) you want to update."
602
+ });
603
+ processContext.exit(1);
604
+ return;
605
+ }
606
+ let config;
607
+ if (options.evaluatorConfig !== undefined) {
608
+ const parsed = parseJsonOrFail(options.evaluatorConfig, "--evaluator-config");
609
+ if (!parsed)
610
+ return;
611
+ config = parsed;
612
+ }
613
+ const auth = await resolveEvalAuthOrFail(options.tenant);
614
+ if (!auth)
615
+ return;
616
+ const url = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evaluators/${encodeURIComponent(evaluatorId)}`;
617
+ const existing = await evalApiRequestOrFail(url, auth.authHeaders, "GET", "Check the evaluator ID and process key.");
618
+ if (!existing)
619
+ return;
620
+ const body = {
621
+ workloadId: existing.workloadId,
622
+ folderKey: existing.folderKey,
623
+ externalId: existing.externalId,
624
+ name: options.name ?? existing.name,
625
+ description: options.description ?? existing.description,
626
+ evaluatorTypeId: options.evaluatorTypeId ?? existing.evaluatorTypeId,
627
+ evaluatorConfig: config ?? existing.evaluatorConfig,
628
+ version: options.version ?? existing.version
629
+ };
630
+ const evaluator = await evalApiRequestOrFail(url, auth.authHeaders, "PUT", "Check the evaluator ID, process key, and provided values.", body);
631
+ if (!evaluator)
632
+ return;
633
+ OutputFormatter.success({
634
+ Result: RESULTS.Success,
635
+ Code: "EvaluatorUpdated",
636
+ Data: formatEvaluator(evaluator)
637
+ });
638
+ });
639
+ evaluatorCmd.command("delete").description("Delete an evaluator.").argument("<evaluatorId>", "Evaluator ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evaluatorId, options) => {
640
+ const auth = await resolveEvalAuthOrFail(options.tenant);
641
+ if (!auth)
642
+ return;
643
+ const result = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evaluators/${encodeURIComponent(evaluatorId)}`, auth.authHeaders, "DELETE", "Check the evaluator ID and process key.");
644
+ if (result === null)
645
+ return;
646
+ OutputFormatter.success({
647
+ Result: RESULTS.Success,
648
+ Code: "EvaluatorDeleted",
649
+ Data: { EvaluatorId: evaluatorId }
650
+ });
651
+ });
652
+ }
653
+
654
+ // src/commands/eval-process-runs.ts
655
+ function formatDuration(ms) {
656
+ if (ms === null)
657
+ return "-";
658
+ if (ms < 1000)
659
+ return `${ms}ms`;
660
+ return `${(ms / 1000).toFixed(1)}s`;
661
+ }
662
+ function formatEvalSetRun(run) {
663
+ return {
664
+ EvalSetRunId: run.id,
665
+ EvalSetId: run.evalSetId,
666
+ Status: run.status,
667
+ Score: run.score ?? "-",
668
+ EvalsExecuted: run.numberOfEvalsExecuted,
669
+ Duration: formatDuration(run.durationMilliseconds),
670
+ CreatedAt: run.createdAt
671
+ };
672
+ }
673
+ function formatEvalRun(run) {
674
+ const snapshot = run.evalSnapshot;
675
+ return {
676
+ EvalRunId: run.id,
677
+ DataPoint: snapshot?.name ?? run.id,
678
+ Status: run.status,
679
+ Result: run.result,
680
+ CreatedAt: run.createdAt
681
+ };
682
+ }
683
+ var LIST_RUNS_EXAMPLES = [
684
+ {
685
+ Description: "List eval set runs for a process",
686
+ Command: "uip eval run list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
687
+ Output: {
688
+ Code: "EvalSetRunList",
689
+ Data: [
690
+ {
691
+ EvalSetRunId: "a1b2c3d4-0000-0000-0000-000000000101",
692
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
693
+ Status: "completed",
694
+ Score: 0.86,
695
+ EvalsExecuted: 5,
696
+ Duration: "42.5s",
697
+ CreatedAt: "2026-08-01T10:00:00Z"
698
+ }
699
+ ]
700
+ }
701
+ }
702
+ ];
703
+ var GET_RUN_EXAMPLES = [
704
+ {
705
+ Description: "Get details of a specific eval set run",
706
+ Command: "uip eval run get a1b2c3d4-0000-0000-0000-000000000101 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
707
+ Output: {
708
+ Code: "EvalSetRunDetails",
709
+ Data: {
710
+ EvalSetRunId: "a1b2c3d4-0000-0000-0000-000000000101",
711
+ Status: "completed",
712
+ Score: 0.86,
713
+ EvalsExecuted: 5,
714
+ Duration: "42.5s"
715
+ }
716
+ }
717
+ }
718
+ ];
719
+ var RESULTS_EXAMPLES = [
720
+ {
721
+ Description: "View per-item results for an eval set run",
722
+ Command: "uip eval run results a1b2c3d4-0000-0000-0000-000000000101 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
723
+ Output: {
724
+ Code: "EvalRunResults",
725
+ Data: [
726
+ {
727
+ EvalRunId: "e1e2e3e4-0000-0000-0000-000000000001",
728
+ DataPoint: "Test Case 1",
729
+ Status: "completed"
730
+ }
731
+ ]
732
+ }
733
+ }
734
+ ];
735
+ function registerEvalProcessRunsCommand(evalCmd) {
736
+ const runCmd = evalCmd.command("run").description("Manage eval runs scoped by process key. " + "Use 'execute-and-evaluate' to submit a new run, and 'run list/get/results' to view results.");
737
+ runCmd.command("list").description("List eval set runs for a process (by deployment process key).").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--limit <n>", "Maximum number of runs to return", "50").option("--offset <n>", "Number of runs to skip", "0").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_RUNS_EXAMPLES).trackedAction(processContext, async (options) => {
738
+ const limit = parseIntOrFail(options.limit ?? "50", "--limit");
739
+ if (limit === null)
740
+ return;
741
+ const offset = parseIntOrFail(options.offset ?? "0", "--offset");
742
+ if (offset === null)
743
+ return;
744
+ const auth = await resolveEvalAuthOrFail(options.tenant);
745
+ if (!auth)
746
+ return;
747
+ const runs = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSetRuns?limit=${limit}&offset=${offset}`, auth.authHeaders, "GET", "Check the process key and try again.");
748
+ if (!runs)
749
+ return;
750
+ OutputFormatter.success({
751
+ Result: RESULTS.Success,
752
+ Code: "EvalSetRunList",
753
+ Data: runs.map(formatEvalSetRun),
754
+ Pagination: new Pagination({
755
+ returned: runs.length,
756
+ limit,
757
+ offset
758
+ })
759
+ });
760
+ });
761
+ runCmd.command("get").description("Get details of a specific eval set run by process key.").argument("<evalSetRunId>", "Eval set run ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").examples(GET_RUN_EXAMPLES).trackedAction(processContext, async (evalSetRunId, options) => {
762
+ const auth = await resolveEvalAuthOrFail(options.tenant);
763
+ if (!auth)
764
+ return;
765
+ const url = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSetRuns/${encodeURIComponent(evalSetRunId)}`;
766
+ const run = await evalApiRequestOrFail(url, auth.authHeaders, "GET", "Check the eval set run ID and process key.");
767
+ if (!run)
768
+ return;
769
+ OutputFormatter.success({
770
+ Result: RESULTS.Success,
771
+ Code: "EvalSetRunDetails",
772
+ Data: formatEvalSetRun(run)
773
+ });
774
+ });
775
+ runCmd.command("results").description("View per-item eval run results for an eval set run by process key.").argument("<evalSetRunId>", "Eval set run ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").examples(RESULTS_EXAMPLES).trackedAction(processContext, async (evalSetRunId, options) => {
776
+ const auth = await resolveEvalAuthOrFail(options.tenant);
777
+ if (!auth)
778
+ return;
779
+ const url = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSetRuns/${encodeURIComponent(evalSetRunId)}/evalRuns`;
780
+ const evalRuns = await evalApiRequestOrFail(url, auth.authHeaders, "GET", "Check the eval set run ID and process key.");
781
+ if (!evalRuns)
782
+ return;
783
+ OutputFormatter.success({
784
+ Result: RESULTS.Success,
785
+ Code: "EvalRunResults",
786
+ Data: evalRuns.map(formatEvalRun)
787
+ });
788
+ });
789
+ }
790
+
791
+ // src/commands/eval-run.ts
792
+ var PAGE_SIZE = 200;
793
+ async function fetchAllPages(baseUrl, headers, instructions) {
794
+ const all = [];
795
+ let skip = 0;
796
+ for (;; ) {
797
+ const page = await evalApiRequestOrFail(`${baseUrl}${baseUrl.includes("?") ? "&" : "?"}skip=${skip}&take=${PAGE_SIZE}`, headers, "GET", instructions);
798
+ if (!page)
799
+ return null;
800
+ all.push(...page);
801
+ if (page.length < PAGE_SIZE)
802
+ break;
803
+ skip += PAGE_SIZE;
804
+ }
805
+ return all;
806
+ }
807
+ function parseJsonArray(value, flag) {
808
+ const [parseErr, parsed] = catchError(() => JSON.parse(value));
809
+ if (parseErr || !Array.isArray(parsed)) {
810
+ OutputFormatter.error({
811
+ Result: RESULTS.Failure,
812
+ Message: `${flag} is not a valid JSON array: ${parseErr?.message ?? "expected an array"}`,
813
+ Instructions: `Example: ${flag} '[{"id":"1","name":"greeting","inputs":{"input":"hello"}}]'`
814
+ });
815
+ return null;
816
+ }
817
+ const nonObjects = parsed.filter((el) => el === null || typeof el !== "object");
818
+ if (nonObjects.length > 0) {
819
+ OutputFormatter.error({
820
+ Result: RESULTS.Failure,
821
+ Message: `${flag} must be an array of objects; found ${nonObjects.length} non-object element(s).`,
822
+ Instructions: `Example: ${flag} '[{"id":"1","name":"greeting","inputs":{"input":"hello"}}]'`
823
+ });
824
+ return null;
825
+ }
826
+ return parsed;
827
+ }
828
+ function validateEvalItems(items) {
829
+ for (let i = 0;i < items.length; i++) {
830
+ const item = items[i];
831
+ if (typeof item.id !== "string" || item.id.length === 0) {
832
+ return `Item at index ${i} is missing a valid "id" (string).`;
833
+ }
834
+ if (typeof item.name !== "string" || item.name.length === 0) {
835
+ return `Item at index ${i} is missing a valid "name" (string).`;
836
+ }
837
+ if (item.inputs === null || item.inputs === undefined || typeof item.inputs !== "object" || Array.isArray(item.inputs)) {
838
+ return `Item at index ${i} is missing a valid "inputs" (object).`;
839
+ }
840
+ }
841
+ return null;
842
+ }
843
+ var ZERO_GUID = "00000000-0000-0000-0000-000000000000";
844
+ async function submitRun(evalSetId, items, evaluators, batchSize, agentsApiBase, authHeaders, folderKey, tenantId, processKey, workloadId) {
845
+ const body = {
846
+ workloadId,
847
+ processKey,
848
+ context: { type: "Offline", evalSetId },
849
+ batchSize,
850
+ executionConfig: {
851
+ workloadType: "OrchestratorJob",
852
+ tenantId,
853
+ folderKey,
854
+ processKey
855
+ },
856
+ evaluatorConfigs: evaluators,
857
+ items
858
+ };
859
+ const raw = await evalApiRequest(`${agentsApiBase}/unifiedEvals/workloads/executeAndEvaluate`, authHeaders, "POST", body);
860
+ if (raw === null || typeof raw !== "object" || !("evalSetRunId" in raw)) {
861
+ throw new Error("API response did not contain evalSetRunId. The run may not have been created.");
862
+ }
863
+ return String(raw.evalSetRunId);
864
+ }
865
+ var EVAL_RUN_EXAMPLES = [
866
+ {
867
+ Description: "Run from an eval set (fetches items and evaluators automatically)",
868
+ Command: "uip eval execute-and-evaluate --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --eval-set-id f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
869
+ Output: {
870
+ Code: "EvalRunSubmitted",
871
+ Data: {
872
+ ProcessKey: "9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
873
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
874
+ EvalSetRunId: "a1b2c3d4-0000-0000-0000-000000000101"
875
+ }
876
+ }
877
+ },
878
+ {
879
+ Description: "Submit with inline items and evaluators",
880
+ Command: `uip eval execute-and-evaluate --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --workload-id a1b2c3d4-0000-0000-0000-000000000001 --items '[{"id":"1","name":"greeting","inputs":{"input":"hello"}}]' --evaluators '[{"id":"ev-1","evaluatorTypeId":"uipath-llm-judge-output-semantic-similarity","evaluatorConfig":{}}]'`,
881
+ Output: {
882
+ Code: "EvalRunSubmitted",
883
+ Data: {
884
+ ProcessKey: "9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
885
+ Folder: "user@uipath.com's workspace",
886
+ EvalSetId: "00000000-0000-0000-0000-000000000000",
887
+ EvalSetRunId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04"
888
+ }
889
+ }
890
+ }
891
+ ];
892
+ async function resolveFromEvalSet(options) {
893
+ const auth = await resolveEvalAuthOrFail(options.tenant);
894
+ if (!auth)
895
+ return;
896
+ const evalSetId = options.evalSetId;
897
+ const base = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}`;
898
+ const evalSet = await evalApiRequestOrFail(`${base}/evalSets/${encodeURIComponent(evalSetId)}`, auth.authHeaders, "GET", "Check the eval set ID and process key.");
899
+ if (!evalSet)
900
+ return;
901
+ const evaluations = await fetchAllPages(`${base}/evalSets/${encodeURIComponent(evalSetId)}/evaluations`, auth.authHeaders, "Failed to list evaluations for the eval set.");
902
+ if (!evaluations)
903
+ return;
904
+ if (evaluations.length === 0) {
905
+ OutputFormatter.error({
906
+ Result: RESULTS.Failure,
907
+ Message: "The eval set has no evaluations (data points). Add evaluations first.",
908
+ Instructions: `uip eval evaluation create --process-key ${options.processKey} --eval-set-id ${evalSetId} --folder-key <folder-key> --name <name> --inputs '<json>'`
909
+ });
910
+ processContext.exit(1);
911
+ return;
912
+ }
913
+ if (evalSet.evaluatorRefs.length === 0) {
914
+ OutputFormatter.error({
915
+ Result: RESULTS.Failure,
916
+ Message: "The eval set has no evaluator refs. Link evaluators first.",
917
+ Instructions: `uip eval eval-set update ${evalSetId} --process-key ${options.processKey} --evaluator-refs <evaluator-id>`
918
+ });
919
+ processContext.exit(1);
920
+ return;
921
+ }
922
+ const allEvaluators = await fetchAllPages(`${base}/evaluators`, auth.authHeaders, "Failed to list evaluators for the process.");
923
+ if (!allEvaluators)
924
+ return;
925
+ const evaluatorsByExternalId = new Map(allEvaluators.map((ev) => [ev.externalId, ev]));
926
+ const evaluatorConfigs = [];
927
+ for (const ref of evalSet.evaluatorRefs) {
928
+ const ev = evaluatorsByExternalId.get(ref);
929
+ if (!ev) {
930
+ OutputFormatter.error({
931
+ Result: RESULTS.Failure,
932
+ Message: `Evaluator with externalId ${ref} not found.`,
933
+ Instructions: "Check the evaluator refs on the eval set."
934
+ });
935
+ processContext.exit(1);
936
+ return;
937
+ }
938
+ evaluatorConfigs.push({
939
+ id: ev.id,
940
+ name: ev.name,
941
+ version: "1.0",
942
+ evaluatorTypeId: ev.evaluatorTypeId,
943
+ evaluatorConfig: ev.evaluatorConfig
944
+ });
945
+ }
946
+ const items = evaluations.map((e) => ({
947
+ id: e.id,
948
+ name: e.name,
949
+ inputs: e.inputs,
950
+ expectedOutput: e.expectedOutput ?? {},
951
+ expectedBehavior: e.expectedBehavior ?? "",
952
+ evaluationCriterias: e.evaluationCriterias ?? {},
953
+ ...e.simulations && e.simulations.length > 0 ? { simulations: e.simulations } : {}
954
+ }));
955
+ let batchSize = evalSet.batchSize ?? 5;
956
+ if (options.batchSize !== undefined) {
957
+ const n = parseIntOrFail(options.batchSize, "--batch-size", 1);
958
+ if (n === null)
959
+ return;
960
+ batchSize = n;
961
+ }
962
+ const folderKey = options.folderKey ?? evalSet.folderKey;
963
+ const [submitErr, evalSetRunId] = await catchError(submitRun(evalSetId, items, evaluatorConfigs, batchSize, auth.agentsApiBase, auth.authHeaders, folderKey, auth.tenantId, options.processKey, evalSet.workloadId));
964
+ if (submitErr) {
965
+ OutputFormatter.error({
966
+ Result: RESULTS.Failure,
967
+ Message: submitErr.message,
968
+ Instructions: "Check that the package is published and the eval set is configured correctly."
969
+ });
970
+ processContext.exit(1);
971
+ return;
972
+ }
973
+ OutputFormatter.success({
974
+ Result: RESULTS.Success,
975
+ Code: "EvalRunSubmitted",
976
+ Data: {
977
+ ProcessKey: options.processKey,
978
+ Folder: folderKey,
979
+ EvalSetId: evalSetId,
980
+ EvalSetRunId: evalSetRunId
981
+ }
982
+ });
983
+ }
984
+ async function resolveInline(options) {
985
+ if (!options.workloadId) {
986
+ OutputFormatter.error({
987
+ Result: RESULTS.Failure,
988
+ Message: "--workload-id is required when using inline --items and --evaluators.",
989
+ Instructions: "Pass --workload-id, or use --eval-set-id without --items/--evaluators to run from an eval set."
990
+ });
991
+ processContext.exit(1);
992
+ return;
993
+ }
994
+ const items = parseJsonArray(options.items, "--items");
995
+ if (items === null) {
996
+ processContext.exit(1);
997
+ return;
998
+ }
999
+ const itemsError = validateEvalItems(items);
1000
+ if (itemsError !== null) {
1001
+ OutputFormatter.error({
1002
+ Result: RESULTS.Failure,
1003
+ Message: itemsError,
1004
+ Instructions: 'Each item must have "id" (string), "name" (string), and "inputs" (object). ' + `Example: --items '[{"id":"1","name":"greeting","inputs":{"input":"hello"}}]'`
1005
+ });
1006
+ processContext.exit(1);
1007
+ return;
1008
+ }
1009
+ const evaluators = parseJsonArray(options.evaluators, "--evaluators");
1010
+ if (evaluators === null) {
1011
+ processContext.exit(1);
1012
+ return;
1013
+ }
1014
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1015
+ if (!auth)
1016
+ return;
1017
+ let folderKey;
1018
+ let folderLabel;
1019
+ if (options.folderKey) {
1020
+ folderKey = options.folderKey;
1021
+ folderLabel = folderKey;
1022
+ } else {
1023
+ const { createApiClient, PersonalWorkspacesApi } = await import("./index-24qchvk6.js");
1024
+ const [wsApiErr, wsApi] = await catchError(createApiClient(PersonalWorkspacesApi, {
1025
+ tenant: options.tenant
1026
+ }));
1027
+ if (wsApiErr) {
1028
+ OutputFormatter.error({
1029
+ Result: RESULTS.Failure,
1030
+ Message: await extractErrorMessage(wsApiErr),
1031
+ Instructions: "Run 'uip login' and retry."
1032
+ });
1033
+ processContext.exit(1);
1034
+ return;
1035
+ }
1036
+ const [wsErr, personalWs] = await catchError(wsApi.personalWorkspacesGetPersonalWorkspace());
1037
+ if (wsErr ?? !personalWs?.key) {
1038
+ OutputFormatter.error({
1039
+ Result: RESULTS.Failure,
1040
+ Message: `Failed to resolve personal workspace: ${wsErr ? await extractErrorMessage(wsErr) : "not found"}`,
1041
+ Instructions: "Ensure your account has a personal workspace enabled."
1042
+ });
1043
+ processContext.exit(1);
1044
+ return;
1045
+ }
1046
+ folderKey = personalWs.key;
1047
+ folderLabel = personalWs.name ?? folderKey;
1048
+ }
1049
+ let batchSize = 5;
1050
+ if (options.batchSize !== undefined) {
1051
+ const n = parseIntOrFail(options.batchSize, "--batch-size");
1052
+ if (n === null)
1053
+ return;
1054
+ batchSize = n;
1055
+ }
1056
+ const evalSetId = options.evalSetId ?? ZERO_GUID;
1057
+ const [submitErr, evalSetRunId] = await catchError(submitRun(evalSetId, items, evaluators, batchSize, auth.agentsApiBase, auth.authHeaders, folderKey, auth.tenantId, options.processKey, options.workloadId));
1058
+ if (submitErr) {
1059
+ OutputFormatter.error({
1060
+ Result: RESULTS.Failure,
1061
+ Message: submitErr.message,
1062
+ Instructions: "Check that the package is published and the eval set ID is correct."
1063
+ });
1064
+ processContext.exit(1);
1065
+ return;
1066
+ }
1067
+ OutputFormatter.success({
1068
+ Result: RESULTS.Success,
1069
+ Code: "EvalRunSubmitted",
1070
+ Data: {
1071
+ ProcessKey: options.processKey,
1072
+ Folder: folderLabel,
1073
+ EvalSetId: evalSetId,
1074
+ EvalSetRunId: evalSetRunId
1075
+ }
1076
+ });
1077
+ }
1078
+ function registerEvalRunCommand(evalCmd) {
1079
+ evalCmd.command("execute-and-evaluate").alias("run-offline-evals").description("Submit a runtime eval run for a process. Pass --eval-set-id alone to fetch items and evaluators from the eval set, or pass --items and --evaluators inline for full control.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find available keys.").option("--workload-id <guid>", "Workload ID (GUID). Required for inline mode.").option("--eval-set-id <id>", "Eval set ID. When used without --items/--evaluators, fetches everything from the eval set.").option("--items <json>", `JSON array of eval items, each with id, name, and inputs (e.g. '[{"id":"1","name":"greeting","inputs":{"input":"hello"}}]')`).option("--evaluators <json>", "JSON array of evaluator configs").option("--batch-size <n>", "Max concurrent evaluation pipelines (default: 5, or eval set's value in eval-set mode)").option("--folder-key <key>", "Folder key (GUID); defaults to personal workspace (inline) or eval set's folder (eval-set mode).").option("--tenant <tenant>", "UiPath tenant name").examples(EVAL_RUN_EXAMPLES).trackedAction(processContext, async (options) => {
1080
+ const hasInline = options.items !== undefined || options.evaluators !== undefined;
1081
+ const hasEvalSet = options.evalSetId !== undefined;
1082
+ if (!hasInline && !hasEvalSet) {
1083
+ OutputFormatter.error({
1084
+ Result: RESULTS.Failure,
1085
+ Message: "Provide either --eval-set-id (to run from an eval set) or --items and --evaluators (for inline mode).",
1086
+ Instructions: "Example: uip eval execute-and-evaluate --process-key <guid> --eval-set-id <guid>"
1087
+ });
1088
+ processContext.exit(1);
1089
+ return;
1090
+ }
1091
+ if (hasInline) {
1092
+ if (!options.items || !options.evaluators) {
1093
+ OutputFormatter.error({
1094
+ Result: RESULTS.Failure,
1095
+ Message: "Both --items and --evaluators are required for inline mode.",
1096
+ Instructions: "Pass both --items and --evaluators, or use --eval-set-id alone to run from an eval set."
1097
+ });
1098
+ processContext.exit(1);
1099
+ return;
1100
+ }
1101
+ await resolveInline(options);
1102
+ } else {
1103
+ await resolveFromEvalSet(options);
1104
+ }
1105
+ });
1106
+ }
1107
+
1108
+ // src/commands/eval-schedule.ts
1109
+ function schedulesUrl(base, processKey) {
1110
+ return `${base}/unifiedEvals/processes/${encodeURIComponent(processKey)}/schedules`;
1111
+ }
1112
+ function scheduleUrl(base, processKey, scheduleId) {
1113
+ return `${schedulesUrl(base, processKey)}/${encodeURIComponent(scheduleId)}`;
1114
+ }
1115
+ var CREATE_EXAMPLES3 = [
1116
+ {
1117
+ Description: "Create an eval schedule that runs daily at 9 AM UTC",
1118
+ Command: 'uip eval schedule create --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --eval-set-id f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04 --cron "0 9 * * *"',
1119
+ Output: {
1120
+ Code: "EvalScheduleCreated",
1121
+ Data: {
1122
+ ScheduleId: "b2c3d4e5-0000-0000-0000-000000000001",
1123
+ ProcessKey: "9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
1124
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
1125
+ CronExpression: "0 9 * * *",
1126
+ Status: "active"
1127
+ }
1128
+ }
1129
+ }
1130
+ ];
1131
+ var LIST_EXAMPLES3 = [
1132
+ {
1133
+ Description: "List all eval schedules for a process",
1134
+ Command: "uip eval schedule list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
1135
+ Output: {
1136
+ Code: "EvalScheduleList",
1137
+ Data: [
1138
+ {
1139
+ ScheduleId: "b2c3d4e5-0000-0000-0000-000000000001",
1140
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
1141
+ CronExpression: "0 9 * * *",
1142
+ Status: "active"
1143
+ }
1144
+ ]
1145
+ }
1146
+ }
1147
+ ];
1148
+ var UPDATE_EXAMPLES3 = [
1149
+ {
1150
+ Description: "Update the cron expression and eval set of a schedule",
1151
+ Command: 'uip eval schedule update b2c3d4e5-0000-0000-0000-000000000001 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --cron "0 */6 * * *" --eval-set-id new-eval-set-guid',
1152
+ Output: {
1153
+ Code: "EvalScheduleUpdated",
1154
+ Data: {
1155
+ ScheduleId: "b2c3d4e5-0000-0000-0000-000000000001",
1156
+ CronExpression: "0 */6 * * *",
1157
+ Status: "active"
1158
+ }
1159
+ }
1160
+ }
1161
+ ];
1162
+ var DELETE_EXAMPLES = [
1163
+ {
1164
+ Description: "Delete an eval schedule",
1165
+ Command: "uip eval schedule delete b2c3d4e5-0000-0000-0000-000000000001 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
1166
+ Output: {
1167
+ Code: "EvalScheduleDeleted",
1168
+ Data: {
1169
+ ScheduleId: "b2c3d4e5-0000-0000-0000-000000000001"
1170
+ }
1171
+ }
1172
+ }
1173
+ ];
1174
+ function formatSchedule(schedule) {
1175
+ return {
1176
+ ScheduleId: schedule.id,
1177
+ WorkloadId: schedule.workloadId,
1178
+ ProcessKey: schedule.processKey,
1179
+ FolderKey: schedule.folderKey,
1180
+ EvalSetId: schedule.evalSetId,
1181
+ CronExpression: schedule.cronExpression,
1182
+ Status: schedule.status,
1183
+ CreatedAt: schedule.createdAt,
1184
+ ...schedule.updatedAt ? { UpdatedAt: schedule.updatedAt } : {}
1185
+ };
1186
+ }
1187
+ function registerEvalScheduleCommand(evalCmd) {
1188
+ const scheduleCmd = evalCmd.command("schedule").description("Manage eval schedules for agent processes. " + "Create, list, update, pause, resume, and delete scheduled evaluation runs.");
1189
+ scheduleCmd.command("create").description("Create a new eval schedule for a process.").requiredOption("--process-key <guid>", "Process key (deployment GUID) — identifies the Orchestrator release.").option("--workload-id <guid>", "Workload ID (GUID). Resolved from the eval set when omitted.").requiredOption("--eval-set-id <guid>", "Eval set ID (GUID) — the evaluation set to run.").requiredOption("--cron <expression>", 'Five-field cron expression in UTC (e.g. "0 9 * * *" for daily at 9 AM).').option("--folder-key <guid>", "Orchestrator folder key (GUID). Resolved from the eval set when omitted.").option("--tenant <tenant>", "UiPath tenant name").examples(CREATE_EXAMPLES3).trackedAction(processContext, async (options) => {
1190
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1191
+ if (!auth)
1192
+ return;
1193
+ let workloadId = options.workloadId;
1194
+ let folderKey = options.folderKey;
1195
+ if (!workloadId || !folderKey) {
1196
+ const evalSet = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(options.evalSetId)}`, auth.authHeaders, "GET", "Check the eval set ID and process key.");
1197
+ if (!evalSet)
1198
+ return;
1199
+ workloadId = workloadId ?? evalSet.workloadId;
1200
+ folderKey = folderKey ?? evalSet.folderKey;
1201
+ }
1202
+ const body = {
1203
+ workloadId,
1204
+ evalSetId: options.evalSetId,
1205
+ cronExpression: options.cron,
1206
+ executionConfig: {
1207
+ workloadType: "OrchestratorJob",
1208
+ tenantId: auth.tenantId,
1209
+ folderKey,
1210
+ processKey: options.processKey
1211
+ }
1212
+ };
1213
+ const schedule = await evalApiRequestOrFail(schedulesUrl(auth.agentsApiBase, options.processKey), auth.authHeaders, "POST", "Check that the process key, eval set, and folder key are correct.", body);
1214
+ if (!schedule)
1215
+ return;
1216
+ OutputFormatter.success({
1217
+ Result: RESULTS.Success,
1218
+ Code: "EvalScheduleCreated",
1219
+ Data: formatSchedule(schedule)
1220
+ });
1221
+ });
1222
+ scheduleCmd.command("list").description("List eval schedules for a process.").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_EXAMPLES3).trackedAction(processContext, async (options) => {
1223
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1224
+ if (!auth)
1225
+ return;
1226
+ const schedules = await evalApiRequestOrFail(schedulesUrl(auth.agentsApiBase, options.processKey), auth.authHeaders, "GET", "Check the process key and try again.");
1227
+ if (!schedules)
1228
+ return;
1229
+ OutputFormatter.success({
1230
+ Result: RESULTS.Success,
1231
+ Code: "EvalScheduleList",
1232
+ Data: schedules.map(formatSchedule)
1233
+ });
1234
+ });
1235
+ scheduleCmd.command("get").description("Get details of a specific eval schedule.").argument("<scheduleId>", "Eval schedule ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (scheduleId, options) => {
1236
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1237
+ if (!auth)
1238
+ return;
1239
+ const schedule = await evalApiRequestOrFail(scheduleUrl(auth.agentsApiBase, options.processKey, scheduleId), auth.authHeaders, "GET", "Check the schedule ID and process key.");
1240
+ if (!schedule)
1241
+ return;
1242
+ OutputFormatter.success({
1243
+ Result: RESULTS.Success,
1244
+ Code: "EvalScheduleDetails",
1245
+ Data: formatSchedule(schedule)
1246
+ });
1247
+ });
1248
+ scheduleCmd.command("update").description("Update an existing eval schedule. Provide at least one field to change.").argument("<scheduleId>", "Eval schedule ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--eval-set-id <guid>", "New eval set ID").option("--cron <expression>", "New five-field cron expression in UTC").option("--tenant <tenant>", "UiPath tenant name").examples(UPDATE_EXAMPLES3).trackedAction(processContext, async (scheduleId, options) => {
1249
+ if (options.evalSetId === undefined && options.cron === undefined) {
1250
+ OutputFormatter.error({
1251
+ Result: RESULTS.Failure,
1252
+ Message: "At least one of --eval-set-id or --cron must be provided.",
1253
+ Instructions: "Provide the field(s) you want to update."
1254
+ });
1255
+ processContext.exit(1);
1256
+ return;
1257
+ }
1258
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1259
+ if (!auth)
1260
+ return;
1261
+ const body = {};
1262
+ if (options.evalSetId !== undefined) {
1263
+ body.evalSetId = options.evalSetId;
1264
+ }
1265
+ if (options.cron !== undefined) {
1266
+ body.cronExpression = options.cron;
1267
+ }
1268
+ const schedule = await evalApiRequestOrFail(scheduleUrl(auth.agentsApiBase, options.processKey, scheduleId), auth.authHeaders, "PUT", "Check the schedule ID, process key, and provided values.", body);
1269
+ if (!schedule)
1270
+ return;
1271
+ OutputFormatter.success({
1272
+ Result: RESULTS.Success,
1273
+ Code: "EvalScheduleUpdated",
1274
+ Data: formatSchedule(schedule)
1275
+ });
1276
+ });
1277
+ scheduleCmd.command("pause").description("Pause an active eval schedule.").argument("<scheduleId>", "Eval schedule ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (scheduleId, options) => {
1278
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1279
+ if (!auth)
1280
+ return;
1281
+ const schedule = await evalApiRequestOrFail(`${scheduleUrl(auth.agentsApiBase, options.processKey, scheduleId)}/pause`, auth.authHeaders, "PUT", "Check the schedule ID and process key.");
1282
+ if (!schedule)
1283
+ return;
1284
+ OutputFormatter.success({
1285
+ Result: RESULTS.Success,
1286
+ Code: "EvalSchedulePaused",
1287
+ Data: formatSchedule(schedule)
1288
+ });
1289
+ });
1290
+ scheduleCmd.command("resume").description("Resume a paused eval schedule.").argument("<scheduleId>", "Eval schedule ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (scheduleId, options) => {
1291
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1292
+ if (!auth)
1293
+ return;
1294
+ const schedule = await evalApiRequestOrFail(`${scheduleUrl(auth.agentsApiBase, options.processKey, scheduleId)}/resume`, auth.authHeaders, "PUT", "Check the schedule ID and process key.");
1295
+ if (!schedule)
1296
+ return;
1297
+ OutputFormatter.success({
1298
+ Result: RESULTS.Success,
1299
+ Code: "EvalScheduleResumed",
1300
+ Data: formatSchedule(schedule)
1301
+ });
1302
+ });
1303
+ scheduleCmd.command("delete").description("Delete an eval schedule.").argument("<scheduleId>", "Eval schedule ID (GUID)").requiredOption("--process-key <guid>", "Process key (deployment GUID).").option("--tenant <tenant>", "UiPath tenant name").examples(DELETE_EXAMPLES).trackedAction(processContext, async (scheduleId, options) => {
1304
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1305
+ if (!auth)
1306
+ return;
1307
+ const result = await evalApiRequestOrFail(scheduleUrl(auth.agentsApiBase, options.processKey, scheduleId), auth.authHeaders, "DELETE", "Check the schedule ID and process key.");
1308
+ if (result === null)
1309
+ return;
1310
+ OutputFormatter.success({
1311
+ Result: RESULTS.Success,
1312
+ Code: "EvalScheduleDeleted",
1313
+ Data: { ScheduleId: scheduleId }
1314
+ });
1315
+ });
1316
+ }
1317
+
1318
+ // src/commands/eval-sets.ts
1319
+ function formatEvalSet(es) {
1320
+ return {
1321
+ EvalSetId: es.id,
1322
+ Name: es.name,
1323
+ Description: es.description ?? "",
1324
+ Version: es.version ?? "-",
1325
+ BatchSize: es.batchSize ?? "-",
1326
+ EvaluatorRefs: es.evaluatorRefs,
1327
+ CreatedAt: es.createdAt,
1328
+ ...es.updatedAt ? { UpdatedAt: es.updatedAt } : {}
1329
+ };
1330
+ }
1331
+ var LIST_EXAMPLES4 = [
1332
+ {
1333
+ Description: "List eval sets for a process",
1334
+ Command: "uip eval eval-set list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
1335
+ Output: {
1336
+ Code: "EvalSetList",
1337
+ Data: [
1338
+ {
1339
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
1340
+ Name: "Smoke Tests",
1341
+ Description: "Basic smoke tests",
1342
+ BatchSize: 10,
1343
+ EvaluatorRefs: ["ev-1", "ev-2"],
1344
+ CreatedAt: "2026-08-01T10:00:00Z"
1345
+ }
1346
+ ]
1347
+ }
1348
+ }
1349
+ ];
1350
+ var UPDATE_EXAMPLES4 = [
1351
+ {
1352
+ Description: "Update an eval set's name and add evaluator refs",
1353
+ Command: 'uip eval eval-set update f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04 --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --name "Updated Name" --evaluator-refs ev-1 ev-2',
1354
+ Output: {
1355
+ Code: "EvalSetUpdated",
1356
+ Data: {
1357
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
1358
+ Name: "Updated Name",
1359
+ EvaluatorRefs: ["ev-1", "ev-2"]
1360
+ }
1361
+ }
1362
+ }
1363
+ ];
1364
+ var CREATE_EXAMPLES4 = [
1365
+ {
1366
+ Description: "Create an eval set with evaluator references",
1367
+ Command: 'uip eval eval-set create --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09 --workload-id a1b2c3d4-0000-0000-0000-000000000001 --folder-key f1f2f3f4-0000-0000-0000-000000000001 --name "Smoke Tests" --evaluator-refs ev-1 ev-2',
1368
+ Output: {
1369
+ Code: "EvalSetCreated",
1370
+ Data: {
1371
+ EvalSetId: "f3a7d219-8b4c-4e62-a951-7d3f6e2c8b04",
1372
+ Name: "Smoke Tests"
1373
+ }
1374
+ }
1375
+ }
1376
+ ];
1377
+ function registerEvalSetsCommand(evalCmd) {
1378
+ const evalSetCmd = evalCmd.command("eval-set").description("Manage eval sets by process key.");
1379
+ evalSetCmd.command("list").description("List eval sets for a process.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--limit <n>", "Maximum number of items to return", "100").option("--offset <n>", "Number of items to skip", "0").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_EXAMPLES4).trackedAction(processContext, async (options) => {
1380
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1381
+ if (!auth)
1382
+ return;
1383
+ const limit = parseIntOrFail(options.limit, "--limit");
1384
+ if (limit === null)
1385
+ return;
1386
+ const offset = parseIntOrFail(options.offset, "--offset");
1387
+ if (offset === null)
1388
+ return;
1389
+ const evalSets = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets?skip=${offset}&take=${limit}`, auth.authHeaders, "GET", "Check that the process key is correct and you have access.");
1390
+ if (!evalSets)
1391
+ return;
1392
+ OutputFormatter.success({
1393
+ Result: RESULTS.Success,
1394
+ Code: "EvalSetList",
1395
+ Data: evalSets.map(formatEvalSet),
1396
+ Pagination: new Pagination({
1397
+ returned: evalSets.length,
1398
+ limit,
1399
+ offset
1400
+ })
1401
+ });
1402
+ });
1403
+ evalSetCmd.command("get").description("Get details of a specific eval set.").argument("<evalSetId>", "Eval set ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evalSetId, options) => {
1404
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1405
+ if (!auth)
1406
+ return;
1407
+ const evalSet = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(evalSetId)}`, auth.authHeaders, "GET", "Check the eval set ID and process key.");
1408
+ if (!evalSet)
1409
+ return;
1410
+ OutputFormatter.success({
1411
+ Result: RESULTS.Success,
1412
+ Code: "EvalSetDetails",
1413
+ Data: formatEvalSet(evalSet)
1414
+ });
1415
+ });
1416
+ evalSetCmd.command("create").description("Create an eval set for a process.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").requiredOption("--workload-id <guid>", "Workload ID (GUID)").requiredOption("--folder-key <guid>", "Folder key (GUID)").requiredOption("--name <name>", "Eval set name").option("--description <text>", "Eval set description").option("--batch-size <n>", "Max concurrent evaluations").option("--timeout-minutes <n>", "Timeout in minutes per evaluation").option("--evaluator-refs <refs...>", "Evaluator ExternalIds to link to this eval set (from 'uip eval evaluator list')").option("--tenant <tenant>", "UiPath tenant name").examples(CREATE_EXAMPLES4).trackedAction(processContext, async (options) => {
1417
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1418
+ if (!auth)
1419
+ return;
1420
+ const body = {
1421
+ workloadId: options.workloadId,
1422
+ folderKey: options.folderKey,
1423
+ externalId: crypto.randomUUID(),
1424
+ name: options.name,
1425
+ description: options.description,
1426
+ evaluatorRefs: options.evaluatorRefs ?? [],
1427
+ version: "1.0"
1428
+ };
1429
+ if (options.batchSize !== undefined) {
1430
+ const n = parseIntOrFail(options.batchSize, "--batch-size", 1);
1431
+ if (n === null)
1432
+ return;
1433
+ body.batchSize = n;
1434
+ }
1435
+ if (options.timeoutMinutes !== undefined) {
1436
+ const n = parseIntOrFail(options.timeoutMinutes, "--timeout-minutes", 1);
1437
+ if (n === null)
1438
+ return;
1439
+ body.timeoutMinutes = n;
1440
+ }
1441
+ const evalSet = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets`, auth.authHeaders, "POST", "Check the process key, workload ID, and folder key.", body);
1442
+ if (!evalSet)
1443
+ return;
1444
+ OutputFormatter.success({
1445
+ Result: RESULTS.Success,
1446
+ Code: "EvalSetCreated",
1447
+ Data: formatEvalSet(evalSet)
1448
+ });
1449
+ });
1450
+ evalSetCmd.command("update").description("Update an existing eval set. Provide at least one field to change.").argument("<evalSetId>", "Eval set ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--name <name>", "New eval set name").option("--description <text>", "New description").option("--batch-size <n>", "Max concurrent evaluations").option("--timeout-minutes <n>", "Timeout in minutes per evaluation").option("--evaluator-refs <refs...>", "Evaluator ExternalIds to link to this eval set (from 'uip eval evaluator list')").option("--tenant <tenant>", "UiPath tenant name").examples(UPDATE_EXAMPLES4).trackedAction(processContext, async (evalSetId, options) => {
1451
+ if (options.name === undefined && options.description === undefined && options.batchSize === undefined && options.timeoutMinutes === undefined && options.evaluatorRefs === undefined) {
1452
+ OutputFormatter.error({
1453
+ Result: RESULTS.Failure,
1454
+ Message: "At least one of --name, --description, --batch-size, --timeout-minutes, or --evaluator-refs must be provided.",
1455
+ Instructions: "Provide the field(s) you want to update."
1456
+ });
1457
+ processContext.exit(1);
1458
+ return;
1459
+ }
1460
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1461
+ if (!auth)
1462
+ return;
1463
+ const url = `${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(evalSetId)}`;
1464
+ const existing = await evalApiRequestOrFail(url, auth.authHeaders, "GET", "Check the eval set ID and process key.");
1465
+ if (!existing)
1466
+ return;
1467
+ const body = {
1468
+ workloadId: existing.workloadId,
1469
+ folderKey: existing.folderKey,
1470
+ externalId: existing.externalId,
1471
+ name: options.name ?? existing.name,
1472
+ version: existing.version ?? "1.0",
1473
+ description: options.description ?? existing.description ?? "",
1474
+ evaluatorRefs: options.evaluatorRefs ?? existing.evaluatorRefs
1475
+ };
1476
+ if (options.batchSize !== undefined) {
1477
+ const n = parseIntOrFail(options.batchSize, "--batch-size", 1);
1478
+ if (n === null)
1479
+ return;
1480
+ body.batchSize = n;
1481
+ } else if (existing.batchSize !== undefined) {
1482
+ body.batchSize = existing.batchSize;
1483
+ }
1484
+ if (options.timeoutMinutes !== undefined) {
1485
+ const n = parseIntOrFail(options.timeoutMinutes, "--timeout-minutes", 1);
1486
+ if (n === null)
1487
+ return;
1488
+ body.timeoutMinutes = n;
1489
+ } else if (existing.timeoutMinutes !== undefined) {
1490
+ body.timeoutMinutes = existing.timeoutMinutes;
1491
+ }
1492
+ const evalSet = await evalApiRequestOrFail(url, auth.authHeaders, "PUT", "Check the eval set ID, process key, and provided values.", body);
1493
+ if (!evalSet)
1494
+ return;
1495
+ OutputFormatter.success({
1496
+ Result: RESULTS.Success,
1497
+ Code: "EvalSetUpdated",
1498
+ Data: formatEvalSet(evalSet)
1499
+ });
1500
+ });
1501
+ evalSetCmd.command("delete").description("Delete an eval set.").argument("<evalSetId>", "Eval set ID (GUID)").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--tenant <tenant>", "UiPath tenant name").trackedAction(processContext, async (evalSetId, options) => {
1502
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1503
+ if (!auth)
1504
+ return;
1505
+ const result = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/evalSets/${encodeURIComponent(evalSetId)}`, auth.authHeaders, "DELETE", "Check the eval set ID and process key.");
1506
+ if (result === null)
1507
+ return;
1508
+ OutputFormatter.success({
1509
+ Result: RESULTS.Success,
1510
+ Code: "EvalSetDeleted",
1511
+ Data: { EvalSetId: evalSetId }
1512
+ });
1513
+ });
1514
+ }
1515
+
1516
+ // src/commands/eval-simulatable-components.ts
1517
+ function formatComponent(c) {
1518
+ return {
1519
+ Id: c.id,
1520
+ Name: c.name,
1521
+ Description: c.description,
1522
+ ComponentType: c.componentType,
1523
+ SubType: c.subType,
1524
+ IsEnabled: c.isEnabled,
1525
+ ...c.outputSchema ? { OutputSchema: c.outputSchema } : {}
1526
+ };
1527
+ }
1528
+ var LIST_EXAMPLES5 = [
1529
+ {
1530
+ Description: "List simulatable components for a process",
1531
+ Command: "uip eval simulatable-components list --process-key 9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09",
1532
+ Output: {
1533
+ Code: "SimulatableComponentList",
1534
+ Data: [
1535
+ {
1536
+ Id: "web-search",
1537
+ Name: "Web Search",
1538
+ Description: "Searches the web",
1539
+ ComponentType: "tool",
1540
+ SubType: "Api",
1541
+ IsEnabled: true
1542
+ }
1543
+ ]
1544
+ }
1545
+ }
1546
+ ];
1547
+ function registerEvalSimulatableComponentsCommand(evalCmd) {
1548
+ const simCompCmd = evalCmd.command("simulatable-components").description("List simulatable components (tools, escalations) for a process.");
1549
+ simCompCmd.command("list").description("List simulatable components for a deployed process.").requiredOption("--process-key <guid>", "Process key (GUID). Use 'uip or processes list' to find keys.").option("--process-version <version>", "Process version (semantic version). Defaults to the latest deployed version.").option("--tenant <tenant>", "UiPath tenant name").examples(LIST_EXAMPLES5).trackedAction(processContext, async (options) => {
1550
+ const auth = await resolveEvalAuthOrFail(options.tenant);
1551
+ if (!auth)
1552
+ return;
1553
+ const params = new URLSearchParams({ workloadType: "Agents" });
1554
+ if (options.processVersion) {
1555
+ params.set("processVersion", options.processVersion);
1556
+ }
1557
+ const response = await evalApiRequestOrFail(`${auth.agentsApiBase}/unifiedEvals/processes/${encodeURIComponent(options.processKey)}/simulatableComponents?${params.toString()}`, auth.authHeaders, "GET", "Check the process key and that the agent is deployed.");
1558
+ if (!response)
1559
+ return;
1560
+ const components = Array.isArray(response.components) ? response.components : [];
1561
+ OutputFormatter.success({
1562
+ Result: RESULTS.Success,
1563
+ Code: "SimulatableComponentList",
1564
+ Data: components.map(formatComponent)
1565
+ });
1566
+ });
1567
+ }
1568
+
1569
+ // src/commands/eval.ts
1570
+ function registerEvalCommand(program) {
1571
+ registerEvalRunCommand(program);
1572
+ registerEvalProcessRunsCommand(program);
1573
+ registerEvalScheduleCommand(program);
1574
+ registerEvalEvaluatorsCommand(program);
1575
+ registerEvalSetsCommand(program);
1576
+ registerEvalEvaluationsCommand(program);
1577
+ registerEvalSimulatableComponentsCommand(program);
1578
+ }
1579
+
1580
+ // src/tool.ts
1581
+ var metadata2 = {
1582
+ name: "eval-tool",
1583
+ version: package_default.version,
1584
+ description: "Run runtime evaluations, manage evaluators, eval sets, evaluations, and schedules.",
1585
+ commandPrefix: "eval"
1586
+ };
1587
+ var registerCommands2 = async (program) => {
1588
+ registerEvalCommand(program);
1589
+ };
1590
+
1591
+ export { metadata2, registerCommands2 };
1592
+
1593
+ //# debugId=EFBD09296909983C64756E2164756E21