@thinkingai/ae-cli 6.0.54 → 6.0.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/README.zh.md +24 -3
- package/dist/{auth-MBWRZTB6.js → auth-U7MNUUEF.js} +37 -13
- package/dist/{automation-NWWVLWG2.js → automation-K32V2NKP.js} +94 -38
- package/dist/{capability-7SSN6PNZ.js → capability-VV7RHB3B.js} +8 -7
- package/dist/{chunk-CT7QDJS6.js → chunk-2IYIYSUO.js} +2 -2
- package/dist/{chunk-PLE722AJ.js → chunk-2UWS5CUP.js} +62 -35
- package/dist/{chunk-ZIQWCZTS.js → chunk-3TQA5RXE.js} +3 -3
- package/dist/{chunk-KFNMQTZI.js → chunk-ELCR235M.js} +26 -19
- package/dist/{chunk-FMNNGNYO.js → chunk-F6NIPIT5.js} +7 -6
- package/dist/{chunk-D6ACFILC.js → chunk-GI5BMEZO.js} +1 -1
- package/dist/chunk-JJJIP2XQ.js +105 -0
- package/dist/{chunk-5XJYLEVC.js → chunk-KHS64JZQ.js} +4 -4
- package/dist/{chunk-UQ77JXYH.js → chunk-LMCYSVVA.js} +12 -13
- package/dist/{chunk-EOYSKMXN.js → chunk-MXV3PYC3.js} +14 -6
- package/dist/{chunk-7QXISROD.js → chunk-P6VFCH3P.js} +4 -2
- package/dist/{chunk-GIK6EDUG.js → chunk-P7KL7IOH.js} +41 -13
- package/dist/{chunk-AP7YP2NX.js → chunk-QSAR3GHY.js} +2 -2
- package/dist/{chunk-S6M42KND.js → chunk-T44M3PLZ.js} +1 -1
- package/dist/{chunk-HVHR4SCP.js → chunk-UUD5RB36.js} +3 -3
- package/dist/{chunk-HBLK3ZHW.js → chunk-W2PTY5NN.js} +3 -3
- package/dist/chunk-X7DRLE62.js +233 -0
- package/dist/{chunk-WNGFSIOB.js → chunk-ZW4VMY5H.js} +103 -6
- package/dist/{community-report-client-PEERQCX2.js → community-report-client-S2K3A3Q6.js} +1 -1
- package/dist/{config-FF6YVJKI.js → config-ASSAAIOI.js} +10 -9
- package/dist/{context-YAE3QXP3.js → context-6RWLVOM5.js} +4 -4
- package/dist/{data-integration-HESZET7E.js → data-integration-EXSUPUDZ.js} +6 -6
- package/dist/evaluation-5YT4QLCL.js +3714 -0
- package/dist/index.js +93 -49
- package/dist/{local-data-upload-client-TWMHFLUA.js → local-data-upload-client-7HCPGIS3.js} +1 -1
- package/dist/{memory-62BJOULV.js → memory-VCYLVPMI.js} +6 -6
- package/dist/{metadata-7CUMWIB4.js → metadata-TIWIOFTU.js} +8 -8
- package/dist/{model-SBV2JFJI.js → model-DPJHSD73.js} +4 -4
- package/dist/{personal-semantic-preference-6SRCYPGE.js → personal-semantic-preference-SBKU2LOK.js} +6 -6
- package/dist/{project-semantic-YYEM4OSU.js → project-semantic-SKNWLH3I.js} +6 -6
- package/dist/{sync-SU33KYX7.js → sync-7PP6KQ75.js} +7 -7
- package/dist/{te-agent-F5QBZSQA.js → te-agent-L2BYNPHM.js} +753 -94
- package/dist/{te-analysis-U4I2W2WC.js → te-analysis-RJB6VJC4.js} +411 -79
- package/dist/{te-community-PFLDLPP4.js → te-community-QN3WMPNT.js} +6 -4
- package/dist/{te-dataops-FOQSRIAZ.js → te-dataops-UKD2XN3E.js} +8 -6
- package/dist/{te-engage-ZX76CNAB.js → te-engage-KPTN4N2U.js} +6 -6
- package/dist/{te-kb-MHDOST62.js → te-kb-SFZI3W5T.js} +7 -7
- package/dist/{te-system-J2FQF5ER.js → te-system-IE7NCKC3.js} +10 -9
- package/dist/{te-team-5UOZPTU3.js → te-team-4KHCFM2I.js} +63 -41
- package/dist/{things-VPPXXLHB.js → things-7JJUVAWL.js} +7 -7
- package/dist/{update-EEUTEI6R.js → update-AX66IFMJ.js} +4 -4
- package/dist/{workspace-O3EEIYQM.js → workspace-Q5NMEKEN.js} +6 -6
- package/package.json +11 -7
- package/skills/ae-agent/SKILL.md +22 -3
- package/skills/ae-agent/references/command_index.md +8 -0
- package/skills/ae-agent/references/resource-package.md +53 -0
- package/skills/ae-analysis/SKILL.md +12 -3
- package/skills/ae-analysis/references/adhoc_export.md +1 -1
- package/skills/ae-analysis/references/agent_review_preflight.md +4 -0
- package/skills/ae-analysis/references/agent_review_submit_to_page.md +4 -0
- package/skills/ae-analysis/references/ai_models/event.md +58 -2
- package/skills/ae-analysis/references/ai_models/rank_list.md +1 -0
- package/skills/ae-analysis/references/ai_models/retention.md +2 -0
- package/skills/ae-analysis/references/ai_models/revenue.md +2 -0
- package/skills/ae-analysis/references/analysis_data_export.md +11 -1
- package/skills/ae-analysis/references/asset_batch_info_export.md +2 -2
- package/skills/ae-analysis/references/asset_batch_sql_export.md +2 -2
- package/skills/ae-analysis/references/asset_export.md +1 -1
- package/skills/ae-analysis/references/bi_panel_page_data_export.md +1 -1
- package/skills/ae-analysis/references/check_export.md +1 -1
- package/skills/ae-analysis/references/command_index.md +34 -25
- package/skills/ae-analysis/references/dashboard_report_data_export.md +2 -0
- package/skills/ae-analysis/references/dashboard_report_get.md +6 -2
- package/skills/ae-analysis/references/dashboard_report_update.md +24 -5
- package/skills/ae-analysis/references/feedback_submit.md +1 -1
- package/skills/ae-analysis/references/governance_recommendation_export.md +10 -2
- package/skills/ae-analysis/references/live_data_export.md +1 -1
- package/skills/ae-analysis/references/metric_create.md +29 -0
- package/skills/ae-analysis/references/metric_get.md +2 -0
- package/skills/ae-analysis/references/metric_update.md +2 -0
- package/skills/ae-analysis/references/operation_record_export.md +1 -1
- package/skills/ae-analysis/references/plan_change_log_export.md +1 -1
- package/skills/ae-analysis/references/plan_export.md +1 -1
- package/skills/ae-analysis/references/project_info_get.md +4 -0
- package/skills/ae-analysis/references/project_member_group_add_members.md +35 -0
- package/skills/ae-analysis/references/project_member_group_create.md +25 -0
- package/skills/ae-analysis/references/project_member_group_delete.md +34 -0
- package/skills/ae-analysis/references/project_member_group_list.md +29 -0
- package/skills/ae-analysis/references/project_member_group_members.md +28 -0
- package/skills/ae-analysis/references/project_member_group_remove_members.md +35 -0
- package/skills/ae-analysis/references/project_member_group_rename.md +26 -0
- package/skills/ae-analysis/references/project_semantic_knowledge_wiki_plan_schema.md +6 -1
- package/skills/ae-analysis/references/report_data_export.md +1 -1
- package/skills/ae-analysis/references/scenario_append.md +25 -0
- package/skills/ae-analysis/references/scenario_collect.md +35 -0
- package/skills/ae-analysis/references/scenario_evaluation.md +30 -0
- package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/build-project-semantic-wiki.mjs +73 -4
- package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/default-compile-schema.md +5 -1
- package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/execution-contract.mjs +173 -0
- package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/generate-build-ir.mjs +109 -64
- package/skills/ae-automation/SKILL.md +2 -2
- package/skills/ae-automation/references/cli.md +18 -1
- package/skills/ae-capability/SKILL.md +7 -3
- package/skills/ae-evaluation/SKILL.md +83 -0
- package/skills/ae-evaluation/references/access-and-members.md +55 -0
- package/skills/ae-evaluation/references/connections.md +78 -0
- package/skills/ae-evaluation/references/datasets.md +57 -0
- package/skills/ae-evaluation/references/evaluators.md +81 -0
- package/skills/ae-evaluation/references/files.md +33 -0
- package/skills/ae-evaluation/references/integrations.md +39 -0
- package/skills/ae-evaluation/references/runs.md +160 -0
- package/skills/ae-evaluation/references/schedules.md +31 -0
- package/skills/ae-metadata/SKILL.md +2 -1
- package/skills/ae-metadata/references/metadata_data_table_download.md +7 -1
- package/skills/ae-metadata/references/metadata_data_table_get.md +2 -0
- package/skills/ae-team/SKILL.md +2 -2
- package/skills/ae-team/references/run-start.md +1 -0
- package/skills/ae-team/references/run-watch.md +7 -5
- package/skills/ae-things/SKILL.md +53 -51
- package/skills/ae-things/references/cli.md +56 -54
- package/skills/ae-things-progress/SKILL.md +20 -20
- package/skills/ae-things-review/SKILL.md +19 -19
- package/dist/chunk-4655PUNX.js +0 -43
|
@@ -0,0 +1,3714 @@
|
|
|
1
|
+
import { createRequire as __createRequire } from 'node:module'; const require = __createRequire(import.meta.url);
|
|
2
|
+
import {
|
|
3
|
+
EvalApiError,
|
|
4
|
+
evalDownloadFile,
|
|
5
|
+
evalRequest,
|
|
6
|
+
resolveEvaluationHost
|
|
7
|
+
} from "./chunk-X7DRLE62.js";
|
|
8
|
+
import {
|
|
9
|
+
getCliToken
|
|
10
|
+
} from "./chunk-P7KL7IOH.js";
|
|
11
|
+
import {
|
|
12
|
+
isAeSandboxRuntime
|
|
13
|
+
} from "./chunk-LMCYSVVA.js";
|
|
14
|
+
import {
|
|
15
|
+
CliValidationError,
|
|
16
|
+
externalCliTokenRequestOptions
|
|
17
|
+
} from "./chunk-ZW4VMY5H.js";
|
|
18
|
+
|
|
19
|
+
// src/core/eval-session.ts
|
|
20
|
+
var SESSION_PATH = "/api/sandbox/agent/evaluation/session";
|
|
21
|
+
async function requestSandboxEvaluationSession(host, cliToken, teProjectId, fetchImpl = fetch) {
|
|
22
|
+
const url = `${resolveEvaluationHost(host)}/agent${SESSION_PATH}`;
|
|
23
|
+
const credentialOptions = externalCliTokenRequestOptions(url);
|
|
24
|
+
const signal = AbortSignal.timeout(3e4);
|
|
25
|
+
let response;
|
|
26
|
+
let body;
|
|
27
|
+
try {
|
|
28
|
+
response = await fetchImpl(url, {
|
|
29
|
+
...credentialOptions,
|
|
30
|
+
method: "POST",
|
|
31
|
+
headers: { Accept: "application/json", "Content-Type": "application/json", "cli-token": cliToken },
|
|
32
|
+
body: JSON.stringify(teProjectId ? { te_project_id: teProjectId } : {}),
|
|
33
|
+
redirect: "error",
|
|
34
|
+
signal
|
|
35
|
+
});
|
|
36
|
+
try {
|
|
37
|
+
body = await response.json();
|
|
38
|
+
} catch (error) {
|
|
39
|
+
if (signal.aborted) throw error;
|
|
40
|
+
body = void 0;
|
|
41
|
+
}
|
|
42
|
+
} catch {
|
|
43
|
+
throw new EvalApiError(signal.aborted ? "Evaluation session request timed out." : "Evaluation session request failed; redirects are not allowed.", {
|
|
44
|
+
code: signal.aborted ? "EVAL_SANDBOX_SESSION_TIMEOUT" : "EVAL_SANDBOX_SESSION_NETWORK_ERROR",
|
|
45
|
+
httpStatus: 0,
|
|
46
|
+
hint: "Verify that the AE Host serves the session endpoint directly without redirects."
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (!response.ok || typeof body?.session_token !== "string" || !body.session_token) {
|
|
50
|
+
throw new EvalApiError(
|
|
51
|
+
typeof body?.error === "string" ? body.error : "Unable to obtain an evaluation session from the sandbox gateway.",
|
|
52
|
+
{
|
|
53
|
+
code: "EVAL_SANDBOX_SESSION_FAILED",
|
|
54
|
+
httpStatus: response.status,
|
|
55
|
+
sensitiveValues: [cliToken, typeof body?.session_token === "string" ? body.session_token : ""],
|
|
56
|
+
hint: response.status === 400 ? "Set AE_EVAL_TE_PROJECT_ID when the CLI token is authorized for multiple TE projects." : "Verify that the sandbox gateway and Agent Evaluation service are configured."
|
|
57
|
+
}
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
return body.session_token;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// src/commands/evaluation/shared.ts
|
|
64
|
+
var EVAL_API_PREFIX = "/agenteval/api/v1";
|
|
65
|
+
function evalFlags() {
|
|
66
|
+
return [
|
|
67
|
+
{
|
|
68
|
+
name: "endpoint",
|
|
69
|
+
type: "string",
|
|
70
|
+
sensitive: true,
|
|
71
|
+
desc: `Deprecated: must match the current AE Host. Independent endpoint overrides are unsupported; use --host to select the AE Host instead.`
|
|
72
|
+
}
|
|
73
|
+
];
|
|
74
|
+
}
|
|
75
|
+
function resolveEvalBaseUrl(ctx, environment = process.env) {
|
|
76
|
+
const baseUrl = resolveEvaluationHost(ctx.host());
|
|
77
|
+
for (const configured of [ctx.str("endpoint").trim(), environment.AE_EVAL_BASE_URL?.trim()]) {
|
|
78
|
+
if (!configured) continue;
|
|
79
|
+
const legacyBaseUrl = configured.replace(/\/+$/, "").replace(/(\/agenteval\/api\/v1|\/agenteval)$/, "");
|
|
80
|
+
if (resolveEvaluationHost(legacyBaseUrl) !== baseUrl) {
|
|
81
|
+
throw new CliValidationError("Independent evaluation endpoint overrides are unsupported.", {
|
|
82
|
+
code: "EVAL_ENDPOINT_OVERRIDE_UNSUPPORTED",
|
|
83
|
+
hint: "Remove --endpoint and AE_EVAL_BASE_URL; select the evaluation AE Host with --host instead."
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return baseUrl;
|
|
88
|
+
}
|
|
89
|
+
function evalPathId(ctx, name = "id") {
|
|
90
|
+
const value = ctx.str(name);
|
|
91
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(value)) {
|
|
92
|
+
throw new CliValidationError(`--${name} must be a resource ID containing only letters, digits, hyphens, or underscores.`, {
|
|
93
|
+
code: "EVAL_RESOURCE_ID_INVALID"
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
return encodeURIComponent(value);
|
|
97
|
+
}
|
|
98
|
+
function evalHeaders(environment = process.env) {
|
|
99
|
+
const accessToken = environment.AE_EVAL_ACCESS_TOKEN?.trim();
|
|
100
|
+
if (!accessToken) {
|
|
101
|
+
throw new EvalApiError("AE_EVAL_ACCESS_TOKEN is not set.", {
|
|
102
|
+
code: "EVAL_ACCESS_TOKEN_MISSING",
|
|
103
|
+
httpStatus: 0,
|
|
104
|
+
hint: "Export AE_EVAL_ACCESS_TOKEN with an Agent Evaluation platform session token (POST /agenteval/api/v1/auth/session on the platform web console)."
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return { Authorization: `Bearer ${accessToken}` };
|
|
108
|
+
}
|
|
109
|
+
async function resolveEvalHeaders(ctx, environment = process.env) {
|
|
110
|
+
if (environment.AE_EVAL_ACCESS_TOKEN?.trim()) return evalHeaders(environment);
|
|
111
|
+
if (!isAeSandboxRuntime()) return evalHeaders(environment);
|
|
112
|
+
const cliToken = await getCliToken(ctx.host());
|
|
113
|
+
const sessionToken = await requestSandboxEvaluationSession(
|
|
114
|
+
ctx.host(),
|
|
115
|
+
cliToken,
|
|
116
|
+
environment.AE_EVAL_TE_PROJECT_ID?.trim()
|
|
117
|
+
);
|
|
118
|
+
return { Authorization: `Bearer ${sessionToken}` };
|
|
119
|
+
}
|
|
120
|
+
async function evalDownload(ctx, path2, output) {
|
|
121
|
+
return evalDownloadFile({
|
|
122
|
+
baseUrl: resolveEvalBaseUrl(ctx),
|
|
123
|
+
headers: await resolveEvalHeaders(ctx),
|
|
124
|
+
method: "GET",
|
|
125
|
+
path: `${EVAL_API_PREFIX}${path2}`
|
|
126
|
+
}, output);
|
|
127
|
+
}
|
|
128
|
+
async function evalApi(ctx, method, path2, params, body, options = {}) {
|
|
129
|
+
return evalRequest({
|
|
130
|
+
baseUrl: resolveEvalBaseUrl(ctx),
|
|
131
|
+
headers: await resolveEvalHeaders(ctx),
|
|
132
|
+
method,
|
|
133
|
+
path: `${EVAL_API_PREFIX}${path2}`,
|
|
134
|
+
params,
|
|
135
|
+
body,
|
|
136
|
+
idempotencyKey: options.idempotencyKey
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
function evalDryRun(ctx, method, path2, params, body, options = {}) {
|
|
140
|
+
return {
|
|
141
|
+
method,
|
|
142
|
+
url: `${resolveEvalBaseUrl(ctx)}${EVAL_API_PREFIX}${path2}`,
|
|
143
|
+
params,
|
|
144
|
+
body,
|
|
145
|
+
...options.idempotencyKey ? { headers: { "Idempotency-Key": options.idempotencyKey } } : {}
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
function idFlag(desc = "Resource UUID") {
|
|
149
|
+
return { name: "id", type: "string", required: true, desc };
|
|
150
|
+
}
|
|
151
|
+
function lockVersionFlag(resource) {
|
|
152
|
+
return {
|
|
153
|
+
name: "lock-version",
|
|
154
|
+
type: "number",
|
|
155
|
+
required: true,
|
|
156
|
+
min: 0,
|
|
157
|
+
desc: `Current ${resource} lock_version (optimistic lock); each successful mutation increments it`
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
function idAndLockFlags(idDesc, resource) {
|
|
161
|
+
return [idFlag(idDesc), lockVersionFlag(resource)];
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// src/commands/evaluation/list-datasets.ts
|
|
165
|
+
var PATH = "/datasets";
|
|
166
|
+
function buildParams(ctx) {
|
|
167
|
+
const params = {};
|
|
168
|
+
const search = ctx.str("search");
|
|
169
|
+
if (search) params.search = search;
|
|
170
|
+
const status = ctx.str("status");
|
|
171
|
+
if (status) params.status = status;
|
|
172
|
+
const cursor = ctx.str("cursor");
|
|
173
|
+
if (cursor) params.cursor = cursor;
|
|
174
|
+
const pageSize = ctx.optionalNum("page-size");
|
|
175
|
+
if (pageSize !== void 0) params.page_size = pageSize;
|
|
176
|
+
return params;
|
|
177
|
+
}
|
|
178
|
+
var listDatasets = {
|
|
179
|
+
service: "evaluation",
|
|
180
|
+
command: "+list_datasets",
|
|
181
|
+
description: "List evaluation datasets via GET /agenteval/api/v1/datasets. Returns a cursor-paginated page of datasets.",
|
|
182
|
+
flags: [
|
|
183
|
+
...evalFlags(),
|
|
184
|
+
{ name: "search", type: "string", desc: "Search text matching dataset name/description" },
|
|
185
|
+
{
|
|
186
|
+
name: "status",
|
|
187
|
+
type: "string",
|
|
188
|
+
desc: "Filter by dataset status: CREATING | READY | IMPORT_FAILED"
|
|
189
|
+
},
|
|
190
|
+
{ name: "cursor", type: "string", desc: "Opaque pagination cursor from the previous page" },
|
|
191
|
+
{
|
|
192
|
+
name: "page-size",
|
|
193
|
+
type: "number",
|
|
194
|
+
min: 1,
|
|
195
|
+
max: 200,
|
|
196
|
+
desc: "Page size (1-200, default 50)"
|
|
197
|
+
}
|
|
198
|
+
],
|
|
199
|
+
risk: "read",
|
|
200
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH, buildParams(ctx)),
|
|
201
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH, buildParams(ctx))
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// src/commands/evaluation/get-dataset.ts
|
|
205
|
+
var PATH2 = "/datasets";
|
|
206
|
+
function pathFor(ctx) {
|
|
207
|
+
return `${PATH2}/${evalPathId(ctx, "id")}`;
|
|
208
|
+
}
|
|
209
|
+
var getDataset = {
|
|
210
|
+
service: "evaluation",
|
|
211
|
+
command: "+get_dataset",
|
|
212
|
+
description: "Get one evaluation dataset by id via GET /agenteval/api/v1/datasets/{datasetId}.",
|
|
213
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
214
|
+
risk: "read",
|
|
215
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor(ctx)),
|
|
216
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor(ctx))
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
// src/commands/evaluation/create-dataset.ts
|
|
220
|
+
var PATH3 = "/datasets";
|
|
221
|
+
function buildBody(ctx) {
|
|
222
|
+
const body = { name: ctx.str("name") };
|
|
223
|
+
const description = ctx.str("description");
|
|
224
|
+
if (description) body.description = description;
|
|
225
|
+
return body;
|
|
226
|
+
}
|
|
227
|
+
var createDataset = {
|
|
228
|
+
service: "evaluation",
|
|
229
|
+
command: "+create_dataset",
|
|
230
|
+
description: "Create a blank evaluation dataset via POST /agenteval/api/v1/datasets. Returns the created dataset.",
|
|
231
|
+
flags: [
|
|
232
|
+
...evalFlags(),
|
|
233
|
+
{ name: "name", type: "string", required: true, desc: "Dataset name (1-128 characters)" },
|
|
234
|
+
{ name: "description", type: "string", desc: "Optional dataset description (<=2000 characters)" }
|
|
235
|
+
],
|
|
236
|
+
risk: "write",
|
|
237
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH3, void 0, buildBody(ctx)),
|
|
238
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH3, void 0, buildBody(ctx))
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
// src/commands/evaluation/import-dataset.ts
|
|
242
|
+
import { promises as fs } from "fs";
|
|
243
|
+
import * as path from "path";
|
|
244
|
+
var PATH4 = "/datasets/imports";
|
|
245
|
+
async function buildForm(ctx) {
|
|
246
|
+
const filePath = path.resolve(ctx.str("file"));
|
|
247
|
+
let content;
|
|
248
|
+
try {
|
|
249
|
+
content = await fs.readFile(filePath);
|
|
250
|
+
} catch {
|
|
251
|
+
throw new Error(`File not found or unreadable: ${ctx.str("file")}`);
|
|
252
|
+
}
|
|
253
|
+
const form = new FormData();
|
|
254
|
+
form.append("name", ctx.str("name"));
|
|
255
|
+
const description = ctx.str("description");
|
|
256
|
+
if (description) form.append("description", description);
|
|
257
|
+
const profile = ctx.str("profile");
|
|
258
|
+
if (profile) form.append("profile", profile);
|
|
259
|
+
form.append("file", new Blob([Uint8Array.from(content)]), path.basename(filePath));
|
|
260
|
+
return form;
|
|
261
|
+
}
|
|
262
|
+
var importDataset = {
|
|
263
|
+
service: "evaluation",
|
|
264
|
+
command: "+import_dataset",
|
|
265
|
+
description: "Import an evaluation dataset from a local CSV/Excel/JSONL file via POST /agenteval/api/v1/datasets/imports (multipart). Requires an Idempotency-Key; returns 202 with the created dataset and staged file.",
|
|
266
|
+
flags: [
|
|
267
|
+
...evalFlags(),
|
|
268
|
+
{ name: "name", type: "string", required: true, desc: "Dataset name (1-128 characters)" },
|
|
269
|
+
{
|
|
270
|
+
name: "file",
|
|
271
|
+
type: "string",
|
|
272
|
+
required: true,
|
|
273
|
+
desc: "Local path to a CSV, Excel, or JSONL file to import"
|
|
274
|
+
},
|
|
275
|
+
{
|
|
276
|
+
name: "profile",
|
|
277
|
+
type: "string",
|
|
278
|
+
desc: "Import profile: GENERIC | AGENT_TASK_CASES_V1 (default GENERIC)"
|
|
279
|
+
},
|
|
280
|
+
{ name: "description", type: "string", desc: "Optional dataset description (<=2000 characters)" },
|
|
281
|
+
{
|
|
282
|
+
name: "idempotency-key",
|
|
283
|
+
type: "string",
|
|
284
|
+
required: true,
|
|
285
|
+
hint: "Required. Use a fresh value per import attempt (max 191 chars) sent as the Idempotency-Key header.",
|
|
286
|
+
desc: "Unique Idempotency-Key header value (1-191 characters)"
|
|
287
|
+
}
|
|
288
|
+
],
|
|
289
|
+
risk: "write",
|
|
290
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH4, void 0, {
|
|
291
|
+
name: ctx.str("name"),
|
|
292
|
+
description: ctx.str("description") || void 0,
|
|
293
|
+
profile: ctx.str("profile") || "GENERIC",
|
|
294
|
+
file: ctx.str("file"),
|
|
295
|
+
contentType: "multipart/form-data"
|
|
296
|
+
}, { idempotencyKey: ctx.str("idempotency-key") }),
|
|
297
|
+
execute: async (ctx) => {
|
|
298
|
+
const form = await buildForm(ctx);
|
|
299
|
+
return evalApi(ctx, "POST", PATH4, void 0, form, {
|
|
300
|
+
idempotencyKey: ctx.str("idempotency-key")
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
// src/commands/evaluation/dataset-import-status.ts
|
|
306
|
+
var PATH5 = "/datasets";
|
|
307
|
+
function pathFor2(ctx) {
|
|
308
|
+
return `${PATH5}/${evalPathId(ctx, "id")}/import-status`;
|
|
309
|
+
}
|
|
310
|
+
var datasetImportStatus = {
|
|
311
|
+
service: "evaluation",
|
|
312
|
+
command: "+dataset_import_status",
|
|
313
|
+
description: "Check the asynchronous import pipeline for a dataset via GET /agenteval/api/v1/datasets/{id}/import-status. Poll this after +import_dataset (which returns 202) until status is READY or IMPORT_FAILED; use +dataset_import_errors for failure details.",
|
|
314
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
315
|
+
risk: "read",
|
|
316
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor2(ctx)),
|
|
317
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor2(ctx))
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
// src/commands/evaluation/dataset-import-errors.ts
|
|
321
|
+
var PATH6 = "/datasets";
|
|
322
|
+
function pathFor3(ctx) {
|
|
323
|
+
return `${PATH6}/${evalPathId(ctx, "id")}/import-errors`;
|
|
324
|
+
}
|
|
325
|
+
var datasetImportErrors = {
|
|
326
|
+
service: "evaluation",
|
|
327
|
+
command: "+dataset_import_errors",
|
|
328
|
+
description: "Fetch row-level errors from a failed dataset import via GET /agenteval/api/v1/datasets/{id}/import-errors. Only meaningful when +dataset_import_status reports IMPORT_FAILED.",
|
|
329
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
330
|
+
risk: "read",
|
|
331
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor3(ctx)),
|
|
332
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor3(ctx))
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
// src/commands/evaluation/update-dataset.ts
|
|
336
|
+
var PATH7 = "/datasets";
|
|
337
|
+
function pathFor4(ctx) {
|
|
338
|
+
return `${PATH7}/${evalPathId(ctx, "id")}`;
|
|
339
|
+
}
|
|
340
|
+
function buildBody2(ctx) {
|
|
341
|
+
const body = { name: ctx.str("name"), lock_version: ctx.num("lock-version") };
|
|
342
|
+
const description = ctx.str("description");
|
|
343
|
+
if (description) body.description = description;
|
|
344
|
+
return body;
|
|
345
|
+
}
|
|
346
|
+
var updateDataset = {
|
|
347
|
+
service: "evaluation",
|
|
348
|
+
command: "+update_dataset",
|
|
349
|
+
description: "Rename or re-describe a dataset via PATCH /agenteval/api/v1/datasets/{id}. Requires the current dataset lock_version from +get_dataset and increments it.",
|
|
350
|
+
flags: [
|
|
351
|
+
...evalFlags(),
|
|
352
|
+
...idAndLockFlags("Dataset UUID", "dataset"),
|
|
353
|
+
{ name: "name", type: "string", required: true, desc: "Dataset name (1-128 characters)" },
|
|
354
|
+
{ name: "description", type: "string", desc: "Optional dataset description (<=2000 characters)" }
|
|
355
|
+
],
|
|
356
|
+
risk: "write",
|
|
357
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor4(ctx), void 0, buildBody2(ctx)),
|
|
358
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor4(ctx), void 0, buildBody2(ctx))
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
// src/commands/evaluation/delete-dataset.ts
|
|
362
|
+
var PATH8 = "/datasets";
|
|
363
|
+
function pathFor5(ctx) {
|
|
364
|
+
return `${PATH8}/${evalPathId(ctx, "id")}`;
|
|
365
|
+
}
|
|
366
|
+
function buildParams2(ctx) {
|
|
367
|
+
return { lock_version: ctx.num("lock-version") };
|
|
368
|
+
}
|
|
369
|
+
var deleteDataset = {
|
|
370
|
+
service: "evaluation",
|
|
371
|
+
command: "+delete_dataset",
|
|
372
|
+
description: "Delete a dataset and its cases via DELETE /agenteval/api/v1/datasets/{id}. Runs referencing the dataset keep their frozen snapshot. Requires the current dataset lock_version as a query parameter.",
|
|
373
|
+
flags: [...evalFlags(), ...idAndLockFlags("Dataset UUID", "dataset")],
|
|
374
|
+
risk: "high-risk-write",
|
|
375
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor5(ctx), buildParams2(ctx)),
|
|
376
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor5(ctx), buildParams2(ctx))
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
// src/commands/evaluation/transfer-dataset-owner.ts
|
|
380
|
+
var PATH9 = "/datasets";
|
|
381
|
+
function pathFor6(ctx) {
|
|
382
|
+
return `${PATH9}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
383
|
+
}
|
|
384
|
+
function buildBody3(ctx) {
|
|
385
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
386
|
+
}
|
|
387
|
+
var transferDatasetOwner = {
|
|
388
|
+
service: "evaluation",
|
|
389
|
+
command: "+transfer_dataset_owner",
|
|
390
|
+
description: "Transfer dataset ownership to another user via POST /agenteval/api/v1/datasets/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current dataset lock_version.",
|
|
391
|
+
flags: [
|
|
392
|
+
...evalFlags(),
|
|
393
|
+
...idAndLockFlags("Dataset UUID", "dataset"),
|
|
394
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
395
|
+
],
|
|
396
|
+
risk: "high-risk-write",
|
|
397
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor6(ctx), void 0, buildBody3(ctx)),
|
|
398
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor6(ctx), void 0, buildBody3(ctx))
|
|
399
|
+
};
|
|
400
|
+
|
|
401
|
+
// src/commands/evaluation/list-dataset-fields.ts
|
|
402
|
+
var PATH10 = "/datasets";
|
|
403
|
+
function pathFor7(ctx) {
|
|
404
|
+
return `${PATH10}/${evalPathId(ctx, "id")}/fields`;
|
|
405
|
+
}
|
|
406
|
+
var listDatasetFields = {
|
|
407
|
+
service: "evaluation",
|
|
408
|
+
command: "+list_dataset_fields",
|
|
409
|
+
description: "List the field schema of a dataset via GET /agenteval/api/v1/datasets/{id}/fields (field_key, data_type, ordinal). Needed to build run field mappings and case data payloads.",
|
|
410
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
411
|
+
risk: "read",
|
|
412
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor7(ctx)),
|
|
413
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor7(ctx))
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
// src/commands/evaluation/create-dataset-field.ts
|
|
417
|
+
var PATH11 = "/datasets";
|
|
418
|
+
function pathFor8(ctx) {
|
|
419
|
+
return `${PATH11}/${evalPathId(ctx, "id")}/fields`;
|
|
420
|
+
}
|
|
421
|
+
function buildBody4(ctx) {
|
|
422
|
+
return { display_name: ctx.str("display-name"), data_type: ctx.str("data-type") };
|
|
423
|
+
}
|
|
424
|
+
var createDatasetField = {
|
|
425
|
+
service: "evaluation",
|
|
426
|
+
command: "+create_dataset_field",
|
|
427
|
+
description: "Add a field to a blank dataset via POST /agenteval/api/v1/datasets/{id}/fields. The new field is appended at the end; reorder with +reorder_dataset_fields.",
|
|
428
|
+
flags: [
|
|
429
|
+
...evalFlags(),
|
|
430
|
+
idFlag("Dataset UUID"),
|
|
431
|
+
{ name: "display-name", type: "string", required: true, desc: "Field display name (1-128 characters); field_key is derived from it" },
|
|
432
|
+
{ name: "data-type", type: "string", required: true, desc: "Field data type: TEXT | NUMBER | BOOLEAN | JSON | DATE | TIME" }
|
|
433
|
+
],
|
|
434
|
+
risk: "write",
|
|
435
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor8(ctx), void 0, buildBody4(ctx)),
|
|
436
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor8(ctx), void 0, buildBody4(ctx))
|
|
437
|
+
};
|
|
438
|
+
|
|
439
|
+
// src/commands/evaluation/batch-create-dataset-fields.ts
|
|
440
|
+
var PATH12 = "/datasets";
|
|
441
|
+
var DATA_TYPES = /* @__PURE__ */ new Set(["TEXT", "NUMBER", "BOOLEAN", "JSON", "DATE", "TIME"]);
|
|
442
|
+
function pathFor9(ctx) {
|
|
443
|
+
return `${PATH12}/${evalPathId(ctx, "id")}/fields/batch`;
|
|
444
|
+
}
|
|
445
|
+
function buildBody5(ctx) {
|
|
446
|
+
const fields = ctx.json("fields-json");
|
|
447
|
+
if (!Array.isArray(fields) || fields.length < 1 || fields.length > 100) {
|
|
448
|
+
throw new CliValidationError("--fields-json must be a JSON array containing 1-100 field definitions");
|
|
449
|
+
}
|
|
450
|
+
for (const field of fields) {
|
|
451
|
+
if (field === null || typeof field !== "object" || Array.isArray(field) || typeof field.display_name !== "string" || !field.display_name || String(field.display_name).length > 128 || !DATA_TYPES.has(String(field.data_type))) {
|
|
452
|
+
throw new CliValidationError(
|
|
453
|
+
"--fields-json items must contain display_name with 1-128 characters and data_type TEXT | NUMBER | BOOLEAN | JSON | DATE | TIME"
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
return { fields };
|
|
458
|
+
}
|
|
459
|
+
var batchCreateDatasetFields = {
|
|
460
|
+
service: "evaluation",
|
|
461
|
+
command: "+batch_create_dataset_fields",
|
|
462
|
+
description: "Create 1-100 dataset fields atomically via POST /agenteval/api/v1/datasets/{id}/fields/batch. The request succeeds completely or creates no fields.",
|
|
463
|
+
flags: [
|
|
464
|
+
...evalFlags(),
|
|
465
|
+
idFlag("Dataset UUID"),
|
|
466
|
+
{
|
|
467
|
+
name: "fields-json",
|
|
468
|
+
type: "json",
|
|
469
|
+
required: true,
|
|
470
|
+
desc: `JSON array of 1-100 field definitions, e.g. '[{"display_name":"Question","data_type":"TEXT"}]'`
|
|
471
|
+
}
|
|
472
|
+
],
|
|
473
|
+
risk: "write",
|
|
474
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor9(ctx), void 0, buildBody5(ctx)),
|
|
475
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor9(ctx), void 0, buildBody5(ctx))
|
|
476
|
+
};
|
|
477
|
+
|
|
478
|
+
// src/commands/evaluation/update-dataset-field.ts
|
|
479
|
+
var PATH13 = "/datasets";
|
|
480
|
+
function pathFor10(ctx) {
|
|
481
|
+
return `${PATH13}/${evalPathId(ctx, "id")}/fields/${evalPathId(ctx, "field-id")}`;
|
|
482
|
+
}
|
|
483
|
+
function buildBody6(ctx) {
|
|
484
|
+
const body = { display_name: ctx.str("display-name"), lock_version: ctx.num("lock-version") };
|
|
485
|
+
const ordinal = ctx.optionalNum("ordinal");
|
|
486
|
+
if (ordinal !== void 0) body.ordinal = ordinal;
|
|
487
|
+
return body;
|
|
488
|
+
}
|
|
489
|
+
var updateDatasetField = {
|
|
490
|
+
service: "evaluation",
|
|
491
|
+
command: "+update_dataset_field",
|
|
492
|
+
description: "Rename or move a dataset field via PATCH /agenteval/api/v1/datasets/{id}/fields/{fieldId}. Requires the dataset lock_version from +get_dataset (not a per-field version) and increments it.",
|
|
493
|
+
flags: [
|
|
494
|
+
...evalFlags(),
|
|
495
|
+
idFlag("Dataset UUID"),
|
|
496
|
+
{ name: "field-id", type: "string", required: true, desc: "Field UUID from +list_dataset_fields" },
|
|
497
|
+
{ name: "display-name", type: "string", required: true, desc: "New field display name (1-128 characters)" },
|
|
498
|
+
{ name: "ordinal", type: "number", min: 0, desc: "New zero-based position of the field" },
|
|
499
|
+
lockVersionFlag("dataset")
|
|
500
|
+
],
|
|
501
|
+
risk: "write",
|
|
502
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor10(ctx), void 0, buildBody6(ctx)),
|
|
503
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor10(ctx), void 0, buildBody6(ctx))
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
// src/commands/evaluation/delete-dataset-field.ts
|
|
507
|
+
var PATH14 = "/datasets";
|
|
508
|
+
function pathFor11(ctx) {
|
|
509
|
+
return `${PATH14}/${evalPathId(ctx, "id")}/fields/${evalPathId(ctx, "field-id")}`;
|
|
510
|
+
}
|
|
511
|
+
function buildBody7(ctx) {
|
|
512
|
+
return { lock_version: ctx.num("lock-version") };
|
|
513
|
+
}
|
|
514
|
+
var deleteDatasetField = {
|
|
515
|
+
service: "evaluation",
|
|
516
|
+
command: "+delete_dataset_field",
|
|
517
|
+
description: "Remove a field from a dataset via DELETE /agenteval/api/v1/datasets/{id}/fields/{fieldId}. Case data for the field is dropped. Requires the dataset lock_version.",
|
|
518
|
+
flags: [
|
|
519
|
+
...evalFlags(),
|
|
520
|
+
idFlag("Dataset UUID"),
|
|
521
|
+
{ name: "field-id", type: "string", required: true, desc: "Field UUID from +list_dataset_fields" },
|
|
522
|
+
lockVersionFlag("dataset")
|
|
523
|
+
],
|
|
524
|
+
risk: "high-risk-write",
|
|
525
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor11(ctx), void 0, buildBody7(ctx)),
|
|
526
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor11(ctx), void 0, buildBody7(ctx))
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
// src/commands/evaluation/reorder-dataset-fields.ts
|
|
530
|
+
var PATH15 = "/datasets";
|
|
531
|
+
function pathFor12(ctx) {
|
|
532
|
+
return `${PATH15}/${evalPathId(ctx, "id")}/fields/reorder`;
|
|
533
|
+
}
|
|
534
|
+
function buildBody8(ctx) {
|
|
535
|
+
const fieldIds = ctx.json("field-ids");
|
|
536
|
+
if (!Array.isArray(fieldIds) || fieldIds.length === 0 || !fieldIds.every((v) => typeof v === "string")) {
|
|
537
|
+
throw new Error(`--field-ids must be a non-empty JSON array of field UUIDs, e.g. '["<uuid-1>","<uuid-2>"]'`);
|
|
538
|
+
}
|
|
539
|
+
return { field_ids: fieldIds, lock_version: ctx.num("lock-version") };
|
|
540
|
+
}
|
|
541
|
+
var reorderDatasetFields = {
|
|
542
|
+
service: "evaluation",
|
|
543
|
+
command: "+reorder_dataset_fields",
|
|
544
|
+
description: "Reorder all fields of a dataset via PUT /agenteval/api/v1/datasets/{id}/fields/reorder. The array must contain every field UUID exactly once, in the desired order. Requires the dataset lock_version.",
|
|
545
|
+
flags: [
|
|
546
|
+
...evalFlags(),
|
|
547
|
+
...idAndLockFlags("Dataset UUID", "dataset"),
|
|
548
|
+
{
|
|
549
|
+
name: "field-ids",
|
|
550
|
+
type: "json",
|
|
551
|
+
required: true,
|
|
552
|
+
desc: `JSON array of all field UUIDs in the new order, e.g. '["<uuid-1>","<uuid-2>"]'`
|
|
553
|
+
}
|
|
554
|
+
],
|
|
555
|
+
risk: "write",
|
|
556
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", pathFor12(ctx), void 0, buildBody8(ctx)),
|
|
557
|
+
execute: async (ctx) => evalApi(ctx, "PUT", pathFor12(ctx), void 0, buildBody8(ctx))
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
// src/commands/evaluation/list-dataset-cases.ts
|
|
561
|
+
var PATH16 = "/datasets";
|
|
562
|
+
function pathFor13(ctx) {
|
|
563
|
+
return `${PATH16}/${evalPathId(ctx, "id")}/cases`;
|
|
564
|
+
}
|
|
565
|
+
function buildParams3(ctx) {
|
|
566
|
+
const params = {};
|
|
567
|
+
const pageSize = ctx.optionalNum("page-size");
|
|
568
|
+
if (pageSize !== void 0) params.page_size = pageSize;
|
|
569
|
+
const cursor = ctx.str("cursor");
|
|
570
|
+
if (cursor) params.cursor = cursor;
|
|
571
|
+
return params;
|
|
572
|
+
}
|
|
573
|
+
var listDatasetCases = {
|
|
574
|
+
service: "evaluation",
|
|
575
|
+
command: "+list_dataset_cases",
|
|
576
|
+
description: "List the cases of a dataset with their field data via GET /agenteval/api/v1/datasets/{id}/cases (cursor-paginated). Use +get_dataset_case for one case in full.",
|
|
577
|
+
flags: [
|
|
578
|
+
...evalFlags(),
|
|
579
|
+
idFlag("Dataset UUID"),
|
|
580
|
+
{ name: "cursor", type: "string", desc: "Pagination cursor from the previous response next_cursor" },
|
|
581
|
+
{ name: "page-size", type: "number", min: 1, max: 200, desc: "Page size (default 50)" }
|
|
582
|
+
],
|
|
583
|
+
risk: "read",
|
|
584
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor13(ctx), buildParams3(ctx)),
|
|
585
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor13(ctx), buildParams3(ctx))
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
// src/commands/evaluation/get-dataset-case.ts
|
|
589
|
+
var PATH17 = "/datasets";
|
|
590
|
+
function pathFor14(ctx) {
|
|
591
|
+
return `${PATH17}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-id")}`;
|
|
592
|
+
}
|
|
593
|
+
var getDatasetCase = {
|
|
594
|
+
service: "evaluation",
|
|
595
|
+
command: "+get_dataset_case",
|
|
596
|
+
description: "Get one dataset case with its full field data via GET /agenteval/api/v1/datasets/{id}/cases/{caseId}. Find caseId via +list_dataset_cases.",
|
|
597
|
+
flags: [
|
|
598
|
+
...evalFlags(),
|
|
599
|
+
idFlag("Dataset UUID"),
|
|
600
|
+
{ name: "case-id", type: "string", required: true, desc: "Case UUID from +list_dataset_cases" }
|
|
601
|
+
],
|
|
602
|
+
risk: "read",
|
|
603
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor14(ctx)),
|
|
604
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor14(ctx))
|
|
605
|
+
};
|
|
606
|
+
|
|
607
|
+
// src/commands/evaluation/create-dataset-case.ts
|
|
608
|
+
var PATH18 = "/datasets";
|
|
609
|
+
function pathFor15(ctx) {
|
|
610
|
+
return `${PATH18}/${evalPathId(ctx, "id")}/cases`;
|
|
611
|
+
}
|
|
612
|
+
function buildBody9(ctx) {
|
|
613
|
+
const data = ctx.json("data");
|
|
614
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
|
615
|
+
throw new Error(`--data must be a JSON object keyed by field_key, e.g. '{"input":"..."}' (see +list_dataset_fields)`);
|
|
616
|
+
}
|
|
617
|
+
return { data };
|
|
618
|
+
}
|
|
619
|
+
var createDatasetCase = {
|
|
620
|
+
service: "evaluation",
|
|
621
|
+
command: "+create_dataset_case",
|
|
622
|
+
description: "Append one case to a dataset via POST /agenteval/api/v1/datasets/{id}/cases. The data object is keyed by field_key from +list_dataset_fields.",
|
|
623
|
+
flags: [
|
|
624
|
+
...evalFlags(),
|
|
625
|
+
idFlag("Dataset UUID"),
|
|
626
|
+
{ name: "data", type: "json", required: true, sensitive: true, desc: `Case data object keyed by field_key, e.g. '{"input":"...","expected":"..."}'` }
|
|
627
|
+
],
|
|
628
|
+
risk: "write",
|
|
629
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor15(ctx), void 0, buildBody9(ctx)),
|
|
630
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor15(ctx), void 0, buildBody9(ctx))
|
|
631
|
+
};
|
|
632
|
+
|
|
633
|
+
// src/commands/evaluation/batch-create-dataset-cases.ts
|
|
634
|
+
var PATH19 = "/datasets";
|
|
635
|
+
function pathFor16(ctx) {
|
|
636
|
+
return `${PATH19}/${evalPathId(ctx, "id")}/cases/batch`;
|
|
637
|
+
}
|
|
638
|
+
function buildBody10(ctx) {
|
|
639
|
+
const cases = ctx.json("cases-json");
|
|
640
|
+
if (!Array.isArray(cases) || cases.length < 1 || cases.length > 1e3) {
|
|
641
|
+
throw new CliValidationError("--cases-json must be a JSON array containing 1-1000 case definitions");
|
|
642
|
+
}
|
|
643
|
+
for (const item of cases) {
|
|
644
|
+
const data = item !== null && typeof item === "object" && !Array.isArray(item) ? item.data : void 0;
|
|
645
|
+
if (data === null || typeof data !== "object" || Array.isArray(data)) {
|
|
646
|
+
throw new CliValidationError(
|
|
647
|
+
`--cases-json items must contain a data object keyed by field_key, e.g. '[{"data":{"input":"..."}}]'`
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
return { cases };
|
|
652
|
+
}
|
|
653
|
+
var batchCreateDatasetCases = {
|
|
654
|
+
service: "evaluation",
|
|
655
|
+
command: "+batch_create_dataset_cases",
|
|
656
|
+
description: "Create 1-1000 dataset cases atomically via POST /agenteval/api/v1/datasets/{id}/cases/batch. The request succeeds completely or creates no cases.",
|
|
657
|
+
flags: [
|
|
658
|
+
...evalFlags(),
|
|
659
|
+
idFlag("Dataset UUID"),
|
|
660
|
+
{
|
|
661
|
+
name: "cases-json",
|
|
662
|
+
type: "json",
|
|
663
|
+
required: true,
|
|
664
|
+
sensitive: true,
|
|
665
|
+
desc: `JSON array of 1-1000 case definitions; each data object is keyed by field_key, e.g. '[{"data":{"input":"..."}}]'`
|
|
666
|
+
}
|
|
667
|
+
],
|
|
668
|
+
risk: "write",
|
|
669
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor16(ctx), void 0, buildBody10(ctx)),
|
|
670
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor16(ctx), void 0, buildBody10(ctx))
|
|
671
|
+
};
|
|
672
|
+
|
|
673
|
+
// src/commands/evaluation/update-dataset-case.ts
|
|
674
|
+
var PATH20 = "/datasets";
|
|
675
|
+
function pathFor17(ctx) {
|
|
676
|
+
return `${PATH20}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-id")}`;
|
|
677
|
+
}
|
|
678
|
+
function buildBody11(ctx) {
|
|
679
|
+
const data = ctx.json("data");
|
|
680
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
|
681
|
+
throw new Error(`--data must be a JSON object keyed by field_key, e.g. '{"input":"..."}'`);
|
|
682
|
+
}
|
|
683
|
+
return { data, lock_version: ctx.num("lock-version") };
|
|
684
|
+
}
|
|
685
|
+
var updateDatasetCase = {
|
|
686
|
+
service: "evaluation",
|
|
687
|
+
command: "+update_dataset_case",
|
|
688
|
+
description: "Replace the field data of one dataset case via PATCH /agenteval/api/v1/datasets/{id}/cases/{caseId}. Requires the case lock_version from +get_dataset_case.",
|
|
689
|
+
flags: [
|
|
690
|
+
...evalFlags(),
|
|
691
|
+
idFlag("Dataset UUID"),
|
|
692
|
+
{ name: "case-id", type: "string", required: true, desc: "Case UUID from +list_dataset_cases" },
|
|
693
|
+
{ name: "data", type: "json", required: true, sensitive: true, desc: "New case data object keyed by field_key" },
|
|
694
|
+
lockVersionFlag("case")
|
|
695
|
+
],
|
|
696
|
+
risk: "write",
|
|
697
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor17(ctx), void 0, buildBody11(ctx)),
|
|
698
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor17(ctx), void 0, buildBody11(ctx))
|
|
699
|
+
};
|
|
700
|
+
|
|
701
|
+
// src/commands/evaluation/delete-dataset-case.ts
|
|
702
|
+
var PATH21 = "/datasets";
|
|
703
|
+
function pathFor18(ctx) {
|
|
704
|
+
return `${PATH21}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-id")}`;
|
|
705
|
+
}
|
|
706
|
+
function buildBody12(ctx) {
|
|
707
|
+
return { lock_version: ctx.num("lock-version") };
|
|
708
|
+
}
|
|
709
|
+
var deleteDatasetCase = {
|
|
710
|
+
service: "evaluation",
|
|
711
|
+
command: "+delete_dataset_case",
|
|
712
|
+
description: "Delete one case from a dataset via DELETE /agenteval/api/v1/datasets/{id}/cases/{caseId}. Requires the case lock_version from +get_dataset_case.",
|
|
713
|
+
flags: [
|
|
714
|
+
...evalFlags(),
|
|
715
|
+
idFlag("Dataset UUID"),
|
|
716
|
+
{ name: "case-id", type: "string", required: true, desc: "Case UUID from +list_dataset_cases" },
|
|
717
|
+
lockVersionFlag("case")
|
|
718
|
+
],
|
|
719
|
+
risk: "high-risk-write",
|
|
720
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor18(ctx), void 0, buildBody12(ctx)),
|
|
721
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor18(ctx), void 0, buildBody12(ctx))
|
|
722
|
+
};
|
|
723
|
+
|
|
724
|
+
// src/commands/evaluation/export-dataset.ts
|
|
725
|
+
var PATH22 = "/datasets";
|
|
726
|
+
function pathFor19(ctx) {
|
|
727
|
+
return `${PATH22}/${evalPathId(ctx, "id")}/exports`;
|
|
728
|
+
}
|
|
729
|
+
function buildBody13(ctx) {
|
|
730
|
+
return { format: ctx.str("export-format") };
|
|
731
|
+
}
|
|
732
|
+
var exportDataset = {
|
|
733
|
+
service: "evaluation",
|
|
734
|
+
command: "+export_dataset",
|
|
735
|
+
description: "Export a dataset to a downloadable file via POST /agenteval/api/v1/datasets/{id}/exports. Returns File metadata; fetch the link with +file_download_url.",
|
|
736
|
+
flags: [
|
|
737
|
+
...evalFlags(),
|
|
738
|
+
idFlag("Dataset UUID"),
|
|
739
|
+
{ name: "export-format", type: "string", required: true, desc: "Export format: csv | jsonl" }
|
|
740
|
+
],
|
|
741
|
+
risk: "write",
|
|
742
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor19(ctx), void 0, buildBody13(ctx)),
|
|
743
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor19(ctx), void 0, buildBody13(ctx))
|
|
744
|
+
};
|
|
745
|
+
|
|
746
|
+
// src/commands/evaluation/list-dataset-versions.ts
|
|
747
|
+
var PATH23 = "/datasets";
|
|
748
|
+
function pathFor20(ctx) {
|
|
749
|
+
return `${PATH23}/${evalPathId(ctx, "id")}/versions`;
|
|
750
|
+
}
|
|
751
|
+
var listDatasetVersions = {
|
|
752
|
+
service: "evaluation",
|
|
753
|
+
command: "+list_dataset_versions",
|
|
754
|
+
description: "List published versions of a dataset via GET /agenteval/api/v1/datasets/{id}/versions. Use a version id with +create_run --dataset-version-id; drafts and published versions are distinct assets.",
|
|
755
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
756
|
+
risk: "read",
|
|
757
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor20(ctx)),
|
|
758
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor20(ctx))
|
|
759
|
+
};
|
|
760
|
+
|
|
761
|
+
// src/commands/evaluation/publish-dataset-version.ts
|
|
762
|
+
var PATH24 = "/datasets";
|
|
763
|
+
function pathFor21(ctx) {
|
|
764
|
+
return `${PATH24}/${evalPathId(ctx, "id")}/versions`;
|
|
765
|
+
}
|
|
766
|
+
var publishDatasetVersion = {
|
|
767
|
+
service: "evaluation",
|
|
768
|
+
command: "+publish_dataset_version",
|
|
769
|
+
description: "Publish an immutable dataset version via POST /agenteval/api/v1/datasets/{id}/versions. Sends {lock_version}. Publishing without changes returns ASSET_VERSION_NO_CHANGES. Published versions can never be overwritten; runs pin them.",
|
|
770
|
+
flags: [...evalFlags(), ...idAndLockFlags("Dataset UUID", "dataset")],
|
|
771
|
+
risk: "write",
|
|
772
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor21(ctx), void 0, { lock_version: ctx.num("lock-version") }),
|
|
773
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor21(ctx), void 0, { lock_version: ctx.num("lock-version") })
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
// src/commands/evaluation/get-dataset-version.ts
|
|
777
|
+
var PATH25 = "/datasets";
|
|
778
|
+
function pathFor22(ctx) {
|
|
779
|
+
return `${PATH25}/${evalPathId(ctx, "id")}/versions/${evalPathId(ctx, "version-id")}`;
|
|
780
|
+
}
|
|
781
|
+
var getDatasetVersion = {
|
|
782
|
+
service: "evaluation",
|
|
783
|
+
command: "+get_dataset_version",
|
|
784
|
+
description: "Get a published dataset version and its frozen fields via GET /agenteval/api/v1/datasets/{id}/versions/{versionId}.",
|
|
785
|
+
flags: [
|
|
786
|
+
...evalFlags(),
|
|
787
|
+
idFlag("Dataset UUID"),
|
|
788
|
+
{ name: "version-id", type: "string", required: true, desc: "Published dataset version UUID" }
|
|
789
|
+
],
|
|
790
|
+
risk: "read",
|
|
791
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor22(ctx)),
|
|
792
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor22(ctx))
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
// src/commands/evaluation/list-dataset-references.ts
|
|
796
|
+
var PATH26 = "/datasets";
|
|
797
|
+
function pathFor23(ctx) {
|
|
798
|
+
return `${PATH26}/${evalPathId(ctx, "id")}/references`;
|
|
799
|
+
}
|
|
800
|
+
var listDatasetReferences = {
|
|
801
|
+
service: "evaluation",
|
|
802
|
+
command: "+list_dataset_references",
|
|
803
|
+
description: "List runs referencing a dataset via GET /agenteval/api/v1/datasets/{id}/references. Check this before deleting a dataset or making version/publish decisions; runs keep their frozen snapshot even after the dataset is deleted.",
|
|
804
|
+
flags: [...evalFlags(), idFlag("Dataset UUID")],
|
|
805
|
+
risk: "read",
|
|
806
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor23(ctx)),
|
|
807
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor23(ctx))
|
|
808
|
+
};
|
|
809
|
+
|
|
810
|
+
// src/commands/evaluation/list-evaluators.ts
|
|
811
|
+
var PATH27 = "/evaluators";
|
|
812
|
+
function buildParams4(ctx) {
|
|
813
|
+
const params = {};
|
|
814
|
+
const search = ctx.str("search");
|
|
815
|
+
if (search) params.search = search;
|
|
816
|
+
const cursor = ctx.str("cursor");
|
|
817
|
+
if (cursor) params.cursor = cursor;
|
|
818
|
+
const pageSize = ctx.optionalNum("page-size");
|
|
819
|
+
if (pageSize !== void 0) params.page_size = pageSize;
|
|
820
|
+
return params;
|
|
821
|
+
}
|
|
822
|
+
var listEvaluators = {
|
|
823
|
+
service: "evaluation",
|
|
824
|
+
command: "+list_evaluators",
|
|
825
|
+
description: "List LLM judge evaluators via GET /agenteval/api/v1/evaluators. Returns a cursor-paginated page.",
|
|
826
|
+
flags: [
|
|
827
|
+
...evalFlags(),
|
|
828
|
+
{ name: "search", type: "string", desc: "Search text matching evaluator name/description" },
|
|
829
|
+
{ name: "cursor", type: "string", desc: "Opaque pagination cursor from the previous page" },
|
|
830
|
+
{ name: "page-size", type: "number", min: 1, max: 200, desc: "Page size (1-200, default 50)" }
|
|
831
|
+
],
|
|
832
|
+
risk: "read",
|
|
833
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH27, buildParams4(ctx)),
|
|
834
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH27, buildParams4(ctx))
|
|
835
|
+
};
|
|
836
|
+
|
|
837
|
+
// src/commands/evaluation/get-evaluator.ts
|
|
838
|
+
var PATH28 = "/evaluators";
|
|
839
|
+
function pathFor24(ctx) {
|
|
840
|
+
return `${PATH28}/${evalPathId(ctx, "id")}`;
|
|
841
|
+
}
|
|
842
|
+
var getEvaluator = {
|
|
843
|
+
service: "evaluation",
|
|
844
|
+
command: "+get_evaluator",
|
|
845
|
+
description: "Get one evaluator by id via GET /agenteval/api/v1/evaluators/{evaluatorId}.",
|
|
846
|
+
flags: [...evalFlags(), idFlag("Evaluator UUID")],
|
|
847
|
+
risk: "read",
|
|
848
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor24(ctx)),
|
|
849
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor24(ctx))
|
|
850
|
+
};
|
|
851
|
+
|
|
852
|
+
// src/commands/evaluation/create-evaluator.ts
|
|
853
|
+
var PATH29 = "/evaluators";
|
|
854
|
+
function buildBody14(ctx) {
|
|
855
|
+
const body = { name: ctx.str("name") };
|
|
856
|
+
const description = ctx.str("description");
|
|
857
|
+
if (description) body.description = description;
|
|
858
|
+
const category = ctx.str("category");
|
|
859
|
+
if (category) body.category = category;
|
|
860
|
+
const prompt = ctx.str("prompt");
|
|
861
|
+
if (prompt) body.prompt = prompt;
|
|
862
|
+
const variables = ctx.json("variables");
|
|
863
|
+
if (variables !== void 0) {
|
|
864
|
+
if (!Array.isArray(variables) || !variables.every((v) => typeof v === "string")) {
|
|
865
|
+
throw new Error(`--variables must be a JSON array of strings, e.g. '["input","output"]'`);
|
|
866
|
+
}
|
|
867
|
+
body.variables = variables;
|
|
868
|
+
}
|
|
869
|
+
const modelConfigId = ctx.str("model-config-id");
|
|
870
|
+
if (modelConfigId) body.model_config_id = modelConfigId;
|
|
871
|
+
const templateCode = ctx.str("template-code");
|
|
872
|
+
if (templateCode) body.template_code = templateCode;
|
|
873
|
+
const templateLocale = ctx.str("template-locale");
|
|
874
|
+
if (templateLocale) body.template_locale = templateLocale;
|
|
875
|
+
return body;
|
|
876
|
+
}
|
|
877
|
+
var createEvaluator = {
|
|
878
|
+
service: "evaluation",
|
|
879
|
+
command: "+create_evaluator",
|
|
880
|
+
description: "Create an LLM judge evaluator via POST /agenteval/api/v1/evaluators. With --model-config-id it is created ACTIVE; without it a draft is created (activate later with +update_evaluator). The prompt references variables with {{name}} placeholders.",
|
|
881
|
+
flags: [
|
|
882
|
+
...evalFlags(),
|
|
883
|
+
{ name: "template-locale", type: "string", pattern: "^(zh|en)$", desc: "Language copied from built-in templates: zh | en (default zh); project templates are unchanged" },
|
|
884
|
+
{ name: "name", type: "string", required: true, desc: "Evaluator name (1-128 characters)" },
|
|
885
|
+
{ name: "description", type: "string", desc: "Optional description (<=2000 characters)" },
|
|
886
|
+
{ name: "category", type: "string", desc: "Optional category label (<=64 characters), e.g. SAFETY" },
|
|
887
|
+
{ name: "prompt", type: "string", desc: "Judge prompt (<=100000 chars) with {{variable}} placeholders matching --variables" },
|
|
888
|
+
{ name: "variables", type: "json", desc: `JSON array of variable names, e.g. '["input","output"]'` },
|
|
889
|
+
{ name: "model-config-id", type: "string", desc: "Model config UUID that powers the judge; omit to create a draft" },
|
|
890
|
+
{ name: "template-code", type: "string", desc: "Optional evaluator template code to initialize from (<=64 chars)" }
|
|
891
|
+
],
|
|
892
|
+
risk: "write",
|
|
893
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH29, void 0, buildBody14(ctx)),
|
|
894
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH29, void 0, buildBody14(ctx))
|
|
895
|
+
};
|
|
896
|
+
|
|
897
|
+
// src/commands/evaluation/update-evaluator.ts
|
|
898
|
+
var PATH30 = "/evaluators";
|
|
899
|
+
function pathFor25(ctx) {
|
|
900
|
+
return `${PATH30}/${evalPathId(ctx, "id")}`;
|
|
901
|
+
}
|
|
902
|
+
function buildBody15(ctx) {
|
|
903
|
+
const body = {
|
|
904
|
+
name: ctx.str("name"),
|
|
905
|
+
model_config_id: ctx.str("model-config-id"),
|
|
906
|
+
lock_version: ctx.num("lock-version")
|
|
907
|
+
};
|
|
908
|
+
const description = ctx.str("description");
|
|
909
|
+
if (description) body.description = description;
|
|
910
|
+
const category = ctx.str("category");
|
|
911
|
+
if (category) body.category = category;
|
|
912
|
+
const prompt = ctx.str("prompt");
|
|
913
|
+
if (prompt) body.prompt = prompt;
|
|
914
|
+
const variables = ctx.json("variables");
|
|
915
|
+
if (variables !== void 0) {
|
|
916
|
+
if (!Array.isArray(variables) || !variables.every((v) => typeof v === "string")) {
|
|
917
|
+
throw new Error(`--variables must be a JSON array of strings, e.g. '["input","output"]'`);
|
|
918
|
+
}
|
|
919
|
+
body.variables = variables;
|
|
920
|
+
}
|
|
921
|
+
return body;
|
|
922
|
+
}
|
|
923
|
+
var updateEvaluator = {
|
|
924
|
+
service: "evaluation",
|
|
925
|
+
command: "+update_evaluator",
|
|
926
|
+
description: "Update an evaluator via PATCH /agenteval/api/v1/evaluators/{id}. Full-replace semantics: pass every field including unchanged ones; drafts become ACTIVE once a model config is bound. Requires the current evaluator lock_version from +get_evaluator.",
|
|
927
|
+
flags: [
|
|
928
|
+
...evalFlags(),
|
|
929
|
+
...idAndLockFlags("Evaluator UUID", "evaluator"),
|
|
930
|
+
{ name: "name", type: "string", required: true, desc: "Evaluator name (1-128 characters)" },
|
|
931
|
+
{ name: "model-config-id", type: "string", required: true, desc: "Model config UUID that powers the judge" },
|
|
932
|
+
{ name: "description", type: "string", desc: "Optional description (<=2000 characters)" },
|
|
933
|
+
{ name: "category", type: "string", desc: "Optional category label (<=64 characters)" },
|
|
934
|
+
{ name: "prompt", type: "string", desc: "Judge prompt (<=100000 chars) with {{variable}} placeholders" },
|
|
935
|
+
{ name: "variables", type: "json", desc: `JSON array of variable names, e.g. '["input","output"]'` }
|
|
936
|
+
],
|
|
937
|
+
risk: "write",
|
|
938
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor25(ctx), void 0, buildBody15(ctx)),
|
|
939
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor25(ctx), void 0, buildBody15(ctx))
|
|
940
|
+
};
|
|
941
|
+
|
|
942
|
+
// src/commands/evaluation/delete-evaluator.ts
|
|
943
|
+
var PATH31 = "/evaluators";
|
|
944
|
+
function pathFor26(ctx) {
|
|
945
|
+
return `${PATH31}/${evalPathId(ctx, "id")}`;
|
|
946
|
+
}
|
|
947
|
+
function buildBody16(ctx) {
|
|
948
|
+
return { lock_version: ctx.num("lock-version") };
|
|
949
|
+
}
|
|
950
|
+
var deleteEvaluator = {
|
|
951
|
+
service: "evaluation",
|
|
952
|
+
command: "+delete_evaluator",
|
|
953
|
+
description: "Delete an evaluator via DELETE /agenteval/api/v1/evaluators/{id}. Runs that already bound it keep their frozen snapshot. Requires the current evaluator lock_version.",
|
|
954
|
+
flags: [...evalFlags(), ...idAndLockFlags("Evaluator UUID", "evaluator")],
|
|
955
|
+
risk: "high-risk-write",
|
|
956
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor26(ctx), void 0, buildBody16(ctx)),
|
|
957
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor26(ctx), void 0, buildBody16(ctx))
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
// src/commands/evaluation/transfer-evaluator-owner.ts
|
|
961
|
+
var PATH32 = "/evaluators";
|
|
962
|
+
function pathFor27(ctx) {
|
|
963
|
+
return `${PATH32}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
964
|
+
}
|
|
965
|
+
function buildBody17(ctx) {
|
|
966
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
967
|
+
}
|
|
968
|
+
var transferEvaluatorOwner = {
|
|
969
|
+
service: "evaluation",
|
|
970
|
+
command: "+transfer_evaluator_owner",
|
|
971
|
+
description: "Transfer evaluator ownership to another user via POST /agenteval/api/v1/evaluators/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current evaluator lock_version.",
|
|
972
|
+
flags: [
|
|
973
|
+
...evalFlags(),
|
|
974
|
+
...idAndLockFlags("Evaluator UUID", "evaluator"),
|
|
975
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
976
|
+
],
|
|
977
|
+
risk: "high-risk-write",
|
|
978
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor27(ctx), void 0, buildBody17(ctx)),
|
|
979
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor27(ctx), void 0, buildBody17(ctx))
|
|
980
|
+
};
|
|
981
|
+
|
|
982
|
+
// src/commands/evaluation/list-evaluator-templates.ts
|
|
983
|
+
var PATH33 = "/evaluator-templates";
|
|
984
|
+
var listEvaluatorTemplates = {
|
|
985
|
+
service: "evaluation",
|
|
986
|
+
command: "+list_evaluator_templates",
|
|
987
|
+
description: "List evaluator templates via GET /agenteval/api/v1/evaluator-templates. Templates provide ready-made judge prompts; reference one by code when creating evaluators with +create_evaluator.",
|
|
988
|
+
flags: [...evalFlags(), { name: "locale", type: "string", pattern: "^(zh|en)$", desc: "Built-in template language: zh | en (server default zh); project templates retain their content" }],
|
|
989
|
+
risk: "read",
|
|
990
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH33, { locale: ctx.str("locale") || void 0 }),
|
|
991
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH33, { locale: ctx.str("locale") || void 0 })
|
|
992
|
+
};
|
|
993
|
+
|
|
994
|
+
// src/commands/evaluation/create-evaluator-template.ts
|
|
995
|
+
var PATH34 = "/evaluator-templates";
|
|
996
|
+
function buildBody18(ctx) {
|
|
997
|
+
const body = {
|
|
998
|
+
name: ctx.str("name"),
|
|
999
|
+
category: ctx.str("category"),
|
|
1000
|
+
prompt: ctx.str("prompt")
|
|
1001
|
+
};
|
|
1002
|
+
const description = ctx.str("description");
|
|
1003
|
+
if (description) body.description = description;
|
|
1004
|
+
const variables = ctx.json("variables");
|
|
1005
|
+
if (variables !== void 0) {
|
|
1006
|
+
if (!Array.isArray(variables) || !variables.every((v) => typeof v === "string")) {
|
|
1007
|
+
throw new Error(`--variables must be a JSON array of strings, e.g. '["input","output"]'`);
|
|
1008
|
+
}
|
|
1009
|
+
body.variables = variables;
|
|
1010
|
+
}
|
|
1011
|
+
return body;
|
|
1012
|
+
}
|
|
1013
|
+
var createEvaluatorTemplate = {
|
|
1014
|
+
service: "evaluation",
|
|
1015
|
+
command: "+create_evaluator_template",
|
|
1016
|
+
description: "Create a project evaluator template via POST /agenteval/api/v1/evaluator-templates. Templates hold reusable judge prompts; users create evaluators from them by template code. Requires ADMIN role.",
|
|
1017
|
+
flags: [
|
|
1018
|
+
...evalFlags(),
|
|
1019
|
+
{ name: "name", type: "string", required: true, desc: "Template name (1-128 characters)" },
|
|
1020
|
+
{ name: "category", type: "string", required: true, desc: "Template category (1-64 characters)" },
|
|
1021
|
+
{ name: "prompt", type: "string", required: true, desc: "Judge prompt (1-100000 chars) with {{variable}} placeholders" },
|
|
1022
|
+
{ name: "description", type: "string", desc: "Optional description (<=2000 characters)" },
|
|
1023
|
+
{ name: "variables", type: "json", desc: `JSON array of variable names, e.g. '["input","output"]'` }
|
|
1024
|
+
],
|
|
1025
|
+
risk: "write",
|
|
1026
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH34, void 0, buildBody18(ctx)),
|
|
1027
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH34, void 0, buildBody18(ctx))
|
|
1028
|
+
};
|
|
1029
|
+
|
|
1030
|
+
// src/commands/evaluation/disable-evaluator-template.ts
|
|
1031
|
+
var PATH35 = "/evaluator-templates";
|
|
1032
|
+
function pathFor28(ctx) {
|
|
1033
|
+
return `${PATH35}/${evalPathId(ctx, "id")}`;
|
|
1034
|
+
}
|
|
1035
|
+
var disableEvaluatorTemplate = {
|
|
1036
|
+
service: "evaluation",
|
|
1037
|
+
command: "+disable_evaluator_template",
|
|
1038
|
+
description: "Disable an evaluator template via DELETE /agenteval/api/v1/evaluator-templates/{id}. Disabled templates can no longer be referenced when creating evaluators. Requires ADMIN role.",
|
|
1039
|
+
flags: [...evalFlags(), idFlag("Evaluator template UUID")],
|
|
1040
|
+
risk: "high-risk-write",
|
|
1041
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor28(ctx)),
|
|
1042
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor28(ctx))
|
|
1043
|
+
};
|
|
1044
|
+
|
|
1045
|
+
// src/commands/evaluation/list-evaluator-versions.ts
|
|
1046
|
+
var PATH36 = "/evaluators";
|
|
1047
|
+
function pathFor29(ctx) {
|
|
1048
|
+
return `${PATH36}/${evalPathId(ctx, "id")}/versions`;
|
|
1049
|
+
}
|
|
1050
|
+
var listEvaluatorVersions = {
|
|
1051
|
+
service: "evaluation",
|
|
1052
|
+
command: "+list_evaluator_versions",
|
|
1053
|
+
description: "List published versions of an evaluator via GET /agenteval/api/v1/evaluators/{id}/versions. Pin a version id in +select_run_evaluators --evaluator-versions-json; never guess version UUIDs.",
|
|
1054
|
+
flags: [...evalFlags(), idFlag("Evaluator UUID")],
|
|
1055
|
+
risk: "read",
|
|
1056
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor29(ctx)),
|
|
1057
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor29(ctx))
|
|
1058
|
+
};
|
|
1059
|
+
|
|
1060
|
+
// src/commands/evaluation/publish-evaluator-version.ts
|
|
1061
|
+
var PATH37 = "/evaluators";
|
|
1062
|
+
function pathFor30(ctx) {
|
|
1063
|
+
return `${PATH37}/${evalPathId(ctx, "id")}/versions`;
|
|
1064
|
+
}
|
|
1065
|
+
var publishEvaluatorVersion = {
|
|
1066
|
+
service: "evaluation",
|
|
1067
|
+
command: "+publish_evaluator_version",
|
|
1068
|
+
description: "Publish an immutable evaluator version via POST /agenteval/api/v1/evaluators/{id}/versions. Sends {lock_version}. Publishing without changes returns ASSET_VERSION_NO_CHANGES. Published versions can never be overwritten; runs pin them.",
|
|
1069
|
+
flags: [...evalFlags(), ...idAndLockFlags("Evaluator UUID", "evaluator")],
|
|
1070
|
+
risk: "write",
|
|
1071
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor30(ctx), void 0, { lock_version: ctx.num("lock-version") }),
|
|
1072
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor30(ctx), void 0, { lock_version: ctx.num("lock-version") })
|
|
1073
|
+
};
|
|
1074
|
+
|
|
1075
|
+
// src/commands/evaluation/optimize-evaluator-prompt.ts
|
|
1076
|
+
var PATH38 = "/evaluators/prompt-optimization";
|
|
1077
|
+
function buildBody19(ctx) {
|
|
1078
|
+
return {
|
|
1079
|
+
model_config_id: ctx.str("model-config-id"),
|
|
1080
|
+
prompt: ctx.str("prompt")
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
var optimizeEvaluatorPrompt = {
|
|
1084
|
+
service: "evaluation",
|
|
1085
|
+
command: "+optimize_evaluator_prompt",
|
|
1086
|
+
description: "Generate an optimized evaluator prompt preview via POST /agenteval/api/v1/evaluators/prompt-optimization. Calls the model and consumes quota (not a pure read). Returns a preview only; it never overwrites an existing evaluator until you apply it explicitly via +update_evaluator.",
|
|
1087
|
+
flags: [
|
|
1088
|
+
...evalFlags(),
|
|
1089
|
+
{ name: "model-config-id", type: "string", required: true, desc: "Model config UUID used to run the optimization" },
|
|
1090
|
+
{
|
|
1091
|
+
name: "prompt",
|
|
1092
|
+
type: "string",
|
|
1093
|
+
required: true,
|
|
1094
|
+
sensitive: true,
|
|
1095
|
+
desc: "Current or draft evaluator prompt to optimize (max 100000 chars); redacted from logs"
|
|
1096
|
+
}
|
|
1097
|
+
],
|
|
1098
|
+
risk: "write",
|
|
1099
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH38, void 0, buildBody19(ctx)),
|
|
1100
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH38, void 0, buildBody19(ctx))
|
|
1101
|
+
};
|
|
1102
|
+
|
|
1103
|
+
// src/commands/evaluation/get-evaluator-health.ts
|
|
1104
|
+
var PATH39 = "/evaluators";
|
|
1105
|
+
function pathFor31(ctx) {
|
|
1106
|
+
return `${PATH39}/${evalPathId(ctx, "id")}/health`;
|
|
1107
|
+
}
|
|
1108
|
+
var getEvaluatorHealth = {
|
|
1109
|
+
service: "evaluation",
|
|
1110
|
+
command: "+get_evaluator_health",
|
|
1111
|
+
description: "Get the warning-only reliability summary of an evaluator via GET /agenteval/api/v1/evaluators/{id}/health (status, parse/provider success rates, score distribution collapse, calibration error count).",
|
|
1112
|
+
flags: [...evalFlags(), idFlag("Evaluator UUID")],
|
|
1113
|
+
risk: "read",
|
|
1114
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor31(ctx)),
|
|
1115
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor31(ctx))
|
|
1116
|
+
};
|
|
1117
|
+
|
|
1118
|
+
// src/commands/evaluation/list-calibration-suites.ts
|
|
1119
|
+
var PATH40 = "/evaluators";
|
|
1120
|
+
function pathFor32(ctx) {
|
|
1121
|
+
return `${PATH40}/${evalPathId(ctx, "id")}/calibration-suites`;
|
|
1122
|
+
}
|
|
1123
|
+
var listCalibrationSuites = {
|
|
1124
|
+
service: "evaluation",
|
|
1125
|
+
command: "+list_calibration_suites",
|
|
1126
|
+
description: "List calibration suites of an evaluator via GET /agenteval/api/v1/evaluators/{id}/calibration-suites. The default suite is the recommended container for Golden Cases; avoid creating extra suites without a reason.",
|
|
1127
|
+
flags: [...evalFlags(), idFlag("Evaluator UUID")],
|
|
1128
|
+
risk: "read",
|
|
1129
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor32(ctx)),
|
|
1130
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor32(ctx))
|
|
1131
|
+
};
|
|
1132
|
+
|
|
1133
|
+
// src/commands/evaluation/create-calibration-suite.ts
|
|
1134
|
+
var PATH41 = "/evaluators";
|
|
1135
|
+
function pathFor33(ctx) {
|
|
1136
|
+
return `${PATH41}/${evalPathId(ctx, "id")}/calibration-suites`;
|
|
1137
|
+
}
|
|
1138
|
+
function buildBody20(ctx) {
|
|
1139
|
+
const body = { name: ctx.str("name") };
|
|
1140
|
+
const description = ctx.str("description");
|
|
1141
|
+
if (description) body.description = description;
|
|
1142
|
+
return body;
|
|
1143
|
+
}
|
|
1144
|
+
var createCalibrationSuite = {
|
|
1145
|
+
service: "evaluation",
|
|
1146
|
+
command: "+create_calibration_suite",
|
|
1147
|
+
description: "Create a calibration suite for an evaluator via POST /agenteval/api/v1/evaluators/{id}/calibration-suites. Prefer the evaluator default suite; create explicit suites only when you need a separate Golden Case set.",
|
|
1148
|
+
flags: [
|
|
1149
|
+
...evalFlags(),
|
|
1150
|
+
idFlag("Evaluator UUID"),
|
|
1151
|
+
{ name: "name", type: "string", required: true, desc: "Suite name (1-128 characters)" },
|
|
1152
|
+
{ name: "description", type: "string", desc: "Optional suite description (max 2000 chars)" }
|
|
1153
|
+
],
|
|
1154
|
+
risk: "write",
|
|
1155
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor33(ctx), void 0, buildBody20(ctx)),
|
|
1156
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor33(ctx), void 0, buildBody20(ctx))
|
|
1157
|
+
};
|
|
1158
|
+
|
|
1159
|
+
// src/commands/evaluation/list-calibration-cases.ts
|
|
1160
|
+
var PATH42 = "/calibration-suites";
|
|
1161
|
+
function pathFor34(ctx) {
|
|
1162
|
+
return `${PATH42}/${evalPathId(ctx, "id")}/cases`;
|
|
1163
|
+
}
|
|
1164
|
+
var listCalibrationCases = {
|
|
1165
|
+
service: "evaluation",
|
|
1166
|
+
command: "+list_calibration_cases",
|
|
1167
|
+
description: "List active Golden Cases of a calibration suite via GET /agenteval/api/v1/calibration-suites/{id}/cases.",
|
|
1168
|
+
flags: [...evalFlags(), idFlag("Calibration suite UUID")],
|
|
1169
|
+
risk: "read",
|
|
1170
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor34(ctx)),
|
|
1171
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor34(ctx))
|
|
1172
|
+
};
|
|
1173
|
+
|
|
1174
|
+
// src/commands/evaluation/create-calibration-case.ts
|
|
1175
|
+
var PATH43 = "/calibration-suites";
|
|
1176
|
+
function pathFor35(ctx) {
|
|
1177
|
+
return `${PATH43}/${evalPathId(ctx, "id")}/cases`;
|
|
1178
|
+
}
|
|
1179
|
+
function buildBody21(ctx) {
|
|
1180
|
+
const variables = ctx.json("variables-json");
|
|
1181
|
+
if (variables === null || typeof variables !== "object" || Array.isArray(variables)) {
|
|
1182
|
+
throw new Error("--variables-json must be a JSON object of template variables");
|
|
1183
|
+
}
|
|
1184
|
+
const tags = ctx.json("tags-json");
|
|
1185
|
+
if (!Array.isArray(tags) || tags.length > 20 || !tags.every((v) => typeof v === "string")) {
|
|
1186
|
+
throw new Error("--tags-json must be a JSON array of at most 20 strings");
|
|
1187
|
+
}
|
|
1188
|
+
return {
|
|
1189
|
+
variables,
|
|
1190
|
+
expected_score: ctx.num("expected-score"),
|
|
1191
|
+
tolerance: ctx.num("tolerance"),
|
|
1192
|
+
tags
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
var createCalibrationCase = {
|
|
1196
|
+
service: "evaluation",
|
|
1197
|
+
command: "+create_calibration_case",
|
|
1198
|
+
description: "Add a Golden Case to a calibration suite via POST /agenteval/api/v1/calibration-suites/{id}/cases. The case stores variables, an expected score (0-1) and a tolerance (0-1) used to calibrate the evaluator.",
|
|
1199
|
+
flags: [
|
|
1200
|
+
...evalFlags(),
|
|
1201
|
+
idFlag("Calibration suite UUID"),
|
|
1202
|
+
{
|
|
1203
|
+
name: "variables-json",
|
|
1204
|
+
type: "json",
|
|
1205
|
+
required: true,
|
|
1206
|
+
desc: "JSON object of template variables rendered into the evaluator template input"
|
|
1207
|
+
},
|
|
1208
|
+
{ name: "expected-score", type: "number", required: true, min: 0, max: 1, desc: "Expected evaluator score (0-1)" },
|
|
1209
|
+
{ name: "tolerance", type: "number", required: true, min: 0, max: 1, desc: "Allowed deviation from the expected score (0-1)" },
|
|
1210
|
+
{
|
|
1211
|
+
name: "tags-json",
|
|
1212
|
+
type: "json",
|
|
1213
|
+
required: true,
|
|
1214
|
+
desc: `JSON array of at most 20 tags (each max 64 chars), e.g. '["golden","regression"]'`
|
|
1215
|
+
}
|
|
1216
|
+
],
|
|
1217
|
+
risk: "write",
|
|
1218
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor35(ctx), void 0, buildBody21(ctx)),
|
|
1219
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor35(ctx), void 0, buildBody21(ctx))
|
|
1220
|
+
};
|
|
1221
|
+
|
|
1222
|
+
// src/commands/evaluation/delete-calibration-case.ts
|
|
1223
|
+
var PATH44 = "/calibration-cases";
|
|
1224
|
+
function pathFor36(ctx) {
|
|
1225
|
+
return `${PATH44}/${evalPathId(ctx, "id")}`;
|
|
1226
|
+
}
|
|
1227
|
+
var deleteCalibrationCase = {
|
|
1228
|
+
service: "evaluation",
|
|
1229
|
+
command: "+delete_calibration_case",
|
|
1230
|
+
description: "Delete a Golden Case via DELETE /agenteval/api/v1/calibration-cases/{id}. Sends {lock_version} in the body; get the current lock_version from +list_calibration_cases.",
|
|
1231
|
+
flags: [...evalFlags(), ...idAndLockFlags("Calibration case UUID", "calibration case")],
|
|
1232
|
+
risk: "high-risk-write",
|
|
1233
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor36(ctx), void 0, { lock_version: ctx.num("lock-version") }),
|
|
1234
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor36(ctx), void 0, { lock_version: ctx.num("lock-version") })
|
|
1235
|
+
};
|
|
1236
|
+
|
|
1237
|
+
// src/commands/evaluation/start-calibration-run.ts
|
|
1238
|
+
var PATH45 = "/calibration-suites";
|
|
1239
|
+
function pathFor37(ctx) {
|
|
1240
|
+
return `${PATH45}/${evalPathId(ctx, "id")}/runs`;
|
|
1241
|
+
}
|
|
1242
|
+
var startCalibrationRun = {
|
|
1243
|
+
service: "evaluation",
|
|
1244
|
+
command: "+start_calibration_run",
|
|
1245
|
+
description: "Queue a calibration run for a suite via POST /agenteval/api/v1/calibration-suites/{id}/runs. Sends no request body; returns the queued calibration run. Poll +get_calibration_run, then +list_calibration_results.",
|
|
1246
|
+
flags: [...evalFlags(), idFlag("Calibration suite UUID")],
|
|
1247
|
+
risk: "write",
|
|
1248
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor37(ctx)),
|
|
1249
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor37(ctx))
|
|
1250
|
+
};
|
|
1251
|
+
|
|
1252
|
+
// src/commands/evaluation/get-calibration-run.ts
|
|
1253
|
+
var PATH46 = "/calibration-runs";
|
|
1254
|
+
function pathFor38(ctx) {
|
|
1255
|
+
return `${PATH46}/${evalPathId(ctx, "id")}`;
|
|
1256
|
+
}
|
|
1257
|
+
var getCalibrationRun = {
|
|
1258
|
+
service: "evaluation",
|
|
1259
|
+
command: "+get_calibration_run",
|
|
1260
|
+
description: "Get a calibration run via GET /agenteval/api/v1/calibration-runs/{id}. Poll until terminal status, then fetch +list_calibration_results.",
|
|
1261
|
+
flags: [...evalFlags(), idFlag("Calibration run UUID")],
|
|
1262
|
+
risk: "read",
|
|
1263
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor38(ctx)),
|
|
1264
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor38(ctx))
|
|
1265
|
+
};
|
|
1266
|
+
|
|
1267
|
+
// src/commands/evaluation/list-calibration-results.ts
|
|
1268
|
+
var PATH47 = "/calibration-runs";
|
|
1269
|
+
function pathFor39(ctx) {
|
|
1270
|
+
return `${PATH47}/${evalPathId(ctx, "id")}/results`;
|
|
1271
|
+
}
|
|
1272
|
+
var listCalibrationResults = {
|
|
1273
|
+
service: "evaluation",
|
|
1274
|
+
command: "+list_calibration_results",
|
|
1275
|
+
description: "List per-case calibration results of a calibration run via GET /agenteval/api/v1/calibration-runs/{id}/results. Each result compares the evaluator score with the Golden Case expected score and tolerance.",
|
|
1276
|
+
flags: [...evalFlags(), idFlag("Calibration run UUID")],
|
|
1277
|
+
risk: "read",
|
|
1278
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor39(ctx)),
|
|
1279
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor39(ctx))
|
|
1280
|
+
};
|
|
1281
|
+
|
|
1282
|
+
// src/commands/evaluation/list-runs.ts
|
|
1283
|
+
var PATH48 = "/evaluation-runs";
|
|
1284
|
+
function buildParams5(ctx) {
|
|
1285
|
+
const params = {};
|
|
1286
|
+
const search = ctx.str("search");
|
|
1287
|
+
if (search) params.search = search;
|
|
1288
|
+
const status = ctx.str("status");
|
|
1289
|
+
if (status) params.status = status;
|
|
1290
|
+
const cursor = ctx.str("cursor");
|
|
1291
|
+
if (cursor) params.cursor = cursor;
|
|
1292
|
+
const limit = ctx.optionalNum("limit");
|
|
1293
|
+
if (limit !== void 0) params.limit = limit;
|
|
1294
|
+
return params;
|
|
1295
|
+
}
|
|
1296
|
+
var listRuns = {
|
|
1297
|
+
service: "evaluation",
|
|
1298
|
+
command: "+list_runs",
|
|
1299
|
+
description: "List evaluation runs via GET /agenteval/api/v1/evaluation-runs. Returns a cursor-paginated page with snapshot names, counts, and average score.",
|
|
1300
|
+
flags: [
|
|
1301
|
+
...evalFlags(),
|
|
1302
|
+
{ name: "search", type: "string", desc: "Search text matching run name/description" },
|
|
1303
|
+
{
|
|
1304
|
+
name: "status",
|
|
1305
|
+
type: "string",
|
|
1306
|
+
desc: "Filter by run status: DRAFT | QUEUED | RUNNING | CANCELLING | COMPLETED | PARTIALLY_FAILED | FAILED | CANCELLED"
|
|
1307
|
+
},
|
|
1308
|
+
{ name: "cursor", type: "string", desc: "Opaque pagination cursor from the previous page" },
|
|
1309
|
+
{ name: "limit", type: "number", min: 1, max: 200, desc: "Page size (1-200, default 50)" }
|
|
1310
|
+
],
|
|
1311
|
+
risk: "read",
|
|
1312
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH48, buildParams5(ctx)),
|
|
1313
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH48, buildParams5(ctx))
|
|
1314
|
+
};
|
|
1315
|
+
|
|
1316
|
+
// src/commands/evaluation/get-run.ts
|
|
1317
|
+
var PATH49 = "/evaluation-runs";
|
|
1318
|
+
function pathFor40(ctx) {
|
|
1319
|
+
return `${PATH49}/${evalPathId(ctx, "id")}`;
|
|
1320
|
+
}
|
|
1321
|
+
var getRun = {
|
|
1322
|
+
service: "evaluation",
|
|
1323
|
+
command: "+get_run",
|
|
1324
|
+
description: "Get one evaluation run by id via GET /agenteval/api/v1/evaluation-runs/{runId}, including its frozen snapshot summary.",
|
|
1325
|
+
flags: [...evalFlags(), idFlag("Run UUID")],
|
|
1326
|
+
risk: "read",
|
|
1327
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor40(ctx)),
|
|
1328
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor40(ctx))
|
|
1329
|
+
};
|
|
1330
|
+
|
|
1331
|
+
// src/commands/evaluation/create-run.ts
|
|
1332
|
+
var PATH50 = "/evaluation-runs";
|
|
1333
|
+
var EXECUTION_MODES = ["LIVE_AGENT", "PRESET_RESULT"];
|
|
1334
|
+
function buildBody22(ctx) {
|
|
1335
|
+
const body = { name: ctx.str("name") };
|
|
1336
|
+
const description = ctx.str("description");
|
|
1337
|
+
if (description) body.description = description;
|
|
1338
|
+
const datasetId = ctx.str("dataset-id");
|
|
1339
|
+
if (datasetId) body.dataset_id = datasetId;
|
|
1340
|
+
const datasetVersionId = ctx.str("dataset-version-id");
|
|
1341
|
+
if (datasetVersionId) body.dataset_version_id = datasetVersionId;
|
|
1342
|
+
const subjectType = ctx.str("subject-type");
|
|
1343
|
+
if (subjectType) body.subject_type = subjectType;
|
|
1344
|
+
const executionMode = ctx.str("execution-mode") || "LIVE_AGENT";
|
|
1345
|
+
if (!EXECUTION_MODES.includes(executionMode)) {
|
|
1346
|
+
throw new Error("--execution-mode must be one of LIVE_AGENT | PRESET_RESULT");
|
|
1347
|
+
}
|
|
1348
|
+
const agentConfigId = ctx.str("agent-config-id");
|
|
1349
|
+
const modelConfigId = ctx.str("model-config-id");
|
|
1350
|
+
if (executionMode === "PRESET_RESULT" && (agentConfigId || modelConfigId)) {
|
|
1351
|
+
throw new Error("--agent-config-id / --model-config-id are not allowed with --execution-mode PRESET_RESULT");
|
|
1352
|
+
}
|
|
1353
|
+
if (agentConfigId) body.agent_config_id = agentConfigId;
|
|
1354
|
+
if (modelConfigId) body.model_config_id = modelConfigId;
|
|
1355
|
+
body.execution_mode = executionMode;
|
|
1356
|
+
const reportToTe = ctx.json("report-to-te");
|
|
1357
|
+
if (reportToTe !== void 0) body.report_to_te = Boolean(reportToTe);
|
|
1358
|
+
const fieldMapping = ctx.json("field-mapping-json");
|
|
1359
|
+
if (fieldMapping !== void 0) body.field_mapping_json = fieldMapping;
|
|
1360
|
+
const resultFilterFields = ctx.json("result-filter-fields-json");
|
|
1361
|
+
if (resultFilterFields !== void 0) body.result_filter_fields_json = resultFilterFields;
|
|
1362
|
+
const passThreshold = ctx.optionalNum("pass-threshold");
|
|
1363
|
+
if (passThreshold !== void 0) body.pass_threshold = passThreshold;
|
|
1364
|
+
const caseSampleSize = ctx.optionalNum("case-sample-size");
|
|
1365
|
+
if (caseSampleSize !== void 0) {
|
|
1366
|
+
if (!Number.isInteger(caseSampleSize) || caseSampleSize < 1 || caseSampleSize > 1e3) {
|
|
1367
|
+
throw new CliValidationError("--case-sample-size must be an integer between 1 and 1000");
|
|
1368
|
+
}
|
|
1369
|
+
body.case_sample_size = caseSampleSize;
|
|
1370
|
+
}
|
|
1371
|
+
const idempotencyKey = ctx.str("idempotency-key");
|
|
1372
|
+
if (idempotencyKey) body.idempotency_key = idempotencyKey;
|
|
1373
|
+
return body;
|
|
1374
|
+
}
|
|
1375
|
+
var createRun = {
|
|
1376
|
+
service: "evaluation",
|
|
1377
|
+
command: "+create_run",
|
|
1378
|
+
description: "Create a draft evaluation run via POST /agenteval/api/v1/evaluation-runs. Returns the draft with lock_version. The run must target a published dataset version (--dataset-version-id from +list_dataset_versions). Configure evaluators, field mapping, and validation before starting.",
|
|
1379
|
+
flags: [
|
|
1380
|
+
...evalFlags(),
|
|
1381
|
+
{ name: "name", type: "string", required: true, desc: "Run name (1-128 characters)" },
|
|
1382
|
+
{ name: "description", type: "string", desc: "Optional run description (<=2000 characters)" },
|
|
1383
|
+
{ name: "dataset-id", type: "string", desc: "Dataset UUID this run evaluates against" },
|
|
1384
|
+
{
|
|
1385
|
+
name: "dataset-version-id",
|
|
1386
|
+
type: "string",
|
|
1387
|
+
desc: "Published dataset version UUID (from +list_dataset_versions); required for PRESET_RESULT and recommended for every run"
|
|
1388
|
+
},
|
|
1389
|
+
{
|
|
1390
|
+
name: "execution-mode",
|
|
1391
|
+
type: "string",
|
|
1392
|
+
desc: "LIVE_AGENT (default) sends the case to the Agent/Model; PRESET_RESULT scores fields already stored in the selected published dataset version"
|
|
1393
|
+
},
|
|
1394
|
+
{
|
|
1395
|
+
name: "subject-type",
|
|
1396
|
+
type: "string",
|
|
1397
|
+
desc: "Subject under test: AGENT | MODEL (default AGENT); only meaningful for LIVE_AGENT"
|
|
1398
|
+
},
|
|
1399
|
+
{ name: "agent-config-id", type: "string", desc: "Agent config UUID (LIVE_AGENT with subject_type=AGENT)" },
|
|
1400
|
+
{ name: "model-config-id", type: "string", desc: "Model config UUID (LIVE_AGENT with subject_type=MODEL)" },
|
|
1401
|
+
{
|
|
1402
|
+
name: "report-to-te",
|
|
1403
|
+
type: "boolean",
|
|
1404
|
+
desc: "When true, report an agent_eval_run_completed event to ThinkingData at terminal status"
|
|
1405
|
+
},
|
|
1406
|
+
{
|
|
1407
|
+
name: "field-mapping-json",
|
|
1408
|
+
type: "json",
|
|
1409
|
+
desc: "FieldMapping JSON (schema_version/agent/evaluators); V6 adds AGENT_EXECUTION_TRACE and AGENT_TOOL_CALLS for LIVE_AGENT Agent process scoring"
|
|
1410
|
+
},
|
|
1411
|
+
{
|
|
1412
|
+
name: "result-filter-fields-json",
|
|
1413
|
+
type: "json",
|
|
1414
|
+
desc: "ResultFilterFields JSON (schema_version/fields, max 5)"
|
|
1415
|
+
},
|
|
1416
|
+
{ name: "pass-threshold", type: "number", min: 0, max: 1, desc: "Optional pass threshold (0-1)" },
|
|
1417
|
+
{
|
|
1418
|
+
name: "case-sample-size",
|
|
1419
|
+
type: "number",
|
|
1420
|
+
min: 1,
|
|
1421
|
+
max: 1e3,
|
|
1422
|
+
desc: "Randomly sample this many dataset cases when the run starts (1-1000); omit to run all cases"
|
|
1423
|
+
},
|
|
1424
|
+
{
|
|
1425
|
+
name: "idempotency-key",
|
|
1426
|
+
type: "string",
|
|
1427
|
+
desc: "Optional Idempotency-Key value (max 128 chars) for duplicate-create protection"
|
|
1428
|
+
}
|
|
1429
|
+
],
|
|
1430
|
+
risk: "write",
|
|
1431
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH50, void 0, buildBody22(ctx)),
|
|
1432
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH50, void 0, buildBody22(ctx))
|
|
1433
|
+
};
|
|
1434
|
+
|
|
1435
|
+
// src/commands/evaluation/update-run.ts
|
|
1436
|
+
var PATH51 = "/evaluation-runs";
|
|
1437
|
+
var EXECUTION_MODES2 = ["LIVE_AGENT", "PRESET_RESULT"];
|
|
1438
|
+
function pathFor41(ctx) {
|
|
1439
|
+
return `${PATH51}/${evalPathId(ctx, "id")}`;
|
|
1440
|
+
}
|
|
1441
|
+
function buildBody23(ctx) {
|
|
1442
|
+
const body = { name: ctx.str("name"), lock_version: ctx.num("lock-version") };
|
|
1443
|
+
const description = ctx.str("description");
|
|
1444
|
+
if (description) body.description = description;
|
|
1445
|
+
const datasetId = ctx.str("dataset-id");
|
|
1446
|
+
if (datasetId) body.dataset_id = datasetId;
|
|
1447
|
+
const datasetVersionId = ctx.str("dataset-version-id");
|
|
1448
|
+
if (datasetVersionId) body.dataset_version_id = datasetVersionId;
|
|
1449
|
+
const subjectType = ctx.str("subject-type");
|
|
1450
|
+
if (subjectType) body.subject_type = subjectType;
|
|
1451
|
+
const executionMode = ctx.str("execution-mode") || "LIVE_AGENT";
|
|
1452
|
+
if (!EXECUTION_MODES2.includes(executionMode)) {
|
|
1453
|
+
throw new Error("--execution-mode must be one of LIVE_AGENT | PRESET_RESULT");
|
|
1454
|
+
}
|
|
1455
|
+
const agentConfigId = ctx.str("agent-config-id");
|
|
1456
|
+
const modelConfigId = ctx.str("model-config-id");
|
|
1457
|
+
if (executionMode === "PRESET_RESULT" && (agentConfigId || modelConfigId)) {
|
|
1458
|
+
throw new Error("--agent-config-id / --model-config-id are not allowed with --execution-mode PRESET_RESULT");
|
|
1459
|
+
}
|
|
1460
|
+
if (agentConfigId) body.agent_config_id = agentConfigId;
|
|
1461
|
+
if (modelConfigId) body.model_config_id = modelConfigId;
|
|
1462
|
+
body.execution_mode = executionMode;
|
|
1463
|
+
const fieldMapping = ctx.json("field-mapping-json");
|
|
1464
|
+
if (fieldMapping !== void 0) body.field_mapping_json = fieldMapping;
|
|
1465
|
+
const resultFilterFields = ctx.json("result-filter-fields-json");
|
|
1466
|
+
if (resultFilterFields !== void 0) body.result_filter_fields_json = resultFilterFields;
|
|
1467
|
+
const passThreshold = ctx.optionalNum("pass-threshold");
|
|
1468
|
+
if (passThreshold !== void 0) body.pass_threshold = passThreshold;
|
|
1469
|
+
const caseSampleSize = ctx.optionalNum("case-sample-size");
|
|
1470
|
+
if (caseSampleSize !== void 0) {
|
|
1471
|
+
if (!Number.isInteger(caseSampleSize) || caseSampleSize < 1 || caseSampleSize > 1e3) {
|
|
1472
|
+
throw new CliValidationError("--case-sample-size must be an integer between 1 and 1000");
|
|
1473
|
+
}
|
|
1474
|
+
body.case_sample_size = caseSampleSize;
|
|
1475
|
+
}
|
|
1476
|
+
return body;
|
|
1477
|
+
}
|
|
1478
|
+
var updateRun = {
|
|
1479
|
+
service: "evaluation",
|
|
1480
|
+
command: "+update_run",
|
|
1481
|
+
description: "Update a draft evaluation run via PATCH /agenteval/api/v1/evaluation-runs/{runId}. Full-replace semantics for provided fields; requires the current lock_version from +get_run and increments it. Runs must target a published dataset version (--dataset-version-id from +list_dataset_versions).",
|
|
1482
|
+
flags: [
|
|
1483
|
+
...evalFlags(),
|
|
1484
|
+
idFlag("Run UUID"),
|
|
1485
|
+
{ name: "name", type: "string", required: true, desc: "Run name (1-128 characters)" },
|
|
1486
|
+
{ name: "description", type: "string", desc: "Optional run description (<=2000 characters)" },
|
|
1487
|
+
{ name: "dataset-id", type: "string", desc: "Dataset UUID this run evaluates against" },
|
|
1488
|
+
{
|
|
1489
|
+
name: "dataset-version-id",
|
|
1490
|
+
type: "string",
|
|
1491
|
+
desc: "Published dataset version UUID (from +list_dataset_versions); required for PRESET_RESULT and recommended for every run"
|
|
1492
|
+
},
|
|
1493
|
+
{
|
|
1494
|
+
name: "execution-mode",
|
|
1495
|
+
type: "string",
|
|
1496
|
+
desc: "LIVE_AGENT (default) sends the case to the Agent/Model; PRESET_RESULT scores fields already stored in the selected published dataset version"
|
|
1497
|
+
},
|
|
1498
|
+
{ name: "subject-type", type: "string", desc: "Subject under test: AGENT | MODEL" },
|
|
1499
|
+
{ name: "agent-config-id", type: "string", desc: "Agent config UUID (LIVE_AGENT with subject_type=AGENT)" },
|
|
1500
|
+
{ name: "model-config-id", type: "string", desc: "Model config UUID (LIVE_AGENT with subject_type=MODEL)" },
|
|
1501
|
+
{ name: "field-mapping-json", type: "json", desc: "FieldMapping JSON (schema_version/agent/evaluators); V6 adds AGENT_EXECUTION_TRACE and AGENT_TOOL_CALLS for LIVE_AGENT Agent process scoring" },
|
|
1502
|
+
{ name: "result-filter-fields-json", type: "json", desc: "ResultFilterFields JSON (schema_version/fields, max 5)" },
|
|
1503
|
+
{ name: "pass-threshold", type: "number", min: 0, max: 1, desc: "Optional pass threshold (0-1)" },
|
|
1504
|
+
{
|
|
1505
|
+
name: "case-sample-size",
|
|
1506
|
+
type: "number",
|
|
1507
|
+
min: 1,
|
|
1508
|
+
max: 1e3,
|
|
1509
|
+
desc: "Randomly sample this many dataset cases when the run starts (1-1000); omit to run all cases"
|
|
1510
|
+
},
|
|
1511
|
+
lockVersionFlag("run")
|
|
1512
|
+
],
|
|
1513
|
+
risk: "write",
|
|
1514
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor41(ctx), void 0, buildBody23(ctx)),
|
|
1515
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor41(ctx), void 0, buildBody23(ctx))
|
|
1516
|
+
};
|
|
1517
|
+
|
|
1518
|
+
// src/commands/evaluation/delete-run.ts
|
|
1519
|
+
var PATH52 = "/evaluation-runs";
|
|
1520
|
+
function pathFor42(ctx) {
|
|
1521
|
+
return `${PATH52}/${evalPathId(ctx, "id")}`;
|
|
1522
|
+
}
|
|
1523
|
+
function buildBody24(ctx) {
|
|
1524
|
+
return { lock_version: ctx.num("lock-version") };
|
|
1525
|
+
}
|
|
1526
|
+
var deleteRun = {
|
|
1527
|
+
service: "evaluation",
|
|
1528
|
+
command: "+delete_run",
|
|
1529
|
+
description: "Delete a draft evaluation run via DELETE /agenteval/api/v1/evaluation-runs/{runId}. Only DRAFT runs can be deleted; requires the current lock_version.",
|
|
1530
|
+
flags: [...evalFlags(), ...idAndLockFlags("Run UUID", "run")],
|
|
1531
|
+
risk: "high-risk-write",
|
|
1532
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor42(ctx), void 0, buildBody24(ctx)),
|
|
1533
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor42(ctx), void 0, buildBody24(ctx))
|
|
1534
|
+
};
|
|
1535
|
+
|
|
1536
|
+
// src/commands/evaluation/select-run-evaluators.ts
|
|
1537
|
+
var PATH53 = "/evaluation-runs";
|
|
1538
|
+
function pathFor43(ctx) {
|
|
1539
|
+
return `${PATH53}/${evalPathId(ctx, "id")}/evaluators`;
|
|
1540
|
+
}
|
|
1541
|
+
function buildBody25(ctx) {
|
|
1542
|
+
const evaluatorIds = ctx.json("evaluator-ids");
|
|
1543
|
+
if (!Array.isArray(evaluatorIds) || evaluatorIds.length === 0 || !evaluatorIds.every((v) => typeof v === "string")) {
|
|
1544
|
+
throw new Error(`--evaluator-ids must be a non-empty JSON array of evaluator UUIDs, e.g. '["<uuid>"]'`);
|
|
1545
|
+
}
|
|
1546
|
+
const body = { evaluator_ids: evaluatorIds, lock_version: ctx.num("lock-version") };
|
|
1547
|
+
const versions = ctx.json("evaluator-versions-json");
|
|
1548
|
+
if (versions !== void 0) {
|
|
1549
|
+
if (!Array.isArray(versions) || !versions.every(
|
|
1550
|
+
(v) => v !== null && typeof v === "object" && !Array.isArray(v) && typeof v.evaluator_id === "string" && typeof v.version_id === "string"
|
|
1551
|
+
)) {
|
|
1552
|
+
throw new Error(
|
|
1553
|
+
'--evaluator-versions-json must be a JSON array of {"evaluator_id":"<uuid>","version_id":"<uuid>"} objects'
|
|
1554
|
+
);
|
|
1555
|
+
}
|
|
1556
|
+
body.evaluator_versions = versions;
|
|
1557
|
+
}
|
|
1558
|
+
return body;
|
|
1559
|
+
}
|
|
1560
|
+
var selectRunEvaluators = {
|
|
1561
|
+
service: "evaluation",
|
|
1562
|
+
command: "+select_run_evaluators",
|
|
1563
|
+
description: "Bind evaluators to a draft run via PUT /agenteval/api/v1/evaluation-runs/{runId}/evaluators. Required before +start_run; at least one active evaluator must be selected. Prefer pinning published evaluator versions from +list_evaluator_versions; never guess version UUIDs. Uses the current lock_version from +get_run and increments it.",
|
|
1564
|
+
flags: [
|
|
1565
|
+
...evalFlags(),
|
|
1566
|
+
idFlag("Run UUID"),
|
|
1567
|
+
{
|
|
1568
|
+
name: "evaluator-ids",
|
|
1569
|
+
type: "json",
|
|
1570
|
+
required: true,
|
|
1571
|
+
desc: `JSON array of evaluator UUIDs to bind, e.g. '["<uuid-1>","<uuid-2>"]' (from +list_evaluators)`
|
|
1572
|
+
},
|
|
1573
|
+
{
|
|
1574
|
+
name: "evaluator-versions-json",
|
|
1575
|
+
type: "json",
|
|
1576
|
+
desc: `Optional JSON array pinning published evaluator versions, e.g. '[{"evaluator_id":"<uuid>","version_id":"<uuid>"}]' (from +list_evaluator_versions)`
|
|
1577
|
+
},
|
|
1578
|
+
{
|
|
1579
|
+
name: "lock-version",
|
|
1580
|
+
type: "number",
|
|
1581
|
+
required: true,
|
|
1582
|
+
min: 0,
|
|
1583
|
+
desc: "Current run lock_version (optimistic lock), from +get_run"
|
|
1584
|
+
}
|
|
1585
|
+
],
|
|
1586
|
+
risk: "write",
|
|
1587
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", pathFor43(ctx), void 0, buildBody25(ctx)),
|
|
1588
|
+
execute: async (ctx) => evalApi(ctx, "PUT", pathFor43(ctx), void 0, buildBody25(ctx))
|
|
1589
|
+
};
|
|
1590
|
+
|
|
1591
|
+
// src/commands/evaluation/list-run-evaluators.ts
|
|
1592
|
+
var PATH54 = "/evaluation-runs";
|
|
1593
|
+
function pathFor44(ctx) {
|
|
1594
|
+
return `${PATH54}/${evalPathId(ctx, "id")}/evaluators`;
|
|
1595
|
+
}
|
|
1596
|
+
var listRunEvaluators = {
|
|
1597
|
+
service: "evaluation",
|
|
1598
|
+
command: "+list_run_evaluators",
|
|
1599
|
+
description: "List the evaluators bound to a run via GET /agenteval/api/v1/evaluation-runs/{runId}/evaluators, including per-evaluator progress and score counts. Bind or replace them with +select_run_evaluators.",
|
|
1600
|
+
flags: [...evalFlags(), idFlag("Run UUID")],
|
|
1601
|
+
risk: "read",
|
|
1602
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor44(ctx)),
|
|
1603
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor44(ctx))
|
|
1604
|
+
};
|
|
1605
|
+
|
|
1606
|
+
// src/commands/evaluation/update-run-field-mapping.ts
|
|
1607
|
+
var PATH55 = "/evaluation-runs";
|
|
1608
|
+
function pathFor45(ctx) {
|
|
1609
|
+
return `${PATH55}/${evalPathId(ctx, "id")}/field-mapping`;
|
|
1610
|
+
}
|
|
1611
|
+
function buildBody26(ctx) {
|
|
1612
|
+
const fieldMapping = ctx.json("field-mapping-json");
|
|
1613
|
+
if (fieldMapping === void 0) {
|
|
1614
|
+
throw new Error("--field-mapping-json is required (schema_version/agent/evaluators, FIELD_MAPPING_V* contract)");
|
|
1615
|
+
}
|
|
1616
|
+
return { field_mapping_json: fieldMapping, lock_version: ctx.num("lock-version") };
|
|
1617
|
+
}
|
|
1618
|
+
var updateRunFieldMapping = {
|
|
1619
|
+
service: "evaluation",
|
|
1620
|
+
command: "+update_run_field_mapping",
|
|
1621
|
+
description: "Set the field mapping on a draft run via PUT /agenteval/api/v1/evaluation-runs/{runId}/field-mapping. Required before +start_run; the agent input and every evaluator variable must be mapped. Uses the current lock_version and increments it.",
|
|
1622
|
+
flags: [
|
|
1623
|
+
...evalFlags(),
|
|
1624
|
+
...idAndLockFlags("Run UUID", "run"),
|
|
1625
|
+
{
|
|
1626
|
+
name: "field-mapping-json",
|
|
1627
|
+
type: "json",
|
|
1628
|
+
required: true,
|
|
1629
|
+
desc: "FieldMapping JSON (schema_version/agent/evaluators); V6 adds AGENT_EXECUTION_TRACE and AGENT_TOOL_CALLS for LIVE_AGENT Agent process scoring"
|
|
1630
|
+
}
|
|
1631
|
+
],
|
|
1632
|
+
risk: "write",
|
|
1633
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", pathFor45(ctx), void 0, buildBody26(ctx)),
|
|
1634
|
+
execute: async (ctx) => evalApi(ctx, "PUT", pathFor45(ctx), void 0, buildBody26(ctx))
|
|
1635
|
+
};
|
|
1636
|
+
|
|
1637
|
+
// src/commands/evaluation/update-run-result-filter-fields.ts
|
|
1638
|
+
var PATH56 = "/evaluation-runs";
|
|
1639
|
+
function pathFor46(ctx) {
|
|
1640
|
+
return `${PATH56}/${evalPathId(ctx, "id")}/result-filter-fields`;
|
|
1641
|
+
}
|
|
1642
|
+
function buildBody27(ctx) {
|
|
1643
|
+
const resultFilterFields = ctx.json("result-filter-fields-json");
|
|
1644
|
+
if (resultFilterFields === void 0) {
|
|
1645
|
+
throw new Error(`--result-filter-fields-json is required; pass an empty filter as '{"fields":[],"schema_version":"RESULT_FILTER_FIELDS_V1"}'`);
|
|
1646
|
+
}
|
|
1647
|
+
return { result_filter_fields_json: resultFilterFields, lock_version: ctx.num("lock-version") };
|
|
1648
|
+
}
|
|
1649
|
+
var updateRunResultFilterFields = {
|
|
1650
|
+
service: "evaluation",
|
|
1651
|
+
command: "+update_run_result_filter_fields",
|
|
1652
|
+
description: "Set result filter fields on a draft run via PUT /agenteval/api/v1/evaluation-runs/{runId}/result-filter-fields. Required before +start_run even when no filter is needed (pass an empty fields array). Uses the current lock_version from +get_run and increments it.",
|
|
1653
|
+
flags: [
|
|
1654
|
+
...evalFlags(),
|
|
1655
|
+
idFlag("Run UUID"),
|
|
1656
|
+
{
|
|
1657
|
+
name: "result-filter-fields-json",
|
|
1658
|
+
type: "json",
|
|
1659
|
+
required: true,
|
|
1660
|
+
desc: `ResultFilterFields JSON (schema_version/fields, max 5). Empty filter: '{"fields":[],"schema_version":"RESULT_FILTER_FIELDS_V1"}'`
|
|
1661
|
+
},
|
|
1662
|
+
{
|
|
1663
|
+
name: "lock-version",
|
|
1664
|
+
type: "number",
|
|
1665
|
+
required: true,
|
|
1666
|
+
min: 0,
|
|
1667
|
+
desc: "Current run lock_version (optimistic lock), from +get_run"
|
|
1668
|
+
}
|
|
1669
|
+
],
|
|
1670
|
+
risk: "write",
|
|
1671
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", pathFor46(ctx), void 0, buildBody27(ctx)),
|
|
1672
|
+
execute: async (ctx) => evalApi(ctx, "PUT", pathFor46(ctx), void 0, buildBody27(ctx))
|
|
1673
|
+
};
|
|
1674
|
+
|
|
1675
|
+
// src/commands/evaluation/validate-run.ts
|
|
1676
|
+
var PATH57 = "/evaluation-runs";
|
|
1677
|
+
function pathFor47(ctx) {
|
|
1678
|
+
return `${PATH57}/${evalPathId(ctx, "id")}/validate`;
|
|
1679
|
+
}
|
|
1680
|
+
var validateRun = {
|
|
1681
|
+
service: "evaluation",
|
|
1682
|
+
command: "+validate_run",
|
|
1683
|
+
description: "Run the same preflight validation as +start_run via POST /agenteval/api/v1/evaluation-runs/{runId}/validate, without starting the run. Sends no request body and needs no lock_version. Returns validation issues (evaluator binding, field mapping, result filter fields, subject connectivity).",
|
|
1684
|
+
flags: [...evalFlags(), idFlag("Run UUID")],
|
|
1685
|
+
risk: "read",
|
|
1686
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor47(ctx)),
|
|
1687
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor47(ctx))
|
|
1688
|
+
};
|
|
1689
|
+
|
|
1690
|
+
// src/commands/evaluation/start-run.ts
|
|
1691
|
+
var PATH58 = "/evaluation-runs";
|
|
1692
|
+
function pathFor48(ctx) {
|
|
1693
|
+
return `${PATH58}/${evalPathId(ctx, "id")}/start`;
|
|
1694
|
+
}
|
|
1695
|
+
function buildBody28(ctx) {
|
|
1696
|
+
return { lock_version: ctx.num("lock-version") };
|
|
1697
|
+
}
|
|
1698
|
+
var startRun = {
|
|
1699
|
+
service: "evaluation",
|
|
1700
|
+
command: "+start_run",
|
|
1701
|
+
description: "Start an evaluation run via POST /agenteval/api/v1/evaluation-runs/{runId}/start. Runs an agent compatibility preflight, then atomically freezes snapshots and queues the run. Requires the current lock_version from +get_run.",
|
|
1702
|
+
flags: [
|
|
1703
|
+
...evalFlags(),
|
|
1704
|
+
idFlag("Run UUID"),
|
|
1705
|
+
{
|
|
1706
|
+
name: "lock-version",
|
|
1707
|
+
type: "number",
|
|
1708
|
+
required: true,
|
|
1709
|
+
min: 0,
|
|
1710
|
+
desc: "Current run lock_version (optimistic lock), from +get_run"
|
|
1711
|
+
}
|
|
1712
|
+
],
|
|
1713
|
+
risk: "write",
|
|
1714
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor48(ctx), void 0, buildBody28(ctx)),
|
|
1715
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor48(ctx), void 0, buildBody28(ctx))
|
|
1716
|
+
};
|
|
1717
|
+
|
|
1718
|
+
// src/commands/evaluation/cancel-run.ts
|
|
1719
|
+
var PATH59 = "/evaluation-runs";
|
|
1720
|
+
function pathFor49(ctx) {
|
|
1721
|
+
return `${PATH59}/${evalPathId(ctx, "id")}/cancel`;
|
|
1722
|
+
}
|
|
1723
|
+
function buildBody29(ctx) {
|
|
1724
|
+
return { lock_version: ctx.num("lock-version") };
|
|
1725
|
+
}
|
|
1726
|
+
var cancelRun = {
|
|
1727
|
+
service: "evaluation",
|
|
1728
|
+
command: "+cancel_run",
|
|
1729
|
+
description: "Request cancellation of an evaluation run via POST /agenteval/api/v1/evaluation-runs/{runId}/cancel. Requires the current lock_version from +get_run.",
|
|
1730
|
+
flags: [
|
|
1731
|
+
...evalFlags(),
|
|
1732
|
+
idFlag("Run UUID"),
|
|
1733
|
+
{
|
|
1734
|
+
name: "lock-version",
|
|
1735
|
+
type: "number",
|
|
1736
|
+
required: true,
|
|
1737
|
+
min: 0,
|
|
1738
|
+
desc: "Current run lock_version (optimistic lock), from +get_run"
|
|
1739
|
+
}
|
|
1740
|
+
],
|
|
1741
|
+
risk: "high-risk-write",
|
|
1742
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor49(ctx), void 0, buildBody29(ctx)),
|
|
1743
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor49(ctx), void 0, buildBody29(ctx))
|
|
1744
|
+
};
|
|
1745
|
+
|
|
1746
|
+
// src/commands/evaluation/run-overview.ts
|
|
1747
|
+
var PATH60 = "/evaluation-runs";
|
|
1748
|
+
function pathFor50(ctx) {
|
|
1749
|
+
return `${PATH60}/${evalPathId(ctx, "id")}/overview`;
|
|
1750
|
+
}
|
|
1751
|
+
var runOverview = {
|
|
1752
|
+
service: "evaluation",
|
|
1753
|
+
command: "+run_overview",
|
|
1754
|
+
description: "Get an evaluation run overview via GET /agenteval/api/v1/evaluation-runs/{runId}/overview: progress, average score, pass rate, score distribution, and result fields.",
|
|
1755
|
+
flags: [...evalFlags(), idFlag("Run UUID")],
|
|
1756
|
+
risk: "read",
|
|
1757
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor50(ctx)),
|
|
1758
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor50(ctx))
|
|
1759
|
+
};
|
|
1760
|
+
|
|
1761
|
+
// src/commands/evaluation/run-cases.ts
|
|
1762
|
+
var PATH61 = "/evaluation-runs";
|
|
1763
|
+
function pathFor51(ctx) {
|
|
1764
|
+
return `${PATH61}/${evalPathId(ctx, "id")}/cases`;
|
|
1765
|
+
}
|
|
1766
|
+
function buildParams6(ctx) {
|
|
1767
|
+
const params = {};
|
|
1768
|
+
const status = ctx.str("status");
|
|
1769
|
+
if (status) params.status = status;
|
|
1770
|
+
const evidenceCompleteness = ctx.str("evidence-completeness");
|
|
1771
|
+
if (evidenceCompleteness) params.evidence_completeness = evidenceCompleteness;
|
|
1772
|
+
const search = ctx.str("search");
|
|
1773
|
+
if (search) params.search = search;
|
|
1774
|
+
const evaluationStatus = ctx.str("evaluation-status");
|
|
1775
|
+
if (evaluationStatus) params.evaluation_status = evaluationStatus;
|
|
1776
|
+
const scoreStatus = ctx.str("score-status");
|
|
1777
|
+
if (scoreStatus) params.score_status = scoreStatus;
|
|
1778
|
+
const evaluatorId = ctx.str("evaluator-id");
|
|
1779
|
+
if (evaluatorId) params.evaluator_id = evaluatorId;
|
|
1780
|
+
const evaluatorIds = ctx.json("evaluator-ids");
|
|
1781
|
+
if (evaluatorIds !== void 0) {
|
|
1782
|
+
if (!Array.isArray(evaluatorIds) || evaluatorIds.length < 1 || evaluatorIds.length > 8 || !evaluatorIds.every((v) => typeof v === "string")) {
|
|
1783
|
+
throw new Error(`--evaluator-ids must be a JSON array of 1-8 run evaluator UUIDs, e.g. '["<uuid>"]'`);
|
|
1784
|
+
}
|
|
1785
|
+
params.evaluator_ids = evaluatorIds.join(",");
|
|
1786
|
+
}
|
|
1787
|
+
const minScore = ctx.optionalNum("min-score");
|
|
1788
|
+
if (minScore !== void 0) params.min_score = minScore;
|
|
1789
|
+
const maxScore = ctx.optionalNum("max-score");
|
|
1790
|
+
if (maxScore !== void 0) params.max_score = maxScore;
|
|
1791
|
+
const scoreGte = ctx.optionalNum("score-gte");
|
|
1792
|
+
if (scoreGte !== void 0) params.score_gte = scoreGte;
|
|
1793
|
+
const scoreLte = ctx.optionalNum("score-lte");
|
|
1794
|
+
if (scoreLte !== void 0) params.score_lte = scoreLte;
|
|
1795
|
+
const fieldFilters = ctx.json("field-filters-json");
|
|
1796
|
+
if (fieldFilters !== void 0) {
|
|
1797
|
+
if (!Array.isArray(fieldFilters) || fieldFilters.length > 5) {
|
|
1798
|
+
throw new Error("--field-filters-json must be a JSON array of at most 5 typed filters");
|
|
1799
|
+
}
|
|
1800
|
+
params.field_filters = JSON.stringify(fieldFilters);
|
|
1801
|
+
}
|
|
1802
|
+
const sort = ctx.str("sort");
|
|
1803
|
+
if (sort) params.sort = sort;
|
|
1804
|
+
const sortEvaluatorId = ctx.str("sort-evaluator-id");
|
|
1805
|
+
if (sortEvaluatorId) params.sort_evaluator_id = sortEvaluatorId;
|
|
1806
|
+
const order = ctx.str("order");
|
|
1807
|
+
if (order) params.order = order;
|
|
1808
|
+
const cursor = ctx.str("cursor");
|
|
1809
|
+
if (cursor) params.cursor = cursor;
|
|
1810
|
+
const limit = ctx.optionalNum("limit");
|
|
1811
|
+
if (limit !== void 0) params.limit = limit;
|
|
1812
|
+
const offset = ctx.optionalNum("offset");
|
|
1813
|
+
if (offset !== void 0) params.offset = offset;
|
|
1814
|
+
return params;
|
|
1815
|
+
}
|
|
1816
|
+
var runCases = {
|
|
1817
|
+
service: "evaluation",
|
|
1818
|
+
command: "+run_cases",
|
|
1819
|
+
description: "List evaluation run case results via GET /agenteval/api/v1/evaluation-runs/{runId}/cases with rich filtering (status, evidence completeness, free-text search, evaluation/score status, evaluator scope, score range, frozen field filters, sort). Returns a cursor-paginated page.",
|
|
1820
|
+
flags: [
|
|
1821
|
+
...evalFlags(),
|
|
1822
|
+
idFlag("Run UUID"),
|
|
1823
|
+
{
|
|
1824
|
+
name: "status",
|
|
1825
|
+
type: "string",
|
|
1826
|
+
desc: "Filter by case status: PENDING | RUNNING | BLOCKED | SUCCEEDED | FAILED | CANCELLED | UNKNOWN"
|
|
1827
|
+
},
|
|
1828
|
+
{
|
|
1829
|
+
name: "evidence-completeness",
|
|
1830
|
+
type: "string",
|
|
1831
|
+
desc: "Filter by evidence completeness: NONE | PARTIAL | COMPLETE | UNKNOWN"
|
|
1832
|
+
},
|
|
1833
|
+
{
|
|
1834
|
+
name: "search",
|
|
1835
|
+
type: "string",
|
|
1836
|
+
maxLength: 128,
|
|
1837
|
+
sensitive: true,
|
|
1838
|
+
desc: "Free-text search (max 128) matching Agent output, public errors, external run/session IDs and runtime fingerprint; redacted from logs"
|
|
1839
|
+
},
|
|
1840
|
+
{
|
|
1841
|
+
name: "evaluation-status",
|
|
1842
|
+
type: "string",
|
|
1843
|
+
desc: "Filter by evaluation status: PENDING | RUNNING | SUCCEEDED | PARTIALLY_FAILED | FAILED | CANCELLED"
|
|
1844
|
+
},
|
|
1845
|
+
{
|
|
1846
|
+
name: "score-status",
|
|
1847
|
+
type: "string",
|
|
1848
|
+
desc: "Filter by score status: PENDING | RUNNING | SUCCEEDED | FAILED | CANCELLED"
|
|
1849
|
+
},
|
|
1850
|
+
{ name: "evaluator-id", type: "string", desc: "Filter by a single run evaluator UUID" },
|
|
1851
|
+
{
|
|
1852
|
+
name: "evaluator-ids",
|
|
1853
|
+
type: "json",
|
|
1854
|
+
desc: `JSON array of 1-8 run evaluator UUIDs, e.g. '["<uuid-1>","<uuid-2>"]' (from +list_run_evaluators)`
|
|
1855
|
+
},
|
|
1856
|
+
{ name: "min-score", type: "number", min: 0, max: 1, desc: "Minimum average score (0-1)" },
|
|
1857
|
+
{ name: "max-score", type: "number", min: 0, max: 1, desc: "Maximum average score (0-1)" },
|
|
1858
|
+
{ name: "score-gte", type: "number", min: 0, max: 1, desc: "Minimum evaluator score (0-1)" },
|
|
1859
|
+
{ name: "score-lte", type: "number", min: 0, max: 1, desc: "Maximum evaluator score (0-1)" },
|
|
1860
|
+
{
|
|
1861
|
+
name: "field-filters-json",
|
|
1862
|
+
type: "json",
|
|
1863
|
+
sensitive: true,
|
|
1864
|
+
desc: `JSON array of up to 5 typed filters over frozen result fields; each needs field_key and may combine state=VALUE with one typed criterion, while numeric filters may use both number_gte and number_lte, e.g. '[{"field_key":"score","number_gte":0.5,"number_lte":1}]'`
|
|
1865
|
+
},
|
|
1866
|
+
{
|
|
1867
|
+
name: "sort",
|
|
1868
|
+
type: "string",
|
|
1869
|
+
desc: "Sort key: MIN_SCORE | AVERAGE_SCORE | EVALUATOR_SCORE (pair EVALUATOR_SCORE with --sort-evaluator-id)"
|
|
1870
|
+
},
|
|
1871
|
+
{ name: "sort-evaluator-id", type: "string", desc: "Run evaluator UUID used when --sort EVALUATOR_SCORE" },
|
|
1872
|
+
{ name: "order", type: "string", desc: "Sort order: asc | desc (default asc)" },
|
|
1873
|
+
{ name: "cursor", type: "string", desc: "Opaque pagination cursor from the previous page" },
|
|
1874
|
+
{ name: "limit", type: "number", min: 1, max: 1e3, desc: "Page size (1-1000, default 50)" },
|
|
1875
|
+
{ name: "offset", type: "number", min: 0, desc: "Non-negative offset into the result set (default 0)" }
|
|
1876
|
+
],
|
|
1877
|
+
risk: "read",
|
|
1878
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor51(ctx), buildParams6(ctx)),
|
|
1879
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor51(ctx), buildParams6(ctx))
|
|
1880
|
+
};
|
|
1881
|
+
|
|
1882
|
+
// src/commands/evaluation/get-run-case.ts
|
|
1883
|
+
var PATH62 = "/evaluation-runs";
|
|
1884
|
+
function pathFor52(ctx) {
|
|
1885
|
+
return `${PATH62}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-run-id")}`;
|
|
1886
|
+
}
|
|
1887
|
+
var getRunCase = {
|
|
1888
|
+
service: "evaluation",
|
|
1889
|
+
command: "+get_run_case",
|
|
1890
|
+
description: "Get one evaluation case result in full detail via GET /agenteval/api/v1/evaluation-runs/{runId}/cases/{caseRunId}: agent input/output, per-evaluator scores, reasons, and error details. Find caseRunId via +run_cases.",
|
|
1891
|
+
flags: [...evalFlags(), idFlag("Run UUID"), { name: "case-run-id", type: "string", required: true, desc: "Case run UUID from +run_cases items[].case_run_id" }],
|
|
1892
|
+
risk: "read",
|
|
1893
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor52(ctx)),
|
|
1894
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor52(ctx))
|
|
1895
|
+
};
|
|
1896
|
+
|
|
1897
|
+
// src/commands/evaluation/export-run-results.ts
|
|
1898
|
+
var PATH63 = "/evaluation-runs";
|
|
1899
|
+
function pathFor53(ctx) {
|
|
1900
|
+
return `${PATH63}/${evalPathId(ctx, "id")}/export`;
|
|
1901
|
+
}
|
|
1902
|
+
function buildParams7(ctx) {
|
|
1903
|
+
const params = {};
|
|
1904
|
+
const format = ctx.str("export-format");
|
|
1905
|
+
if (format) params.format = format;
|
|
1906
|
+
return params;
|
|
1907
|
+
}
|
|
1908
|
+
var exportRunResults = {
|
|
1909
|
+
service: "evaluation",
|
|
1910
|
+
command: "+export_run_results",
|
|
1911
|
+
description: "Queue an asynchronous export of an evaluation run result via POST /agenteval/api/v1/evaluation-runs/{runId}/export. Requires an Idempotency-Key; returns 202 with File metadata (poll +get_file until COMPLETED, then use file download --output <path>).",
|
|
1912
|
+
flags: [
|
|
1913
|
+
...evalFlags(),
|
|
1914
|
+
idFlag("Run UUID"),
|
|
1915
|
+
{ name: "export-format", type: "string", desc: "Export format: CSV | JSONL (default CSV)" },
|
|
1916
|
+
{
|
|
1917
|
+
name: "idempotency-key",
|
|
1918
|
+
type: "string",
|
|
1919
|
+
required: true,
|
|
1920
|
+
hint: "Required. Use a fresh value per export attempt (max 191 chars) sent as the Idempotency-Key header.",
|
|
1921
|
+
desc: "Unique Idempotency-Key header value (1-191 characters)"
|
|
1922
|
+
}
|
|
1923
|
+
],
|
|
1924
|
+
risk: "write",
|
|
1925
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor53(ctx), buildParams7(ctx), void 0, {
|
|
1926
|
+
idempotencyKey: ctx.str("idempotency-key")
|
|
1927
|
+
}),
|
|
1928
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor53(ctx), buildParams7(ctx), void 0, {
|
|
1929
|
+
idempotencyKey: ctx.str("idempotency-key")
|
|
1930
|
+
})
|
|
1931
|
+
};
|
|
1932
|
+
|
|
1933
|
+
// src/commands/evaluation/export-run-low-scores.ts
|
|
1934
|
+
var PATH64 = "/evaluation-runs";
|
|
1935
|
+
function pathFor54(ctx) {
|
|
1936
|
+
return `${PATH64}/${evalPathId(ctx, "id")}/low-score-export`;
|
|
1937
|
+
}
|
|
1938
|
+
function buildParams8(ctx) {
|
|
1939
|
+
const params = { max_score: ctx.num("max-score") };
|
|
1940
|
+
const format = ctx.str("export-format");
|
|
1941
|
+
if (format) params.format = format;
|
|
1942
|
+
const minScore = ctx.optionalNum("min-score");
|
|
1943
|
+
if (minScore !== void 0) params.min_score = minScore;
|
|
1944
|
+
const status = ctx.str("status");
|
|
1945
|
+
if (status) params.status = status;
|
|
1946
|
+
return params;
|
|
1947
|
+
}
|
|
1948
|
+
var exportRunLowScores = {
|
|
1949
|
+
service: "evaluation",
|
|
1950
|
+
command: "+export_run_low_scores",
|
|
1951
|
+
description: "Queue an asynchronous export of low-scoring case results via POST /agenteval/api/v1/evaluation-runs/{runId}/low-score-export. Requires --max-score and an Idempotency-Key; returns 202 with File metadata (poll +get_file until COMPLETED, then use file download --output <path>).",
|
|
1952
|
+
flags: [
|
|
1953
|
+
...evalFlags(),
|
|
1954
|
+
idFlag("Run UUID"),
|
|
1955
|
+
{ name: "max-score", type: "number", required: true, min: 0, max: 1, desc: "Upper score bound (0-1); only cases scoring <= this value are exported" },
|
|
1956
|
+
{ name: "min-score", type: "number", min: 0, max: 1, desc: "Optional lower score bound (0-1)" },
|
|
1957
|
+
{ name: "status", type: "string", desc: "Optional case status filter, e.g. SUCCEEDED | FAILED" },
|
|
1958
|
+
{ name: "export-format", type: "string", desc: "Export format: CSV | JSONL (default CSV)" },
|
|
1959
|
+
{
|
|
1960
|
+
name: "idempotency-key",
|
|
1961
|
+
type: "string",
|
|
1962
|
+
required: true,
|
|
1963
|
+
hint: "Required. Use a fresh value per export attempt (max 191 chars) sent as the Idempotency-Key header.",
|
|
1964
|
+
desc: "Unique Idempotency-Key header value (1-191 characters)"
|
|
1965
|
+
}
|
|
1966
|
+
],
|
|
1967
|
+
risk: "write",
|
|
1968
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor54(ctx), buildParams8(ctx), void 0, {
|
|
1969
|
+
idempotencyKey: ctx.str("idempotency-key")
|
|
1970
|
+
}),
|
|
1971
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor54(ctx), buildParams8(ctx), void 0, {
|
|
1972
|
+
idempotencyKey: ctx.str("idempotency-key")
|
|
1973
|
+
})
|
|
1974
|
+
};
|
|
1975
|
+
|
|
1976
|
+
// src/commands/evaluation/file-download-url.ts
|
|
1977
|
+
var PATH65 = "/files";
|
|
1978
|
+
function pathFor55(ctx) {
|
|
1979
|
+
return `${PATH65}/${evalPathId(ctx, "id")}/download-url`;
|
|
1980
|
+
}
|
|
1981
|
+
var fileDownloadUrl = {
|
|
1982
|
+
service: "evaluation",
|
|
1983
|
+
command: "+file_download_url",
|
|
1984
|
+
description: "Create a temporary download URL for a staged file via POST /agenteval/api/v1/files/{fileId}/download-url. Returns download_url and expires_in_seconds; local storage URLs refer to the server filesystem. Use file download for authenticated remote downloads.",
|
|
1985
|
+
flags: [...evalFlags(), idFlag("File UUID")],
|
|
1986
|
+
risk: "read",
|
|
1987
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor55(ctx)),
|
|
1988
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor55(ctx))
|
|
1989
|
+
};
|
|
1990
|
+
|
|
1991
|
+
// src/commands/evaluation/get-file.ts
|
|
1992
|
+
var PATH66 = "/files";
|
|
1993
|
+
function pathFor56(ctx) {
|
|
1994
|
+
return `${PATH66}/${evalPathId(ctx, "id")}`;
|
|
1995
|
+
}
|
|
1996
|
+
var getFile = {
|
|
1997
|
+
service: "evaluation",
|
|
1998
|
+
command: "+get_file",
|
|
1999
|
+
description: "Get file metadata via GET /agenteval/api/v1/files/{id}. Completes the export chain: +export_run_results / +export_dataset return a file id, +get_file reads its metadata, +file_download_url mints a temporary download URL.",
|
|
2000
|
+
flags: [...evalFlags(), idFlag("File UUID")],
|
|
2001
|
+
risk: "read",
|
|
2002
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor56(ctx)),
|
|
2003
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor56(ctx))
|
|
2004
|
+
};
|
|
2005
|
+
|
|
2006
|
+
// src/commands/evaluation/create-next-task-version-draft.ts
|
|
2007
|
+
var PATH67 = "/evaluation-runs";
|
|
2008
|
+
function pathFor57(ctx) {
|
|
2009
|
+
return `${PATH67}/${evalPathId(ctx, "id")}/new-version`;
|
|
2010
|
+
}
|
|
2011
|
+
var createNextTaskVersionDraft = {
|
|
2012
|
+
service: "evaluation",
|
|
2013
|
+
command: "+create_next_task_version_draft",
|
|
2014
|
+
description: "Create the next draft copy from the task version pinned by a run via POST /agenteval/api/v1/evaluation-runs/{id}/new-version. Sends no request body; returns the new draft run.",
|
|
2015
|
+
flags: [...evalFlags(), idFlag("Source run UUID")],
|
|
2016
|
+
risk: "write",
|
|
2017
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor57(ctx)),
|
|
2018
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor57(ctx))
|
|
2019
|
+
};
|
|
2020
|
+
|
|
2021
|
+
// src/commands/evaluation/list-evaluation-task-versions.ts
|
|
2022
|
+
var PATH68 = "/evaluation-tasks";
|
|
2023
|
+
function pathFor58(ctx) {
|
|
2024
|
+
return `${PATH68}/${evalPathId(ctx, "task-id")}/versions`;
|
|
2025
|
+
}
|
|
2026
|
+
var listEvaluationTaskVersions = {
|
|
2027
|
+
service: "evaluation",
|
|
2028
|
+
command: "+list_evaluation_task_versions",
|
|
2029
|
+
description: "List published versions of an evaluation task via GET /agenteval/api/v1/evaluation-tasks/{taskId}/versions. Each version pins a dataset version; runs pin one immutable version.",
|
|
2030
|
+
flags: [
|
|
2031
|
+
...evalFlags(),
|
|
2032
|
+
{ name: "task-id", type: "string", required: true, desc: "Evaluation task UUID" }
|
|
2033
|
+
],
|
|
2034
|
+
risk: "read",
|
|
2035
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor58(ctx)),
|
|
2036
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor58(ctx))
|
|
2037
|
+
};
|
|
2038
|
+
|
|
2039
|
+
// src/commands/evaluation/retry-failed-run-cases.ts
|
|
2040
|
+
var PATH69 = "/evaluation-runs";
|
|
2041
|
+
function pathFor59(ctx) {
|
|
2042
|
+
return `${PATH69}/${evalPathId(ctx, "id")}/retry-failed-cases`;
|
|
2043
|
+
}
|
|
2044
|
+
var retryFailedRunCases = {
|
|
2045
|
+
service: "evaluation",
|
|
2046
|
+
command: "+retry_failed_run_cases",
|
|
2047
|
+
description: "Requeue Agent-stage FAILED, BLOCKED and UNKNOWN cases of a run via POST /agenteval/api/v1/evaluation-runs/{id}/retry-failed-cases. Sends {lock_version}. Successful cases are preserved; the immutable run snapshot is reused, the execution epoch increases and aggregates are recomputed.",
|
|
2048
|
+
flags: [...evalFlags(), ...idAndLockFlags("Run UUID", "run")],
|
|
2049
|
+
risk: "write",
|
|
2050
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor59(ctx), void 0, { lock_version: ctx.num("lock-version") }),
|
|
2051
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor59(ctx), void 0, { lock_version: ctx.num("lock-version") })
|
|
2052
|
+
};
|
|
2053
|
+
|
|
2054
|
+
// src/commands/evaluation/get-run-case-trace.ts
|
|
2055
|
+
var PATH70 = "/evaluation-runs";
|
|
2056
|
+
function pathFor60(ctx) {
|
|
2057
|
+
return `${PATH70}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-run-id")}/trace`;
|
|
2058
|
+
}
|
|
2059
|
+
var getRunCaseTrace = {
|
|
2060
|
+
service: "evaluation",
|
|
2061
|
+
command: "+get_run_case_trace",
|
|
2062
|
+
description: "Get the immutable normalized execution trace of a case via GET /agenteval/api/v1/evaluation-runs/{id}/cases/{caseRunId}/trace. Only available for LIVE_AGENT runs with subject AGENT; capture status is independent of case success. Traces carry at most 500 spans and may be truncated (PARTIAL is not a complete trajectory). Never echo hidden chain-of-thought, credentials, cookies, auth headers or unredacted tool input/output into reports.",
|
|
2063
|
+
flags: [
|
|
2064
|
+
...evalFlags(),
|
|
2065
|
+
{ name: "id", type: "string", required: true, desc: "Run UUID" },
|
|
2066
|
+
{ name: "case-run-id", type: "string", required: true, desc: "Case run UUID" }
|
|
2067
|
+
],
|
|
2068
|
+
risk: "read",
|
|
2069
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor60(ctx)),
|
|
2070
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor60(ctx))
|
|
2071
|
+
};
|
|
2072
|
+
|
|
2073
|
+
// src/commands/evaluation/retry-run-case-trace.ts
|
|
2074
|
+
var PATH71 = "/evaluation-runs";
|
|
2075
|
+
function pathFor61(ctx) {
|
|
2076
|
+
return `${PATH71}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-run-id")}/trace:retry`;
|
|
2077
|
+
}
|
|
2078
|
+
var retryRunCaseTrace = {
|
|
2079
|
+
service: "evaluation",
|
|
2080
|
+
command: "+retry_run_case_trace",
|
|
2081
|
+
description: "ADMIN-only retry of a failed AI Monitor-backed trace capture via POST /agenteval/api/v1/evaluation-runs/{id}/cases/{caseRunId}/trace:retry. Sends no request body; only retryable captures are accepted.",
|
|
2082
|
+
flags: [
|
|
2083
|
+
...evalFlags(),
|
|
2084
|
+
{ name: "id", type: "string", required: true, desc: "Run UUID" },
|
|
2085
|
+
{ name: "case-run-id", type: "string", required: true, desc: "Case run UUID" }
|
|
2086
|
+
],
|
|
2087
|
+
risk: "write",
|
|
2088
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor61(ctx)),
|
|
2089
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor61(ctx))
|
|
2090
|
+
};
|
|
2091
|
+
|
|
2092
|
+
// src/commands/evaluation/run-distribution.ts
|
|
2093
|
+
var PATH72 = "/evaluation-runs";
|
|
2094
|
+
function pathFor62(ctx) {
|
|
2095
|
+
return `${PATH72}/${evalPathId(ctx, "id")}/distribution`;
|
|
2096
|
+
}
|
|
2097
|
+
function buildParams9(ctx) {
|
|
2098
|
+
const params = {};
|
|
2099
|
+
const runEvaluatorId = ctx.str("run-evaluator-id");
|
|
2100
|
+
if (runEvaluatorId) params.run_evaluator_id = runEvaluatorId;
|
|
2101
|
+
return params;
|
|
2102
|
+
}
|
|
2103
|
+
var runDistribution = {
|
|
2104
|
+
service: "evaluation",
|
|
2105
|
+
command: "+run_distribution",
|
|
2106
|
+
description: "Get the five fixed score buckets of a run via GET /agenteval/api/v1/evaluation-runs/{id}/distribution. Pass --run-evaluator-id to scope the buckets to one run evaluator binding.",
|
|
2107
|
+
flags: [
|
|
2108
|
+
...evalFlags(),
|
|
2109
|
+
idFlag("Run UUID"),
|
|
2110
|
+
{ name: "run-evaluator-id", type: "string", desc: "Run evaluator UUID to scope the distribution to" }
|
|
2111
|
+
],
|
|
2112
|
+
risk: "read",
|
|
2113
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor62(ctx), buildParams9(ctx)),
|
|
2114
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor62(ctx), buildParams9(ctx))
|
|
2115
|
+
};
|
|
2116
|
+
|
|
2117
|
+
// src/commands/evaluation/copy-run-case.ts
|
|
2118
|
+
var PATH73 = "/evaluation-runs";
|
|
2119
|
+
function pathFor63(ctx) {
|
|
2120
|
+
return `${PATH73}/${evalPathId(ctx, "id")}/cases/${evalPathId(ctx, "case-run-id")}/copy`;
|
|
2121
|
+
}
|
|
2122
|
+
var copyRunCase = {
|
|
2123
|
+
service: "evaluation",
|
|
2124
|
+
command: "+copy_run_case",
|
|
2125
|
+
description: "Copy a case result as sanitized structured data via POST /agenteval/api/v1/evaluation-runs/{id}/cases/{caseRunId}/copy. Sends no request body; the response is a sanitized CaseRunResult safe for reuse in dataset cases or reports.",
|
|
2126
|
+
flags: [
|
|
2127
|
+
...evalFlags(),
|
|
2128
|
+
{ name: "id", type: "string", required: true, desc: "Run UUID" },
|
|
2129
|
+
{ name: "case-run-id", type: "string", required: true, desc: "Case run UUID" }
|
|
2130
|
+
],
|
|
2131
|
+
risk: "read",
|
|
2132
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor63(ctx)),
|
|
2133
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor63(ctx))
|
|
2134
|
+
};
|
|
2135
|
+
|
|
2136
|
+
// src/commands/evaluation/transfer-run-owner.ts
|
|
2137
|
+
var PATH74 = "/evaluation-runs";
|
|
2138
|
+
function pathFor64(ctx) {
|
|
2139
|
+
return `${PATH74}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
2140
|
+
}
|
|
2141
|
+
function buildBody30(ctx) {
|
|
2142
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
2143
|
+
}
|
|
2144
|
+
var transferRunOwner = {
|
|
2145
|
+
service: "evaluation",
|
|
2146
|
+
command: "+transfer_run_owner",
|
|
2147
|
+
description: "Transfer evaluation run ownership to another user via POST /agenteval/api/v1/evaluation-runs/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current run lock_version. Use +list_project_members to discover real user UUIDs.",
|
|
2148
|
+
flags: [
|
|
2149
|
+
...evalFlags(),
|
|
2150
|
+
...idAndLockFlags("Run UUID", "run"),
|
|
2151
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
2152
|
+
],
|
|
2153
|
+
risk: "high-risk-write",
|
|
2154
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor64(ctx), void 0, buildBody30(ctx)),
|
|
2155
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor64(ctx), void 0, buildBody30(ctx))
|
|
2156
|
+
};
|
|
2157
|
+
|
|
2158
|
+
// src/commands/evaluation/list-schedules.ts
|
|
2159
|
+
var PATH75 = "/evaluation-run-schedules";
|
|
2160
|
+
function buildParams10(ctx) {
|
|
2161
|
+
const params = {};
|
|
2162
|
+
const status = ctx.str("status");
|
|
2163
|
+
if (status) params.status = status;
|
|
2164
|
+
const cursor = ctx.str("cursor");
|
|
2165
|
+
if (cursor) params.cursor = cursor;
|
|
2166
|
+
const limit = ctx.optionalNum("limit");
|
|
2167
|
+
if (limit !== void 0) params.limit = limit;
|
|
2168
|
+
return params;
|
|
2169
|
+
}
|
|
2170
|
+
var listSchedules = {
|
|
2171
|
+
service: "evaluation",
|
|
2172
|
+
command: "+list_schedules",
|
|
2173
|
+
description: "List evaluation run schedules via GET /agenteval/api/v1/evaluation-run-schedules. Returns a cursor-paginated page with next occurrence and status.",
|
|
2174
|
+
flags: [
|
|
2175
|
+
...evalFlags(),
|
|
2176
|
+
{
|
|
2177
|
+
name: "status",
|
|
2178
|
+
type: "string",
|
|
2179
|
+
desc: "Filter by schedule status: PENDING | PROCESSING | COMPLETED | FAILED | CANCELLED"
|
|
2180
|
+
},
|
|
2181
|
+
{ name: "cursor", type: "string", desc: "Opaque pagination cursor from the previous page" },
|
|
2182
|
+
{ name: "limit", type: "number", min: 1, max: 200, desc: "Page size (1-200, default 50)" }
|
|
2183
|
+
],
|
|
2184
|
+
risk: "read",
|
|
2185
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH75, buildParams10(ctx)),
|
|
2186
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH75, buildParams10(ctx))
|
|
2187
|
+
};
|
|
2188
|
+
|
|
2189
|
+
// src/commands/evaluation/create-schedule.ts
|
|
2190
|
+
var PATH76 = "/evaluation-run-schedules";
|
|
2191
|
+
function buildBody31(ctx) {
|
|
2192
|
+
const body = {
|
|
2193
|
+
run_id: ctx.str("run-id"),
|
|
2194
|
+
frequency: ctx.str("frequency"),
|
|
2195
|
+
time_zone: ctx.str("time-zone")
|
|
2196
|
+
};
|
|
2197
|
+
const minuteOfHour = ctx.optionalNum("minute-of-hour");
|
|
2198
|
+
if (minuteOfHour !== void 0) body.minute_of_hour = minuteOfHour;
|
|
2199
|
+
const hourOfDay = ctx.optionalNum("hour-of-day");
|
|
2200
|
+
if (hourOfDay !== void 0) body.hour_of_day = hourOfDay;
|
|
2201
|
+
const dayOfWeek = ctx.optionalNum("day-of-week");
|
|
2202
|
+
if (dayOfWeek !== void 0) body.day_of_week = dayOfWeek;
|
|
2203
|
+
const dayOfMonth = ctx.optionalNum("day-of-month");
|
|
2204
|
+
if (dayOfMonth !== void 0) body.day_of_month = dayOfMonth;
|
|
2205
|
+
return body;
|
|
2206
|
+
}
|
|
2207
|
+
var createSchedule = {
|
|
2208
|
+
service: "evaluation",
|
|
2209
|
+
command: "+create_schedule",
|
|
2210
|
+
description: "Create a recurring evaluation schedule from an existing run template via POST /agenteval/api/v1/evaluation-run-schedules. Each occurrence creates and starts a new immutable run.",
|
|
2211
|
+
flags: [
|
|
2212
|
+
...evalFlags(),
|
|
2213
|
+
{ name: "run-id", type: "string", required: true, desc: "Template run UUID to schedule" },
|
|
2214
|
+
{
|
|
2215
|
+
name: "frequency",
|
|
2216
|
+
type: "string",
|
|
2217
|
+
required: true,
|
|
2218
|
+
desc: "HOURLY | DAILY | WEEKLY | MONTHLY. HOURLY runs at minute 0 and must omit minute fields"
|
|
2219
|
+
},
|
|
2220
|
+
{
|
|
2221
|
+
name: "time-zone",
|
|
2222
|
+
type: "string",
|
|
2223
|
+
required: true,
|
|
2224
|
+
desc: "IANA time zone, e.g. Asia/Shanghai"
|
|
2225
|
+
},
|
|
2226
|
+
{
|
|
2227
|
+
name: "minute-of-hour",
|
|
2228
|
+
type: "number",
|
|
2229
|
+
min: 0,
|
|
2230
|
+
max: 59,
|
|
2231
|
+
desc: "Minute of the hour. Required for DAILY/WEEKLY/MONTHLY (send 0 for hourly)"
|
|
2232
|
+
},
|
|
2233
|
+
{ name: "hour-of-day", type: "number", min: 0, max: 23, desc: "Hour of the day (0-23)" },
|
|
2234
|
+
{
|
|
2235
|
+
name: "day-of-week",
|
|
2236
|
+
type: "number",
|
|
2237
|
+
min: 1,
|
|
2238
|
+
max: 7,
|
|
2239
|
+
desc: "Day of the week for WEEKLY: Monday is 1, Sunday is 7"
|
|
2240
|
+
},
|
|
2241
|
+
{ name: "day-of-month", type: "number", min: 1, max: 31, desc: "Day of the month for MONTHLY (1-31)" }
|
|
2242
|
+
],
|
|
2243
|
+
risk: "write",
|
|
2244
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH76, void 0, buildBody31(ctx)),
|
|
2245
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH76, void 0, buildBody31(ctx))
|
|
2246
|
+
};
|
|
2247
|
+
|
|
2248
|
+
// src/commands/evaluation/update-schedule.ts
|
|
2249
|
+
var PATH77 = "/evaluation-run-schedules";
|
|
2250
|
+
function pathFor65(ctx) {
|
|
2251
|
+
return `${PATH77}/${evalPathId(ctx, "id")}`;
|
|
2252
|
+
}
|
|
2253
|
+
function buildBody32(ctx) {
|
|
2254
|
+
const body = {
|
|
2255
|
+
frequency: ctx.str("frequency"),
|
|
2256
|
+
time_zone: ctx.str("time-zone"),
|
|
2257
|
+
lock_version: ctx.num("lock-version")
|
|
2258
|
+
};
|
|
2259
|
+
const minuteOfHour = ctx.optionalNum("minute-of-hour");
|
|
2260
|
+
if (minuteOfHour !== void 0) body.minute_of_hour = minuteOfHour;
|
|
2261
|
+
const hourOfDay = ctx.optionalNum("hour-of-day");
|
|
2262
|
+
if (hourOfDay !== void 0) body.hour_of_day = hourOfDay;
|
|
2263
|
+
const dayOfWeek = ctx.optionalNum("day-of-week");
|
|
2264
|
+
if (dayOfWeek !== void 0) body.day_of_week = dayOfWeek;
|
|
2265
|
+
const dayOfMonth = ctx.optionalNum("day-of-month");
|
|
2266
|
+
if (dayOfMonth !== void 0) body.day_of_month = dayOfMonth;
|
|
2267
|
+
return body;
|
|
2268
|
+
}
|
|
2269
|
+
var updateSchedule = {
|
|
2270
|
+
service: "evaluation",
|
|
2271
|
+
command: "+update_schedule",
|
|
2272
|
+
description: "Update a run schedule via PATCH /agenteval/api/v1/evaluation-run-schedules/{id}: change frequency, time zone, or occurrence time. Requires the current schedule lock_version from +list_schedules.",
|
|
2273
|
+
flags: [
|
|
2274
|
+
...evalFlags(),
|
|
2275
|
+
...idAndLockFlags("Schedule UUID", "schedule"),
|
|
2276
|
+
{
|
|
2277
|
+
name: "frequency",
|
|
2278
|
+
type: "string",
|
|
2279
|
+
required: true,
|
|
2280
|
+
desc: "HOURLY | DAILY | WEEKLY | MONTHLY. HOURLY runs at minute 0 and must omit minute fields"
|
|
2281
|
+
},
|
|
2282
|
+
{ name: "time-zone", type: "string", required: true, desc: "IANA time zone, e.g. Asia/Shanghai" },
|
|
2283
|
+
{ name: "minute-of-hour", type: "number", min: 0, max: 59, desc: "Minute of the hour. Required for DAILY/WEEKLY/MONTHLY" },
|
|
2284
|
+
{ name: "hour-of-day", type: "number", min: 0, max: 23, desc: "Hour of the day (0-23)" },
|
|
2285
|
+
{ name: "day-of-week", type: "number", min: 1, max: 7, desc: "Day of the week for WEEKLY: Monday is 1, Sunday is 7" },
|
|
2286
|
+
{ name: "day-of-month", type: "number", min: 1, max: 31, desc: "Day of the month for MONTHLY (1-31)" }
|
|
2287
|
+
],
|
|
2288
|
+
risk: "write",
|
|
2289
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor65(ctx), void 0, buildBody32(ctx)),
|
|
2290
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor65(ctx), void 0, buildBody32(ctx))
|
|
2291
|
+
};
|
|
2292
|
+
|
|
2293
|
+
// src/commands/evaluation/delete-schedule.ts
|
|
2294
|
+
var PATH78 = "/evaluation-run-schedules";
|
|
2295
|
+
function pathFor66(ctx) {
|
|
2296
|
+
return `${PATH78}/${evalPathId(ctx, "id")}`;
|
|
2297
|
+
}
|
|
2298
|
+
function buildBody33(ctx) {
|
|
2299
|
+
return { lock_version: ctx.num("lock-version") };
|
|
2300
|
+
}
|
|
2301
|
+
var deleteSchedule = {
|
|
2302
|
+
service: "evaluation",
|
|
2303
|
+
command: "+delete_schedule",
|
|
2304
|
+
description: "Delete a scheduled evaluation in any status via DELETE /agenteval/api/v1/evaluation-run-schedules/{id}. An occurrence already claimed by a worker may finish, but no future occurrence is created. Requires the current lock_version from +list_schedules.",
|
|
2305
|
+
flags: [
|
|
2306
|
+
...evalFlags(),
|
|
2307
|
+
idFlag("Schedule UUID"),
|
|
2308
|
+
{
|
|
2309
|
+
name: "lock-version",
|
|
2310
|
+
type: "number",
|
|
2311
|
+
required: true,
|
|
2312
|
+
min: 0,
|
|
2313
|
+
desc: "Current schedule lock_version (optimistic lock), from +list_schedules"
|
|
2314
|
+
}
|
|
2315
|
+
],
|
|
2316
|
+
risk: "high-risk-write",
|
|
2317
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor66(ctx), void 0, buildBody33(ctx)),
|
|
2318
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor66(ctx), void 0, buildBody33(ctx))
|
|
2319
|
+
};
|
|
2320
|
+
|
|
2321
|
+
// src/commands/evaluation/get-te-report-config.ts
|
|
2322
|
+
var PATH79 = "/te-report-config";
|
|
2323
|
+
var getTeReportConfig = {
|
|
2324
|
+
service: "evaluation",
|
|
2325
|
+
command: "+get_te_report_config",
|
|
2326
|
+
description: "Get the project TE reporting config via GET /agenteval/api/v1/te-report-config. Config is null when unset; the response also carries the deployment default base URL. Read its lock_version before +save_te_report_config.",
|
|
2327
|
+
flags: [...evalFlags()],
|
|
2328
|
+
risk: "read",
|
|
2329
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH79),
|
|
2330
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH79)
|
|
2331
|
+
};
|
|
2332
|
+
|
|
2333
|
+
// src/commands/evaluation/save-te-report-config.ts
|
|
2334
|
+
var PATH80 = "/te-report-config";
|
|
2335
|
+
function buildBody34(ctx) {
|
|
2336
|
+
const body = {
|
|
2337
|
+
base_url: ctx.str("base-url"),
|
|
2338
|
+
app_id: ctx.str("app-id")
|
|
2339
|
+
};
|
|
2340
|
+
const lockVersion = ctx.optionalNum("lock-version");
|
|
2341
|
+
if (lockVersion !== void 0) body.lock_version = lockVersion;
|
|
2342
|
+
return body;
|
|
2343
|
+
}
|
|
2344
|
+
var saveTeReportConfig = {
|
|
2345
|
+
service: "evaluation",
|
|
2346
|
+
command: "+save_te_report_config",
|
|
2347
|
+
description: "Create or update the project TE reporting config via PUT /agenteval/api/v1/te-report-config. Omit --lock-version on first create; on update pass the current lock_version from +get_te_report_config (0 is a valid version).",
|
|
2348
|
+
flags: [
|
|
2349
|
+
...evalFlags(),
|
|
2350
|
+
{ name: "base-url", type: "string", required: true, desc: "ThinkingData base URL (absolute URI, max 1000 chars)" },
|
|
2351
|
+
{ name: "app-id", type: "string", required: true, desc: "ThinkingData app id (max 128 chars)" },
|
|
2352
|
+
{
|
|
2353
|
+
name: "lock-version",
|
|
2354
|
+
type: "number",
|
|
2355
|
+
min: 0,
|
|
2356
|
+
desc: "Current config lock_version when updating an existing config (from +get_te_report_config); omit on first create"
|
|
2357
|
+
}
|
|
2358
|
+
],
|
|
2359
|
+
risk: "write",
|
|
2360
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", PATH80, void 0, buildBody34(ctx)),
|
|
2361
|
+
execute: async (ctx) => evalApi(ctx, "PUT", PATH80, void 0, buildBody34(ctx))
|
|
2362
|
+
};
|
|
2363
|
+
|
|
2364
|
+
// src/commands/evaluation/get-ai-monitor-connection.ts
|
|
2365
|
+
var PATH81 = "/ai-monitor-connection";
|
|
2366
|
+
var getAiMonitorConnection = {
|
|
2367
|
+
service: "evaluation",
|
|
2368
|
+
command: "+get_ai_monitor_connection",
|
|
2369
|
+
description: "Get the project AI Monitor connection via GET /agenteval/api/v1/ai-monitor-connection. Connection is null when unset. Read its lock_version before +save_ai_monitor_connection.",
|
|
2370
|
+
flags: [...evalFlags()],
|
|
2371
|
+
risk: "read",
|
|
2372
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH81),
|
|
2373
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH81)
|
|
2374
|
+
};
|
|
2375
|
+
|
|
2376
|
+
// src/commands/evaluation/save-ai-monitor-connection.ts
|
|
2377
|
+
var PATH82 = "/ai-monitor-connection";
|
|
2378
|
+
var REQUIRED_CLAIM_KEYS = ["user_id", "te_user_id", "name", "company_id"];
|
|
2379
|
+
function buildBody35(ctx) {
|
|
2380
|
+
const claims = ctx.json("session-claims-json");
|
|
2381
|
+
if (claims === null || typeof claims !== "object" || Array.isArray(claims) || !REQUIRED_CLAIM_KEYS.every((key) => typeof claims[key] === "string")) {
|
|
2382
|
+
throw new Error(
|
|
2383
|
+
"--session-claims-json must be a JSON object with string user_id, te_user_id, name and company_id (is_root/platform_access optional)"
|
|
2384
|
+
);
|
|
2385
|
+
}
|
|
2386
|
+
const body = {
|
|
2387
|
+
base_url: ctx.str("base-url"),
|
|
2388
|
+
credential_id: ctx.str("credential-id"),
|
|
2389
|
+
session_claims: claims
|
|
2390
|
+
};
|
|
2391
|
+
const lockVersion = ctx.optionalNum("lock-version");
|
|
2392
|
+
if (lockVersion !== void 0) body.lock_version = lockVersion;
|
|
2393
|
+
return body;
|
|
2394
|
+
}
|
|
2395
|
+
function redactedDryRunBody(ctx) {
|
|
2396
|
+
const body = buildBody35(ctx);
|
|
2397
|
+
if (body.session_claims !== void 0) body.session_claims = "***redacted***";
|
|
2398
|
+
return body;
|
|
2399
|
+
}
|
|
2400
|
+
var saveAiMonitorConnection = {
|
|
2401
|
+
service: "evaluation",
|
|
2402
|
+
command: "+save_ai_monitor_connection",
|
|
2403
|
+
description: "Create or update the project AI Monitor connection via PUT /agenteval/api/v1/ai-monitor-connection. The credential must be of type AI_MONITOR_SESSION. Omit --lock-version on first create; on update pass the current lock_version from +get_ai_monitor_connection (0 is a valid version).",
|
|
2404
|
+
flags: [
|
|
2405
|
+
...evalFlags(),
|
|
2406
|
+
{ name: "base-url", type: "string", required: true, desc: "AI Monitor base URL (absolute URI, max 1000 chars)" },
|
|
2407
|
+
{
|
|
2408
|
+
name: "credential-id",
|
|
2409
|
+
type: "string",
|
|
2410
|
+
required: true,
|
|
2411
|
+
desc: "Credential UUID of type AI_MONITOR_SESSION (from +list_credentials)"
|
|
2412
|
+
},
|
|
2413
|
+
{
|
|
2414
|
+
name: "session-claims-json",
|
|
2415
|
+
type: "json",
|
|
2416
|
+
required: true,
|
|
2417
|
+
sensitive: true,
|
|
2418
|
+
desc: "Session claims JSON with required string keys user_id, te_user_id, name, company_id (optional is_root, platform_access)"
|
|
2419
|
+
},
|
|
2420
|
+
{
|
|
2421
|
+
name: "lock-version",
|
|
2422
|
+
type: "number",
|
|
2423
|
+
min: 0,
|
|
2424
|
+
desc: "Current connection lock_version when updating an existing connection (from +get_ai_monitor_connection); omit on first create"
|
|
2425
|
+
}
|
|
2426
|
+
],
|
|
2427
|
+
risk: "write",
|
|
2428
|
+
dryRun: (ctx) => evalDryRun(ctx, "PUT", PATH82, void 0, redactedDryRunBody(ctx)),
|
|
2429
|
+
execute: async (ctx) => evalApi(ctx, "PUT", PATH82, void 0, buildBody35(ctx))
|
|
2430
|
+
};
|
|
2431
|
+
|
|
2432
|
+
// src/commands/evaluation/verify-ai-monitor-connection.ts
|
|
2433
|
+
var PATH83 = "/ai-monitor-connection/verify";
|
|
2434
|
+
var verifyAiMonitorConnection = {
|
|
2435
|
+
service: "evaluation",
|
|
2436
|
+
command: "+verify_ai_monitor_connection",
|
|
2437
|
+
description: "Verify the saved AI Monitor connection by reaching the upstream via POST /agenteval/api/v1/ai-monitor-connection/verify. Sends no request body; returns the upstream trace total.",
|
|
2438
|
+
flags: [...evalFlags()],
|
|
2439
|
+
risk: "read",
|
|
2440
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH83),
|
|
2441
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH83)
|
|
2442
|
+
};
|
|
2443
|
+
|
|
2444
|
+
// src/commands/evaluation/import-ai-monitor-traces.ts
|
|
2445
|
+
var PATH84 = "/datasets/ai-monitor-imports";
|
|
2446
|
+
function buildBody36(ctx) {
|
|
2447
|
+
const body = { name: ctx.str("name") };
|
|
2448
|
+
const description = ctx.str("description");
|
|
2449
|
+
if (description) body.description = description;
|
|
2450
|
+
const selection = ctx.json("selection-json");
|
|
2451
|
+
if (selection === null || typeof selection !== "object" || Array.isArray(selection)) {
|
|
2452
|
+
throw new Error("--selection-json must be a JSON object");
|
|
2453
|
+
}
|
|
2454
|
+
const mode = selection.mode;
|
|
2455
|
+
if (mode !== "FILTER" && mode !== "TRACE_IDS") {
|
|
2456
|
+
throw new Error("--selection-json mode must be FILTER or TRACE_IDS");
|
|
2457
|
+
}
|
|
2458
|
+
if (mode === "FILTER") {
|
|
2459
|
+
const startTime = selection.start_time;
|
|
2460
|
+
const endTime = selection.end_time;
|
|
2461
|
+
const isDate = (value) => {
|
|
2462
|
+
if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
|
|
2463
|
+
const timestamp = Date.parse(`${value}T00:00:00Z`);
|
|
2464
|
+
return !Number.isNaN(timestamp) && new Date(timestamp).toISOString().slice(0, 10) === value;
|
|
2465
|
+
};
|
|
2466
|
+
if (!isDate(startTime) || !isDate(endTime)) {
|
|
2467
|
+
throw new Error("--selection-json with mode FILTER requires start_time and end_time as YYYY-MM-DD dates");
|
|
2468
|
+
}
|
|
2469
|
+
if (Date.parse(`${startTime}T00:00:00Z`) > Date.parse(`${endTime}T00:00:00Z`)) {
|
|
2470
|
+
throw new Error("--selection-json end_time must not precede start_time");
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
if (mode === "TRACE_IDS") {
|
|
2474
|
+
const traceIds = selection.trace_ids;
|
|
2475
|
+
if (!Array.isArray(traceIds) || traceIds.length < 1 || traceIds.length > 1e3 || !traceIds.every((v) => typeof v === "string")) {
|
|
2476
|
+
throw new Error("--selection-json with mode TRACE_IDS requires trace_ids: 1-1000 strings");
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
body.selection = selection;
|
|
2480
|
+
return body;
|
|
2481
|
+
}
|
|
2482
|
+
var importAiMonitorTraces = {
|
|
2483
|
+
service: "evaluation",
|
|
2484
|
+
command: "+import_ai_monitor_traces",
|
|
2485
|
+
description: "Create a dataset import job from AI Monitor traces via POST /agenteval/api/v1/datasets/ai-monitor-imports. Returns the created dataset and import job; the import is asynchronous, so poll +dataset_import_status with the returned dataset id instead of treating the 201 as completion.",
|
|
2486
|
+
flags: [
|
|
2487
|
+
...evalFlags(),
|
|
2488
|
+
{ name: "name", type: "string", required: true, desc: "Target dataset name (max 128 chars)" },
|
|
2489
|
+
{ name: "description", type: "string", desc: "Optional dataset description (max 2000 chars)" },
|
|
2490
|
+
{
|
|
2491
|
+
name: "selection-json",
|
|
2492
|
+
type: "json",
|
|
2493
|
+
required: true,
|
|
2494
|
+
sensitive: true,
|
|
2495
|
+
desc: 'Import selection JSON (redacted from logs): {"mode":"FILTER","start_time":"YYYY-MM-DD","end_time":"YYYY-MM-DD"} (dates required, optional agent, channel, search_term) or {"mode":"TRACE_IDS","trace_ids":["<id>"]} (1-1000 ids)'
|
|
2496
|
+
},
|
|
2497
|
+
{
|
|
2498
|
+
name: "idempotency-key",
|
|
2499
|
+
type: "string",
|
|
2500
|
+
desc: "Optional Idempotency-Key header (max 191 chars) for duplicate-import protection"
|
|
2501
|
+
}
|
|
2502
|
+
],
|
|
2503
|
+
risk: "write",
|
|
2504
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH84, void 0, buildBody36(ctx), {
|
|
2505
|
+
idempotencyKey: ctx.str("idempotency-key") || void 0
|
|
2506
|
+
}),
|
|
2507
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH84, void 0, buildBody36(ctx), {
|
|
2508
|
+
idempotencyKey: ctx.str("idempotency-key") || void 0
|
|
2509
|
+
})
|
|
2510
|
+
};
|
|
2511
|
+
|
|
2512
|
+
// src/commands/evaluation/list-te-claude-catalog.ts
|
|
2513
|
+
var PATH85 = "/agents/te-claude/catalog";
|
|
2514
|
+
function buildBody37(ctx) {
|
|
2515
|
+
const body = { endpoint_url: ctx.str("endpoint-url") };
|
|
2516
|
+
const credentialId = ctx.str("credential-id");
|
|
2517
|
+
if (credentialId) body.credential_id = credentialId;
|
|
2518
|
+
return body;
|
|
2519
|
+
}
|
|
2520
|
+
var listTeClaudeCatalog = {
|
|
2521
|
+
service: "evaluation",
|
|
2522
|
+
command: "+list_te_claude_catalog",
|
|
2523
|
+
description: "List TE Claude agents and models via POST /agenteval/api/v1/agents/te-claude/catalog. TE sessions use the login access token automatically; non-TE authentication may provide a legacy credential. Use the returned ids when creating TE_CLAUDE_WEB model configs and agent configs.",
|
|
2524
|
+
flags: [
|
|
2525
|
+
...evalFlags(),
|
|
2526
|
+
{ name: "endpoint-url", type: "string", required: true, desc: "TE Claude endpoint URL (absolute URI)" },
|
|
2527
|
+
{
|
|
2528
|
+
name: "credential-id",
|
|
2529
|
+
type: "string",
|
|
2530
|
+
desc: "Optional legacy credential UUID for non-TE authentication; TE sessions use the login access token automatically"
|
|
2531
|
+
}
|
|
2532
|
+
],
|
|
2533
|
+
risk: "read",
|
|
2534
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH85, void 0, buildBody37(ctx)),
|
|
2535
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH85, void 0, buildBody37(ctx))
|
|
2536
|
+
};
|
|
2537
|
+
|
|
2538
|
+
// src/commands/evaluation/list-project-members.ts
|
|
2539
|
+
var PATH86 = "/projects/members";
|
|
2540
|
+
var listProjectMembers = {
|
|
2541
|
+
service: "evaluation",
|
|
2542
|
+
command: "+list_project_members",
|
|
2543
|
+
description: "List project members via GET /agenteval/api/v1/projects/members. Requires ADMIN role. Includes external_id, login status and root | agent_admin | member role. Use the returned user UUIDs as --owner-user-id for +transfer_run_owner and other transfer commands.",
|
|
2544
|
+
flags: [...evalFlags()],
|
|
2545
|
+
risk: "read",
|
|
2546
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH86),
|
|
2547
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH86)
|
|
2548
|
+
};
|
|
2549
|
+
|
|
2550
|
+
// src/commands/evaluation/list-model-configs.ts
|
|
2551
|
+
var PATH87 = "/model-configs";
|
|
2552
|
+
var listModelConfigs = {
|
|
2553
|
+
service: "evaluation",
|
|
2554
|
+
command: "+list_model_configs",
|
|
2555
|
+
description: "List model configurations via GET /agenteval/api/v1/model-configs. Returns all model configs (LLM judge backends).",
|
|
2556
|
+
flags: [...evalFlags()],
|
|
2557
|
+
risk: "read",
|
|
2558
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH87),
|
|
2559
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH87)
|
|
2560
|
+
};
|
|
2561
|
+
|
|
2562
|
+
// src/commands/evaluation/get-model-config.ts
|
|
2563
|
+
var PATH88 = "/model-configs";
|
|
2564
|
+
function pathFor67(ctx) {
|
|
2565
|
+
return `${PATH88}/${evalPathId(ctx, "id")}`;
|
|
2566
|
+
}
|
|
2567
|
+
var getModelConfig = {
|
|
2568
|
+
service: "evaluation",
|
|
2569
|
+
command: "+get_model_config",
|
|
2570
|
+
description: "Get one model configuration by id via GET /agenteval/api/v1/model-configs/{modelConfigId}.",
|
|
2571
|
+
flags: [...evalFlags(), idFlag("Model config UUID")],
|
|
2572
|
+
risk: "read",
|
|
2573
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor67(ctx)),
|
|
2574
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor67(ctx))
|
|
2575
|
+
};
|
|
2576
|
+
|
|
2577
|
+
// src/commands/evaluation/create-model-config.ts
|
|
2578
|
+
var PATH89 = "/model-configs";
|
|
2579
|
+
function buildBody38(ctx) {
|
|
2580
|
+
const body = {
|
|
2581
|
+
name: ctx.str("name"),
|
|
2582
|
+
provider: ctx.str("provider"),
|
|
2583
|
+
model_name: ctx.str("model-name"),
|
|
2584
|
+
base_url: ctx.str("base-url")
|
|
2585
|
+
};
|
|
2586
|
+
const parameters = ctx.json("parameters-json");
|
|
2587
|
+
if (parameters !== void 0) body.parameters = parameters;
|
|
2588
|
+
const capabilities = ctx.json("capabilities-json");
|
|
2589
|
+
if (capabilities !== void 0) body.capabilities = capabilities;
|
|
2590
|
+
const credentialId = ctx.str("credential-id");
|
|
2591
|
+
if (credentialId) body.credential_id = credentialId;
|
|
2592
|
+
return body;
|
|
2593
|
+
}
|
|
2594
|
+
var createModelConfig = {
|
|
2595
|
+
service: "evaluation",
|
|
2596
|
+
command: "+create_model_config",
|
|
2597
|
+
description: "Create an LLM model configuration via POST /agenteval/api/v1/model-configs. Capabilities must set chat_completions=true and max_context_tokens>=1. Validate reachability with +test_model_config_connectivity before use. Creates a personal resource; sharing and administrative operations use separate permission checks.",
|
|
2598
|
+
flags: [
|
|
2599
|
+
...evalFlags(),
|
|
2600
|
+
{ name: "name", type: "string", required: true, desc: "Model config name (1-128 characters)" },
|
|
2601
|
+
{
|
|
2602
|
+
name: "provider",
|
|
2603
|
+
type: "string",
|
|
2604
|
+
required: true,
|
|
2605
|
+
desc: "Provider protocol: OPENAI_COMPATIBLE_CHAT | ANTHROPIC_CHAT | GEMINI_CHAT | AZURE_OPENAI_CHAT | TE_CLAUDE_WEB"
|
|
2606
|
+
},
|
|
2607
|
+
{ name: "model-name", type: "string", required: true, desc: "Model name sent to the provider (1-128 characters)" },
|
|
2608
|
+
{ name: "base-url", type: "string", required: true, desc: "Provider base URL; /v1 is appended automatically per provider convention" },
|
|
2609
|
+
{
|
|
2610
|
+
name: "parameters-json",
|
|
2611
|
+
type: "json",
|
|
2612
|
+
desc: "ModelParameters JSON (MODEL_PARAMETERS_V2: temperature/top_p/seed/timeout_ms/max_retries/structured_output); output limits are delegated to the provider"
|
|
2613
|
+
},
|
|
2614
|
+
{
|
|
2615
|
+
name: "capabilities-json",
|
|
2616
|
+
type: "json",
|
|
2617
|
+
desc: "ModelCapabilities JSON (schema_version/chat_completions/json_schema/seed/usage/max_context_tokens), MODEL_CAPABILITIES_V1"
|
|
2618
|
+
},
|
|
2619
|
+
{
|
|
2620
|
+
name: "credential-id",
|
|
2621
|
+
type: "string",
|
|
2622
|
+
desc: "Credential UUID from +list_credentials; TE_CLAUDE_WEB under TE authentication binds the login credential automatically"
|
|
2623
|
+
}
|
|
2624
|
+
],
|
|
2625
|
+
risk: "write",
|
|
2626
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH89, void 0, buildBody38(ctx)),
|
|
2627
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH89, void 0, buildBody38(ctx))
|
|
2628
|
+
};
|
|
2629
|
+
|
|
2630
|
+
// src/commands/evaluation/update-model-config.ts
|
|
2631
|
+
var PATH90 = "/model-configs";
|
|
2632
|
+
function pathFor68(ctx) {
|
|
2633
|
+
return `${PATH90}/${evalPathId(ctx, "id")}`;
|
|
2634
|
+
}
|
|
2635
|
+
function buildBody39(ctx) {
|
|
2636
|
+
const body = {
|
|
2637
|
+
name: ctx.str("name"),
|
|
2638
|
+
provider: ctx.str("provider"),
|
|
2639
|
+
model_name: ctx.str("model-name"),
|
|
2640
|
+
base_url: ctx.str("base-url"),
|
|
2641
|
+
lock_version: ctx.num("lock-version")
|
|
2642
|
+
};
|
|
2643
|
+
const parameters = ctx.json("parameters-json");
|
|
2644
|
+
if (parameters !== void 0) body.parameters = parameters;
|
|
2645
|
+
const capabilities = ctx.json("capabilities-json");
|
|
2646
|
+
if (capabilities !== void 0) body.capabilities = capabilities;
|
|
2647
|
+
const credentialId = ctx.str("credential-id");
|
|
2648
|
+
if (credentialId) body.credential_id = credentialId;
|
|
2649
|
+
return body;
|
|
2650
|
+
}
|
|
2651
|
+
var updateModelConfig = {
|
|
2652
|
+
service: "evaluation",
|
|
2653
|
+
command: "+update_model_config",
|
|
2654
|
+
description: "Update a model configuration via PATCH /agenteval/api/v1/model-configs/{id}. Full-replace semantics: pass every field, not only the ones changing (e.g. raise timeout_ms inside --parameters-json). Requires the current lock_version from +get_model_config.",
|
|
2655
|
+
flags: [
|
|
2656
|
+
...evalFlags(),
|
|
2657
|
+
...idAndLockFlags("Model config UUID", "model config"),
|
|
2658
|
+
{ name: "name", type: "string", required: true, desc: "Model config name (1-128 characters)" },
|
|
2659
|
+
{
|
|
2660
|
+
name: "provider",
|
|
2661
|
+
type: "string",
|
|
2662
|
+
required: true,
|
|
2663
|
+
desc: "Provider protocol: OPENAI_COMPATIBLE_CHAT | ANTHROPIC_CHAT | GEMINI_CHAT | AZURE_OPENAI_CHAT | TE_CLAUDE_WEB"
|
|
2664
|
+
},
|
|
2665
|
+
{ name: "model-name", type: "string", required: true, desc: "Model name sent to the provider" },
|
|
2666
|
+
{ name: "base-url", type: "string", required: true, desc: "Provider base URL" },
|
|
2667
|
+
{
|
|
2668
|
+
name: "parameters-json",
|
|
2669
|
+
type: "json",
|
|
2670
|
+
desc: "ModelParameters JSON (MODEL_PARAMETERS_V2), e.g. temperature/top_p/seed/timeout_ms/max_retries; output limits are provider-controlled"
|
|
2671
|
+
},
|
|
2672
|
+
{
|
|
2673
|
+
name: "capabilities-json",
|
|
2674
|
+
type: "json",
|
|
2675
|
+
desc: "ModelCapabilities JSON (MODEL_CAPABILITIES_V1); chat_completions=true and max_context_tokens>=1 are required"
|
|
2676
|
+
},
|
|
2677
|
+
{
|
|
2678
|
+
name: "credential-id",
|
|
2679
|
+
type: "string",
|
|
2680
|
+
desc: "Credential UUID used to authenticate to the provider; TE_CLAUDE_WEB under TE authentication binds the login credential automatically"
|
|
2681
|
+
}
|
|
2682
|
+
],
|
|
2683
|
+
risk: "write",
|
|
2684
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor68(ctx), void 0, buildBody39(ctx)),
|
|
2685
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor68(ctx), void 0, buildBody39(ctx))
|
|
2686
|
+
};
|
|
2687
|
+
|
|
2688
|
+
// src/commands/evaluation/delete-model-config.ts
|
|
2689
|
+
var PATH91 = "/model-configs";
|
|
2690
|
+
function pathFor69(ctx) {
|
|
2691
|
+
return `${PATH91}/${evalPathId(ctx, "id")}`;
|
|
2692
|
+
}
|
|
2693
|
+
function buildBody40(ctx) {
|
|
2694
|
+
return { lock_version: ctx.num("lock-version") };
|
|
2695
|
+
}
|
|
2696
|
+
var deleteModelConfig = {
|
|
2697
|
+
service: "evaluation",
|
|
2698
|
+
command: "+delete_model_config",
|
|
2699
|
+
description: "Delete a model configuration via DELETE /agenteval/api/v1/model-configs/{id}. Evaluators and run templates referencing it are unaffected (frozen snapshots) but new bindings fail. Requires the current lock_version.",
|
|
2700
|
+
flags: [...evalFlags(), ...idAndLockFlags("Model config UUID", "model config")],
|
|
2701
|
+
risk: "high-risk-write",
|
|
2702
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor69(ctx), void 0, buildBody40(ctx)),
|
|
2703
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor69(ctx), void 0, buildBody40(ctx))
|
|
2704
|
+
};
|
|
2705
|
+
|
|
2706
|
+
// src/commands/evaluation/test-model-config-connectivity.ts
|
|
2707
|
+
var PATH92 = "/model-configs";
|
|
2708
|
+
function pathFor70(ctx) {
|
|
2709
|
+
return `${PATH92}/${evalPathId(ctx, "id")}/test-connectivity`;
|
|
2710
|
+
}
|
|
2711
|
+
var testModelConfigConnectivity = {
|
|
2712
|
+
service: "evaluation",
|
|
2713
|
+
command: "+test_model_config_connectivity",
|
|
2714
|
+
description: "Probe a model configuration with a minimal chat request via POST /agenteval/api/v1/model-configs/{id}/test-connectivity. Verifies base URL, credential, and model name before the config is used by evaluators or runs.",
|
|
2715
|
+
flags: [...evalFlags(), idFlag("Model config UUID")],
|
|
2716
|
+
risk: "read",
|
|
2717
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor70(ctx)),
|
|
2718
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor70(ctx))
|
|
2719
|
+
};
|
|
2720
|
+
|
|
2721
|
+
// src/commands/evaluation/transfer-model-config-owner.ts
|
|
2722
|
+
var PATH93 = "/model-configs";
|
|
2723
|
+
function pathFor71(ctx) {
|
|
2724
|
+
return `${PATH93}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
2725
|
+
}
|
|
2726
|
+
function buildBody41(ctx) {
|
|
2727
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
2728
|
+
}
|
|
2729
|
+
var transferModelConfigOwner = {
|
|
2730
|
+
service: "evaluation",
|
|
2731
|
+
command: "+transfer_model_config_owner",
|
|
2732
|
+
description: "Transfer model config ownership to another user via POST /agenteval/api/v1/model-configs/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current model config lock_version.",
|
|
2733
|
+
flags: [
|
|
2734
|
+
...evalFlags(),
|
|
2735
|
+
...idAndLockFlags("Model config UUID", "model config"),
|
|
2736
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
2737
|
+
],
|
|
2738
|
+
risk: "high-risk-write",
|
|
2739
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor71(ctx), void 0, buildBody41(ctx)),
|
|
2740
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor71(ctx), void 0, buildBody41(ctx))
|
|
2741
|
+
};
|
|
2742
|
+
|
|
2743
|
+
// src/commands/evaluation/list-agents.ts
|
|
2744
|
+
var PATH94 = "/agents";
|
|
2745
|
+
var listAgents = {
|
|
2746
|
+
service: "evaluation",
|
|
2747
|
+
command: "+list_agents",
|
|
2748
|
+
description: "List agents under test via GET /agenteval/api/v1/agents. Returns all configured agent connectors.",
|
|
2749
|
+
flags: [...evalFlags()],
|
|
2750
|
+
risk: "read",
|
|
2751
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH94),
|
|
2752
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH94)
|
|
2753
|
+
};
|
|
2754
|
+
|
|
2755
|
+
// src/commands/evaluation/get-agent.ts
|
|
2756
|
+
var PATH95 = "/agents";
|
|
2757
|
+
function pathFor72(ctx) {
|
|
2758
|
+
return `${PATH95}/${evalPathId(ctx, "id")}`;
|
|
2759
|
+
}
|
|
2760
|
+
var getAgent = {
|
|
2761
|
+
service: "evaluation",
|
|
2762
|
+
command: "+get_agent",
|
|
2763
|
+
description: "Get one agent configuration by id via GET /agenteval/api/v1/agents/{agentId}.",
|
|
2764
|
+
flags: [...evalFlags(), idFlag("Agent UUID")],
|
|
2765
|
+
risk: "read",
|
|
2766
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor72(ctx)),
|
|
2767
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor72(ctx))
|
|
2768
|
+
};
|
|
2769
|
+
|
|
2770
|
+
// src/commands/evaluation/create-agent.ts
|
|
2771
|
+
var PATH96 = "/agents";
|
|
2772
|
+
function buildBody42(ctx) {
|
|
2773
|
+
const body = {
|
|
2774
|
+
name: ctx.str("name"),
|
|
2775
|
+
endpoint_url: ctx.str("endpoint-url"),
|
|
2776
|
+
input_schema: ctx.json("input-schema-json"),
|
|
2777
|
+
config: ctx.json("config-json")
|
|
2778
|
+
};
|
|
2779
|
+
const connectorType = ctx.str("connector-type");
|
|
2780
|
+
if (connectorType) body.connector_type = connectorType;
|
|
2781
|
+
const credentialId = ctx.str("credential-id");
|
|
2782
|
+
if (credentialId) body.credential_id = credentialId;
|
|
2783
|
+
return body;
|
|
2784
|
+
}
|
|
2785
|
+
function validateBody(body) {
|
|
2786
|
+
if (typeof body.input_schema !== "object" || body.input_schema === null) {
|
|
2787
|
+
throw new Error("--input-schema-json is required (AGENT_INPUT_SCHEMA_V1 object with type/properties/required)");
|
|
2788
|
+
}
|
|
2789
|
+
if (typeof body.config !== "object" || body.config === null) {
|
|
2790
|
+
throw new Error("--config-json is required (AGENT_CONFIG_V* object matching the connector type)");
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
var createAgent = {
|
|
2794
|
+
service: "evaluation",
|
|
2795
|
+
command: "+create_agent",
|
|
2796
|
+
description: "Create an agent under test via POST /agenteval/api/v1/agents. Validate reachability with +test_agent_connectivity before binding the agent to runs. Creates a personal resource; sharing and administrative operations use separate permission checks.",
|
|
2797
|
+
flags: [
|
|
2798
|
+
...evalFlags(),
|
|
2799
|
+
{ name: "name", type: "string", required: true, desc: "Agent config name (1-128 characters)" },
|
|
2800
|
+
{
|
|
2801
|
+
name: "connector-type",
|
|
2802
|
+
type: "string",
|
|
2803
|
+
desc: "Connector protocol: HTTP_JSON_V1 | OPENCODE_SERVER | OPENCLAW_GATEWAY_V1 | HERMES_RUNS_V1 | TE_CLAUDE_WEB_V1"
|
|
2804
|
+
},
|
|
2805
|
+
{ name: "endpoint-url", type: "string", required: true, desc: "Agent endpoint URL (max 1000 characters)" },
|
|
2806
|
+
{
|
|
2807
|
+
name: "input-schema-json",
|
|
2808
|
+
type: "json",
|
|
2809
|
+
required: true,
|
|
2810
|
+
desc: 'AgentInputSchema JSON (AGENT_INPUT_SCHEMA_V1): {"type":"object","properties":{...},"required":[...]}'
|
|
2811
|
+
},
|
|
2812
|
+
{
|
|
2813
|
+
name: "config-json",
|
|
2814
|
+
type: "json",
|
|
2815
|
+
required: true,
|
|
2816
|
+
desc: "AgentConfig JSON (AGENT_CONFIG_V*) matching the connector type; see an existing agent via +get_agent for a template"
|
|
2817
|
+
},
|
|
2818
|
+
{
|
|
2819
|
+
name: "credential-id",
|
|
2820
|
+
type: "string",
|
|
2821
|
+
desc: "Credential UUID from +list_credentials; TE_CLAUDE_WEB_V1 under TE authentication binds the login credential automatically"
|
|
2822
|
+
}
|
|
2823
|
+
],
|
|
2824
|
+
risk: "write",
|
|
2825
|
+
dryRun: (ctx) => {
|
|
2826
|
+
const body = buildBody42(ctx);
|
|
2827
|
+
validateBody(body);
|
|
2828
|
+
return evalDryRun(ctx, "POST", PATH96, void 0, body);
|
|
2829
|
+
},
|
|
2830
|
+
execute: async (ctx) => {
|
|
2831
|
+
const body = buildBody42(ctx);
|
|
2832
|
+
validateBody(body);
|
|
2833
|
+
return evalApi(ctx, "POST", PATH96, void 0, body);
|
|
2834
|
+
}
|
|
2835
|
+
};
|
|
2836
|
+
|
|
2837
|
+
// src/commands/evaluation/update-agent.ts
|
|
2838
|
+
var PATH97 = "/agents";
|
|
2839
|
+
function pathFor73(ctx) {
|
|
2840
|
+
return `${PATH97}/${evalPathId(ctx, "id")}`;
|
|
2841
|
+
}
|
|
2842
|
+
function buildBody43(ctx) {
|
|
2843
|
+
const body = {
|
|
2844
|
+
name: ctx.str("name"),
|
|
2845
|
+
endpoint_url: ctx.str("endpoint-url"),
|
|
2846
|
+
input_schema: ctx.json("input-schema-json"),
|
|
2847
|
+
config: ctx.json("config-json"),
|
|
2848
|
+
lock_version: ctx.num("lock-version")
|
|
2849
|
+
};
|
|
2850
|
+
const connectorType = ctx.str("connector-type");
|
|
2851
|
+
if (connectorType) body.connector_type = connectorType;
|
|
2852
|
+
const credentialId = ctx.str("credential-id");
|
|
2853
|
+
if (credentialId) body.credential_id = credentialId;
|
|
2854
|
+
return body;
|
|
2855
|
+
}
|
|
2856
|
+
function validateBody2(body) {
|
|
2857
|
+
if (typeof body.input_schema !== "object" || body.input_schema === null) {
|
|
2858
|
+
throw new Error("--input-schema-json is required (AGENT_INPUT_SCHEMA_V1 object)");
|
|
2859
|
+
}
|
|
2860
|
+
if (typeof body.config !== "object" || body.config === null) {
|
|
2861
|
+
throw new Error("--config-json is required (AGENT_CONFIG_V* object matching the connector type)");
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
var updateAgent = {
|
|
2865
|
+
service: "evaluation",
|
|
2866
|
+
command: "+update_agent",
|
|
2867
|
+
description: "Update an agent under test via PATCH /agenteval/api/v1/agents/{id}. Full-replace semantics: pass every field including unchanged ones. Requires the current agent lock_version from +get_agent.",
|
|
2868
|
+
flags: [
|
|
2869
|
+
...evalFlags(),
|
|
2870
|
+
...idAndLockFlags("Agent UUID", "agent"),
|
|
2871
|
+
{ name: "name", type: "string", required: true, desc: "Agent config name (1-128 characters)" },
|
|
2872
|
+
{
|
|
2873
|
+
name: "connector-type",
|
|
2874
|
+
type: "string",
|
|
2875
|
+
desc: "Connector protocol: HTTP_JSON_V1 | OPENCODE_SERVER | OPENCLAW_GATEWAY_V1 | HERMES_RUNS_V1 | TE_CLAUDE_WEB_V1"
|
|
2876
|
+
},
|
|
2877
|
+
{ name: "endpoint-url", type: "string", required: true, desc: "Agent endpoint URL (max 1000 characters)" },
|
|
2878
|
+
{ name: "input-schema-json", type: "json", required: true, desc: "AgentInputSchema JSON (AGENT_INPUT_SCHEMA_V1)" },
|
|
2879
|
+
{ name: "config-json", type: "json", required: true, desc: "AgentConfig JSON (AGENT_CONFIG_V*) matching the connector type" },
|
|
2880
|
+
{
|
|
2881
|
+
name: "credential-id",
|
|
2882
|
+
type: "string",
|
|
2883
|
+
desc: "Credential UUID used to authenticate to the agent; TE_CLAUDE_WEB_V1 under TE authentication binds the login credential automatically"
|
|
2884
|
+
}
|
|
2885
|
+
],
|
|
2886
|
+
risk: "write",
|
|
2887
|
+
dryRun: (ctx) => {
|
|
2888
|
+
const body = buildBody43(ctx);
|
|
2889
|
+
validateBody2(body);
|
|
2890
|
+
return evalDryRun(ctx, "PATCH", pathFor73(ctx), void 0, body);
|
|
2891
|
+
},
|
|
2892
|
+
execute: async (ctx) => {
|
|
2893
|
+
const body = buildBody43(ctx);
|
|
2894
|
+
validateBody2(body);
|
|
2895
|
+
return evalApi(ctx, "PATCH", pathFor73(ctx), void 0, body);
|
|
2896
|
+
}
|
|
2897
|
+
};
|
|
2898
|
+
|
|
2899
|
+
// src/commands/evaluation/delete-agent.ts
|
|
2900
|
+
var PATH98 = "/agents";
|
|
2901
|
+
function pathFor74(ctx) {
|
|
2902
|
+
return `${PATH98}/${evalPathId(ctx, "id")}`;
|
|
2903
|
+
}
|
|
2904
|
+
function buildBody44(ctx) {
|
|
2905
|
+
return { lock_version: ctx.num("lock-version") };
|
|
2906
|
+
}
|
|
2907
|
+
var deleteAgent = {
|
|
2908
|
+
service: "evaluation",
|
|
2909
|
+
command: "+delete_agent",
|
|
2910
|
+
description: "Delete an agent under test via DELETE /agenteval/api/v1/agents/{id}. Runs referencing the agent keep their frozen snapshot. Requires the current agent lock_version.",
|
|
2911
|
+
flags: [...evalFlags(), ...idAndLockFlags("Agent UUID", "agent")],
|
|
2912
|
+
risk: "high-risk-write",
|
|
2913
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor74(ctx), void 0, buildBody44(ctx)),
|
|
2914
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor74(ctx), void 0, buildBody44(ctx))
|
|
2915
|
+
};
|
|
2916
|
+
|
|
2917
|
+
// src/commands/evaluation/test-agent-connectivity.ts
|
|
2918
|
+
var PATH99 = "/agents";
|
|
2919
|
+
function pathFor75(ctx) {
|
|
2920
|
+
return `${PATH99}/${evalPathId(ctx, "id")}/test-connectivity`;
|
|
2921
|
+
}
|
|
2922
|
+
var testAgentConnectivity = {
|
|
2923
|
+
service: "evaluation",
|
|
2924
|
+
command: "+test_agent_connectivity",
|
|
2925
|
+
description: "Run a compatibility preflight against an agent via POST /agenteval/api/v1/agents/{id}/test-connectivity: connection, authentication, and protocol version. Fix any reported issue before starting runs.",
|
|
2926
|
+
flags: [...evalFlags(), idFlag("Agent UUID")],
|
|
2927
|
+
risk: "read",
|
|
2928
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor75(ctx)),
|
|
2929
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor75(ctx))
|
|
2930
|
+
};
|
|
2931
|
+
|
|
2932
|
+
// src/commands/evaluation/agent-snapshot-preview.ts
|
|
2933
|
+
var PATH100 = "/agents";
|
|
2934
|
+
function pathFor76(ctx) {
|
|
2935
|
+
return `${PATH100}/${evalPathId(ctx, "id")}/snapshot-preview`;
|
|
2936
|
+
}
|
|
2937
|
+
var agentSnapshotPreview = {
|
|
2938
|
+
service: "evaluation",
|
|
2939
|
+
command: "+agent_snapshot_preview",
|
|
2940
|
+
description: "Preview the immutable agent snapshot a run would freeze at start via GET /agenteval/api/v1/agents/{id}/snapshot-preview. Use to verify what a run template actually captured.",
|
|
2941
|
+
flags: [...evalFlags(), idFlag("Agent UUID")],
|
|
2942
|
+
risk: "read",
|
|
2943
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor76(ctx)),
|
|
2944
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor76(ctx))
|
|
2945
|
+
};
|
|
2946
|
+
|
|
2947
|
+
// src/commands/evaluation/transfer-agent-owner.ts
|
|
2948
|
+
var PATH101 = "/agents";
|
|
2949
|
+
function pathFor77(ctx) {
|
|
2950
|
+
return `${PATH101}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
2951
|
+
}
|
|
2952
|
+
function buildBody45(ctx) {
|
|
2953
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
2954
|
+
}
|
|
2955
|
+
var transferAgentOwner = {
|
|
2956
|
+
service: "evaluation",
|
|
2957
|
+
command: "+transfer_agent_owner",
|
|
2958
|
+
description: "Transfer agent ownership to another user via POST /agenteval/api/v1/agents/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current agent lock_version.",
|
|
2959
|
+
flags: [
|
|
2960
|
+
...evalFlags(),
|
|
2961
|
+
...idAndLockFlags("Agent UUID", "agent"),
|
|
2962
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
2963
|
+
],
|
|
2964
|
+
risk: "high-risk-write",
|
|
2965
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor77(ctx), void 0, buildBody45(ctx)),
|
|
2966
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor77(ctx), void 0, buildBody45(ctx))
|
|
2967
|
+
};
|
|
2968
|
+
|
|
2969
|
+
// src/commands/evaluation/list-credentials.ts
|
|
2970
|
+
var PATH102 = "/credentials";
|
|
2971
|
+
var listCredentials = {
|
|
2972
|
+
service: "evaluation",
|
|
2973
|
+
command: "+list_credentials",
|
|
2974
|
+
description: "List platform credentials via GET /agenteval/api/v1/credentials. Secrets are never returned; use +get_credential for one entry. Requires ADMIN role.",
|
|
2975
|
+
flags: [...evalFlags()],
|
|
2976
|
+
risk: "read",
|
|
2977
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH102),
|
|
2978
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH102)
|
|
2979
|
+
};
|
|
2980
|
+
|
|
2981
|
+
// src/commands/evaluation/get-credential.ts
|
|
2982
|
+
var PATH103 = "/credentials";
|
|
2983
|
+
function pathFor78(ctx) {
|
|
2984
|
+
return `${PATH103}/${evalPathId(ctx, "id")}`;
|
|
2985
|
+
}
|
|
2986
|
+
var getCredential = {
|
|
2987
|
+
service: "evaluation",
|
|
2988
|
+
command: "+get_credential",
|
|
2989
|
+
description: "Get one credential by UUID via GET /agenteval/api/v1/credentials/{id}. The secret value is never returned. Requires ADMIN role.",
|
|
2990
|
+
flags: [...evalFlags(), idFlag("Credential UUID")],
|
|
2991
|
+
risk: "read",
|
|
2992
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", pathFor78(ctx)),
|
|
2993
|
+
execute: async (ctx) => evalApi(ctx, "GET", pathFor78(ctx))
|
|
2994
|
+
};
|
|
2995
|
+
|
|
2996
|
+
// src/commands/evaluation/create-credential.ts
|
|
2997
|
+
var PATH104 = "/credentials";
|
|
2998
|
+
var CREDENTIAL_TYPES = ["BEARER_TOKEN", "API_KEY_HEADER", "AI_MONITOR_SESSION"];
|
|
2999
|
+
function buildBody46(ctx) {
|
|
3000
|
+
const type = ctx.str("type");
|
|
3001
|
+
if (!CREDENTIAL_TYPES.includes(type)) {
|
|
3002
|
+
throw new Error("--type must be one of BEARER_TOKEN | API_KEY_HEADER | AI_MONITOR_SESSION");
|
|
3003
|
+
}
|
|
3004
|
+
const body = {
|
|
3005
|
+
name: ctx.str("name"),
|
|
3006
|
+
type,
|
|
3007
|
+
secret: ctx.str("secret")
|
|
3008
|
+
};
|
|
3009
|
+
const header = ctx.str("header-name");
|
|
3010
|
+
if (header) body.header_name = header;
|
|
3011
|
+
const dataToken = ctx.str("data-token");
|
|
3012
|
+
const reportSource = ctx.str("report-source");
|
|
3013
|
+
if (type === "AI_MONITOR_SESSION") {
|
|
3014
|
+
if (!dataToken || !reportSource) {
|
|
3015
|
+
throw new Error("--data-token and --report-source are required for AI_MONITOR_SESSION credentials");
|
|
3016
|
+
}
|
|
3017
|
+
body.data_token = dataToken;
|
|
3018
|
+
body.report_source = reportSource;
|
|
3019
|
+
} else if (dataToken || reportSource) {
|
|
3020
|
+
throw new Error("--data-token / --report-source are only supported for AI_MONITOR_SESSION credentials");
|
|
3021
|
+
}
|
|
3022
|
+
return body;
|
|
3023
|
+
}
|
|
3024
|
+
function redactedDryRunBody2(ctx) {
|
|
3025
|
+
const body = buildBody46(ctx);
|
|
3026
|
+
const redacted = { ...body };
|
|
3027
|
+
if (typeof redacted.secret === "string" && redacted.secret) redacted.secret = "***redacted***";
|
|
3028
|
+
if (typeof redacted.data_token === "string" && redacted.data_token) redacted.data_token = "***redacted***";
|
|
3029
|
+
if (typeof redacted.report_source === "string" && redacted.report_source) redacted.report_source = "***redacted***";
|
|
3030
|
+
return redacted;
|
|
3031
|
+
}
|
|
3032
|
+
var createCredential = {
|
|
3033
|
+
service: "evaluation",
|
|
3034
|
+
command: "+create_credential",
|
|
3035
|
+
description: "Create a credential via POST /agenteval/api/v1/credentials. The secret is stored server-side and never echoed back; reference the credential by UUID when creating model configs, agents, or AI Monitor connections. Requires ADMIN role.",
|
|
3036
|
+
flags: [
|
|
3037
|
+
...evalFlags(),
|
|
3038
|
+
{ name: "name", type: "string", required: true, desc: "Credential name (1-128 characters)" },
|
|
3039
|
+
{
|
|
3040
|
+
name: "type",
|
|
3041
|
+
type: "string",
|
|
3042
|
+
required: true,
|
|
3043
|
+
desc: "Credential type: BEARER_TOKEN | API_KEY_HEADER | AI_MONITOR_SESSION"
|
|
3044
|
+
},
|
|
3045
|
+
{
|
|
3046
|
+
name: "secret",
|
|
3047
|
+
type: "string",
|
|
3048
|
+
required: true,
|
|
3049
|
+
sensitive: true,
|
|
3050
|
+
desc: "Secret value (max 8192 chars); sent once to the current AE Host and never returned by the platform"
|
|
3051
|
+
},
|
|
3052
|
+
{ name: "header-name", type: "string", desc: "Header name for API_KEY_HEADER credentials (max 64 chars)" },
|
|
3053
|
+
{
|
|
3054
|
+
name: "data-token",
|
|
3055
|
+
type: "string",
|
|
3056
|
+
sensitive: true,
|
|
3057
|
+
desc: "AI Monitor data token (max 512 chars); required for AI_MONITOR_SESSION credentials"
|
|
3058
|
+
},
|
|
3059
|
+
{
|
|
3060
|
+
name: "report-source",
|
|
3061
|
+
type: "string",
|
|
3062
|
+
sensitive: true,
|
|
3063
|
+
desc: "AI Monitor report source (max 128 chars); required for AI_MONITOR_SESSION credentials"
|
|
3064
|
+
}
|
|
3065
|
+
],
|
|
3066
|
+
risk: "write",
|
|
3067
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH104, void 0, redactedDryRunBody2(ctx)),
|
|
3068
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH104, void 0, buildBody46(ctx))
|
|
3069
|
+
};
|
|
3070
|
+
|
|
3071
|
+
// src/commands/evaluation/update-credential.ts
|
|
3072
|
+
var PATH105 = "/credentials";
|
|
3073
|
+
function pathFor79(ctx) {
|
|
3074
|
+
return `${PATH105}/${evalPathId(ctx, "id")}`;
|
|
3075
|
+
}
|
|
3076
|
+
function buildBody47(ctx) {
|
|
3077
|
+
return { name: ctx.str("name"), lock_version: ctx.num("lock-version") };
|
|
3078
|
+
}
|
|
3079
|
+
var updateCredential = {
|
|
3080
|
+
service: "evaluation",
|
|
3081
|
+
command: "+update_credential",
|
|
3082
|
+
description: "Rename a credential via PATCH /agenteval/api/v1/credentials/{id}. To replace the secret use +rotate_credential. Requires the current credential lock_version.",
|
|
3083
|
+
flags: [
|
|
3084
|
+
...evalFlags(),
|
|
3085
|
+
...idAndLockFlags("Credential UUID", "credential"),
|
|
3086
|
+
{ name: "name", type: "string", required: true, desc: "New credential name (1-128 characters)" }
|
|
3087
|
+
],
|
|
3088
|
+
risk: "write",
|
|
3089
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor79(ctx), void 0, buildBody47(ctx)),
|
|
3090
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor79(ctx), void 0, buildBody47(ctx))
|
|
3091
|
+
};
|
|
3092
|
+
|
|
3093
|
+
// src/commands/evaluation/rotate-credential.ts
|
|
3094
|
+
var PATH106 = "/credentials";
|
|
3095
|
+
function pathFor80(ctx) {
|
|
3096
|
+
return `${PATH106}/${evalPathId(ctx, "id")}/rotate`;
|
|
3097
|
+
}
|
|
3098
|
+
function buildBody48(ctx) {
|
|
3099
|
+
return { secret: ctx.str("secret"), lock_version: ctx.num("lock-version") };
|
|
3100
|
+
}
|
|
3101
|
+
var rotateCredential = {
|
|
3102
|
+
service: "evaluation",
|
|
3103
|
+
command: "+rotate_credential",
|
|
3104
|
+
description: "Replace the secret of a credential via POST /agenteval/api/v1/credentials/{id}/rotate. Model configs and agents referencing it pick up the new secret on their next call. Requires the current credential lock_version.",
|
|
3105
|
+
flags: [
|
|
3106
|
+
...evalFlags(),
|
|
3107
|
+
...idAndLockFlags("Credential UUID", "credential"),
|
|
3108
|
+
{
|
|
3109
|
+
name: "secret",
|
|
3110
|
+
type: "string",
|
|
3111
|
+
required: true,
|
|
3112
|
+
sensitive: true,
|
|
3113
|
+
desc: "New secret value (max 8192 chars); sent once to the current AE Host and never returned by the platform"
|
|
3114
|
+
}
|
|
3115
|
+
],
|
|
3116
|
+
risk: "write",
|
|
3117
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor80(ctx), void 0, { secret: "***redacted***", lock_version: ctx.num("lock-version") }),
|
|
3118
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor80(ctx), void 0, buildBody48(ctx))
|
|
3119
|
+
};
|
|
3120
|
+
|
|
3121
|
+
// src/commands/evaluation/disable-credential.ts
|
|
3122
|
+
var PATH107 = "/credentials";
|
|
3123
|
+
function pathFor81(ctx) {
|
|
3124
|
+
return `${PATH107}/${evalPathId(ctx, "id")}/disable`;
|
|
3125
|
+
}
|
|
3126
|
+
function buildBody49(ctx) {
|
|
3127
|
+
return { lock_version: ctx.num("lock-version") };
|
|
3128
|
+
}
|
|
3129
|
+
var disableCredential = {
|
|
3130
|
+
service: "evaluation",
|
|
3131
|
+
command: "+disable_credential",
|
|
3132
|
+
description: "Disable a credential via POST /agenteval/api/v1/credentials/{id}/disable. Disabled credentials fail connectivity and are rejected for new bindings. Requires the current credential lock_version.",
|
|
3133
|
+
flags: [...evalFlags(), ...idAndLockFlags("Credential UUID", "credential")],
|
|
3134
|
+
risk: "high-risk-write",
|
|
3135
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor81(ctx), void 0, buildBody49(ctx)),
|
|
3136
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor81(ctx), void 0, buildBody49(ctx))
|
|
3137
|
+
};
|
|
3138
|
+
|
|
3139
|
+
// src/commands/evaluation/enable-credential.ts
|
|
3140
|
+
var PATH108 = "/credentials";
|
|
3141
|
+
function pathFor82(ctx) {
|
|
3142
|
+
return `${PATH108}/${evalPathId(ctx, "id")}/enable`;
|
|
3143
|
+
}
|
|
3144
|
+
function buildBody50(ctx) {
|
|
3145
|
+
return { lock_version: ctx.num("lock-version") };
|
|
3146
|
+
}
|
|
3147
|
+
var enableCredential = {
|
|
3148
|
+
service: "evaluation",
|
|
3149
|
+
command: "+enable_credential",
|
|
3150
|
+
description: "Re-enable a disabled credential via POST /agenteval/api/v1/credentials/{id}/enable. Requires the current credential lock_version.",
|
|
3151
|
+
flags: [...evalFlags(), ...idAndLockFlags("Credential UUID", "credential")],
|
|
3152
|
+
risk: "high-risk-write",
|
|
3153
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor82(ctx), void 0, buildBody50(ctx)),
|
|
3154
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor82(ctx), void 0, buildBody50(ctx))
|
|
3155
|
+
};
|
|
3156
|
+
|
|
3157
|
+
// src/commands/evaluation/delete-credential.ts
|
|
3158
|
+
var PATH109 = "/credentials";
|
|
3159
|
+
function pathFor83(ctx) {
|
|
3160
|
+
return `${PATH109}/${evalPathId(ctx, "id")}`;
|
|
3161
|
+
}
|
|
3162
|
+
function buildBody51(ctx) {
|
|
3163
|
+
return { lock_version: ctx.num("lock-version") };
|
|
3164
|
+
}
|
|
3165
|
+
var deleteCredential = {
|
|
3166
|
+
service: "evaluation",
|
|
3167
|
+
command: "+delete_credential",
|
|
3168
|
+
description: "Delete a credential via DELETE /agenteval/api/v1/credentials/{id}. Model configs and agents still referencing it will fail until rebound. Requires the current credential lock_version.",
|
|
3169
|
+
flags: [...evalFlags(), ...idAndLockFlags("Credential UUID", "credential")],
|
|
3170
|
+
risk: "high-risk-write",
|
|
3171
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor83(ctx), void 0, buildBody51(ctx)),
|
|
3172
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor83(ctx), void 0, buildBody51(ctx))
|
|
3173
|
+
};
|
|
3174
|
+
|
|
3175
|
+
// src/commands/evaluation/transfer-credential-owner.ts
|
|
3176
|
+
var PATH110 = "/credentials";
|
|
3177
|
+
function pathFor84(ctx) {
|
|
3178
|
+
return `${PATH110}/${evalPathId(ctx, "id")}/transfer-owner`;
|
|
3179
|
+
}
|
|
3180
|
+
function buildBody52(ctx) {
|
|
3181
|
+
return { owner_user_id: ctx.str("owner-user-id"), lock_version: ctx.num("lock-version") };
|
|
3182
|
+
}
|
|
3183
|
+
var transferCredentialOwner = {
|
|
3184
|
+
service: "evaluation",
|
|
3185
|
+
command: "+transfer_credential_owner",
|
|
3186
|
+
description: "Transfer credential ownership to another user via POST /agenteval/api/v1/credentials/{id}/transfer-owner. Changes resource permissions; requires ADMIN role and the current credential lock_version.",
|
|
3187
|
+
flags: [
|
|
3188
|
+
...evalFlags(),
|
|
3189
|
+
...idAndLockFlags("Credential UUID", "credential"),
|
|
3190
|
+
{ name: "owner-user-id", type: "string", required: true, desc: "User UUID of the new owner" }
|
|
3191
|
+
],
|
|
3192
|
+
risk: "high-risk-write",
|
|
3193
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor84(ctx), void 0, buildBody52(ctx)),
|
|
3194
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor84(ctx), void 0, buildBody52(ctx))
|
|
3195
|
+
};
|
|
3196
|
+
|
|
3197
|
+
// src/commands/evaluation/change-dataset-access-scope.ts
|
|
3198
|
+
function pathFor85(ctx) {
|
|
3199
|
+
return `/datasets/${evalPathId(ctx, "id")}/access-scope`;
|
|
3200
|
+
}
|
|
3201
|
+
function buildBody53(ctx) {
|
|
3202
|
+
const accessScope = ctx.str("access-scope");
|
|
3203
|
+
if (!["PERSONAL", "COMPANY"].includes(accessScope)) {
|
|
3204
|
+
throw new CliValidationError("--access-scope must be PERSONAL or COMPANY");
|
|
3205
|
+
}
|
|
3206
|
+
return { access_scope: accessScope, lock_version: ctx.num("lock-version") };
|
|
3207
|
+
}
|
|
3208
|
+
var changeDatasetAccessScope = {
|
|
3209
|
+
service: "evaluation",
|
|
3210
|
+
resource: "dataset",
|
|
3211
|
+
command: "access-scope",
|
|
3212
|
+
description: "Change dataset visibility within the current project. Requires an administrator and the current lock_version. COMPANY assets cannot depend on private configurations or credentials.",
|
|
3213
|
+
flags: [
|
|
3214
|
+
...evalFlags(),
|
|
3215
|
+
...idAndLockFlags("Dataset UUID", "dataset"),
|
|
3216
|
+
{ name: "access-scope", type: "string", required: true, pattern: "^(PERSONAL|COMPANY)$", desc: "Target visibility: PERSONAL | COMPANY; SYSTEM is read-only" }
|
|
3217
|
+
],
|
|
3218
|
+
risk: "high-risk-write",
|
|
3219
|
+
preflight: (ctx) => {
|
|
3220
|
+
buildBody53(ctx);
|
|
3221
|
+
},
|
|
3222
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor85(ctx), void 0, buildBody53(ctx)),
|
|
3223
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor85(ctx), void 0, buildBody53(ctx))
|
|
3224
|
+
};
|
|
3225
|
+
|
|
3226
|
+
// src/commands/evaluation/change-evaluator-access-scope.ts
|
|
3227
|
+
function pathFor86(ctx) {
|
|
3228
|
+
return `/evaluators/${evalPathId(ctx, "id")}/access-scope`;
|
|
3229
|
+
}
|
|
3230
|
+
function buildBody54(ctx) {
|
|
3231
|
+
const accessScope = ctx.str("access-scope");
|
|
3232
|
+
if (!["PERSONAL", "COMPANY"].includes(accessScope)) {
|
|
3233
|
+
throw new CliValidationError("--access-scope must be PERSONAL or COMPANY");
|
|
3234
|
+
}
|
|
3235
|
+
return { access_scope: accessScope, lock_version: ctx.num("lock-version") };
|
|
3236
|
+
}
|
|
3237
|
+
var changeEvaluatorAccessScope = {
|
|
3238
|
+
service: "evaluation",
|
|
3239
|
+
resource: "evaluator",
|
|
3240
|
+
command: "access-scope",
|
|
3241
|
+
description: "Change evaluator visibility within the current project. Requires an administrator and the current lock_version. COMPANY assets cannot depend on private configurations or credentials.",
|
|
3242
|
+
flags: [
|
|
3243
|
+
...evalFlags(),
|
|
3244
|
+
...idAndLockFlags("Evaluator UUID", "evaluator"),
|
|
3245
|
+
{ name: "access-scope", type: "string", required: true, pattern: "^(PERSONAL|COMPANY)$", desc: "Target visibility: PERSONAL | COMPANY; SYSTEM is read-only" }
|
|
3246
|
+
],
|
|
3247
|
+
risk: "high-risk-write",
|
|
3248
|
+
preflight: (ctx) => {
|
|
3249
|
+
buildBody54(ctx);
|
|
3250
|
+
},
|
|
3251
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor86(ctx), void 0, buildBody54(ctx)),
|
|
3252
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor86(ctx), void 0, buildBody54(ctx))
|
|
3253
|
+
};
|
|
3254
|
+
|
|
3255
|
+
// src/commands/evaluation/change-model-config-access-scope.ts
|
|
3256
|
+
function pathFor87(ctx) {
|
|
3257
|
+
return `/model-configs/${evalPathId(ctx, "id")}/access-scope`;
|
|
3258
|
+
}
|
|
3259
|
+
function buildBody55(ctx) {
|
|
3260
|
+
const accessScope = ctx.str("access-scope");
|
|
3261
|
+
if (!["PERSONAL", "COMPANY"].includes(accessScope)) {
|
|
3262
|
+
throw new CliValidationError("--access-scope must be PERSONAL or COMPANY");
|
|
3263
|
+
}
|
|
3264
|
+
return { access_scope: accessScope, lock_version: ctx.num("lock-version") };
|
|
3265
|
+
}
|
|
3266
|
+
var changeModelConfigAccessScope = {
|
|
3267
|
+
service: "evaluation",
|
|
3268
|
+
resource: "model-config",
|
|
3269
|
+
command: "access-scope",
|
|
3270
|
+
description: "Change model-config visibility within the current project. Requires an administrator and the current lock_version. COMPANY assets cannot depend on private configurations or credentials.",
|
|
3271
|
+
flags: [
|
|
3272
|
+
...evalFlags(),
|
|
3273
|
+
...idAndLockFlags("Model config UUID", "model-config"),
|
|
3274
|
+
{ name: "access-scope", type: "string", required: true, pattern: "^(PERSONAL|COMPANY)$", desc: "Target visibility: PERSONAL | COMPANY; SYSTEM is read-only" }
|
|
3275
|
+
],
|
|
3276
|
+
risk: "high-risk-write",
|
|
3277
|
+
preflight: (ctx) => {
|
|
3278
|
+
buildBody55(ctx);
|
|
3279
|
+
},
|
|
3280
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor87(ctx), void 0, buildBody55(ctx)),
|
|
3281
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor87(ctx), void 0, buildBody55(ctx))
|
|
3282
|
+
};
|
|
3283
|
+
|
|
3284
|
+
// src/commands/evaluation/change-agent-access-scope.ts
|
|
3285
|
+
function pathFor88(ctx) {
|
|
3286
|
+
return `/agents/${evalPathId(ctx, "id")}/access-scope`;
|
|
3287
|
+
}
|
|
3288
|
+
function buildBody56(ctx) {
|
|
3289
|
+
const accessScope = ctx.str("access-scope");
|
|
3290
|
+
if (!["PERSONAL", "COMPANY"].includes(accessScope)) {
|
|
3291
|
+
throw new CliValidationError("--access-scope must be PERSONAL or COMPANY");
|
|
3292
|
+
}
|
|
3293
|
+
return { access_scope: accessScope, lock_version: ctx.num("lock-version") };
|
|
3294
|
+
}
|
|
3295
|
+
var changeAgentAccessScope = {
|
|
3296
|
+
service: "evaluation",
|
|
3297
|
+
resource: "agent",
|
|
3298
|
+
command: "access-scope",
|
|
3299
|
+
description: "Change agent visibility within the current project. Requires an administrator and the current lock_version. COMPANY assets cannot depend on private configurations or credentials.",
|
|
3300
|
+
flags: [
|
|
3301
|
+
...evalFlags(),
|
|
3302
|
+
...idAndLockFlags("Agent UUID", "agent"),
|
|
3303
|
+
{ name: "access-scope", type: "string", required: true, pattern: "^(PERSONAL|COMPANY)$", desc: "Target visibility: PERSONAL | COMPANY; SYSTEM is read-only" }
|
|
3304
|
+
],
|
|
3305
|
+
risk: "high-risk-write",
|
|
3306
|
+
preflight: (ctx) => {
|
|
3307
|
+
buildBody56(ctx);
|
|
3308
|
+
},
|
|
3309
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor88(ctx), void 0, buildBody56(ctx)),
|
|
3310
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor88(ctx), void 0, buildBody56(ctx))
|
|
3311
|
+
};
|
|
3312
|
+
|
|
3313
|
+
// src/commands/evaluation/enable-evaluator-template.ts
|
|
3314
|
+
function pathFor89(ctx) {
|
|
3315
|
+
return `/evaluator-templates/${evalPathId(ctx, "id")}/enable`;
|
|
3316
|
+
}
|
|
3317
|
+
var enableEvaluatorTemplate = {
|
|
3318
|
+
service: "evaluation",
|
|
3319
|
+
resource: "evaluator-template",
|
|
3320
|
+
command: "enable",
|
|
3321
|
+
description: "Re-enable a project evaluator template. Administrator only; built-in templates cannot be enabled here. Repeated requests succeed without a request body.",
|
|
3322
|
+
flags: [
|
|
3323
|
+
...evalFlags(),
|
|
3324
|
+
idFlag("Project evaluator template UUID")
|
|
3325
|
+
],
|
|
3326
|
+
risk: "write",
|
|
3327
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", pathFor89(ctx)),
|
|
3328
|
+
execute: async (ctx) => evalApi(ctx, "POST", pathFor89(ctx))
|
|
3329
|
+
};
|
|
3330
|
+
|
|
3331
|
+
// src/commands/evaluation/list-project-member-candidates.ts
|
|
3332
|
+
var PATH111 = "/projects/members/candidates";
|
|
3333
|
+
var listProjectMemberCandidates = {
|
|
3334
|
+
service: "evaluation",
|
|
3335
|
+
resource: "project-member",
|
|
3336
|
+
command: "list-candidates",
|
|
3337
|
+
description: "List available TE company members for project admission. Administrator only; returns external_id, login_name and display_name, excluding frozen accounts and existing members.",
|
|
3338
|
+
flags: [...evalFlags()],
|
|
3339
|
+
risk: "read",
|
|
3340
|
+
dryRun: (ctx) => evalDryRun(ctx, "GET", PATH111),
|
|
3341
|
+
execute: async (ctx) => evalApi(ctx, "GET", PATH111)
|
|
3342
|
+
};
|
|
3343
|
+
|
|
3344
|
+
// src/commands/evaluation/batch-add-project-members.ts
|
|
3345
|
+
var PATH112 = "/projects/members/batch-add";
|
|
3346
|
+
function buildBody57(ctx) {
|
|
3347
|
+
const members = ctx.json("members-json");
|
|
3348
|
+
if (!Array.isArray(members) || members.length === 0 || members.some((m) => !m || typeof m !== "object" || Array.isArray(m) || typeof m.external_id !== "string" || !m.external_id.trim() || m.external_id.length > 191 || m.display_name !== void 0 && (typeof m.display_name !== "string" || m.display_name.length > 128) || Object.keys(m).some((key) => !["external_id", "display_name"].includes(key)))) {
|
|
3349
|
+
throw new CliValidationError("--members-json must be a nonempty array of {external_id, display_name?}");
|
|
3350
|
+
}
|
|
3351
|
+
return { members };
|
|
3352
|
+
}
|
|
3353
|
+
var batchAddProjectMembers = {
|
|
3354
|
+
service: "evaluation",
|
|
3355
|
+
resource: "project-member",
|
|
3356
|
+
command: "batch-add",
|
|
3357
|
+
description: "Admit TE identities to the current project whitelist. Administrator only; existing members are skipped. Use external IDs from project-member list-candidates.",
|
|
3358
|
+
flags: [
|
|
3359
|
+
...evalFlags(),
|
|
3360
|
+
{ name: "members-json", type: "json", required: true, sensitive: true, desc: "Nonempty array of {external_id, display_name?}; external_id is a TE identity, not a member UUID" }
|
|
3361
|
+
],
|
|
3362
|
+
risk: "high-risk-write",
|
|
3363
|
+
preflight: (ctx) => {
|
|
3364
|
+
buildBody57(ctx);
|
|
3365
|
+
},
|
|
3366
|
+
dryRun: (ctx) => evalDryRun(ctx, "POST", PATH112, void 0, buildBody57(ctx)),
|
|
3367
|
+
execute: async (ctx) => evalApi(ctx, "POST", PATH112, void 0, buildBody57(ctx))
|
|
3368
|
+
};
|
|
3369
|
+
|
|
3370
|
+
// src/commands/evaluation/change-project-member-role.ts
|
|
3371
|
+
function pathFor90(ctx) {
|
|
3372
|
+
return `/projects/members/${evalPathId(ctx, "id")}/role`;
|
|
3373
|
+
}
|
|
3374
|
+
function buildBody58(ctx) {
|
|
3375
|
+
const value = ctx.str("role");
|
|
3376
|
+
if (!["agent_admin", "member"].includes(value)) {
|
|
3377
|
+
throw new CliValidationError("--role must be agent_admin or member");
|
|
3378
|
+
}
|
|
3379
|
+
return { role: value };
|
|
3380
|
+
}
|
|
3381
|
+
var changeProjectMemberRole = {
|
|
3382
|
+
service: "evaluation",
|
|
3383
|
+
resource: "project-member",
|
|
3384
|
+
command: "change-role",
|
|
3385
|
+
description: "Change a project role. Administrator only; root members are protected.",
|
|
3386
|
+
flags: [
|
|
3387
|
+
...evalFlags(),
|
|
3388
|
+
idFlag("Project member user UUID from +list_project_members"),
|
|
3389
|
+
{ name: "role", type: "string", required: true, pattern: "^(agent_admin|member)$", desc: "Target role: agent_admin | member" }
|
|
3390
|
+
],
|
|
3391
|
+
risk: "high-risk-write",
|
|
3392
|
+
preflight: (ctx) => {
|
|
3393
|
+
buildBody58(ctx);
|
|
3394
|
+
},
|
|
3395
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor90(ctx), void 0, buildBody58(ctx)),
|
|
3396
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor90(ctx), void 0, buildBody58(ctx))
|
|
3397
|
+
};
|
|
3398
|
+
|
|
3399
|
+
// src/commands/evaluation/set-project-member-status.ts
|
|
3400
|
+
function pathFor91(ctx) {
|
|
3401
|
+
return `/projects/members/${evalPathId(ctx, "id")}`;
|
|
3402
|
+
}
|
|
3403
|
+
function buildBody59(ctx) {
|
|
3404
|
+
const value = ctx.str("status");
|
|
3405
|
+
if (!["ACTIVE", "DISABLED"].includes(value)) {
|
|
3406
|
+
throw new CliValidationError("--status must be ACTIVE or DISABLED");
|
|
3407
|
+
}
|
|
3408
|
+
return { status: value };
|
|
3409
|
+
}
|
|
3410
|
+
var setProjectMemberStatus = {
|
|
3411
|
+
service: "evaluation",
|
|
3412
|
+
resource: "project-member",
|
|
3413
|
+
command: "set-status",
|
|
3414
|
+
description: "Enable or disable a login across every project in the tenant. Administrator only; identities with a root role are protected.",
|
|
3415
|
+
flags: [
|
|
3416
|
+
...evalFlags(),
|
|
3417
|
+
idFlag("Project member user UUID from +list_project_members"),
|
|
3418
|
+
{ name: "status", type: "string", required: true, pattern: "^(ACTIVE|DISABLED)$", desc: "Target status: ACTIVE | DISABLED" }
|
|
3419
|
+
],
|
|
3420
|
+
risk: "high-risk-write",
|
|
3421
|
+
preflight: (ctx) => {
|
|
3422
|
+
buildBody59(ctx);
|
|
3423
|
+
},
|
|
3424
|
+
dryRun: (ctx) => evalDryRun(ctx, "PATCH", pathFor91(ctx), void 0, buildBody59(ctx)),
|
|
3425
|
+
execute: async (ctx) => evalApi(ctx, "PATCH", pathFor91(ctx), void 0, buildBody59(ctx))
|
|
3426
|
+
};
|
|
3427
|
+
|
|
3428
|
+
// src/commands/evaluation/remove-project-member.ts
|
|
3429
|
+
function pathFor92(ctx) {
|
|
3430
|
+
return `/projects/members/${evalPathId(ctx, "id")}`;
|
|
3431
|
+
}
|
|
3432
|
+
var removeProjectMember = {
|
|
3433
|
+
service: "evaluation",
|
|
3434
|
+
resource: "project-member",
|
|
3435
|
+
command: "remove",
|
|
3436
|
+
description: "Revoke membership and future login access to the current project until explicitly re-added. Administrator only; root members are protected. Other projects are unaffected.",
|
|
3437
|
+
flags: [
|
|
3438
|
+
...evalFlags(),
|
|
3439
|
+
idFlag("Project member user UUID from +list_project_members")
|
|
3440
|
+
],
|
|
3441
|
+
risk: "high-risk-write",
|
|
3442
|
+
dryRun: (ctx) => evalDryRun(ctx, "DELETE", pathFor92(ctx)),
|
|
3443
|
+
execute: async (ctx) => evalApi(ctx, "DELETE", pathFor92(ctx))
|
|
3444
|
+
};
|
|
3445
|
+
|
|
3446
|
+
// src/commands/evaluation/download-file.ts
|
|
3447
|
+
import { resolve as resolve2 } from "path";
|
|
3448
|
+
function pathFor93(ctx) {
|
|
3449
|
+
return `/files/${evalPathId(ctx, "id")}/content`;
|
|
3450
|
+
}
|
|
3451
|
+
var downloadFile = {
|
|
3452
|
+
service: "evaluation",
|
|
3453
|
+
resource: "file",
|
|
3454
|
+
command: "download",
|
|
3455
|
+
description: "Download a completed, unexpired file through the authenticated API for local or S3 storage. Saves bytes to a new file; returns path and byte_size. Existing files are never overwritten.",
|
|
3456
|
+
flags: [
|
|
3457
|
+
...evalFlags(),
|
|
3458
|
+
idFlag("File UUID from an import or export operation"),
|
|
3459
|
+
{ name: "output", type: "string", required: true, minLength: 1, desc: "Destination file path; parent directory must exist and the file must not exist" }
|
|
3460
|
+
],
|
|
3461
|
+
risk: "read",
|
|
3462
|
+
dryRun: (ctx) => ({ ...evalDryRun(ctx, "GET", pathFor93(ctx)), output: resolve2(ctx.str("output")) }),
|
|
3463
|
+
execute: async (ctx) => evalDownload(ctx, pathFor93(ctx), ctx.str("output"))
|
|
3464
|
+
};
|
|
3465
|
+
|
|
3466
|
+
// src/commands/evaluation/index.ts
|
|
3467
|
+
var commands = [
|
|
3468
|
+
changeDatasetAccessScope,
|
|
3469
|
+
changeEvaluatorAccessScope,
|
|
3470
|
+
changeModelConfigAccessScope,
|
|
3471
|
+
changeAgentAccessScope,
|
|
3472
|
+
enableEvaluatorTemplate,
|
|
3473
|
+
listProjectMemberCandidates,
|
|
3474
|
+
batchAddProjectMembers,
|
|
3475
|
+
changeProjectMemberRole,
|
|
3476
|
+
setProjectMemberStatus,
|
|
3477
|
+
removeProjectMember,
|
|
3478
|
+
downloadFile,
|
|
3479
|
+
listDatasets,
|
|
3480
|
+
getDataset,
|
|
3481
|
+
createDataset,
|
|
3482
|
+
importDataset,
|
|
3483
|
+
datasetImportStatus,
|
|
3484
|
+
datasetImportErrors,
|
|
3485
|
+
updateDataset,
|
|
3486
|
+
deleteDataset,
|
|
3487
|
+
transferDatasetOwner,
|
|
3488
|
+
listDatasetFields,
|
|
3489
|
+
createDatasetField,
|
|
3490
|
+
batchCreateDatasetFields,
|
|
3491
|
+
updateDatasetField,
|
|
3492
|
+
deleteDatasetField,
|
|
3493
|
+
reorderDatasetFields,
|
|
3494
|
+
listDatasetCases,
|
|
3495
|
+
getDatasetCase,
|
|
3496
|
+
createDatasetCase,
|
|
3497
|
+
batchCreateDatasetCases,
|
|
3498
|
+
updateDatasetCase,
|
|
3499
|
+
deleteDatasetCase,
|
|
3500
|
+
exportDataset,
|
|
3501
|
+
listDatasetVersions,
|
|
3502
|
+
publishDatasetVersion,
|
|
3503
|
+
getDatasetVersion,
|
|
3504
|
+
listDatasetReferences,
|
|
3505
|
+
listEvaluators,
|
|
3506
|
+
getEvaluator,
|
|
3507
|
+
createEvaluator,
|
|
3508
|
+
updateEvaluator,
|
|
3509
|
+
deleteEvaluator,
|
|
3510
|
+
transferEvaluatorOwner,
|
|
3511
|
+
listEvaluatorTemplates,
|
|
3512
|
+
createEvaluatorTemplate,
|
|
3513
|
+
disableEvaluatorTemplate,
|
|
3514
|
+
listEvaluatorVersions,
|
|
3515
|
+
publishEvaluatorVersion,
|
|
3516
|
+
optimizeEvaluatorPrompt,
|
|
3517
|
+
getEvaluatorHealth,
|
|
3518
|
+
listCalibrationSuites,
|
|
3519
|
+
createCalibrationSuite,
|
|
3520
|
+
listCalibrationCases,
|
|
3521
|
+
createCalibrationCase,
|
|
3522
|
+
deleteCalibrationCase,
|
|
3523
|
+
startCalibrationRun,
|
|
3524
|
+
getCalibrationRun,
|
|
3525
|
+
listCalibrationResults,
|
|
3526
|
+
listRuns,
|
|
3527
|
+
getRun,
|
|
3528
|
+
createRun,
|
|
3529
|
+
updateRun,
|
|
3530
|
+
deleteRun,
|
|
3531
|
+
selectRunEvaluators,
|
|
3532
|
+
listRunEvaluators,
|
|
3533
|
+
updateRunFieldMapping,
|
|
3534
|
+
updateRunResultFilterFields,
|
|
3535
|
+
validateRun,
|
|
3536
|
+
startRun,
|
|
3537
|
+
cancelRun,
|
|
3538
|
+
runOverview,
|
|
3539
|
+
runCases,
|
|
3540
|
+
getRunCase,
|
|
3541
|
+
exportRunResults,
|
|
3542
|
+
exportRunLowScores,
|
|
3543
|
+
fileDownloadUrl,
|
|
3544
|
+
getFile,
|
|
3545
|
+
createNextTaskVersionDraft,
|
|
3546
|
+
listEvaluationTaskVersions,
|
|
3547
|
+
retryFailedRunCases,
|
|
3548
|
+
getRunCaseTrace,
|
|
3549
|
+
retryRunCaseTrace,
|
|
3550
|
+
runDistribution,
|
|
3551
|
+
copyRunCase,
|
|
3552
|
+
transferRunOwner,
|
|
3553
|
+
listSchedules,
|
|
3554
|
+
createSchedule,
|
|
3555
|
+
updateSchedule,
|
|
3556
|
+
deleteSchedule,
|
|
3557
|
+
getTeReportConfig,
|
|
3558
|
+
saveTeReportConfig,
|
|
3559
|
+
getAiMonitorConnection,
|
|
3560
|
+
saveAiMonitorConnection,
|
|
3561
|
+
verifyAiMonitorConnection,
|
|
3562
|
+
importAiMonitorTraces,
|
|
3563
|
+
listTeClaudeCatalog,
|
|
3564
|
+
listProjectMembers,
|
|
3565
|
+
listModelConfigs,
|
|
3566
|
+
getModelConfig,
|
|
3567
|
+
createModelConfig,
|
|
3568
|
+
updateModelConfig,
|
|
3569
|
+
deleteModelConfig,
|
|
3570
|
+
testModelConfigConnectivity,
|
|
3571
|
+
transferModelConfigOwner,
|
|
3572
|
+
listAgents,
|
|
3573
|
+
getAgent,
|
|
3574
|
+
createAgent,
|
|
3575
|
+
updateAgent,
|
|
3576
|
+
deleteAgent,
|
|
3577
|
+
testAgentConnectivity,
|
|
3578
|
+
agentSnapshotPreview,
|
|
3579
|
+
transferAgentOwner,
|
|
3580
|
+
listCredentials,
|
|
3581
|
+
getCredential,
|
|
3582
|
+
createCredential,
|
|
3583
|
+
updateCredential,
|
|
3584
|
+
rotateCredential,
|
|
3585
|
+
disableCredential,
|
|
3586
|
+
enableCredential,
|
|
3587
|
+
deleteCredential,
|
|
3588
|
+
transferCredentialOwner
|
|
3589
|
+
];
|
|
3590
|
+
var evaluation_default = commands;
|
|
3591
|
+
export {
|
|
3592
|
+
agentSnapshotPreview,
|
|
3593
|
+
batchAddProjectMembers,
|
|
3594
|
+
batchCreateDatasetCases,
|
|
3595
|
+
batchCreateDatasetFields,
|
|
3596
|
+
cancelRun,
|
|
3597
|
+
changeAgentAccessScope,
|
|
3598
|
+
changeDatasetAccessScope,
|
|
3599
|
+
changeEvaluatorAccessScope,
|
|
3600
|
+
changeModelConfigAccessScope,
|
|
3601
|
+
changeProjectMemberRole,
|
|
3602
|
+
copyRunCase,
|
|
3603
|
+
createAgent,
|
|
3604
|
+
createCalibrationCase,
|
|
3605
|
+
createCalibrationSuite,
|
|
3606
|
+
createCredential,
|
|
3607
|
+
createDataset,
|
|
3608
|
+
createDatasetCase,
|
|
3609
|
+
createDatasetField,
|
|
3610
|
+
createEvaluator,
|
|
3611
|
+
createEvaluatorTemplate,
|
|
3612
|
+
createModelConfig,
|
|
3613
|
+
createNextTaskVersionDraft,
|
|
3614
|
+
createRun,
|
|
3615
|
+
createSchedule,
|
|
3616
|
+
datasetImportErrors,
|
|
3617
|
+
datasetImportStatus,
|
|
3618
|
+
evaluation_default as default,
|
|
3619
|
+
deleteAgent,
|
|
3620
|
+
deleteCalibrationCase,
|
|
3621
|
+
deleteCredential,
|
|
3622
|
+
deleteDataset,
|
|
3623
|
+
deleteDatasetCase,
|
|
3624
|
+
deleteDatasetField,
|
|
3625
|
+
deleteEvaluator,
|
|
3626
|
+
deleteModelConfig,
|
|
3627
|
+
deleteRun,
|
|
3628
|
+
deleteSchedule,
|
|
3629
|
+
disableCredential,
|
|
3630
|
+
disableEvaluatorTemplate,
|
|
3631
|
+
downloadFile,
|
|
3632
|
+
enableCredential,
|
|
3633
|
+
enableEvaluatorTemplate,
|
|
3634
|
+
exportDataset,
|
|
3635
|
+
exportRunLowScores,
|
|
3636
|
+
exportRunResults,
|
|
3637
|
+
fileDownloadUrl,
|
|
3638
|
+
getAgent,
|
|
3639
|
+
getAiMonitorConnection,
|
|
3640
|
+
getCalibrationRun,
|
|
3641
|
+
getCredential,
|
|
3642
|
+
getDataset,
|
|
3643
|
+
getDatasetCase,
|
|
3644
|
+
getDatasetVersion,
|
|
3645
|
+
getEvaluator,
|
|
3646
|
+
getEvaluatorHealth,
|
|
3647
|
+
getFile,
|
|
3648
|
+
getModelConfig,
|
|
3649
|
+
getRun,
|
|
3650
|
+
getRunCase,
|
|
3651
|
+
getRunCaseTrace,
|
|
3652
|
+
getTeReportConfig,
|
|
3653
|
+
importAiMonitorTraces,
|
|
3654
|
+
importDataset,
|
|
3655
|
+
listAgents,
|
|
3656
|
+
listCalibrationCases,
|
|
3657
|
+
listCalibrationResults,
|
|
3658
|
+
listCalibrationSuites,
|
|
3659
|
+
listCredentials,
|
|
3660
|
+
listDatasetCases,
|
|
3661
|
+
listDatasetFields,
|
|
3662
|
+
listDatasetReferences,
|
|
3663
|
+
listDatasetVersions,
|
|
3664
|
+
listDatasets,
|
|
3665
|
+
listEvaluationTaskVersions,
|
|
3666
|
+
listEvaluatorTemplates,
|
|
3667
|
+
listEvaluatorVersions,
|
|
3668
|
+
listEvaluators,
|
|
3669
|
+
listModelConfigs,
|
|
3670
|
+
listProjectMemberCandidates,
|
|
3671
|
+
listProjectMembers,
|
|
3672
|
+
listRunEvaluators,
|
|
3673
|
+
listRuns,
|
|
3674
|
+
listSchedules,
|
|
3675
|
+
listTeClaudeCatalog,
|
|
3676
|
+
optimizeEvaluatorPrompt,
|
|
3677
|
+
publishDatasetVersion,
|
|
3678
|
+
publishEvaluatorVersion,
|
|
3679
|
+
removeProjectMember,
|
|
3680
|
+
reorderDatasetFields,
|
|
3681
|
+
retryFailedRunCases,
|
|
3682
|
+
retryRunCaseTrace,
|
|
3683
|
+
rotateCredential,
|
|
3684
|
+
runCases,
|
|
3685
|
+
runDistribution,
|
|
3686
|
+
runOverview,
|
|
3687
|
+
saveAiMonitorConnection,
|
|
3688
|
+
saveTeReportConfig,
|
|
3689
|
+
selectRunEvaluators,
|
|
3690
|
+
setProjectMemberStatus,
|
|
3691
|
+
startCalibrationRun,
|
|
3692
|
+
startRun,
|
|
3693
|
+
testAgentConnectivity,
|
|
3694
|
+
testModelConfigConnectivity,
|
|
3695
|
+
transferAgentOwner,
|
|
3696
|
+
transferCredentialOwner,
|
|
3697
|
+
transferDatasetOwner,
|
|
3698
|
+
transferEvaluatorOwner,
|
|
3699
|
+
transferModelConfigOwner,
|
|
3700
|
+
transferRunOwner,
|
|
3701
|
+
updateAgent,
|
|
3702
|
+
updateCredential,
|
|
3703
|
+
updateDataset,
|
|
3704
|
+
updateDatasetCase,
|
|
3705
|
+
updateDatasetField,
|
|
3706
|
+
updateEvaluator,
|
|
3707
|
+
updateModelConfig,
|
|
3708
|
+
updateRun,
|
|
3709
|
+
updateRunFieldMapping,
|
|
3710
|
+
updateRunResultFilterFields,
|
|
3711
|
+
updateSchedule,
|
|
3712
|
+
validateRun,
|
|
3713
|
+
verifyAiMonitorConnection
|
|
3714
|
+
};
|