@guandata/decidex 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +22 -0
  2. package/package.json +1 -1
  3. package/src/cli.js +126 -14
package/README.md CHANGED
@@ -158,6 +158,28 @@ decidex eval stop AGENT_ID RUN_ID
158
158
 
159
159
  CLI 按最新 `origin/main` 校验该文件:只接受 `caseIds`、`targetModelSelection` 和 `judgeModelSelection`。使用 `--cases all` 时由 CLI 获取并覆盖 `caseIds`;Agent 没有评测案例时会在本地停止。
160
160
 
161
+ 查询、接管或批量监控已经启动的评测:
162
+
163
+ ```bash
164
+ decidex eval runs list AGENT_ID
165
+ decidex eval runs get AGENT_ID RUN_ID
166
+ decidex eval runs wait AGENT_ID RUN_ID --follow --output result.json
167
+ decidex eval runs watch evaluation-runs.json --parallel 3 --follow --output summary.json
168
+ ```
169
+
170
+ `evaluation-runs.json`:
171
+
172
+ ```json
173
+ {
174
+ "runs": [
175
+ { "label": "A/S07", "agentId": "AGENT_ID", "runId": "RUN_ID" },
176
+ { "label": "B/S07", "agentId": "AGENT_ID", "runId": "RUN_ID" }
177
+ ]
178
+ }
179
+ ```
180
+
181
+ `watch` 只在状态或计数变化时向 stderr 输出进度,最终 JSON 保留每个 Run 的完整 Case 结果;任一 Run 失败、停止或监控出错时退出码为 1。
182
+
161
183
  ## 组织
162
184
 
163
185
  ```bash
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@guandata/decidex",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "CLI for building and operating Decidex ontologies, agents, skills, sessions, and evaluations.",
5
5
  "license": "UNLICENSED",
6
6
  "type": "module",
package/src/cli.js CHANGED
@@ -52,7 +52,11 @@ Usage:
52
52
  decidex eval cases export AGENT_ID FILE
53
53
  decidex eval cases plan AGENT_ID FILE [--prune]
54
54
  decidex eval cases apply AGENT_ID FILE [--prune]
55
- decidex eval run AGENT_ID FILE [--cases all] [--wait] [--output FILE] [--timeout SECONDS]
55
+ decidex eval run AGENT_ID FILE [--cases all] [--wait] [--follow] [--output FILE] [--timeout SECONDS]
56
+ decidex eval runs list AGENT_ID [--cursor CURSOR]
57
+ decidex eval runs get AGENT_ID RUN_ID
58
+ decidex eval runs wait AGENT_ID RUN_ID [--follow] [--output FILE] [--timeout SECONDS]
59
+ decidex eval runs watch FILE [--parallel N] [--follow] [--output FILE] [--timeout SECONDS]
56
60
  decidex eval stop AGENT_ID RUN_ID
57
61
 
58
62
  decidex ontology list
@@ -111,13 +115,19 @@ Runtime Pod termination details require backend support.`,
111
115
  decidex skill delete ID`,
112
116
  eval: `Usage:
113
117
  decidex eval cases list|export|plan|apply ...
114
- decidex eval run AGENT_ID FILE [--cases all] [--wait] [--output FILE]
118
+ decidex eval run AGENT_ID FILE [--cases all] [--wait] [--follow] [--output FILE]
119
+ decidex eval runs list|get|wait|watch ...
115
120
  decidex eval stop AGENT_ID RUN_ID`,
116
121
  "eval cases": `Usage:
117
122
  decidex eval cases list AGENT_ID
118
123
  decidex eval cases export AGENT_ID FILE
119
124
  decidex eval cases plan AGENT_ID FILE [--prune]
120
125
  decidex eval cases apply AGENT_ID FILE [--prune]`,
126
+ "eval runs": `Usage:
127
+ decidex eval runs list AGENT_ID [--cursor CURSOR]
128
+ decidex eval runs get AGENT_ID RUN_ID
129
+ decidex eval runs wait AGENT_ID RUN_ID [--follow] [--output FILE] [--timeout SECONDS]
130
+ decidex eval runs watch FILE [--parallel N] [--follow] [--output FILE] [--timeout SECONDS]`,
121
131
  ontology: `Usage:
122
132
  decidex ontology list|get|export|plan|apply ...`,
123
133
  api: `Usage:
@@ -1035,10 +1045,123 @@ function evaluationRunSpec(value) {
1035
1045
  return value;
1036
1046
  }
1037
1047
 
1048
+ function evaluationRunPath(agentId, runId) {
1049
+ return `/api/personal-agents/${encodeURIComponent(agentId)}/evaluation-runs/${encodeURIComponent(runId)}`;
1050
+ }
1051
+
1052
+ function evaluationProgress(run, label) {
1053
+ const counts = run.counts ?? {};
1054
+ return `[${label}] ${run.status} evaluated=${counts.evaluated ?? 0}/${counts.total ?? run.caseCount ?? "?"} passed=${counts.passed ?? 0} notPassed=${counts.notPassed ?? 0} failed=${counts.evaluationFailed ?? 0}`;
1055
+ }
1056
+
1057
+ async function waitEvaluationRun(agentId, runId, values, label = runId) {
1058
+ let lastProgress;
1059
+ try {
1060
+ return await poll(
1061
+ (deadline) => request(evaluationRunPath(agentId, runId), { deadline, values }),
1062
+ (result) => ["COMPLETED", "STOPPED", "FAILED"].includes(result.run.status),
1063
+ positiveSeconds(values.timeout),
1064
+ values.follow
1065
+ ? (result) => {
1066
+ const progress = evaluationProgress(result.run, label);
1067
+ if (progress !== lastProgress) process.stderr.write(`[decidex] ${progress}\n`);
1068
+ lastProgress = progress;
1069
+ }
1070
+ : undefined,
1071
+ );
1072
+ } catch (error) {
1073
+ error.message = `${error.message}; agentId=${agentId}; runId=${runId}; 可运行 decidex eval stop ${agentId} ${runId}`;
1074
+ throw error;
1075
+ }
1076
+ }
1077
+
1078
+ function evaluationRunTargets(value) {
1079
+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some((key) => key !== "runs") || !Array.isArray(value.runs) || value.runs.length === 0) {
1080
+ throw new Error("watch 文件必须是包含非空 runs 数组的 JSON 对象");
1081
+ }
1082
+ const seen = new Set();
1083
+ return value.runs.map((item, index) => {
1084
+ if (!item || typeof item !== "object" || Array.isArray(item)) throw new Error(`runs[${index}] 必须是对象`);
1085
+ if (Object.keys(item).some((key) => !new Set(["label", "agentId", "runId"]).has(key))) throw new Error(`runs[${index}] 包含不支持的字段`);
1086
+ for (const field of ["agentId", "runId"]) if (typeof item[field] !== "string" || !item[field].trim()) throw new Error(`runs[${index}].${field} 不能为空`);
1087
+ if (item.label !== undefined && (typeof item.label !== "string" || !item.label.trim())) throw new Error(`runs[${index}].label 不能为空`);
1088
+ const key = `${item.agentId}\0${item.runId}`;
1089
+ if (seen.has(key)) throw new Error(`watch 文件包含重复 Run:${item.agentId}/${item.runId}`);
1090
+ seen.add(key);
1091
+ return { label: item.label ?? item.runId, agentId: item.agentId, runId: item.runId };
1092
+ });
1093
+ }
1094
+
1095
+ function positiveInteger(value, name) {
1096
+ const number = Number(value);
1097
+ if (!Number.isInteger(number) || number <= 0) throw new Error(`${name} 必须是正整数`);
1098
+ return number;
1099
+ }
1100
+
1101
+ async function evalRunsCommand(args) {
1102
+ const { values, positionals } = commandArgs(args, {
1103
+ cursor: { type: "string" },
1104
+ follow: { type: "boolean", default: false },
1105
+ output: { type: "string" },
1106
+ parallel: { type: "string", default: "3" },
1107
+ timeout: { type: "string", default: "1800" },
1108
+ });
1109
+ const [action, agentId, runId] = positionals;
1110
+ if (action === "list") {
1111
+ expectPositionals(positionals, 2, "eval runs list AGENT_ID");
1112
+ const query = values.cursor ? `?cursor=${encodeURIComponent(values.cursor)}` : "";
1113
+ return print(await request(`/api/personal-agents/${encodeURIComponent(agentId)}/evaluation-runs${query}`, { values }));
1114
+ }
1115
+ if (action === "get") {
1116
+ expectPositionals(positionals, 3, "eval runs get AGENT_ID RUN_ID");
1117
+ return print(await request(evaluationRunPath(agentId, runId), { values }));
1118
+ }
1119
+ if (action === "wait") {
1120
+ expectPositionals(positionals, 3, "eval runs wait AGENT_ID RUN_ID");
1121
+ const result = await waitEvaluationRun(agentId, runId, values);
1122
+ await emit({ schemaVersion: 1, agentId, runId, run: result.run }, values.output);
1123
+ if (result.run.status !== "COMPLETED") process.exitCode = 1;
1124
+ return;
1125
+ }
1126
+ if (action === "watch") {
1127
+ expectPositionals(positionals, 2, "eval runs watch FILE");
1128
+ const targets = evaluationRunTargets(await readJsonFile(required(agentId, "FILE")));
1129
+ const parallel = Math.min(positiveInteger(values.parallel, "parallel"), targets.length);
1130
+ const results = new Array(targets.length);
1131
+ let nextIndex = 0;
1132
+ async function worker() {
1133
+ while (nextIndex < targets.length) {
1134
+ const index = nextIndex++;
1135
+ const target = targets[index];
1136
+ try {
1137
+ const result = await waitEvaluationRun(target.agentId, target.runId, values, target.label);
1138
+ results[index] = { ...target, run: result.run, monitorError: null };
1139
+ } catch (error) {
1140
+ results[index] = { ...target, run: null, monitorError: error.message };
1141
+ }
1142
+ }
1143
+ }
1144
+ await Promise.all(Array.from({ length: parallel }, worker));
1145
+ const summary = {
1146
+ total: results.length,
1147
+ completed: results.filter((item) => item.run?.status === "COMPLETED").length,
1148
+ stopped: results.filter((item) => item.run?.status === "STOPPED").length,
1149
+ failed: results.filter((item) => item.run?.status === "FAILED").length,
1150
+ errors: results.filter((item) => item.monitorError).length,
1151
+ };
1152
+ await emit({ schemaVersion: 1, summary, runs: results }, values.output);
1153
+ if (summary.completed !== summary.total) process.exitCode = 1;
1154
+ return;
1155
+ }
1156
+ throw new Error(`未知 eval runs 命令:${action || "(空)"}`);
1157
+ }
1158
+
1038
1159
  async function evalCommand(action, args) {
1039
1160
  if (action === "cases") return evalCasesCommand(args);
1161
+ if (action === "runs") return evalRunsCommand(args);
1040
1162
  const { values, positionals } = commandArgs(args, {
1041
1163
  cases: { type: "string" },
1164
+ follow: { type: "boolean", default: false },
1042
1165
  output: { type: "string" },
1043
1166
  timeout: { type: "string", default: "1800" },
1044
1167
  wait: { type: "boolean", default: false },
@@ -1058,19 +1181,8 @@ async function evalCommand(action, args) {
1058
1181
  }
1059
1182
  const started = await request(`/api/personal-agents/${agentId}/evaluation-runs`, { method: "POST", body: evaluationRunSpec(body), values });
1060
1183
  if (!values.wait) return emit(started, values.output);
1061
- const runId = encodeURIComponent(started.run.id);
1062
1184
  process.stderr.write(`[decidex] evaluation run=${started.run.id}\n`);
1063
- let finished;
1064
- try {
1065
- finished = await poll(
1066
- (deadline) => request(`/api/personal-agents/${agentId}/evaluation-runs/${runId}`, { deadline, values }),
1067
- (result) => ["COMPLETED", "STOPPED", "FAILED"].includes(result.run.status),
1068
- positiveSeconds(values.timeout),
1069
- );
1070
- } catch (error) {
1071
- error.message = `${error.message}; runId=${started.run.id}; 可运行 decidex eval stop ${positionals[0]} ${started.run.id}`;
1072
- throw error;
1073
- }
1185
+ const finished = await waitEvaluationRun(positionals[0], started.run.id, values);
1074
1186
  await emit(finished, values.output);
1075
1187
  if (finished.run.status !== "COMPLETED") process.exitCode = 1;
1076
1188
  }