@guandata/decidex 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/package.json +1 -1
- package/src/cli.js +126 -14
package/README.md
CHANGED
|
@@ -158,6 +158,28 @@ decidex eval stop AGENT_ID RUN_ID
|
|
|
158
158
|
|
|
159
159
|
CLI 按最新 `origin/main` 校验该文件:只接受 `caseIds`、`targetModelSelection` 和 `judgeModelSelection`。使用 `--cases all` 时由 CLI 获取并覆盖 `caseIds`;Agent 没有评测案例时会在本地停止。
|
|
160
160
|
|
|
161
|
+
查询、接管或批量监控已经启动的评测:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
decidex eval runs list AGENT_ID
|
|
165
|
+
decidex eval runs get AGENT_ID RUN_ID
|
|
166
|
+
decidex eval runs wait AGENT_ID RUN_ID --follow --output result.json
|
|
167
|
+
decidex eval runs watch evaluation-runs.json --parallel 3 --follow --output summary.json
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`evaluation-runs.json`:
|
|
171
|
+
|
|
172
|
+
```json
|
|
173
|
+
{
|
|
174
|
+
"runs": [
|
|
175
|
+
{ "label": "A/S07", "agentId": "AGENT_ID", "runId": "RUN_ID" },
|
|
176
|
+
{ "label": "B/S07", "agentId": "AGENT_ID", "runId": "RUN_ID" }
|
|
177
|
+
]
|
|
178
|
+
}
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
`watch` 只在状态或计数变化时向 stderr 输出进度,最终 JSON 保留每个 Run 的完整 Case 结果;任一 Run 失败、停止或监控出错时退出码为 1。
|
|
182
|
+
|
|
161
183
|
## 组织
|
|
162
184
|
|
|
163
185
|
```bash
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -52,7 +52,11 @@ Usage:
|
|
|
52
52
|
decidex eval cases export AGENT_ID FILE
|
|
53
53
|
decidex eval cases plan AGENT_ID FILE [--prune]
|
|
54
54
|
decidex eval cases apply AGENT_ID FILE [--prune]
|
|
55
|
-
decidex eval run AGENT_ID FILE [--cases all] [--wait] [--output FILE] [--timeout SECONDS]
|
|
55
|
+
decidex eval run AGENT_ID FILE [--cases all] [--wait] [--follow] [--output FILE] [--timeout SECONDS]
|
|
56
|
+
decidex eval runs list AGENT_ID [--cursor CURSOR]
|
|
57
|
+
decidex eval runs get AGENT_ID RUN_ID
|
|
58
|
+
decidex eval runs wait AGENT_ID RUN_ID [--follow] [--output FILE] [--timeout SECONDS]
|
|
59
|
+
decidex eval runs watch FILE [--parallel N] [--follow] [--output FILE] [--timeout SECONDS]
|
|
56
60
|
decidex eval stop AGENT_ID RUN_ID
|
|
57
61
|
|
|
58
62
|
decidex ontology list
|
|
@@ -111,13 +115,19 @@ Runtime Pod termination details require backend support.`,
|
|
|
111
115
|
decidex skill delete ID`,
|
|
112
116
|
eval: `Usage:
|
|
113
117
|
decidex eval cases list|export|plan|apply ...
|
|
114
|
-
decidex eval run AGENT_ID FILE [--cases all] [--wait] [--output FILE]
|
|
118
|
+
decidex eval run AGENT_ID FILE [--cases all] [--wait] [--follow] [--output FILE]
|
|
119
|
+
decidex eval runs list|get|wait|watch ...
|
|
115
120
|
decidex eval stop AGENT_ID RUN_ID`,
|
|
116
121
|
"eval cases": `Usage:
|
|
117
122
|
decidex eval cases list AGENT_ID
|
|
118
123
|
decidex eval cases export AGENT_ID FILE
|
|
119
124
|
decidex eval cases plan AGENT_ID FILE [--prune]
|
|
120
125
|
decidex eval cases apply AGENT_ID FILE [--prune]`,
|
|
126
|
+
"eval runs": `Usage:
|
|
127
|
+
decidex eval runs list AGENT_ID [--cursor CURSOR]
|
|
128
|
+
decidex eval runs get AGENT_ID RUN_ID
|
|
129
|
+
decidex eval runs wait AGENT_ID RUN_ID [--follow] [--output FILE] [--timeout SECONDS]
|
|
130
|
+
decidex eval runs watch FILE [--parallel N] [--follow] [--output FILE] [--timeout SECONDS]`,
|
|
121
131
|
ontology: `Usage:
|
|
122
132
|
decidex ontology list|get|export|plan|apply ...`,
|
|
123
133
|
api: `Usage:
|
|
@@ -1035,10 +1045,123 @@ function evaluationRunSpec(value) {
|
|
|
1035
1045
|
return value;
|
|
1036
1046
|
}
|
|
1037
1047
|
|
|
1048
|
+
function evaluationRunPath(agentId, runId) {
|
|
1049
|
+
return `/api/personal-agents/${encodeURIComponent(agentId)}/evaluation-runs/${encodeURIComponent(runId)}`;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
function evaluationProgress(run, label) {
|
|
1053
|
+
const counts = run.counts ?? {};
|
|
1054
|
+
return `[${label}] ${run.status} evaluated=${counts.evaluated ?? 0}/${counts.total ?? run.caseCount ?? "?"} passed=${counts.passed ?? 0} notPassed=${counts.notPassed ?? 0} failed=${counts.evaluationFailed ?? 0}`;
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
async function waitEvaluationRun(agentId, runId, values, label = runId) {
|
|
1058
|
+
let lastProgress;
|
|
1059
|
+
try {
|
|
1060
|
+
return await poll(
|
|
1061
|
+
(deadline) => request(evaluationRunPath(agentId, runId), { deadline, values }),
|
|
1062
|
+
(result) => ["COMPLETED", "STOPPED", "FAILED"].includes(result.run.status),
|
|
1063
|
+
positiveSeconds(values.timeout),
|
|
1064
|
+
values.follow
|
|
1065
|
+
? (result) => {
|
|
1066
|
+
const progress = evaluationProgress(result.run, label);
|
|
1067
|
+
if (progress !== lastProgress) process.stderr.write(`[decidex] ${progress}\n`);
|
|
1068
|
+
lastProgress = progress;
|
|
1069
|
+
}
|
|
1070
|
+
: undefined,
|
|
1071
|
+
);
|
|
1072
|
+
} catch (error) {
|
|
1073
|
+
error.message = `${error.message}; agentId=${agentId}; runId=${runId}; 可运行 decidex eval stop ${agentId} ${runId}`;
|
|
1074
|
+
throw error;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
function evaluationRunTargets(value) {
|
|
1079
|
+
if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some((key) => key !== "runs") || !Array.isArray(value.runs) || value.runs.length === 0) {
|
|
1080
|
+
throw new Error("watch 文件必须是包含非空 runs 数组的 JSON 对象");
|
|
1081
|
+
}
|
|
1082
|
+
const seen = new Set();
|
|
1083
|
+
return value.runs.map((item, index) => {
|
|
1084
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) throw new Error(`runs[${index}] 必须是对象`);
|
|
1085
|
+
if (Object.keys(item).some((key) => !new Set(["label", "agentId", "runId"]).has(key))) throw new Error(`runs[${index}] 包含不支持的字段`);
|
|
1086
|
+
for (const field of ["agentId", "runId"]) if (typeof item[field] !== "string" || !item[field].trim()) throw new Error(`runs[${index}].${field} 不能为空`);
|
|
1087
|
+
if (item.label !== undefined && (typeof item.label !== "string" || !item.label.trim())) throw new Error(`runs[${index}].label 不能为空`);
|
|
1088
|
+
const key = `${item.agentId}\0${item.runId}`;
|
|
1089
|
+
if (seen.has(key)) throw new Error(`watch 文件包含重复 Run:${item.agentId}/${item.runId}`);
|
|
1090
|
+
seen.add(key);
|
|
1091
|
+
return { label: item.label ?? item.runId, agentId: item.agentId, runId: item.runId };
|
|
1092
|
+
});
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
function positiveInteger(value, name) {
|
|
1096
|
+
const number = Number(value);
|
|
1097
|
+
if (!Number.isInteger(number) || number <= 0) throw new Error(`${name} 必须是正整数`);
|
|
1098
|
+
return number;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
async function evalRunsCommand(args) {
|
|
1102
|
+
const { values, positionals } = commandArgs(args, {
|
|
1103
|
+
cursor: { type: "string" },
|
|
1104
|
+
follow: { type: "boolean", default: false },
|
|
1105
|
+
output: { type: "string" },
|
|
1106
|
+
parallel: { type: "string", default: "3" },
|
|
1107
|
+
timeout: { type: "string", default: "1800" },
|
|
1108
|
+
});
|
|
1109
|
+
const [action, agentId, runId] = positionals;
|
|
1110
|
+
if (action === "list") {
|
|
1111
|
+
expectPositionals(positionals, 2, "eval runs list AGENT_ID");
|
|
1112
|
+
const query = values.cursor ? `?cursor=${encodeURIComponent(values.cursor)}` : "";
|
|
1113
|
+
return print(await request(`/api/personal-agents/${encodeURIComponent(agentId)}/evaluation-runs${query}`, { values }));
|
|
1114
|
+
}
|
|
1115
|
+
if (action === "get") {
|
|
1116
|
+
expectPositionals(positionals, 3, "eval runs get AGENT_ID RUN_ID");
|
|
1117
|
+
return print(await request(evaluationRunPath(agentId, runId), { values }));
|
|
1118
|
+
}
|
|
1119
|
+
if (action === "wait") {
|
|
1120
|
+
expectPositionals(positionals, 3, "eval runs wait AGENT_ID RUN_ID");
|
|
1121
|
+
const result = await waitEvaluationRun(agentId, runId, values);
|
|
1122
|
+
await emit({ schemaVersion: 1, agentId, runId, run: result.run }, values.output);
|
|
1123
|
+
if (result.run.status !== "COMPLETED") process.exitCode = 1;
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
if (action === "watch") {
|
|
1127
|
+
expectPositionals(positionals, 2, "eval runs watch FILE");
|
|
1128
|
+
const targets = evaluationRunTargets(await readJsonFile(required(agentId, "FILE")));
|
|
1129
|
+
const parallel = Math.min(positiveInteger(values.parallel, "parallel"), targets.length);
|
|
1130
|
+
const results = new Array(targets.length);
|
|
1131
|
+
let nextIndex = 0;
|
|
1132
|
+
async function worker() {
|
|
1133
|
+
while (nextIndex < targets.length) {
|
|
1134
|
+
const index = nextIndex++;
|
|
1135
|
+
const target = targets[index];
|
|
1136
|
+
try {
|
|
1137
|
+
const result = await waitEvaluationRun(target.agentId, target.runId, values, target.label);
|
|
1138
|
+
results[index] = { ...target, run: result.run, monitorError: null };
|
|
1139
|
+
} catch (error) {
|
|
1140
|
+
results[index] = { ...target, run: null, monitorError: error.message };
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
await Promise.all(Array.from({ length: parallel }, worker));
|
|
1145
|
+
const summary = {
|
|
1146
|
+
total: results.length,
|
|
1147
|
+
completed: results.filter((item) => item.run?.status === "COMPLETED").length,
|
|
1148
|
+
stopped: results.filter((item) => item.run?.status === "STOPPED").length,
|
|
1149
|
+
failed: results.filter((item) => item.run?.status === "FAILED").length,
|
|
1150
|
+
errors: results.filter((item) => item.monitorError).length,
|
|
1151
|
+
};
|
|
1152
|
+
await emit({ schemaVersion: 1, summary, runs: results }, values.output);
|
|
1153
|
+
if (summary.completed !== summary.total) process.exitCode = 1;
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
throw new Error(`未知 eval runs 命令:${action || "(空)"}`);
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1038
1159
|
async function evalCommand(action, args) {
|
|
1039
1160
|
if (action === "cases") return evalCasesCommand(args);
|
|
1161
|
+
if (action === "runs") return evalRunsCommand(args);
|
|
1040
1162
|
const { values, positionals } = commandArgs(args, {
|
|
1041
1163
|
cases: { type: "string" },
|
|
1164
|
+
follow: { type: "boolean", default: false },
|
|
1042
1165
|
output: { type: "string" },
|
|
1043
1166
|
timeout: { type: "string", default: "1800" },
|
|
1044
1167
|
wait: { type: "boolean", default: false },
|
|
@@ -1058,19 +1181,8 @@ async function evalCommand(action, args) {
|
|
|
1058
1181
|
}
|
|
1059
1182
|
const started = await request(`/api/personal-agents/${agentId}/evaluation-runs`, { method: "POST", body: evaluationRunSpec(body), values });
|
|
1060
1183
|
if (!values.wait) return emit(started, values.output);
|
|
1061
|
-
const runId = encodeURIComponent(started.run.id);
|
|
1062
1184
|
process.stderr.write(`[decidex] evaluation run=${started.run.id}\n`);
|
|
1063
|
-
|
|
1064
|
-
try {
|
|
1065
|
-
finished = await poll(
|
|
1066
|
-
(deadline) => request(`/api/personal-agents/${agentId}/evaluation-runs/${runId}`, { deadline, values }),
|
|
1067
|
-
(result) => ["COMPLETED", "STOPPED", "FAILED"].includes(result.run.status),
|
|
1068
|
-
positiveSeconds(values.timeout),
|
|
1069
|
-
);
|
|
1070
|
-
} catch (error) {
|
|
1071
|
-
error.message = `${error.message}; runId=${started.run.id}; 可运行 decidex eval stop ${positionals[0]} ${started.run.id}`;
|
|
1072
|
-
throw error;
|
|
1073
|
-
}
|
|
1185
|
+
const finished = await waitEvaluationRun(positionals[0], started.run.id, values);
|
|
1074
1186
|
await emit(finished, values.output);
|
|
1075
1187
|
if (finished.run.status !== "COMPLETED") process.exitCode = 1;
|
|
1076
1188
|
}
|