@fuxi-ailabs/dsh-eval-ux 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -2
- package/dist/client.js +50542 -0
- package/dist/index.js +1182 -0
- package/package.json +68 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,1182 @@
|
|
|
1
|
+
// src/config.ts
|
|
2
|
+
var defaultConfig = {
|
|
3
|
+
manualSkillPaths: []
|
|
4
|
+
};
|
|
5
|
+
function expandHome(value, home = process.env.HOME ?? "~") {
|
|
6
|
+
if (value === "~") return home;
|
|
7
|
+
if (value.startsWith("~/")) return `${home}${value.slice(1)}`;
|
|
8
|
+
return value;
|
|
9
|
+
}
|
|
10
|
+
function normalizeConfig(input) {
|
|
11
|
+
const paths = Array.isArray(input?.manualSkillPaths) ? input.manualSkillPaths.filter((item) => typeof item === "string" && item.trim().length > 0) : [];
|
|
12
|
+
return {
|
|
13
|
+
manualSkillPaths: [...new Set(paths.map((item) => expandHome(item.trim())))]
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// src/host/endpoint.ts
|
|
18
|
+
import { readFile as readFile4, readdir as readdir2, rm as rm2, stat as stat4 } from "node:fs/promises";
|
|
19
|
+
import os3 from "node:os";
|
|
20
|
+
import path4 from "node:path";
|
|
21
|
+
|
|
22
|
+
// src/host/catalog.ts
|
|
23
|
+
import { createHash } from "node:crypto";
|
|
24
|
+
import { readFile as readFile2, readdir, stat as stat2 } from "node:fs/promises";
|
|
25
|
+
import path2 from "node:path";
|
|
26
|
+
|
|
27
|
+
// src/host/manual-paths.ts
|
|
28
|
+
import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
|
|
29
|
+
import os from "node:os";
|
|
30
|
+
import path from "node:path";
|
|
31
|
+
function storeFile() {
|
|
32
|
+
return path.join(process.env.DSH_HOME ?? path.join(os.homedir(), ".dsh"), "dsh-eval-ux", "manual-paths.json");
|
|
33
|
+
}
|
|
34
|
+
function legacyStoreFile() {
|
|
35
|
+
return path.join(process.env.DSH_HOME ?? path.join(os.homedir(), ".dsh"), "dsh-sift-workbench", "manual-paths.json");
|
|
36
|
+
}
|
|
37
|
+
async function readPaths(file) {
|
|
38
|
+
try {
|
|
39
|
+
const parsed = JSON.parse(await readFile(file, "utf8"));
|
|
40
|
+
return Array.isArray(parsed.paths) ? parsed.paths.filter((item) => typeof item === "string" && item.trim().length > 0) : [];
|
|
41
|
+
} catch {
|
|
42
|
+
return void 0;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
async function readUserPaths() {
|
|
46
|
+
const current = await readPaths(storeFile());
|
|
47
|
+
if (current) return current;
|
|
48
|
+
const legacy = await readPaths(legacyStoreFile());
|
|
49
|
+
if (!legacy?.length) return legacy ?? [];
|
|
50
|
+
try {
|
|
51
|
+
await writeUserPaths(legacy);
|
|
52
|
+
} catch {
|
|
53
|
+
}
|
|
54
|
+
return legacy;
|
|
55
|
+
}
|
|
56
|
+
async function writeUserPaths(paths) {
|
|
57
|
+
const file = storeFile();
|
|
58
|
+
await mkdir(path.dirname(file), { recursive: true });
|
|
59
|
+
const temp = `${file}.${process.pid}.tmp`;
|
|
60
|
+
await writeFile(temp, JSON.stringify({ paths }, null, 2), "utf8");
|
|
61
|
+
await rename(temp, file);
|
|
62
|
+
}
|
|
63
|
+
function unique(paths) {
|
|
64
|
+
return [...new Set(paths)];
|
|
65
|
+
}
|
|
66
|
+
async function readManualPathEntries(config) {
|
|
67
|
+
const configPaths = config.manualSkillPaths.map((item) => path.resolve(expandHome(item)));
|
|
68
|
+
const userPaths = (await readUserPaths()).map((item) => path.resolve(expandHome(item)));
|
|
69
|
+
return [
|
|
70
|
+
...unique(configPaths).map((item) => ({ path: item, source: "config" })),
|
|
71
|
+
...unique(userPaths.filter((item) => !configPaths.includes(item))).map((item) => ({ path: item, source: "user" }))
|
|
72
|
+
];
|
|
73
|
+
}
|
|
74
|
+
async function addManualPath(rawPath) {
|
|
75
|
+
const value = rawPath.trim();
|
|
76
|
+
if (!value) throw new Error("\u8DEF\u5F84\u4E0D\u80FD\u4E3A\u7A7A");
|
|
77
|
+
const resolved = path.resolve(expandHome(value));
|
|
78
|
+
const info = await stat(resolved).catch(() => void 0);
|
|
79
|
+
if (!info) throw new Error(`\u8DEF\u5F84\u4E0D\u5B58\u5728: ${resolved}`);
|
|
80
|
+
if (!info.isDirectory() && !info.isFile()) throw new Error(`\u4E0D\u662F\u6587\u4EF6\u6216\u76EE\u5F55: ${resolved}`);
|
|
81
|
+
if (info.isFile() && !resolved.toLowerCase().endsWith(".md")) throw new Error("\u6587\u4EF6\u8DEF\u5F84\u5FC5\u987B\u662F Markdown");
|
|
82
|
+
const userPaths = await readUserPaths();
|
|
83
|
+
const normalized = unique([...userPaths.map((item) => path.resolve(expandHome(item))), resolved]);
|
|
84
|
+
await writeUserPaths(normalized);
|
|
85
|
+
return normalized.map((item) => ({ path: item, source: "user" }));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// src/host/catalog.ts
|
|
89
|
+
function stableId(source, filePath, name2) {
|
|
90
|
+
return createHash("sha256").update(`${source}\0${filePath ?? ""}\0${name2}`).digest("hex").slice(0, 20);
|
|
91
|
+
}
|
|
92
|
+
function toView(summary) {
|
|
93
|
+
return {
|
|
94
|
+
id: stableId(summary.source || "unknown", summary.path, summary.name),
|
|
95
|
+
name: summary.name,
|
|
96
|
+
description: summary.description || "",
|
|
97
|
+
source: summary.source || "unknown",
|
|
98
|
+
provider: summary.provider || "unknown",
|
|
99
|
+
...summary.path ? { path: summary.path } : {},
|
|
100
|
+
...summary.resourceBase?.kind === "directory" && summary.resourceBase.path ? { resourceBasePath: summary.resourceBase.path, resourceBaseKind: "directory" } : summary.resourceBase?.kind ? { resourceBaseKind: summary.resourceBase.kind } : {},
|
|
101
|
+
modelInvocable: summary.invocation?.modelInvocable !== false,
|
|
102
|
+
userInvocable: summary.invocation?.userInvocable !== false
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function uniq(items) {
|
|
106
|
+
const byId = /* @__PURE__ */ new Map();
|
|
107
|
+
for (const item of items) byId.set(item.id, item);
|
|
108
|
+
return [...byId.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
109
|
+
}
|
|
110
|
+
function skillPathKey(skill) {
|
|
111
|
+
const candidate = skill.path ?? (skill.resourceBaseKind === "directory" && skill.resourceBasePath ? path2.join(skill.resourceBasePath, "SKILL.md") : void 0);
|
|
112
|
+
if (!candidate) return void 0;
|
|
113
|
+
const normalized = path2.normalize(path2.resolve(candidate));
|
|
114
|
+
return process.platform === "darwin" || process.platform === "win32" ? normalized.toLowerCase() : normalized;
|
|
115
|
+
}
|
|
116
|
+
function uniqByPath(items) {
|
|
117
|
+
const seen = /* @__PURE__ */ new Set();
|
|
118
|
+
const result = [];
|
|
119
|
+
for (const skill of uniq(items)) {
|
|
120
|
+
const key = skillPathKey(skill);
|
|
121
|
+
if (key && seen.has(key)) continue;
|
|
122
|
+
if (key) seen.add(key);
|
|
123
|
+
result.push(skill);
|
|
124
|
+
}
|
|
125
|
+
return result;
|
|
126
|
+
}
|
|
127
|
+
function isSystem(source) {
|
|
128
|
+
return source === "bundled";
|
|
129
|
+
}
|
|
130
|
+
function isManualSource(source) {
|
|
131
|
+
return source === "custom";
|
|
132
|
+
}
|
|
133
|
+
function isProject(source) {
|
|
134
|
+
return source.startsWith("project-");
|
|
135
|
+
}
|
|
136
|
+
function isUser(source) {
|
|
137
|
+
return !isSystem(source) && !isManualSource(source) && !isProject(source);
|
|
138
|
+
}
|
|
139
|
+
function scalar(frontmatter, key) {
|
|
140
|
+
const match = frontmatter.match(new RegExp(`^${key}\\s*:\\s*(.+)$`, "m"));
|
|
141
|
+
if (!match) return "";
|
|
142
|
+
let value = match[1].trim();
|
|
143
|
+
if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
|
|
144
|
+
value = value.slice(1, -1);
|
|
145
|
+
}
|
|
146
|
+
return value;
|
|
147
|
+
}
|
|
148
|
+
function parseManualSkill(filePath, content) {
|
|
149
|
+
const match = content.match(/^---\s*\n([\s\S]*?)\n---(?:\n|$)/);
|
|
150
|
+
if (!match) return void 0;
|
|
151
|
+
const frontmatter = match[1];
|
|
152
|
+
const name2 = scalar(frontmatter, "name") || path2.basename(path2.dirname(filePath));
|
|
153
|
+
if (!name2) return void 0;
|
|
154
|
+
const description = scalar(frontmatter, "description");
|
|
155
|
+
return {
|
|
156
|
+
id: stableId("manual-path", filePath, name2),
|
|
157
|
+
name: name2,
|
|
158
|
+
description,
|
|
159
|
+
source: "manual-path",
|
|
160
|
+
provider: "configured-path",
|
|
161
|
+
path: filePath,
|
|
162
|
+
resourceBasePath: path2.dirname(filePath),
|
|
163
|
+
resourceBaseKind: "directory",
|
|
164
|
+
modelInvocable: true,
|
|
165
|
+
userInvocable: true
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
async function exists(filePath) {
|
|
169
|
+
try {
|
|
170
|
+
await stat2(filePath);
|
|
171
|
+
return true;
|
|
172
|
+
} catch {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async function skillFilesAt(configuredPath, warnings) {
|
|
177
|
+
let info;
|
|
178
|
+
try {
|
|
179
|
+
info = await stat2(configuredPath);
|
|
180
|
+
} catch {
|
|
181
|
+
warnings.push(`\u624B\u52A8\u8DEF\u5F84\u4E0D\u5B58\u5728: ${configuredPath}`);
|
|
182
|
+
return [];
|
|
183
|
+
}
|
|
184
|
+
if (info.isFile()) return configuredPath.toLowerCase().endsWith(".md") ? [configuredPath] : [];
|
|
185
|
+
if (!info.isDirectory()) return [];
|
|
186
|
+
const direct = path2.join(configuredPath, "SKILL.md");
|
|
187
|
+
if (await exists(direct)) return [direct];
|
|
188
|
+
const files = [];
|
|
189
|
+
let entries = [];
|
|
190
|
+
try {
|
|
191
|
+
entries = await readdir(configuredPath);
|
|
192
|
+
} catch (error) {
|
|
193
|
+
warnings.push(`\u8BFB\u53D6\u624B\u52A8\u8DEF\u5F84\u5931\u8D25 ${configuredPath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
194
|
+
return [];
|
|
195
|
+
}
|
|
196
|
+
for (const entry of entries.sort()) {
|
|
197
|
+
const candidate = path2.join(configuredPath, entry, "SKILL.md");
|
|
198
|
+
if (await exists(candidate)) files.push(candidate);
|
|
199
|
+
}
|
|
200
|
+
return files;
|
|
201
|
+
}
|
|
202
|
+
async function readManualSkills(paths, warnings) {
|
|
203
|
+
const skills = [];
|
|
204
|
+
for (const configuredPath of paths) {
|
|
205
|
+
for (const filePath of await skillFilesAt(configuredPath, warnings)) {
|
|
206
|
+
try {
|
|
207
|
+
const content = await readFile2(filePath, "utf8");
|
|
208
|
+
const skill = parseManualSkill(filePath, content);
|
|
209
|
+
if (skill) skills.push(skill);
|
|
210
|
+
else warnings.push(`\u624B\u52A8 skill \u7F3A\u5C11\u6709\u6548 frontmatter: ${filePath}`);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
warnings.push(`\u8BFB\u53D6\u624B\u52A8 skill \u5931\u8D25 ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return uniq(skills);
|
|
217
|
+
}
|
|
218
|
+
function withTimeout(promise, timeoutMs, message) {
|
|
219
|
+
return new Promise((resolve, reject) => {
|
|
220
|
+
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
221
|
+
promise.then(
|
|
222
|
+
(value) => {
|
|
223
|
+
clearTimeout(timer);
|
|
224
|
+
resolve(value);
|
|
225
|
+
},
|
|
226
|
+
(error) => {
|
|
227
|
+
clearTimeout(timer);
|
|
228
|
+
reject(error);
|
|
229
|
+
}
|
|
230
|
+
);
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
function skillsService(ctx) {
|
|
234
|
+
const skills = ctx.get("skills");
|
|
235
|
+
if (!skills || typeof skills.list !== "function") {
|
|
236
|
+
throw new Error("\u5BBF\u4E3B\u672A\u6302\u8F7D skills \u670D\u52A1");
|
|
237
|
+
}
|
|
238
|
+
return skills;
|
|
239
|
+
}
|
|
240
|
+
function currentAgentScope(ctx, warnings) {
|
|
241
|
+
try {
|
|
242
|
+
const agents = ctx.get("agents");
|
|
243
|
+
return agents && typeof agents.list === "function" ? agents.list()[0] : void 0;
|
|
244
|
+
} catch (error) {
|
|
245
|
+
warnings.push(`\u8BFB\u53D6 live agent \u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
246
|
+
return void 0;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async function currentCatalogSkills(skills, scope, cwd, warnings) {
|
|
250
|
+
try {
|
|
251
|
+
return await withTimeout(
|
|
252
|
+
skills.list({ cwd, ...scope ? { scope } : {} }),
|
|
253
|
+
4e3,
|
|
254
|
+
"\u8BFB\u53D6\u5F53\u524D skill catalog \u8D85\u65F6"
|
|
255
|
+
);
|
|
256
|
+
} catch (error) {
|
|
257
|
+
warnings.push(`\u8BFB\u53D6\u5F53\u524D skill catalog \u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
258
|
+
return [];
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async function buildSkillCatalog(ctx, config) {
|
|
262
|
+
const fetchedAt = Date.now();
|
|
263
|
+
const warnings = [];
|
|
264
|
+
let skills;
|
|
265
|
+
try {
|
|
266
|
+
skills = skillsService(ctx);
|
|
267
|
+
} catch (error) {
|
|
268
|
+
const manualPaths2 = await readManualPathEntries(config);
|
|
269
|
+
const manual2 = await readManualSkills(manualPaths2.map((entry) => entry.path), warnings);
|
|
270
|
+
return {
|
|
271
|
+
ok: false,
|
|
272
|
+
error: error instanceof Error ? error.message : String(error),
|
|
273
|
+
warnings,
|
|
274
|
+
system: [],
|
|
275
|
+
user: [],
|
|
276
|
+
manual: manual2,
|
|
277
|
+
manualPaths: manualPaths2,
|
|
278
|
+
workspaces: [],
|
|
279
|
+
fetchedAt
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
let workspaces = [];
|
|
283
|
+
try {
|
|
284
|
+
const registry = ctx.get("workspaceRegistry");
|
|
285
|
+
workspaces = registry && typeof registry.list === "function" ? registry.list() : [];
|
|
286
|
+
} catch (error) {
|
|
287
|
+
warnings.push(`\u8BFB\u53D6 workspace \u5217\u8868\u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
288
|
+
}
|
|
289
|
+
const scope = currentAgentScope(ctx, warnings);
|
|
290
|
+
const globalCwd = workspaces[0]?.path ?? process.cwd();
|
|
291
|
+
const summaries = await currentCatalogSkills(skills, scope, globalCwd, warnings);
|
|
292
|
+
const views = summaries.map(toView);
|
|
293
|
+
const workspaceGroups = await Promise.all(workspaces.map(async (workspace) => {
|
|
294
|
+
const base = {
|
|
295
|
+
id: `workspace:${workspace.id}`,
|
|
296
|
+
kind: "workspace",
|
|
297
|
+
title: workspace.title || workspace.path,
|
|
298
|
+
workspaceId: workspace.id,
|
|
299
|
+
skills: []
|
|
300
|
+
};
|
|
301
|
+
try {
|
|
302
|
+
const listed = await withTimeout(
|
|
303
|
+
skills.list({ cwd: workspace.path, ...scope ? { scope } : {} }),
|
|
304
|
+
4e3,
|
|
305
|
+
`\u8BFB\u53D6\u5DE5\u4F5C\u533A ${workspace.title || workspace.path} \u8D85\u65F6`
|
|
306
|
+
);
|
|
307
|
+
return {
|
|
308
|
+
...base,
|
|
309
|
+
skills: uniq(listed.filter((skill) => isProject(skill.source)).map(toView))
|
|
310
|
+
};
|
|
311
|
+
} catch (error) {
|
|
312
|
+
warnings.push(`\u8BFB\u53D6\u5DE5\u4F5C\u533A ${workspace.title || workspace.path} \u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
313
|
+
return base;
|
|
314
|
+
}
|
|
315
|
+
}));
|
|
316
|
+
const manualPaths = await readManualPathEntries(config);
|
|
317
|
+
const knownSkills = uniq([
|
|
318
|
+
...views.filter((skill) => !isManualSource(skill.source)),
|
|
319
|
+
...workspaceGroups.flatMap((group) => group.skills)
|
|
320
|
+
]);
|
|
321
|
+
const knownPaths = new Set(knownSkills.map(skillPathKey).filter((key) => Boolean(key)));
|
|
322
|
+
const manual = uniqByPath([
|
|
323
|
+
...views.filter((skill) => isManualSource(skill.source)),
|
|
324
|
+
...await readManualSkills(manualPaths.map((entry) => entry.path), warnings)
|
|
325
|
+
]).filter((skill) => {
|
|
326
|
+
const key = skillPathKey(skill);
|
|
327
|
+
return !key || !knownPaths.has(key);
|
|
328
|
+
});
|
|
329
|
+
return {
|
|
330
|
+
ok: true,
|
|
331
|
+
warnings,
|
|
332
|
+
system: uniq(views.filter((skill) => isSystem(skill.source))),
|
|
333
|
+
user: uniq(views.filter((skill) => isUser(skill.source))),
|
|
334
|
+
manual,
|
|
335
|
+
manualPaths,
|
|
336
|
+
workspaces: workspaceGroups,
|
|
337
|
+
fetchedAt
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// src/host/eval-suite.ts
|
|
342
|
+
import { readFile as readFile3, rename as rename2, rm, stat as stat3, writeFile as writeFile2 } from "node:fs/promises";
|
|
343
|
+
import os2 from "node:os";
|
|
344
|
+
import path3 from "node:path";
|
|
345
|
+
function isRecord(value) {
|
|
346
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
347
|
+
}
|
|
348
|
+
function stringValue(value) {
|
|
349
|
+
return typeof value === "string" ? value : "";
|
|
350
|
+
}
|
|
351
|
+
function stringArray(value) {
|
|
352
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
353
|
+
}
|
|
354
|
+
function assertionViews(value) {
|
|
355
|
+
if (!Array.isArray(value)) return [];
|
|
356
|
+
return value.filter(isRecord).map((item) => ({
|
|
357
|
+
name: stringValue(item.name),
|
|
358
|
+
type: stringValue(item.type),
|
|
359
|
+
params: isRecord(item.params) ? item.params : {},
|
|
360
|
+
...stringValue(item.tier) ? { tier: stringValue(item.tier) } : {}
|
|
361
|
+
}));
|
|
362
|
+
}
|
|
363
|
+
function evalCaseView(value, index) {
|
|
364
|
+
if (!isRecord(value)) return void 0;
|
|
365
|
+
const prompt = stringValue(value.prompt);
|
|
366
|
+
if (!prompt) return void 0;
|
|
367
|
+
return {
|
|
368
|
+
id: typeof value.id === "number" ? value.id : index + 1,
|
|
369
|
+
name: stringValue(value.name),
|
|
370
|
+
prompt,
|
|
371
|
+
expectedOutput: stringValue(value.expected_output),
|
|
372
|
+
level: stringValue(value.level) || "L1",
|
|
373
|
+
expectations: stringArray(value.expectations),
|
|
374
|
+
assertions: assertionViews(value.assertions),
|
|
375
|
+
rationale: stringValue(value.rationale),
|
|
376
|
+
confidence: typeof value.confidence === "number" ? value.confidence : null,
|
|
377
|
+
flags: stringArray(value.flags)
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
function skillDirectory(skillPath) {
|
|
381
|
+
const resolved = path3.resolve(skillPath);
|
|
382
|
+
return path3.basename(resolved).toLowerCase().endsWith(".md") ? path3.dirname(resolved) : resolved;
|
|
383
|
+
}
|
|
384
|
+
function defaultPhaseOutput(skillPath, phase) {
|
|
385
|
+
const home = process.env.DSH_HOME ?? path3.join(os2.homedir(), ".dsh");
|
|
386
|
+
const leaf = path3.basename(skillPath).toLowerCase().endsWith(".md") ? path3.basename(path3.dirname(skillPath)) : path3.basename(skillPath);
|
|
387
|
+
const safe = leaf.replace(/[^\p{L}\p{N}._-]+/gu, "-").replace(/^-+|-+$/g, "") || "skill";
|
|
388
|
+
return path3.join(home, "eval-runs", safe, phase);
|
|
389
|
+
}
|
|
390
|
+
async function exists2(filePath) {
|
|
391
|
+
return stat3(filePath).then(() => true).catch(() => false);
|
|
392
|
+
}
|
|
393
|
+
async function readJson(filePath) {
|
|
394
|
+
try {
|
|
395
|
+
return JSON.parse(await readFile3(filePath, "utf8"));
|
|
396
|
+
} catch {
|
|
397
|
+
return void 0;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
async function readEvalSuite(skillPath) {
|
|
401
|
+
const resolvedSkill = path3.resolve(skillPath);
|
|
402
|
+
const directory = skillDirectory(resolvedSkill);
|
|
403
|
+
const evalsPath = path3.join(directory, "evals", "evals.json");
|
|
404
|
+
const baselinePath = path3.join(directory, "evals", "baseline.json");
|
|
405
|
+
const baselineMetaPath = path3.join(directory, "evals", "baseline.meta.json");
|
|
406
|
+
const coveragePath = path3.join(directory, "evals", "coverage-report.json");
|
|
407
|
+
const managedEvalOutput = defaultPhaseOutput(resolvedSkill, "eval");
|
|
408
|
+
const legacyEvalOutput = path3.resolve("output");
|
|
409
|
+
const managedBenchmark = path3.join(managedEvalOutput, "benchmark.json");
|
|
410
|
+
const legacyBenchmark = path3.join(legacyEvalOutput, "benchmark.json");
|
|
411
|
+
const evalOutputDir = await exists2(managedBenchmark) ? managedEvalOutput : await exists2(legacyBenchmark) ? legacyEvalOutput : managedEvalOutput;
|
|
412
|
+
const benchmarkPath = path3.join(evalOutputDir, "benchmark.json");
|
|
413
|
+
const gatePath = path3.join(evalOutputDir, "gate.json");
|
|
414
|
+
const reportPath = path3.join(evalOutputDir, "report.html");
|
|
415
|
+
const scorecardPath = path3.join(evalOutputDir, "scorecard.json");
|
|
416
|
+
const triggerOutputDir = defaultPhaseOutput(resolvedSkill, "trigger");
|
|
417
|
+
const triggerReportJson = path3.join(triggerOutputDir, "trigger-report.json");
|
|
418
|
+
const triggerReportHtml = path3.join(triggerOutputDir, "trigger-report.html");
|
|
419
|
+
const artifacts = {
|
|
420
|
+
evalOutputDir,
|
|
421
|
+
benchmarkPath,
|
|
422
|
+
benchmarkExists: await exists2(benchmarkPath),
|
|
423
|
+
gatePath,
|
|
424
|
+
gateExists: await exists2(gatePath),
|
|
425
|
+
reportPath,
|
|
426
|
+
reportExists: await exists2(reportPath),
|
|
427
|
+
scorecardPath,
|
|
428
|
+
scorecardExists: await exists2(scorecardPath),
|
|
429
|
+
triggerOutputDir,
|
|
430
|
+
triggerReportJson,
|
|
431
|
+
triggerReportHtml,
|
|
432
|
+
triggerReportExists: await exists2(triggerReportJson)
|
|
433
|
+
};
|
|
434
|
+
const base = {
|
|
435
|
+
ok: true,
|
|
436
|
+
exists: false,
|
|
437
|
+
skillPath: resolvedSkill,
|
|
438
|
+
evalsPath,
|
|
439
|
+
skillName: "",
|
|
440
|
+
total: 0,
|
|
441
|
+
assertions: 0,
|
|
442
|
+
expectations: 0,
|
|
443
|
+
levels: {},
|
|
444
|
+
flags: [],
|
|
445
|
+
attention: 0,
|
|
446
|
+
cases: [],
|
|
447
|
+
triggerProbes: { total: 0, shouldTrigger: 0, shouldNotTrigger: 0 },
|
|
448
|
+
baseline: { exists: false, path: baselinePath },
|
|
449
|
+
artifacts
|
|
450
|
+
};
|
|
451
|
+
let fileInfo;
|
|
452
|
+
try {
|
|
453
|
+
fileInfo = await stat3(evalsPath);
|
|
454
|
+
} catch {
|
|
455
|
+
return base;
|
|
456
|
+
}
|
|
457
|
+
const data = await readJson(evalsPath);
|
|
458
|
+
if (!isRecord(data)) {
|
|
459
|
+
return { ...base, ok: false, error: "evals.json \u65E0\u6CD5\u89E3\u6790", ...fileInfo ? { mtimeMs: fileInfo.mtimeMs } : {} };
|
|
460
|
+
}
|
|
461
|
+
const rawCases = Array.isArray(data.evals) ? data.evals : [];
|
|
462
|
+
const cases = rawCases.map(evalCaseView).filter((item) => item !== void 0);
|
|
463
|
+
const rawProbes = Array.isArray(data.trigger_probes) ? data.trigger_probes.filter(isRecord) : [];
|
|
464
|
+
const shouldTrigger = rawProbes.filter((probe) => probe.expect_trigger === true).length;
|
|
465
|
+
const triggerProbes = {
|
|
466
|
+
total: rawProbes.length,
|
|
467
|
+
shouldTrigger,
|
|
468
|
+
shouldNotTrigger: rawProbes.length - shouldTrigger
|
|
469
|
+
};
|
|
470
|
+
const levels = {};
|
|
471
|
+
for (const item of cases) levels[item.level] = (levels[item.level] ?? 0) + 1;
|
|
472
|
+
const flags = [...new Set(cases.flatMap((item) => item.flags))];
|
|
473
|
+
const attention = cases.filter((item) => item.flags.some((flag) => flag.startsWith("degraded:")) || item.assertions.length === 0).length;
|
|
474
|
+
const baselineInfo = await readJson(baselineMetaPath);
|
|
475
|
+
const coverageData = await readJson(coveragePath);
|
|
476
|
+
const scorecardData = await readJson(scorecardPath);
|
|
477
|
+
const benchmarkData = await readJson(benchmarkPath);
|
|
478
|
+
const coveragePoints = isRecord(coverageData) && Array.isArray(coverageData.points) ? coverageData.points : void 0;
|
|
479
|
+
const coverageItems = coveragePoints?.filter(isRecord).map((point) => ({
|
|
480
|
+
label: stringValue(point.label),
|
|
481
|
+
sourceQuote: stringValue(point.source_quote),
|
|
482
|
+
coveredBy: Array.isArray(point.covered_by) ? point.covered_by.filter((id) => typeof id === "number" && Number.isInteger(id)) : []
|
|
483
|
+
}));
|
|
484
|
+
const lastEval = isRecord(benchmarkData) ? (() => {
|
|
485
|
+
const metadata = isRecord(benchmarkData.metadata) ? benchmarkData.metadata : {};
|
|
486
|
+
const summary = isRecord(benchmarkData.run_summary) ? benchmarkData.run_summary : {};
|
|
487
|
+
const configurations = Object.entries(summary).filter(([name2, value]) => name2 !== "delta" && isRecord(value)).map(([name2, value]) => {
|
|
488
|
+
const row = value;
|
|
489
|
+
const passRate = isRecord(row.pass_rate) && typeof row.pass_rate.mean === "number" ? row.pass_rate.mean : null;
|
|
490
|
+
const tokens = isRecord(row.tokens) && typeof row.tokens.mean === "number" ? row.tokens.mean : null;
|
|
491
|
+
const durationSeconds = isRecord(row.time_seconds) && typeof row.time_seconds.mean === "number" ? row.time_seconds.mean : null;
|
|
492
|
+
return { name: name2, passRate, tokens, durationSeconds };
|
|
493
|
+
});
|
|
494
|
+
const comparisonRaw = isRecord(benchmarkData.comparison_summary) ? benchmarkData.comparison_summary : void 0;
|
|
495
|
+
const comparison = comparisonRaw ? {
|
|
496
|
+
labelA: stringValue(comparisonRaw.label_a) || "A",
|
|
497
|
+
labelB: stringValue(comparisonRaw.label_b) || "B",
|
|
498
|
+
valid: typeof comparisonRaw.valid === "number" ? comparisonRaw.valid : 0,
|
|
499
|
+
aWins: typeof comparisonRaw.a_wins === "number" ? comparisonRaw.a_wins : 0,
|
|
500
|
+
bWins: typeof comparisonRaw.b_wins === "number" ? comparisonRaw.b_wins : 0,
|
|
501
|
+
ties: typeof comparisonRaw.ties === "number" ? comparisonRaw.ties : 0,
|
|
502
|
+
winRate: typeof comparisonRaw.win_rate === "number" ? comparisonRaw.win_rate : null
|
|
503
|
+
} : void 0;
|
|
504
|
+
return {
|
|
505
|
+
...stringValue(metadata.timestamp) ? { timestamp: stringValue(metadata.timestamp) } : {},
|
|
506
|
+
evalsCount: Array.isArray(metadata.evals_run) ? metadata.evals_run.length : 0,
|
|
507
|
+
...typeof metadata.runs_per_configuration === "number" ? { runsPerConfiguration: metadata.runs_per_configuration } : {},
|
|
508
|
+
configurations,
|
|
509
|
+
...comparison ? { comparison } : {}
|
|
510
|
+
};
|
|
511
|
+
})() : void 0;
|
|
512
|
+
const scorecard = isRecord(scorecardData) ? {
|
|
513
|
+
score: typeof scorecardData.score === "number" ? scorecardData.score : 0,
|
|
514
|
+
maxScore: typeof scorecardData.max_score === "number" ? scorecardData.max_score : 0,
|
|
515
|
+
category: stringValue(scorecardData.category),
|
|
516
|
+
checks: Array.isArray(scorecardData.checks) ? scorecardData.checks.filter(isRecord).map((check) => ({
|
|
517
|
+
name: stringValue(check.name),
|
|
518
|
+
passed: check.passed === true,
|
|
519
|
+
message: stringValue(check.message)
|
|
520
|
+
})) : []
|
|
521
|
+
} : void 0;
|
|
522
|
+
return {
|
|
523
|
+
...base,
|
|
524
|
+
exists: true,
|
|
525
|
+
mtimeMs: fileInfo.mtimeMs,
|
|
526
|
+
skillName: stringValue(data.skill_name),
|
|
527
|
+
total: cases.length,
|
|
528
|
+
assertions: cases.reduce((sum, item) => sum + item.assertions.length, 0),
|
|
529
|
+
expectations: cases.reduce((sum, item) => sum + item.expectations.length, 0),
|
|
530
|
+
levels,
|
|
531
|
+
flags,
|
|
532
|
+
attention,
|
|
533
|
+
cases,
|
|
534
|
+
triggerProbes,
|
|
535
|
+
...scorecard ? { scorecard } : {},
|
|
536
|
+
...lastEval ? { lastEval } : {},
|
|
537
|
+
baseline: {
|
|
538
|
+
exists: await stat3(baselinePath).then(() => true).catch(() => false),
|
|
539
|
+
path: baselinePath,
|
|
540
|
+
...isRecord(baselineInfo) && stringValue(baselineInfo.frozen_at) ? { frozenAt: stringValue(baselineInfo.frozen_at) } : {},
|
|
541
|
+
...isRecord(baselineInfo) && stringValue(baselineInfo.target_skill_version) ? { skillVersion: stringValue(baselineInfo.target_skill_version) } : {}
|
|
542
|
+
},
|
|
543
|
+
...coveragePoints ? {
|
|
544
|
+
coverage: {
|
|
545
|
+
exists: true,
|
|
546
|
+
path: coveragePath,
|
|
547
|
+
points: coverageItems?.length ?? 0,
|
|
548
|
+
gaps: coverageItems?.filter((point) => point.coveredBy.length === 0).length ?? 0,
|
|
549
|
+
items: coverageItems
|
|
550
|
+
}
|
|
551
|
+
} : {}
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
async function deleteEvalSuite(skillPath) {
|
|
555
|
+
const directory = skillDirectory(skillPath);
|
|
556
|
+
const evalsDir = path3.join(directory, "evals");
|
|
557
|
+
const candidates = [
|
|
558
|
+
"evals.json",
|
|
559
|
+
"baseline.json",
|
|
560
|
+
"baseline.meta.json",
|
|
561
|
+
"coverage-report.json",
|
|
562
|
+
"coverage-report.html"
|
|
563
|
+
];
|
|
564
|
+
const deleted = [];
|
|
565
|
+
for (const name2 of candidates) {
|
|
566
|
+
const file = path3.join(evalsDir, name2);
|
|
567
|
+
try {
|
|
568
|
+
await stat3(file);
|
|
569
|
+
await rm(file, { force: true });
|
|
570
|
+
deleted.push(file);
|
|
571
|
+
} catch {
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
return { ok: true, deleted, evalsDir };
|
|
575
|
+
}
|
|
576
|
+
async function deleteEvalCase(skillPath, id) {
|
|
577
|
+
const directory = skillDirectory(skillPath);
|
|
578
|
+
const evalsPath = path3.join(directory, "evals", "evals.json");
|
|
579
|
+
const data = await readJson(evalsPath);
|
|
580
|
+
if (!isRecord(data) || !Array.isArray(data.evals)) {
|
|
581
|
+
return { ok: false, removed: 0, remaining: 0, error: "evals.json \u4E0D\u5B58\u5728\u6216\u65E0\u6CD5\u89E3\u6790" };
|
|
582
|
+
}
|
|
583
|
+
const before = data.evals.length;
|
|
584
|
+
const next = data.evals.filter((item) => !isRecord(item) || item.id !== id);
|
|
585
|
+
const removed = before - next.length;
|
|
586
|
+
if (removed === 0) return { ok: false, removed: 0, remaining: before, error: `\u627E\u4E0D\u5230\u9898\u76EE #${id}` };
|
|
587
|
+
const updated = { ...data, evals: next };
|
|
588
|
+
const temp = `${evalsPath}.${process.pid}.tmp`;
|
|
589
|
+
await writeFile2(temp, JSON.stringify(updated, null, 2), "utf8");
|
|
590
|
+
await rename2(temp, evalsPath);
|
|
591
|
+
return { ok: true, removed, remaining: next.length };
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
// src/host/endpoint.ts
|
|
595
|
+
var SKILL_CATALOG_ENDPOINT_PATH = "/api/dsh-eval-ux/catalog";
|
|
596
|
+
var MANUAL_PATHS_ENDPOINT_PATH = "/api/dsh-eval-ux/manual-paths";
|
|
597
|
+
var OPERATIONS_ENDPOINT_PATH = "/api/dsh-eval-ux/operations";
|
|
598
|
+
var OPERATIONS_STREAM_ENDPOINT_PATH = "/api/dsh-eval-ux/operations/stream";
|
|
599
|
+
var EVAL_SUITE_ENDPOINT_PATH = "/api/dsh-eval-ux/eval-suite";
|
|
600
|
+
var EVAL_CASE_ENDPOINT_PATH = "/api/dsh-eval-ux/eval-case";
|
|
601
|
+
var RUN_HISTORY_ENDPOINT_PATH = "/api/dsh-eval-ux/run-history";
|
|
602
|
+
var REPORT_ENDPOINT_PATH = "/api/dsh-eval-ux/report";
|
|
603
|
+
function json(payload, status = 200) {
|
|
604
|
+
return new Response(JSON.stringify(payload), {
|
|
605
|
+
status,
|
|
606
|
+
headers: {
|
|
607
|
+
"content-type": "application/json; charset=utf-8",
|
|
608
|
+
"cache-control": "no-store",
|
|
609
|
+
"x-content-type-options": "nosniff"
|
|
610
|
+
}
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
function failure(error, warnings = []) {
|
|
614
|
+
return {
|
|
615
|
+
ok: false,
|
|
616
|
+
error: error instanceof Error ? error.message : String(error),
|
|
617
|
+
warnings,
|
|
618
|
+
system: [],
|
|
619
|
+
user: [],
|
|
620
|
+
manual: [],
|
|
621
|
+
manualPaths: [],
|
|
622
|
+
workspaces: [],
|
|
623
|
+
fetchedAt: Date.now()
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
async function handleSkillCatalogRequest(ctx, config, _request) {
|
|
627
|
+
try {
|
|
628
|
+
const payload = await buildSkillCatalog(ctx, config);
|
|
629
|
+
return json(payload, payload.ok ? 200 : 503);
|
|
630
|
+
} catch (error) {
|
|
631
|
+
return json(failure(error), 500);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async function handleManualPathsRequest(ctx, config, request) {
|
|
635
|
+
if (request.method !== "POST") return json(failure("\u53EA\u652F\u6301 POST"), 405);
|
|
636
|
+
try {
|
|
637
|
+
const body = await request.json();
|
|
638
|
+
if (typeof body.path !== "string") throw new Error("path \u5FC5\u987B\u662F\u5B57\u7B26\u4E32");
|
|
639
|
+
await addManualPath(body.path);
|
|
640
|
+
const payload = await buildSkillCatalog(ctx, config);
|
|
641
|
+
return json(payload, payload.ok ? 200 : 503);
|
|
642
|
+
} catch (error) {
|
|
643
|
+
return json(failure(error), 400);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
async function handleReportRequest(request) {
|
|
647
|
+
const url = new URL(request.url);
|
|
648
|
+
const skillPath = url.searchParams.get("skillPath")?.trim();
|
|
649
|
+
if (!skillPath) return json({ ok: false, error: "skillPath \u5FC5\u586B" }, 400);
|
|
650
|
+
const suite = await readEvalSuite(skillPath);
|
|
651
|
+
if (!suite.artifacts.reportExists) return json({ ok: false, error: "report.html \u5C1A\u672A\u751F\u6210" }, 404);
|
|
652
|
+
try {
|
|
653
|
+
const html = await readFile4(suite.artifacts.reportPath, "utf8");
|
|
654
|
+
return new Response(html, {
|
|
655
|
+
status: 200,
|
|
656
|
+
headers: {
|
|
657
|
+
"content-type": "text/html; charset=utf-8",
|
|
658
|
+
"cache-control": "no-store",
|
|
659
|
+
"content-security-policy": "sandbox allow-scripts allow-downloads; default-src 'self' data: blob: 'unsafe-inline' 'unsafe-eval'",
|
|
660
|
+
"x-content-type-options": "nosniff"
|
|
661
|
+
}
|
|
662
|
+
});
|
|
663
|
+
} catch (error) {
|
|
664
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error) }, 500);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
function registerSkillCatalogEndpoints(ctx, config) {
|
|
668
|
+
if (typeof ctx.inject !== "function") return;
|
|
669
|
+
ctx.inject(["connection"], (connectionCtx) => {
|
|
670
|
+
const connection = connectionCtx.get("connection");
|
|
671
|
+
if (!connection?.fetch) return;
|
|
672
|
+
connectionCtx.effect?.(() => {
|
|
673
|
+
const registered = [
|
|
674
|
+
connection.fetch.register({
|
|
675
|
+
path: SKILL_CATALOG_ENDPOINT_PATH,
|
|
676
|
+
methods: ["GET"],
|
|
677
|
+
requestBody: "buffered",
|
|
678
|
+
fetch: (request) => handleSkillCatalogRequest(connectionCtx, config, request)
|
|
679
|
+
}),
|
|
680
|
+
connection.fetch.register({
|
|
681
|
+
path: MANUAL_PATHS_ENDPOINT_PATH,
|
|
682
|
+
methods: ["POST"],
|
|
683
|
+
requestBody: "buffered",
|
|
684
|
+
fetch: (request) => handleManualPathsRequest(connectionCtx, config, request)
|
|
685
|
+
}),
|
|
686
|
+
connection.fetch.register({
|
|
687
|
+
path: OPERATIONS_STREAM_ENDPOINT_PATH,
|
|
688
|
+
methods: ["POST"],
|
|
689
|
+
requestBody: "buffered",
|
|
690
|
+
fetch: (request) => handleOperationStreamRequest(connectionCtx, request)
|
|
691
|
+
}),
|
|
692
|
+
connection.fetch.register({
|
|
693
|
+
path: OPERATIONS_ENDPOINT_PATH,
|
|
694
|
+
methods: ["POST"],
|
|
695
|
+
requestBody: "buffered",
|
|
696
|
+
fetch: (request) => handleOperationRequest(connectionCtx, request)
|
|
697
|
+
}),
|
|
698
|
+
connection.fetch.register({
|
|
699
|
+
path: REPORT_ENDPOINT_PATH,
|
|
700
|
+
methods: ["GET"],
|
|
701
|
+
requestBody: "buffered",
|
|
702
|
+
fetch: (request) => handleReportRequest(request)
|
|
703
|
+
}),
|
|
704
|
+
connection.fetch.register({
|
|
705
|
+
path: RUN_HISTORY_ENDPOINT_PATH,
|
|
706
|
+
methods: ["GET", "DELETE"],
|
|
707
|
+
requestBody: "buffered",
|
|
708
|
+
fetch: (request) => handleRunHistoryRequest(request)
|
|
709
|
+
}),
|
|
710
|
+
connection.fetch.register({
|
|
711
|
+
path: EVAL_SUITE_ENDPOINT_PATH,
|
|
712
|
+
methods: ["GET", "DELETE"],
|
|
713
|
+
requestBody: "buffered",
|
|
714
|
+
fetch: (request) => handleEvalSuiteRequest(request)
|
|
715
|
+
}),
|
|
716
|
+
connection.fetch.register({
|
|
717
|
+
path: EVAL_CASE_ENDPOINT_PATH,
|
|
718
|
+
methods: ["DELETE"],
|
|
719
|
+
requestBody: "buffered",
|
|
720
|
+
fetch: (request) => handleEvalCaseRequest(request)
|
|
721
|
+
})
|
|
722
|
+
];
|
|
723
|
+
return () => {
|
|
724
|
+
for (const route of registered) void route.then((dispose) => dispose());
|
|
725
|
+
};
|
|
726
|
+
}, "dsh-eval-ux: skill catalog endpoints");
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
function stringField(body, name2) {
|
|
730
|
+
const value = body[name2];
|
|
731
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
732
|
+
}
|
|
733
|
+
function numberField(body, name2) {
|
|
734
|
+
const value = body[name2];
|
|
735
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : void 0;
|
|
736
|
+
}
|
|
737
|
+
function boolField(body, name2) {
|
|
738
|
+
return typeof body[name2] === "boolean" ? body[name2] : void 0;
|
|
739
|
+
}
|
|
740
|
+
function skillDirectory2(skillPath) {
|
|
741
|
+
return path4.basename(skillPath).toLowerCase().endsWith(".md") ? path4.dirname(skillPath) : skillPath;
|
|
742
|
+
}
|
|
743
|
+
function deriveEvalsPath(skillPath) {
|
|
744
|
+
return path4.join(skillDirectory2(skillPath), "evals", "evals.json");
|
|
745
|
+
}
|
|
746
|
+
function deriveBenchmarkPath(skillPath) {
|
|
747
|
+
return path4.join(skillDirectory2(skillPath), "evals", "benchmark.json");
|
|
748
|
+
}
|
|
749
|
+
function defaultRunOutput(skillPath) {
|
|
750
|
+
const home = process.env.DSH_HOME ?? path4.join(os3.homedir(), ".dsh");
|
|
751
|
+
const leaf = path4.basename(skillDirectory2(skillPath)).replace(/[^\p{L}\p{N}._-]+/gu, "-").replace(/^-+|-+$/g, "") || "skill";
|
|
752
|
+
return path4.join(home, "eval-runs", leaf, "run");
|
|
753
|
+
}
|
|
754
|
+
function preview(value, max = 1600) {
|
|
755
|
+
const text = typeof value === "string" ? value : JSON.stringify(value, null, 2);
|
|
756
|
+
if (typeof text !== "string") return "";
|
|
757
|
+
return text.length <= max ? text : `${text.slice(0, max)}
|
|
758
|
+
\u2026`;
|
|
759
|
+
}
|
|
760
|
+
async function listRunDirs(outputDir) {
|
|
761
|
+
try {
|
|
762
|
+
const entries = await readdir2(outputDir, { withFileTypes: true });
|
|
763
|
+
const candidates = await Promise.all(entries.filter((entry) => entry.isDirectory() && entry.name.startsWith("run-")).map(async (entry) => {
|
|
764
|
+
const full = path4.join(outputDir, entry.name);
|
|
765
|
+
const info = await stat4(full).catch(() => void 0);
|
|
766
|
+
return info ? { full, mtimeMs: info.mtimeMs } : void 0;
|
|
767
|
+
}));
|
|
768
|
+
return candidates.filter((item) => item !== void 0).sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
769
|
+
} catch {
|
|
770
|
+
return [];
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
async function readRunArtifactSummary(output, preferredRunDir) {
|
|
774
|
+
const outputDir = path4.resolve(output);
|
|
775
|
+
const runDir = preferredRunDir ?? (await listRunDirs(outputDir))[0]?.full;
|
|
776
|
+
if (!runDir) return void 0;
|
|
777
|
+
const runInfo = await stat4(runDir).catch(() => void 0);
|
|
778
|
+
let debug = {};
|
|
779
|
+
try {
|
|
780
|
+
const value = JSON.parse(await readFile4(path4.join(runDir, "run_debug.json"), "utf8"));
|
|
781
|
+
if (value && typeof value === "object" && !Array.isArray(value)) debug = value;
|
|
782
|
+
} catch {
|
|
783
|
+
}
|
|
784
|
+
const transcriptPath = await stat4(path4.join(runDir, "transcript.jsonl")).then(() => path4.join(runDir, "transcript.jsonl")).catch(() => void 0);
|
|
785
|
+
const tools = [];
|
|
786
|
+
let sessionCwd;
|
|
787
|
+
if (transcriptPath) {
|
|
788
|
+
try {
|
|
789
|
+
const lines = (await readFile4(transcriptPath, "utf8")).split("\n");
|
|
790
|
+
const byId = /* @__PURE__ */ new Map();
|
|
791
|
+
for (const line of lines) {
|
|
792
|
+
if (!line.trim()) continue;
|
|
793
|
+
let event;
|
|
794
|
+
try {
|
|
795
|
+
event = JSON.parse(line);
|
|
796
|
+
} catch {
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
if (event.type === "session" && typeof event.cwd === "string") {
|
|
800
|
+
sessionCwd = event.cwd;
|
|
801
|
+
continue;
|
|
802
|
+
}
|
|
803
|
+
if (event.type === "tool_call") {
|
|
804
|
+
const id = typeof event.callId === "string" ? event.callId : `tool-${tools.length + 1}`;
|
|
805
|
+
const tool = {
|
|
806
|
+
id,
|
|
807
|
+
name: typeof event.tool === "string" ? event.tool : "tool",
|
|
808
|
+
status: "running",
|
|
809
|
+
input: preview(event.input),
|
|
810
|
+
result: ""
|
|
811
|
+
};
|
|
812
|
+
tools.push(tool);
|
|
813
|
+
byId.set(id, tool);
|
|
814
|
+
} else if (event.type === "tool_result") {
|
|
815
|
+
const id = typeof event.callId === "string" ? event.callId : "";
|
|
816
|
+
const tool = byId.get(id);
|
|
817
|
+
if (tool) {
|
|
818
|
+
tool.status = typeof event.status === "string" ? event.status : "completed";
|
|
819
|
+
tool.result = preview(event.result);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
} catch {
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
const rawToolCalls = debug.tool_calls && typeof debug.tool_calls === "object" && !Array.isArray(debug.tool_calls) ? Object.entries(debug.tool_calls).filter(([, count]) => typeof count === "number").map(([name2, count]) => ({ name: name2, count })) : [];
|
|
827
|
+
return {
|
|
828
|
+
runId: path4.basename(runDir),
|
|
829
|
+
...runInfo ? { updatedAt: runInfo.mtimeMs } : {},
|
|
830
|
+
outputDir,
|
|
831
|
+
runDir,
|
|
832
|
+
...transcriptPath ? { transcriptPath } : {},
|
|
833
|
+
...sessionCwd ? { sessionCwd } : {},
|
|
834
|
+
...typeof debug.executor === "string" ? { executor: debug.executor } : {},
|
|
835
|
+
...typeof debug.task === "string" ? { task: debug.task } : {},
|
|
836
|
+
...typeof debug.duration_ms === "number" ? { durationMs: debug.duration_ms } : {},
|
|
837
|
+
...typeof debug.total_tokens === "number" ? { totalTokens: debug.total_tokens } : {},
|
|
838
|
+
...typeof debug.num_turns === "number" ? { numTurns: debug.num_turns } : {},
|
|
839
|
+
...typeof debug.stop_reason === "string" ? { stopReason: debug.stop_reason } : {},
|
|
840
|
+
...typeof debug.result_text === "string" ? { resultText: debug.result_text } : {},
|
|
841
|
+
toolCalls: rawToolCalls,
|
|
842
|
+
tools
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
async function readRunHistory(skillPath) {
|
|
846
|
+
const skillRoot = path4.resolve(skillDirectory2(skillPath));
|
|
847
|
+
const managedOutput = defaultRunOutput(skillPath);
|
|
848
|
+
const legacyOutput = path4.resolve("output");
|
|
849
|
+
const candidates = [.../* @__PURE__ */ new Set([managedOutput, legacyOutput])];
|
|
850
|
+
const records = [];
|
|
851
|
+
for (const outputDir of candidates) {
|
|
852
|
+
const dirs = await listRunDirs(outputDir);
|
|
853
|
+
const found = (await Promise.all(dirs.map((dir) => readRunArtifactSummary(outputDir, dir.full)))).filter((record) => record !== void 0);
|
|
854
|
+
if (outputDir === managedOutput) {
|
|
855
|
+
records.push(...found.map((record) => ({
|
|
856
|
+
...record,
|
|
857
|
+
source: "managed",
|
|
858
|
+
historyId: `managed:${record.runId ?? path4.basename(record.runDir ?? "run")}`
|
|
859
|
+
})));
|
|
860
|
+
} else {
|
|
861
|
+
records.push(...found.filter((record) => record.sessionCwd !== void 0 && path4.resolve(record.sessionCwd) === skillRoot).map((record) => ({
|
|
862
|
+
...record,
|
|
863
|
+
source: "legacy",
|
|
864
|
+
historyId: `legacy:${record.runId ?? path4.basename(record.runDir ?? "run")}`
|
|
865
|
+
})));
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
return records.sort((a, b) => (b.updatedAt ?? 0) - (a.updatedAt ?? 0));
|
|
869
|
+
}
|
|
870
|
+
async function handleRunHistoryRequest(request) {
|
|
871
|
+
const url = new URL(request.url);
|
|
872
|
+
const skillPath = url.searchParams.get("skillPath")?.trim();
|
|
873
|
+
if (!skillPath) return json({ ok: false, error: "skillPath \u5FC5\u586B" }, 400);
|
|
874
|
+
const outputDir = defaultRunOutput(skillPath);
|
|
875
|
+
if (request.method === "GET") {
|
|
876
|
+
return json({ ok: true, skillPath, outputDir, records: await readRunHistory(skillPath) });
|
|
877
|
+
}
|
|
878
|
+
if (request.method !== "DELETE") return json({ ok: false, error: "\u53EA\u652F\u6301 GET / DELETE" }, 405);
|
|
879
|
+
const runId = url.searchParams.get("runId")?.trim();
|
|
880
|
+
const source = url.searchParams.get("source") === "legacy" ? "legacy" : "managed";
|
|
881
|
+
try {
|
|
882
|
+
if (runId) {
|
|
883
|
+
if (!/^run-[A-Za-z0-9._-]+$/.test(runId)) return json({ ok: false, error: "runId \u975E\u6CD5" }, 400);
|
|
884
|
+
const targetRoot = source === "legacy" ? path4.resolve("output") : outputDir;
|
|
885
|
+
await rm2(path4.join(targetRoot, runId), { recursive: true, force: true });
|
|
886
|
+
} else {
|
|
887
|
+
const legacyRecords = (await readRunHistory(skillPath)).filter((record) => record.source === "legacy" && record.runId);
|
|
888
|
+
await Promise.all([
|
|
889
|
+
rm2(outputDir, { recursive: true, force: true }),
|
|
890
|
+
...legacyRecords.map((record) => rm2(path4.join(path4.resolve("output"), record.runId), { recursive: true, force: true }))
|
|
891
|
+
]);
|
|
892
|
+
}
|
|
893
|
+
return json({ ok: true, skillPath, outputDir, records: await readRunHistory(skillPath) });
|
|
894
|
+
} catch (error) {
|
|
895
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error) }, 500);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
var activeOperations = /* @__PURE__ */ new Map();
|
|
899
|
+
async function handleOperationStreamRequest(ctx, request) {
|
|
900
|
+
if (request.method !== "POST") return json({ ok: false, error: "\u53EA\u652F\u6301 POST" }, 405);
|
|
901
|
+
let body;
|
|
902
|
+
try {
|
|
903
|
+
const parsed = await request.json();
|
|
904
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("\u8BF7\u6C42\u4F53\u5FC5\u987B\u662F\u5BF9\u8C61");
|
|
905
|
+
body = parsed;
|
|
906
|
+
} catch (error) {
|
|
907
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error) }, 400);
|
|
908
|
+
}
|
|
909
|
+
const action = typeof body.action === "string" ? body.action : "";
|
|
910
|
+
const skillPath = typeof body.skillPath === "string" ? body.skillPath.trim() : "";
|
|
911
|
+
if (!action || !skillPath) return json({ ok: false, error: "action \u548C skillPath \u5FC5\u586B" }, 400);
|
|
912
|
+
const operationKey = path4.resolve(skillPath);
|
|
913
|
+
if (activeOperations.has(operationKey)) {
|
|
914
|
+
return json({ ok: false, error: "\u8BE5 Skill \u5DF2\u6709\u64CD\u4F5C\u6B63\u5728\u8FD0\u884C\uFF0C\u8BF7\u7B49\u5F85\u5B8C\u6210\u6216\u5148\u53D6\u6D88" }, 409);
|
|
915
|
+
}
|
|
916
|
+
const controller = new AbortController();
|
|
917
|
+
activeOperations.set(operationKey, controller);
|
|
918
|
+
const encodedBody = JSON.stringify(body);
|
|
919
|
+
const encoder = new TextEncoder();
|
|
920
|
+
const stream = new ReadableStream({
|
|
921
|
+
start(streamController) {
|
|
922
|
+
let closed = false;
|
|
923
|
+
const send = (event) => {
|
|
924
|
+
if (closed) return;
|
|
925
|
+
try {
|
|
926
|
+
streamController.enqueue(encoder.encode(`${JSON.stringify(event)}
|
|
927
|
+
`));
|
|
928
|
+
} catch {
|
|
929
|
+
closed = true;
|
|
930
|
+
}
|
|
931
|
+
};
|
|
932
|
+
void (async () => {
|
|
933
|
+
send({ type: "start", action, skillPath, startedAt: Date.now() });
|
|
934
|
+
try {
|
|
935
|
+
const response = await handleOperationRequest(ctx, new Request(request.url, {
|
|
936
|
+
method: "POST",
|
|
937
|
+
headers: { "content-type": "application/json" },
|
|
938
|
+
body: encodedBody
|
|
939
|
+
}), {
|
|
940
|
+
signal: controller.signal,
|
|
941
|
+
onOutput: (streamName, chunk) => send({ type: "output", stream: streamName, text: chunk })
|
|
942
|
+
});
|
|
943
|
+
const payload = await response.json();
|
|
944
|
+
if (!response.ok || payload.ok === false) send({ type: "error", result: payload, error: String(payload.error ?? `HTTP ${response.status}`) });
|
|
945
|
+
else send({ type: "done", result: payload });
|
|
946
|
+
} catch (error) {
|
|
947
|
+
send({ type: "error", error: error instanceof Error ? error.message : String(error) });
|
|
948
|
+
} finally {
|
|
949
|
+
activeOperations.delete(operationKey);
|
|
950
|
+
closed = true;
|
|
951
|
+
try {
|
|
952
|
+
streamController.close();
|
|
953
|
+
} catch {
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
})();
|
|
957
|
+
},
|
|
958
|
+
cancel() {
|
|
959
|
+
controller.abort();
|
|
960
|
+
activeOperations.delete(operationKey);
|
|
961
|
+
}
|
|
962
|
+
});
|
|
963
|
+
return new Response(stream, {
|
|
964
|
+
status: 200,
|
|
965
|
+
headers: {
|
|
966
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
967
|
+
"cache-control": "no-store, no-transform",
|
|
968
|
+
"x-content-type-options": "nosniff"
|
|
969
|
+
}
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
async function handleOperationRequest(ctx, request, hooks) {
|
|
973
|
+
if (request.method !== "POST") return json(failure("\u53EA\u652F\u6301 POST"), 405);
|
|
974
|
+
try {
|
|
975
|
+
const body = await request.json();
|
|
976
|
+
const action = stringField(body, "action");
|
|
977
|
+
const skillPath = stringField(body, "skillPath");
|
|
978
|
+
if (!action || !skillPath) throw new Error("action \u548C skillPath \u5FC5\u586B");
|
|
979
|
+
const service = ctx.get("evalSuite");
|
|
980
|
+
if (!service) throw new Error("evalSuite \u670D\u52A1\u4E0D\u53EF\u7528\uFF0C\u8BF7\u5148\u5B89\u88C5 dsh-eval-suite");
|
|
981
|
+
const common = {
|
|
982
|
+
...stringField(body, "cwd") ? { cwd: stringField(body, "cwd") } : {},
|
|
983
|
+
...stringField(body, "model") ? { model: stringField(body, "model") } : {},
|
|
984
|
+
...numberField(body, "timeoutMs") ? { timeoutMs: numberField(body, "timeoutMs") } : {},
|
|
985
|
+
...hooks?.onOutput ? { onOutput: hooks.onOutput } : {},
|
|
986
|
+
...hooks?.signal ? { signal: hooks.signal } : {}
|
|
987
|
+
};
|
|
988
|
+
let result;
|
|
989
|
+
let runOutputDir;
|
|
990
|
+
if (action === "init") {
|
|
991
|
+
result = await service.init({
|
|
992
|
+
skillPath,
|
|
993
|
+
...common,
|
|
994
|
+
...stringField(body, "output") ? { output: stringField(body, "output") } : {},
|
|
995
|
+
generatePrompts: boolField(body, "generatePrompts"),
|
|
996
|
+
numEvals: numberField(body, "numEvals"),
|
|
997
|
+
append: boolField(body, "append"),
|
|
998
|
+
fromTranscript: stringField(body, "fromTranscript"),
|
|
999
|
+
focus: stringField(body, "focus"),
|
|
1000
|
+
triggerProbes: boolField(body, "triggerProbes"),
|
|
1001
|
+
kind: stringField(body, "kind")
|
|
1002
|
+
});
|
|
1003
|
+
} else if (action === "add") {
|
|
1004
|
+
result = await service.add({
|
|
1005
|
+
skillPath,
|
|
1006
|
+
...common,
|
|
1007
|
+
idea: stringField(body, "idea"),
|
|
1008
|
+
prompt: stringField(body, "prompt"),
|
|
1009
|
+
...Array.isArray(body.expectation) ? { expectation: body.expectation.filter((item) => typeof item === "string") } : {},
|
|
1010
|
+
level: stringField(body, "level"),
|
|
1011
|
+
kind: stringField(body, "kind"),
|
|
1012
|
+
dryRun: boolField(body, "dryRun")
|
|
1013
|
+
});
|
|
1014
|
+
} else if (action === "coverage") {
|
|
1015
|
+
result = await service.coverage({
|
|
1016
|
+
skillPath,
|
|
1017
|
+
...common,
|
|
1018
|
+
output: stringField(body, "output"),
|
|
1019
|
+
evalsPath: stringField(body, "evalsPath"),
|
|
1020
|
+
augment: boolField(body, "augment"),
|
|
1021
|
+
kind: stringField(body, "kind")
|
|
1022
|
+
});
|
|
1023
|
+
} else if (action === "inspect") {
|
|
1024
|
+
result = await service.inspect({
|
|
1025
|
+
evalsPath: stringField(body, "evalsPath") ?? deriveEvalsPath(skillPath),
|
|
1026
|
+
...common,
|
|
1027
|
+
json: true
|
|
1028
|
+
});
|
|
1029
|
+
} else if (action === "freeze") {
|
|
1030
|
+
result = await service.freeze({
|
|
1031
|
+
evalsPath: stringField(body, "evalsPath") ?? deriveEvalsPath(skillPath),
|
|
1032
|
+
...common,
|
|
1033
|
+
name: stringField(body, "name"),
|
|
1034
|
+
output: stringField(body, "output"),
|
|
1035
|
+
skillVersion: stringField(body, "skillVersion")
|
|
1036
|
+
});
|
|
1037
|
+
} else if (action === "eval") {
|
|
1038
|
+
result = await service.evaluate({
|
|
1039
|
+
skillPath,
|
|
1040
|
+
...common,
|
|
1041
|
+
...stringField(body, "executor") ? { executor: stringField(body, "executor") } : {},
|
|
1042
|
+
evalsPath: stringField(body, "evalsPath") ?? deriveEvalsPath(skillPath),
|
|
1043
|
+
skillB: stringField(body, "skillB"),
|
|
1044
|
+
labelA: stringField(body, "labelA"),
|
|
1045
|
+
labelB: stringField(body, "labelB"),
|
|
1046
|
+
kind: stringField(body, "kind"),
|
|
1047
|
+
runs: numberField(body, "runs"),
|
|
1048
|
+
workers: numberField(body, "workers"),
|
|
1049
|
+
timeoutSeconds: numberField(body, "timeoutSeconds"),
|
|
1050
|
+
maxTurns: numberField(body, "maxTurns"),
|
|
1051
|
+
maxRetries: numberField(body, "maxRetries"),
|
|
1052
|
+
scoring: stringField(body, "scoring"),
|
|
1053
|
+
compliance: boolField(body, "compliance"),
|
|
1054
|
+
calibration: boolField(body, "calibration"),
|
|
1055
|
+
securityGate: boolField(body, "securityGate"),
|
|
1056
|
+
output: stringField(body, "output"),
|
|
1057
|
+
format: stringField(body, "format"),
|
|
1058
|
+
evalSamples: stringField(body, "evalSamples"),
|
|
1059
|
+
evalIds: stringField(body, "evalIds"),
|
|
1060
|
+
changedFrom: stringField(body, "changedFrom")
|
|
1061
|
+
});
|
|
1062
|
+
} else if (action === "trigger") {
|
|
1063
|
+
result = await service.trigger({
|
|
1064
|
+
skillPath,
|
|
1065
|
+
...common,
|
|
1066
|
+
...stringField(body, "executor") ? { executor: stringField(body, "executor") } : {},
|
|
1067
|
+
evalsPath: stringField(body, "evalsPath") ?? deriveEvalsPath(skillPath),
|
|
1068
|
+
output: stringField(body, "output"),
|
|
1069
|
+
runs: numberField(body, "runs"),
|
|
1070
|
+
timeoutSeconds: numberField(body, "timeoutSeconds"),
|
|
1071
|
+
maxTurns: numberField(body, "maxTurns")
|
|
1072
|
+
});
|
|
1073
|
+
} else if (action === "run") {
|
|
1074
|
+
const task = stringField(body, "task");
|
|
1075
|
+
if (!task) throw new Error("run \u9700\u8981 task");
|
|
1076
|
+
runOutputDir = stringField(body, "output") ?? defaultRunOutput(skillPath);
|
|
1077
|
+
result = await service.execute({
|
|
1078
|
+
skillPath,
|
|
1079
|
+
task,
|
|
1080
|
+
...common,
|
|
1081
|
+
cwd: stringField(body, "cwd"),
|
|
1082
|
+
runs: numberField(body, "runs"),
|
|
1083
|
+
timeoutSeconds: numberField(body, "timeoutSeconds"),
|
|
1084
|
+
maxTurns: numberField(body, "maxTurns"),
|
|
1085
|
+
executor: stringField(body, "executor"),
|
|
1086
|
+
output: runOutputDir,
|
|
1087
|
+
securityGate: boolField(body, "securityGate")
|
|
1088
|
+
});
|
|
1089
|
+
} else if (action === "gate") {
|
|
1090
|
+
result = await service.gate({
|
|
1091
|
+
benchmarkPath: stringField(body, "benchmarkPath") ?? deriveBenchmarkPath(skillPath),
|
|
1092
|
+
...common,
|
|
1093
|
+
minWinRate: numberField(body, "minWinRate"),
|
|
1094
|
+
minValid: numberField(body, "minValid"),
|
|
1095
|
+
programmaticGuard: boolField(body, "programmaticGuard"),
|
|
1096
|
+
candidate: stringField(body, "candidate"),
|
|
1097
|
+
mustPass: stringField(body, "mustPass"),
|
|
1098
|
+
baseline: stringField(body, "baseline"),
|
|
1099
|
+
strictDrift: boolField(body, "strictDrift"),
|
|
1100
|
+
output: stringField(body, "output")
|
|
1101
|
+
});
|
|
1102
|
+
} else if (action === "report") {
|
|
1103
|
+
result = await service.report({
|
|
1104
|
+
workspace: skillDirectory2(skillPath),
|
|
1105
|
+
...common,
|
|
1106
|
+
benchmarkPath: stringField(body, "benchmarkPath"),
|
|
1107
|
+
previousPath: stringField(body, "previousPath"),
|
|
1108
|
+
scorecardPath: stringField(body, "scorecardPath")
|
|
1109
|
+
});
|
|
1110
|
+
} else {
|
|
1111
|
+
throw new Error(`\u672A\u77E5 action: ${action}`);
|
|
1112
|
+
}
|
|
1113
|
+
if (runOutputDir) {
|
|
1114
|
+
const runSummary = await readRunArtifactSummary(runOutputDir);
|
|
1115
|
+
return json({ ...result && typeof result === "object" ? result : {}, ...runSummary ? { runSummary } : {} });
|
|
1116
|
+
}
|
|
1117
|
+
return json(result);
|
|
1118
|
+
} catch (error) {
|
|
1119
|
+
return json(failure(error), 400);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
async function handleEvalSuiteRequest(request) {
|
|
1123
|
+
const url = new URL(request.url);
|
|
1124
|
+
const skillPath = url.searchParams.get("skillPath")?.trim();
|
|
1125
|
+
if (!skillPath) return json({ ok: false, error: "skillPath \u5FC5\u586B", exists: false }, 400);
|
|
1126
|
+
try {
|
|
1127
|
+
if (request.method === "DELETE") return json(await deleteEvalSuite(skillPath));
|
|
1128
|
+
return json(await readEvalSuite(skillPath));
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error), exists: false }, 500);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
async function handleEvalCaseRequest(request) {
|
|
1134
|
+
if (request.method !== "DELETE") return json({ ok: false, error: "\u53EA\u652F\u6301 DELETE" }, 405);
|
|
1135
|
+
const url = new URL(request.url);
|
|
1136
|
+
const skillPath = url.searchParams.get("skillPath")?.trim();
|
|
1137
|
+
const id = Number(url.searchParams.get("id"));
|
|
1138
|
+
if (!skillPath || !Number.isInteger(id)) return json({ ok: false, error: "skillPath \u548C id \u5FC5\u586B" }, 400);
|
|
1139
|
+
try {
|
|
1140
|
+
return json(await deleteEvalCase(skillPath, id));
|
|
1141
|
+
} catch (error) {
|
|
1142
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error) }, 500);
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// src/index.ts
|
|
1147
|
+
var name = "dsh-eval-ux";
|
|
1148
|
+
var inject = [];
|
|
1149
|
+
function apply(ctx, config) {
|
|
1150
|
+
const resolved = normalizeConfig(config);
|
|
1151
|
+
registerSkillCatalogEndpoints(ctx, resolved);
|
|
1152
|
+
process.stderr.write(`[dsh-eval-ux] ready \u2014 catalog enabled \xB7 manualPaths=${resolved.manualSkillPaths.length}
|
|
1153
|
+
`);
|
|
1154
|
+
}
|
|
1155
|
+
export {
|
|
1156
|
+
EVAL_CASE_ENDPOINT_PATH,
|
|
1157
|
+
EVAL_SUITE_ENDPOINT_PATH,
|
|
1158
|
+
MANUAL_PATHS_ENDPOINT_PATH,
|
|
1159
|
+
OPERATIONS_ENDPOINT_PATH,
|
|
1160
|
+
OPERATIONS_STREAM_ENDPOINT_PATH,
|
|
1161
|
+
REPORT_ENDPOINT_PATH,
|
|
1162
|
+
RUN_HISTORY_ENDPOINT_PATH,
|
|
1163
|
+
SKILL_CATALOG_ENDPOINT_PATH,
|
|
1164
|
+
apply,
|
|
1165
|
+
buildSkillCatalog,
|
|
1166
|
+
defaultConfig,
|
|
1167
|
+
deleteEvalCase,
|
|
1168
|
+
deleteEvalSuite,
|
|
1169
|
+
handleEvalCaseRequest,
|
|
1170
|
+
handleEvalSuiteRequest,
|
|
1171
|
+
handleManualPathsRequest,
|
|
1172
|
+
handleOperationRequest,
|
|
1173
|
+
handleOperationStreamRequest,
|
|
1174
|
+
handleReportRequest,
|
|
1175
|
+
handleRunHistoryRequest,
|
|
1176
|
+
handleSkillCatalogRequest,
|
|
1177
|
+
inject,
|
|
1178
|
+
name,
|
|
1179
|
+
normalizeConfig,
|
|
1180
|
+
readEvalSuite,
|
|
1181
|
+
registerSkillCatalogEndpoints
|
|
1182
|
+
};
|