dsh-eval-guard 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/neural-pendulum/SPEC.md +470 -0
- package/assets/neural-pendulum/TASK.md +360 -0
- package/assets/neural-pendulum/hidden/test_infra.cpp +631 -0
- package/assets/neural-pendulum/include/nn/nn.hpp +88 -0
- package/assets/neural-pendulum/sim.cpp +126 -0
- package/assets/neural-pendulum/sim_eval +0 -0
- package/assets/neural-pendulum/workspace.zip +0 -0
- package/eval/neural-pendulum.py +15 -0
- package/lib/client/index.js +41 -4
- package/lib/client/index.js.map +1 -1
- package/lib/index.js +454 -60
- package/neural-cifar-eval/README.md +57 -0
- package/neural-cifar-eval/cifar_cache.py +201 -0
- package/neural-cifar-eval/common.py +67 -0
- package/neural-cifar-eval/fetch_cifar.py +31 -0
- package/neural-cifar-eval/hidden/.gitkeep +1 -0
- package/neural-cifar-eval/ops_eval.py +502 -0
- package/neural-cifar-eval/run_eval.py +208 -0
- package/neural-pendulum-eval/run_eval.py +302 -0
- package/neural-pendulum-eval/sim_eval +0 -0
- package/package.json +4 -2
- package/eval/__pycache__/common.cpython-310.pyc +0 -0
- package/eval/__pycache__/neural-cifar.cpython-310.pyc +0 -0
- package/eval/__pycache__/neural-fp32.cpython-310.pyc +0 -0
package/lib/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { execFileSync, spawn } from "node:child_process";
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
8
9
|
//#region src/config.ts
|
|
9
10
|
/**
|
|
10
11
|
* Plugin configuration. The Settings page writes this document through the
|
|
@@ -27,7 +28,9 @@ const Config = z.object({
|
|
|
27
28
|
pipProxyUrl: z.string().default("").volatile(),
|
|
28
29
|
cifarProxyUrl: z.string().default("").volatile(),
|
|
29
30
|
cifarMirror: z.string().default("toronto").volatile(),
|
|
30
|
-
customCifarUrl: z.string().default("").volatile()
|
|
31
|
+
customCifarUrl: z.string().default("").volatile(),
|
|
32
|
+
parallelEval: z.boolean().default(false).volatile(),
|
|
33
|
+
taskId: z.string().default("").volatile()
|
|
31
34
|
});
|
|
32
35
|
/** Clamp and normalize a live configuration object. */
|
|
33
36
|
function resolveConfig(value) {
|
|
@@ -47,7 +50,9 @@ function resolveConfig(value) {
|
|
|
47
50
|
pipProxyUrl: typeof value?.pipProxyUrl === "string" ? value.pipProxyUrl : "",
|
|
48
51
|
cifarProxyUrl: typeof value?.cifarProxyUrl === "string" ? value.cifarProxyUrl : "",
|
|
49
52
|
cifarMirror: typeof value?.cifarMirror === "string" && value.cifarMirror !== "" ? value.cifarMirror : "toronto",
|
|
50
|
-
customCifarUrl: typeof value?.customCifarUrl === "string" ? value.customCifarUrl : ""
|
|
53
|
+
customCifarUrl: typeof value?.customCifarUrl === "string" ? value.customCifarUrl : "",
|
|
54
|
+
parallelEval: value?.parallelEval === true,
|
|
55
|
+
taskId: typeof value?.taskId === "string" ? value.taskId : ""
|
|
51
56
|
};
|
|
52
57
|
}
|
|
53
58
|
//#endregion
|
|
@@ -86,6 +91,8 @@ const NEURAL_FP32_TASK = "neural-fp32";
|
|
|
86
91
|
const NEURAL_FP32_TASK_LABEL = "Neural-FP32 · 运算与性能";
|
|
87
92
|
const NEURAL_CIFAR_TASK = "neural-cifar";
|
|
88
93
|
const NEURAL_CIFAR_TASK_LABEL = "Neural-CIFAR · 从零训练图像分类";
|
|
94
|
+
const NEURAL_PENDULUM_TASK = "neural-pendulum";
|
|
95
|
+
const NEURAL_PENDULUM_TASK_LABEL = "Neural-Pendulum · 从零训练三节摆控制器";
|
|
89
96
|
/** Common pip indexes offered by the settings page. */
|
|
90
97
|
const CIFAR_MIRRORS = [{
|
|
91
98
|
id: "toronto",
|
|
@@ -465,6 +472,15 @@ const SYSTEM_READ_DIRS = [
|
|
|
465
472
|
"/sys",
|
|
466
473
|
"/var/lib"
|
|
467
474
|
];
|
|
475
|
+
/**
|
|
476
|
+
* Individual read-only system files commonly needed by benign inspection tools
|
|
477
|
+
* such as uptime/who. Keep this list narrow; do not expose /run as a directory.
|
|
478
|
+
*/
|
|
479
|
+
const SYSTEM_READ_FILES = [
|
|
480
|
+
"/run/utmp",
|
|
481
|
+
"/run/utmpx",
|
|
482
|
+
"/var/log/wtmp"
|
|
483
|
+
];
|
|
468
484
|
/** Python stdlib/venv directories that may live next to a workspace. */
|
|
469
485
|
function pythonRuntimeDirs(workspace) {
|
|
470
486
|
const dirs = /* @__PURE__ */ new Set();
|
|
@@ -587,6 +603,10 @@ function writePolicy(workspace, rules) {
|
|
|
587
603
|
}
|
|
588
604
|
} catch {}
|
|
589
605
|
lines.push(line("read", "/proc"));
|
|
606
|
+
for (const file of SYSTEM_READ_FILES) {
|
|
607
|
+
lines.push(line("read", file));
|
|
608
|
+
for (const ancestor of ancestorsOf(file)) lines.push(line("exact", ancestor));
|
|
609
|
+
}
|
|
590
610
|
lines.push(line("exact", "/"));
|
|
591
611
|
for (const ancestor of ancestorsOf(workspace)) lines.push(line("exact", ancestor));
|
|
592
612
|
for (const ancestor of ancestorsOf(runtime.node)) lines.push(line("exact", ancestor));
|
|
@@ -641,12 +661,26 @@ function bwrapArgs(workspace, cwd, rules, runtime) {
|
|
|
641
661
|
const args = [
|
|
642
662
|
"--unshare-all",
|
|
643
663
|
"--die-with-parent",
|
|
644
|
-
"--new-session"
|
|
664
|
+
"--new-session",
|
|
665
|
+
"--as-pid-1"
|
|
645
666
|
];
|
|
646
667
|
for (const system of SYSTEM_READ_DIRS) try {
|
|
647
668
|
if (fs.statSync(system).isDirectory()) args.push("--ro-bind", system, system);
|
|
648
669
|
} catch {}
|
|
649
670
|
args.push("--proc", "/proc", "--dev", "/dev");
|
|
671
|
+
const systemFiles = SYSTEM_READ_FILES.filter((file) => {
|
|
672
|
+
try {
|
|
673
|
+
return fs.statSync(file).isFile();
|
|
674
|
+
} catch {
|
|
675
|
+
return false;
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
if (systemFiles.some((file) => file.startsWith("/run/"))) args.push("--dir", "/run");
|
|
679
|
+
if (systemFiles.some((file) => file.startsWith("/var/log/"))) {
|
|
680
|
+
args.push("--dir", "/var");
|
|
681
|
+
args.push("--dir", "/var/log");
|
|
682
|
+
}
|
|
683
|
+
for (const file of systemFiles) args.push("--ro-bind", file, file);
|
|
650
684
|
args.push("--bind", workspace, workspace);
|
|
651
685
|
const tmp = path.join(workspace, ".tmp");
|
|
652
686
|
ensureDir(tmp);
|
|
@@ -779,12 +813,15 @@ var Guard = class {
|
|
|
779
813
|
startedAt = /* @__PURE__ */ new Map();
|
|
780
814
|
runs = /* @__PURE__ */ new Map();
|
|
781
815
|
sessionRuns = /* @__PURE__ */ new Map();
|
|
782
|
-
pending = /* @__PURE__ */ new Map();
|
|
783
816
|
offsets = /* @__PURE__ */ new Map();
|
|
784
817
|
knownAgents = /* @__PURE__ */ new Set();
|
|
785
818
|
toolCache = [];
|
|
786
819
|
disposers = [];
|
|
787
820
|
task;
|
|
821
|
+
/** Optional hook invoked when any run binds to an agent. */
|
|
822
|
+
onRunAssociated;
|
|
823
|
+
/** Optional automatic task binding for every main conversation. */
|
|
824
|
+
onAutoRun;
|
|
788
825
|
/** @param ctx - host context; @param raw - live config; @param task - the one task pack. */
|
|
789
826
|
constructor(ctx, services, raw, task) {
|
|
790
827
|
this.ctx = ctx;
|
|
@@ -840,7 +877,9 @@ var Guard = class {
|
|
|
840
877
|
pipProxyUrl: read("pipProxyUrl"),
|
|
841
878
|
cifarProxyUrl: read("cifarProxyUrl"),
|
|
842
879
|
cifarMirror: read("cifarMirror"),
|
|
843
|
-
customCifarUrl: read("customCifarUrl")
|
|
880
|
+
customCifarUrl: read("customCifarUrl"),
|
|
881
|
+
parallelEval: read("parallelEval"),
|
|
882
|
+
taskId: read("taskId")
|
|
844
883
|
});
|
|
845
884
|
}
|
|
846
885
|
/** Apply a new configuration document. */
|
|
@@ -1014,11 +1053,9 @@ var Guard = class {
|
|
|
1014
1053
|
if (this.enabled) this.forceApproval(agent.session);
|
|
1015
1054
|
const cwd = agent.session.header?.meta?.cwd;
|
|
1016
1055
|
if (cwd !== void 0) {
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
this.associate(pending.runId, agent);
|
|
1021
|
-
}
|
|
1056
|
+
if (this.enabled) try {
|
|
1057
|
+
this.onAutoRun?.(agent, cwd);
|
|
1058
|
+
} catch {}
|
|
1022
1059
|
}
|
|
1023
1060
|
}
|
|
1024
1061
|
/** An agent went away. */
|
|
@@ -1036,7 +1073,12 @@ var Guard = class {
|
|
|
1036
1073
|
}
|
|
1037
1074
|
}
|
|
1038
1075
|
}
|
|
1039
|
-
/**
|
|
1076
|
+
/** Bind an already-registered run to an agent. */
|
|
1077
|
+
attachRun(agent, run) {
|
|
1078
|
+
this.runs.set(run.runId, run);
|
|
1079
|
+
this.associate(run.runId, agent);
|
|
1080
|
+
}
|
|
1081
|
+
/** Associate one run with the new session. */
|
|
1040
1082
|
associate(runId, agent) {
|
|
1041
1083
|
const run = this.runs.get(runId);
|
|
1042
1084
|
if (run === void 0) return;
|
|
@@ -1049,6 +1091,9 @@ var Guard = class {
|
|
|
1049
1091
|
this.sessionRuns.set(agent.id, runId);
|
|
1050
1092
|
this.writeRun(run);
|
|
1051
1093
|
this.writeUsage(runId, this.zeroUsage(run.startedAt));
|
|
1094
|
+
try {
|
|
1095
|
+
this.onRunAssociated?.(agent, run);
|
|
1096
|
+
} catch {}
|
|
1052
1097
|
}
|
|
1053
1098
|
/** Session event intake. */
|
|
1054
1099
|
onSessionEvent(session, event) {
|
|
@@ -1283,10 +1328,9 @@ var Guard = class {
|
|
|
1283
1328
|
} catch {}
|
|
1284
1329
|
}
|
|
1285
1330
|
}
|
|
1286
|
-
/** Remove old sandbox policy files
|
|
1331
|
+
/** Remove old sandbox policy files. */
|
|
1287
1332
|
cleanupTemp() {
|
|
1288
1333
|
const now = Date.now();
|
|
1289
|
-
for (const [workspace, pending] of [...this.pending]) if (pending.expiresAt <= now) this.pending.delete(workspace);
|
|
1290
1334
|
const tmp = path.join(guardRoot(), "tmp");
|
|
1291
1335
|
try {
|
|
1292
1336
|
for (const entry of fs.readdirSync(tmp)) {
|
|
@@ -1367,10 +1411,12 @@ var Guard = class {
|
|
|
1367
1411
|
this.writeRun(run);
|
|
1368
1412
|
this.writeUsage(run.runId, this.zeroUsage(null));
|
|
1369
1413
|
fs.writeFileSync(path.join(guardRoot(), "runs", run.runId, "violations.jsonl"), "", "utf8");
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1414
|
+
}
|
|
1415
|
+
/** Resolve the registered run bound to one agent session. */
|
|
1416
|
+
runForSession(sessionId) {
|
|
1417
|
+
if (sessionId === void 0) return void 0;
|
|
1418
|
+
const runId = this.sessionRuns.get(sessionId);
|
|
1419
|
+
return runId === void 0 ? void 0 : this.runs.get(runId);
|
|
1374
1420
|
}
|
|
1375
1421
|
/** Build the prompt for one task with only installed tools listed. */
|
|
1376
1422
|
promptFor(workspacePath, source, task = this.task) {
|
|
@@ -1752,6 +1798,14 @@ const TASKS = [
|
|
|
1752
1798
|
timeLimitMinutes: 1440,
|
|
1753
1799
|
timeLimitEnabled: false,
|
|
1754
1800
|
needsSetup: true
|
|
1801
|
+
},
|
|
1802
|
+
{
|
|
1803
|
+
id: NEURAL_PENDULUM_TASK,
|
|
1804
|
+
label: NEURAL_PENDULUM_TASK_LABEL,
|
|
1805
|
+
description: "从零实现纯 C++17 神经网络基础设施,用三个关节角度控制预编译三节摆起摆并稳定保持;评测固定初始状态,统计 t_can。",
|
|
1806
|
+
timeLimitMinutes: 1440,
|
|
1807
|
+
timeLimitEnabled: false,
|
|
1808
|
+
needsSetup: true
|
|
1755
1809
|
}
|
|
1756
1810
|
];
|
|
1757
1811
|
/** The tank task definition (original prompt; only halftime mechanics removed). */
|
|
@@ -2056,6 +2110,34 @@ const NEURAL_CIFAR_DEFINITION = {
|
|
|
2056
2110
|
].join("\n");
|
|
2057
2111
|
}
|
|
2058
2112
|
};
|
|
2113
|
+
/** Neural-Pendulum swing-up task. */
|
|
2114
|
+
const NEURAL_PENDULUM_DEFINITION = {
|
|
2115
|
+
id: NEURAL_PENDULUM_TASK,
|
|
2116
|
+
label: NEURAL_PENDULUM_TASK_LABEL,
|
|
2117
|
+
description: TASKS[11].description,
|
|
2118
|
+
asset: path.join(packageRoot(), "assets", "neural-pendulum", "workspace.zip"),
|
|
2119
|
+
timeLimitMinutes: 1440,
|
|
2120
|
+
timeLimitEnabled: false,
|
|
2121
|
+
needsSetup: true,
|
|
2122
|
+
prompt() {
|
|
2123
|
+
return [
|
|
2124
|
+
"你是一名控制与神经网络工程师。任务:从零实现纯 C++17、仅 CPU、fp32 的神经网络基础设施,并用它训练一个控制器,控制预编译的三节摆从接近全向下起摆,使三根杆全部进入竖直向上容差并尽可能长时间保持。",
|
|
2125
|
+
"",
|
|
2126
|
+
"请先完整阅读 TASK.md;它是权威规则。硬约束:仅 C++17 标准库、仅 CPU、fp32;不使用 Python/OpenCL/CUDA/BLAS/Eigen/第三方库;必须从提交源码构建出 build/control 与 build/libnn.a;基础设施需实现工作区公开头文件 include/nn/nn.hpp 的契约,隐藏测试由评测侧独立编译链接。",
|
|
2127
|
+
"",
|
|
2128
|
+
"环境细节:joint1 与 joint3 驱动,joint2 被动;控制频率 100 Hz;每回合最多 3000 步(30 s),达到上限后继续 step 返回 {\"error\":\"episode_finished\"},必须 reset。环境只返回三个关节角度(弧度,不除 pi),不返回 reward/loss/cost/done;环境不提供杆长、质量、惯量、角速度或 tip。训练 reset 带小范围随机初始角度和角速度;评测使用固定初始状态,无随机扰动。",
|
|
2129
|
+
"",
|
|
2130
|
+
"优化目标:从接近全向下起摆,使三根杆全部进入 0.05 rad 的竖直向上容差,并尽可能长时间保持;评测指标为 t_can。损失函数、奖励、优化器、网络结构和训练方法由你自行设计,不规定任何训练方案。",
|
|
2131
|
+
"",
|
|
2132
|
+
"评测会从干净副本重新构建提交源码,编译并运行宿主侧隐藏基础设施测试,然后在固定初始状态上驱动 build/control,统计连续全向上保持时间 t_can。不要把任何未提交的二进制、临时构建产物或外部进程当作交付依赖。",
|
|
2133
|
+
"使用宿主工具 eval_submit 提交当前源码快照,使用 eval_test 请求宿主评测最新提交。提交快照、事件、测试结果和最终分数全部保存在工作区之外的宿主目录;不要尝试读取或修改这些数据,也不要依赖任何工作区内的测试结果文件。最终评测使用最新提交快照,按宿主事件时间积分计算分数。",
|
|
2134
|
+
"",
|
|
2135
|
+
"工作方式:先完成最小可构建的基础设施和 control 协议;再持续做控制/训练实验并记录可复现结果;最终保证工作区从零构建、隐藏基础设施测试通过、control 可被评测驱动,并留下 NOTES.md/RESULTS.md。",
|
|
2136
|
+
"",
|
|
2137
|
+
"不要向我提问,也不要等待确认,独立完成。"
|
|
2138
|
+
].join("\n");
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2059
2141
|
/** All task definitions keyed by id. */
|
|
2060
2142
|
const TASK_DEFINITIONS = {
|
|
2061
2143
|
[TANK_TASK]: TANK_DEFINITION,
|
|
@@ -2068,7 +2150,8 @@ const TASK_DEFINITIONS = {
|
|
|
2068
2150
|
[FEATUREBENCH_TASK]: FEATUREBENCH_DEFINITION,
|
|
2069
2151
|
[NEURAL_CPP_TASK]: NEURAL_CPP_DEFINITION,
|
|
2070
2152
|
[NEURAL_FP32_TASK]: NEURAL_FP32_DEFINITION,
|
|
2071
|
-
[NEURAL_CIFAR_TASK]: NEURAL_CIFAR_DEFINITION
|
|
2153
|
+
[NEURAL_CIFAR_TASK]: NEURAL_CIFAR_DEFINITION,
|
|
2154
|
+
[NEURAL_PENDULUM_TASK]: NEURAL_PENDULUM_DEFINITION
|
|
2072
2155
|
};
|
|
2073
2156
|
var TaskService = class {
|
|
2074
2157
|
guard;
|
|
@@ -2079,6 +2162,272 @@ var TaskService = class {
|
|
|
2079
2162
|
this.registry = registry;
|
|
2080
2163
|
this.jobs = new JobManager();
|
|
2081
2164
|
}
|
|
2165
|
+
/** Host-side directory for one workspace's authoritative task state. */
|
|
2166
|
+
hostDir(workspaceId) {
|
|
2167
|
+
return path.join(guardRoot(), "neural-pendulum", workspaceId.replace(/[^A-Za-z0-9_.-]/g, "_"));
|
|
2168
|
+
}
|
|
2169
|
+
hostEvents(workspaceId) {
|
|
2170
|
+
return path.join(this.hostDir(workspaceId), "events.jsonl");
|
|
2171
|
+
}
|
|
2172
|
+
appendHostEvent(workspaceId, event) {
|
|
2173
|
+
const dir = this.hostDir(workspaceId);
|
|
2174
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
2175
|
+
fs.appendFileSync(this.hostEvents(workspaceId), JSON.stringify(event) + "\n", "utf8");
|
|
2176
|
+
}
|
|
2177
|
+
hostLatestSubmission(workspaceId) {
|
|
2178
|
+
const root = path.join(this.hostDir(workspaceId), "submissions");
|
|
2179
|
+
if (!fs.existsSync(root)) return void 0;
|
|
2180
|
+
const dirs = fs.readdirSync(root, { withFileTypes: true }).filter((entry) => entry.isDirectory());
|
|
2181
|
+
if (dirs.length === 0) return void 0;
|
|
2182
|
+
const id = dirs.map((entry) => entry.name).sort((a, b) => {
|
|
2183
|
+
const am = fs.statSync(path.join(root, a)).mtimeMs;
|
|
2184
|
+
return fs.statSync(path.join(root, b)).mtimeMs - am;
|
|
2185
|
+
})[0];
|
|
2186
|
+
return {
|
|
2187
|
+
id,
|
|
2188
|
+
path: path.join(root, id)
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
copySnapshot(source, destination) {
|
|
2192
|
+
const excluded = /* @__PURE__ */ new Set([
|
|
2193
|
+
"build",
|
|
2194
|
+
".git",
|
|
2195
|
+
".eval",
|
|
2196
|
+
".submissions",
|
|
2197
|
+
"node_modules",
|
|
2198
|
+
"__pycache__",
|
|
2199
|
+
".venv"
|
|
2200
|
+
]);
|
|
2201
|
+
let count = 0;
|
|
2202
|
+
const walk = (dir) => {
|
|
2203
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
2204
|
+
if (excluded.has(entry.name)) continue;
|
|
2205
|
+
const from = path.join(dir, entry.name);
|
|
2206
|
+
const to = path.join(destination, path.relative(source, from));
|
|
2207
|
+
if (entry.isDirectory()) {
|
|
2208
|
+
fs.mkdirSync(to, { recursive: true });
|
|
2209
|
+
walk(from);
|
|
2210
|
+
} else if (entry.isFile()) {
|
|
2211
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
2212
|
+
fs.copyFileSync(from, to);
|
|
2213
|
+
count++;
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
};
|
|
2217
|
+
fs.mkdirSync(destination, { recursive: true });
|
|
2218
|
+
walk(source);
|
|
2219
|
+
return count;
|
|
2220
|
+
}
|
|
2221
|
+
/** Host-side submit: snapshot the workspace into the run directory. */
|
|
2222
|
+
hostSubmit(run) {
|
|
2223
|
+
const done = path.join(this.hostDir(run.workspaceId), "done");
|
|
2224
|
+
if (fs.existsSync(done)) return {
|
|
2225
|
+
ok: false,
|
|
2226
|
+
error: "task already completed"
|
|
2227
|
+
};
|
|
2228
|
+
const id = Date.now().toString(36) + "-" + Math.random().toString(36).slice(2, 8);
|
|
2229
|
+
const metaPath = path.join(this.hostDir(run.workspaceId), "meta.json");
|
|
2230
|
+
if (!fs.existsSync(metaPath)) {
|
|
2231
|
+
fs.mkdirSync(this.hostDir(run.workspaceId), { recursive: true });
|
|
2232
|
+
fs.writeFileSync(metaPath, JSON.stringify({
|
|
2233
|
+
started_at: run.createdAt / 1e3,
|
|
2234
|
+
t_tgt: 10
|
|
2235
|
+
}), "utf8");
|
|
2236
|
+
}
|
|
2237
|
+
const destination = path.join(this.hostDir(run.workspaceId), "submissions", id);
|
|
2238
|
+
if (fs.existsSync(destination)) return {
|
|
2239
|
+
ok: false,
|
|
2240
|
+
error: "submission id collision"
|
|
2241
|
+
};
|
|
2242
|
+
const files = this.copySnapshot(run.workspacePath, destination);
|
|
2243
|
+
const hash = createHash("sha256");
|
|
2244
|
+
const hashDir = (dir) => {
|
|
2245
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
2246
|
+
const full = path.join(dir, entry.name);
|
|
2247
|
+
if (entry.isDirectory()) hashDir(full);
|
|
2248
|
+
else if (entry.isFile()) {
|
|
2249
|
+
hash.update(path.relative(destination, full));
|
|
2250
|
+
hash.update(fs.readFileSync(full));
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
};
|
|
2254
|
+
hashDir(destination);
|
|
2255
|
+
this.appendHostEvent(run.workspaceId, {
|
|
2256
|
+
event: "submit",
|
|
2257
|
+
time: Date.now() / 1e3,
|
|
2258
|
+
submission_id: id,
|
|
2259
|
+
source_hash: "sha256:" + hash.digest("hex"),
|
|
2260
|
+
files
|
|
2261
|
+
});
|
|
2262
|
+
return {
|
|
2263
|
+
ok: true,
|
|
2264
|
+
submission: id,
|
|
2265
|
+
files
|
|
2266
|
+
};
|
|
2267
|
+
}
|
|
2268
|
+
async runHostEval(workspaceId) {
|
|
2269
|
+
const latest = this.hostLatestSubmission(workspaceId);
|
|
2270
|
+
if (latest === void 0) return {
|
|
2271
|
+
stage: "no_submission",
|
|
2272
|
+
t_can: 0,
|
|
2273
|
+
completed: false,
|
|
2274
|
+
error: ""
|
|
2275
|
+
};
|
|
2276
|
+
const temp = fs.mkdtempSync(path.join(guardRoot(), "pend-eval-"));
|
|
2277
|
+
try {
|
|
2278
|
+
const submissions = path.join(temp, ".eval", "submissions");
|
|
2279
|
+
fs.mkdirSync(submissions, { recursive: true });
|
|
2280
|
+
fs.cpSync(latest.path, path.join(submissions, latest.id), { recursive: true });
|
|
2281
|
+
const events = this.hostEvents(workspaceId);
|
|
2282
|
+
fs.writeFileSync(path.join(temp, ".eval", "events.jsonl"), fs.existsSync(events) ? fs.readFileSync(events) : "", "utf8");
|
|
2283
|
+
fs.writeFileSync(path.join(temp, ".eval", "meta.json"), JSON.stringify({
|
|
2284
|
+
started_at: Date.now() / 1e3,
|
|
2285
|
+
t_tgt: 10
|
|
2286
|
+
}), "utf8");
|
|
2287
|
+
const resultFile = path.join(temp, "result.json");
|
|
2288
|
+
const script = path.join(packageRoot(), "neural-pendulum-eval", "run_eval.py");
|
|
2289
|
+
const env = {
|
|
2290
|
+
...process.env,
|
|
2291
|
+
EVAL_RESULT_FILE: resultFile,
|
|
2292
|
+
PYTHONPATH: "",
|
|
2293
|
+
PYTHONHOME: ""
|
|
2294
|
+
};
|
|
2295
|
+
const child = spawn("python3", [
|
|
2296
|
+
"-X",
|
|
2297
|
+
"utf8",
|
|
2298
|
+
script,
|
|
2299
|
+
"--workspace",
|
|
2300
|
+
temp
|
|
2301
|
+
], { env });
|
|
2302
|
+
let stderr = "";
|
|
2303
|
+
child.stderr?.on("data", (chunk) => {
|
|
2304
|
+
stderr += String(chunk);
|
|
2305
|
+
});
|
|
2306
|
+
await new Promise((resolve, reject) => {
|
|
2307
|
+
child.on("error", reject);
|
|
2308
|
+
child.on("close", () => resolve());
|
|
2309
|
+
});
|
|
2310
|
+
if (!fs.existsSync(resultFile)) return {
|
|
2311
|
+
stage: "controller",
|
|
2312
|
+
t_can: 0,
|
|
2313
|
+
completed: false,
|
|
2314
|
+
error: stderr || "evaluator failed"
|
|
2315
|
+
};
|
|
2316
|
+
const value = JSON.parse(fs.readFileSync(resultFile, "utf8"));
|
|
2317
|
+
const error = String(value.error ?? "");
|
|
2318
|
+
let stage = "controller";
|
|
2319
|
+
if (error.includes("cmake") || error.includes("build")) stage = "compile";
|
|
2320
|
+
else if (error.includes("selftest") || value.infra_ok === false) stage = "infra";
|
|
2321
|
+
else if (error.includes("controller")) stage = "controller";
|
|
2322
|
+
else if (value.success === true) stage = "completed";
|
|
2323
|
+
else if (value.infra_ok === true) stage = "scoring";
|
|
2324
|
+
else stage = "simulation";
|
|
2325
|
+
return {
|
|
2326
|
+
stage,
|
|
2327
|
+
t_can: Number(value.t_can ?? 0),
|
|
2328
|
+
completed: value.success === true,
|
|
2329
|
+
error
|
|
2330
|
+
};
|
|
2331
|
+
} finally {
|
|
2332
|
+
fs.rmSync(temp, {
|
|
2333
|
+
recursive: true,
|
|
2334
|
+
force: true
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
/** Host-side test: evaluate the latest host submission and write the event. */
|
|
2339
|
+
async hostTest(run) {
|
|
2340
|
+
const done = path.join(this.hostDir(run.workspaceId), "done");
|
|
2341
|
+
if (fs.existsSync(done)) return {
|
|
2342
|
+
ok: false,
|
|
2343
|
+
stage: "completed",
|
|
2344
|
+
error: "task already completed"
|
|
2345
|
+
};
|
|
2346
|
+
const latest = this.hostLatestSubmission(run.workspaceId);
|
|
2347
|
+
if (latest === void 0) {
|
|
2348
|
+
this.appendHostEvent(run.workspaceId, {
|
|
2349
|
+
event: "test",
|
|
2350
|
+
time: Date.now() / 1e3,
|
|
2351
|
+
submission_id: "",
|
|
2352
|
+
stage: "no_submission",
|
|
2353
|
+
t_can: 0,
|
|
2354
|
+
completed: false,
|
|
2355
|
+
error: ""
|
|
2356
|
+
});
|
|
2357
|
+
return {
|
|
2358
|
+
ok: true,
|
|
2359
|
+
stage: "no_submission",
|
|
2360
|
+
t_can: 0,
|
|
2361
|
+
completed: false,
|
|
2362
|
+
error: ""
|
|
2363
|
+
};
|
|
2364
|
+
}
|
|
2365
|
+
const value = await this.runHostEval(run.workspaceId);
|
|
2366
|
+
const stage = String(value.stage ?? "simulation");
|
|
2367
|
+
const tCan = Number(value.t_can ?? 0);
|
|
2368
|
+
const completed = stage === "completed" && Number.isFinite(tCan);
|
|
2369
|
+
this.appendHostEvent(run.workspaceId, {
|
|
2370
|
+
event: "test",
|
|
2371
|
+
time: Date.now() / 1e3,
|
|
2372
|
+
submission_id: latest.id,
|
|
2373
|
+
stage,
|
|
2374
|
+
t_can: tCan,
|
|
2375
|
+
completed,
|
|
2376
|
+
error: String(value.error ?? "")
|
|
2377
|
+
});
|
|
2378
|
+
if (completed) fs.writeFileSync(done, String(Date.now() / 1e3), "utf8");
|
|
2379
|
+
return {
|
|
2380
|
+
ok: true,
|
|
2381
|
+
stage,
|
|
2382
|
+
t_can: tCan,
|
|
2383
|
+
completed,
|
|
2384
|
+
error: String(value.error ?? "")
|
|
2385
|
+
};
|
|
2386
|
+
}
|
|
2387
|
+
/** Build a temporary workspace containing the host-side authoritative state. */
|
|
2388
|
+
stageHostWorkspace(workspaceId) {
|
|
2389
|
+
if (this.hostLatestSubmission(workspaceId) === void 0) return void 0;
|
|
2390
|
+
const temp = fs.mkdtempSync(path.join(guardRoot(), "pend-eval-view-"));
|
|
2391
|
+
const state = path.join(temp, ".eval");
|
|
2392
|
+
fs.mkdirSync(path.join(state, "submissions"), { recursive: true });
|
|
2393
|
+
const root = path.join(this.hostDir(workspaceId), "submissions");
|
|
2394
|
+
for (const entry of fs.readdirSync(root, { withFileTypes: true })) if (entry.isDirectory()) fs.cpSync(path.join(root, entry.name), path.join(state, "submissions", entry.name), { recursive: true });
|
|
2395
|
+
const events = this.hostEvents(workspaceId);
|
|
2396
|
+
fs.writeFileSync(path.join(state, "events.jsonl"), fs.existsSync(events) ? fs.readFileSync(events) : "", "utf8");
|
|
2397
|
+
const meta = path.join(this.hostDir(workspaceId), "meta.json");
|
|
2398
|
+
fs.writeFileSync(path.join(state, "meta.json"), fs.existsSync(meta) ? fs.readFileSync(meta) : JSON.stringify({
|
|
2399
|
+
started_at: Date.now() / 1e3,
|
|
2400
|
+
t_tgt: 10
|
|
2401
|
+
}), "utf8");
|
|
2402
|
+
return temp;
|
|
2403
|
+
}
|
|
2404
|
+
/** Bind every main conversation to the configured active task, no install needed. */
|
|
2405
|
+
autoBind(agent, cwd) {
|
|
2406
|
+
if (!this.guard.active) return false;
|
|
2407
|
+
const taskId = this.guard.configuration.taskId;
|
|
2408
|
+
if (taskId === "") return false;
|
|
2409
|
+
const definition = TASK_DEFINITIONS[taskId];
|
|
2410
|
+
if (definition === void 0) return false;
|
|
2411
|
+
const workspaceId = (this.registry.list?.() ?? []).find((workspace) => path.resolve(workspace.path) === path.resolve(cwd))?.id ?? "path-" + createHash("sha1").update(cwd).digest("hex").slice(0, 16);
|
|
2412
|
+
const runId = this.newRunId();
|
|
2413
|
+
const prompt = this.promptNotice(this.guard.promptFor(cwd, this.materialZip(taskId), definition));
|
|
2414
|
+
const run = {
|
|
2415
|
+
runId,
|
|
2416
|
+
task: definition.id,
|
|
2417
|
+
workspaceId,
|
|
2418
|
+
workspacePath: cwd,
|
|
2419
|
+
sessionId: null,
|
|
2420
|
+
createdAt: Date.now(),
|
|
2421
|
+
startedAt: null,
|
|
2422
|
+
endedAt: null,
|
|
2423
|
+
status: "prepared",
|
|
2424
|
+
prompt,
|
|
2425
|
+
timeLimitMinutes: this.guard.configuration.timeLimitMinutes
|
|
2426
|
+
};
|
|
2427
|
+
this.guard.registerRun(run);
|
|
2428
|
+
this.guard.attachRun(agent, run);
|
|
2429
|
+
return true;
|
|
2430
|
+
}
|
|
2082
2431
|
/** The clone URL for a task, honouring the selected source. */
|
|
2083
2432
|
cloneUrl(definition, options) {
|
|
2084
2433
|
const repo = definition.repoUrl ?? "";
|
|
@@ -2268,26 +2617,7 @@ var TaskService = class {
|
|
|
2268
2617
|
onFailure: cleanupInstall,
|
|
2269
2618
|
onSuccess: () => {
|
|
2270
2619
|
if (definition.needsSetup) this.cleanupWorkspace(checked.path);
|
|
2271
|
-
|
|
2272
|
-
const prompt = this.guard.promptFor(checked.path, this.materialZip(taskId), definition);
|
|
2273
|
-
const run = {
|
|
2274
|
-
runId,
|
|
2275
|
-
task: definition.id,
|
|
2276
|
-
workspaceId,
|
|
2277
|
-
workspacePath: checked.path,
|
|
2278
|
-
sessionId: null,
|
|
2279
|
-
createdAt: Date.now(),
|
|
2280
|
-
startedAt: null,
|
|
2281
|
-
endedAt: null,
|
|
2282
|
-
status: "prepared",
|
|
2283
|
-
prompt,
|
|
2284
|
-
timeLimitMinutes: this.guard.configuration.timeLimitMinutes
|
|
2285
|
-
};
|
|
2286
|
-
this.guard.registerRun(run);
|
|
2287
|
-
return {
|
|
2288
|
-
runId,
|
|
2289
|
-
prompt
|
|
2290
|
-
};
|
|
2620
|
+
return { prompt: this.promptNotice(this.guard.promptFor(checked.path, this.materialZip(taskId), definition)) };
|
|
2291
2621
|
}
|
|
2292
2622
|
});
|
|
2293
2623
|
if (job.status === "failed") cleanupInstall();
|
|
@@ -2307,11 +2637,16 @@ var TaskService = class {
|
|
|
2307
2637
|
ok: false,
|
|
2308
2638
|
error: checked.error
|
|
2309
2639
|
};
|
|
2640
|
+
if (!this.guard.active) return {
|
|
2641
|
+
ok: false,
|
|
2642
|
+
error: "请先启用评测守护。"
|
|
2643
|
+
};
|
|
2310
2644
|
const script = path.join(packageRoot(), "eval", taskId + ".py");
|
|
2311
2645
|
if (!fs.existsSync(script)) return {
|
|
2312
2646
|
ok: false,
|
|
2313
2647
|
error: "评测脚本不存在:" + script
|
|
2314
2648
|
};
|
|
2649
|
+
const evalWorkspace = (taskId === "neural-pendulum" ? this.stageHostWorkspace(workspaceId) : void 0) ?? checked.path;
|
|
2315
2650
|
return {
|
|
2316
2651
|
ok: true,
|
|
2317
2652
|
jobId: this.jobs.start({
|
|
@@ -2324,18 +2659,24 @@ var TaskService = class {
|
|
|
2324
2659
|
"utf8",
|
|
2325
2660
|
script,
|
|
2326
2661
|
"--workspace",
|
|
2327
|
-
|
|
2662
|
+
evalWorkspace
|
|
2328
2663
|
],
|
|
2329
2664
|
cwd: packageRoot(),
|
|
2330
2665
|
env: {
|
|
2331
2666
|
PYTHONPATH: "",
|
|
2332
2667
|
PYTHONHOME: "",
|
|
2333
2668
|
EVAL_CIFAR_PROXY: this.proxyValue(this.guard.configuration.cifarProxyUrl),
|
|
2334
|
-
EVAL_CIFAR_URL: this.cifarUrl(this.guard.configuration)
|
|
2669
|
+
EVAL_CIFAR_URL: this.cifarUrl(this.guard.configuration),
|
|
2670
|
+
EVAL_SUBMISSIONS_ROOT: path.join(guardRoot(), "submissions", workspaceId.replace(/[^A-Za-z0-9_.-]/g, "_"))
|
|
2335
2671
|
}
|
|
2336
2672
|
}).id
|
|
2337
2673
|
};
|
|
2338
2674
|
}
|
|
2675
|
+
/** Add the optional shared-machine notice to a task prompt. */
|
|
2676
|
+
promptNotice(prompt) {
|
|
2677
|
+
if (!this.guard.configuration.parallelEval) return prompt;
|
|
2678
|
+
return prompt + "\n\n注意:当前机器可能同时运行其他 agent 的评测/训练任务。运行环境的 CPU、内存、IO 与可用并发会因其他 agent 的行为而波动;请自行适应这种波动,不要假设环境稳定,也不要依赖固定墙钟时间或固定可用资源来判断进度。";
|
|
2679
|
+
}
|
|
2339
2680
|
/** Copy the packaged workspace.zip to a stable material path for install/manual use. */
|
|
2340
2681
|
materialZip(taskId = TANK_TASK) {
|
|
2341
2682
|
const definition = TASK_DEFINITIONS[taskId];
|
|
@@ -2394,25 +2735,9 @@ var TaskService = class {
|
|
|
2394
2735
|
error: "任务准备失败:" + (error instanceof Error ? error.message : String(error))
|
|
2395
2736
|
};
|
|
2396
2737
|
}
|
|
2397
|
-
const
|
|
2398
|
-
const prompt = promptOverride ?? this.guard.promptFor(checked.path, this.materialZip(taskId), definition);
|
|
2399
|
-
const run = {
|
|
2400
|
-
runId,
|
|
2401
|
-
task: definition.id,
|
|
2402
|
-
workspaceId,
|
|
2403
|
-
workspacePath: checked.path,
|
|
2404
|
-
sessionId: null,
|
|
2405
|
-
createdAt: Date.now(),
|
|
2406
|
-
startedAt: null,
|
|
2407
|
-
endedAt: null,
|
|
2408
|
-
status: "prepared",
|
|
2409
|
-
prompt,
|
|
2410
|
-
timeLimitMinutes: this.guard.configuration.timeLimitMinutes
|
|
2411
|
-
};
|
|
2412
|
-
this.guard.registerRun(run);
|
|
2738
|
+
const prompt = this.promptNotice(promptOverride ?? this.guard.promptFor(checked.path, this.materialZip(taskId), definition));
|
|
2413
2739
|
return {
|
|
2414
2740
|
ok: true,
|
|
2415
|
-
runId,
|
|
2416
2741
|
workspaceId,
|
|
2417
2742
|
workspacePath: checked.path,
|
|
2418
2743
|
prompt
|
|
@@ -2423,7 +2748,7 @@ var TaskService = class {
|
|
|
2423
2748
|
const definition = TASK_DEFINITIONS[taskId];
|
|
2424
2749
|
if (definition === void 0) return "";
|
|
2425
2750
|
const workspace = this.registry.get(workspaceId);
|
|
2426
|
-
return this.guard.promptFor(workspace?.path ?? "", this.materialZip(taskId), definition);
|
|
2751
|
+
return this.promptNotice(this.guard.promptFor(workspace?.path ?? "", this.materialZip(taskId), definition));
|
|
2427
2752
|
}
|
|
2428
2753
|
/** Filesystem paths for the manual channel. */
|
|
2429
2754
|
manualPaths(workspaceId, taskId = TANK_TASK) {
|
|
@@ -2435,7 +2760,7 @@ var TaskService = class {
|
|
|
2435
2760
|
const workspace = this.registry.get(workspaceId);
|
|
2436
2761
|
return {
|
|
2437
2762
|
source: this.materialZip(taskId),
|
|
2438
|
-
prompt: this.guard.promptFor(workspace?.path ?? "", this.materialZip(taskId), definition)
|
|
2763
|
+
prompt: this.promptNotice(this.guard.promptFor(workspace?.path ?? "", this.materialZip(taskId), definition))
|
|
2439
2764
|
};
|
|
2440
2765
|
}
|
|
2441
2766
|
/** A unique run id. */
|
|
@@ -2657,6 +2982,70 @@ function registerRoutes(ctx, guard, tasks) {
|
|
|
2657
2982
|
});
|
|
2658
2983
|
}
|
|
2659
2984
|
//#endregion
|
|
2985
|
+
//#region src/submission-tools.ts
|
|
2986
|
+
const OUTPUT_SCHEMA = {
|
|
2987
|
+
type: "object",
|
|
2988
|
+
additionalProperties: true,
|
|
2989
|
+
properties: {
|
|
2990
|
+
ok: { type: "boolean" },
|
|
2991
|
+
submission: { type: "string" },
|
|
2992
|
+
files: { type: "number" },
|
|
2993
|
+
stage: { type: "string" },
|
|
2994
|
+
t_can: { type: "number" },
|
|
2995
|
+
completed: { type: "boolean" },
|
|
2996
|
+
error: { type: "string" }
|
|
2997
|
+
},
|
|
2998
|
+
required: ["ok"]
|
|
2999
|
+
};
|
|
3000
|
+
function render(_args, value) {
|
|
3001
|
+
return [{
|
|
3002
|
+
type: "text",
|
|
3003
|
+
text: JSON.stringify(value)
|
|
3004
|
+
}];
|
|
3005
|
+
}
|
|
3006
|
+
function resolve(guard, exec) {
|
|
3007
|
+
if (!guard.active) throw new Error("eval guard is disabled");
|
|
3008
|
+
const run = guard.runForSession(exec.agent?.id);
|
|
3009
|
+
if (run === void 0) throw new Error("eval task run is not active for this session");
|
|
3010
|
+
if (run.task !== "neural-pendulum") throw new Error("this tool is only available for the neural-pendulum task");
|
|
3011
|
+
return run;
|
|
3012
|
+
}
|
|
3013
|
+
/** Install the task tools into one agent's own scoped registry. */
|
|
3014
|
+
function registerAgentSubmissionTools(agent, guard, tasks) {
|
|
3015
|
+
const tools = agent.ctx?.tools;
|
|
3016
|
+
if (tools === void 0) return;
|
|
3017
|
+
const submit = {
|
|
3018
|
+
name: "eval_submit",
|
|
3019
|
+
description: "Submit the current workspace source as a new immutable snapshot for the active Neural-Pendulum run. Submission itself does not change the score.",
|
|
3020
|
+
parameters: {
|
|
3021
|
+
type: "object",
|
|
3022
|
+
properties: {}
|
|
3023
|
+
},
|
|
3024
|
+
output: {
|
|
3025
|
+
schema: OUTPUT_SCHEMA,
|
|
3026
|
+
render
|
|
3027
|
+
},
|
|
3028
|
+
timeoutMs: 6e4,
|
|
3029
|
+
execute: async (_args, exec) => tasks.hostSubmit(resolve(guard, exec))
|
|
3030
|
+
};
|
|
3031
|
+
const test = {
|
|
3032
|
+
name: "eval_test",
|
|
3033
|
+
description: "Run a host-side evaluation of the latest submitted snapshot and return stage, t_can, completed and error. Only successful tests update the score.",
|
|
3034
|
+
parameters: {
|
|
3035
|
+
type: "object",
|
|
3036
|
+
properties: {}
|
|
3037
|
+
},
|
|
3038
|
+
output: {
|
|
3039
|
+
schema: OUTPUT_SCHEMA,
|
|
3040
|
+
render
|
|
3041
|
+
},
|
|
3042
|
+
timeoutMs: 9e5,
|
|
3043
|
+
execute: async (_args, exec) => tasks.hostTest(resolve(guard, exec))
|
|
3044
|
+
};
|
|
3045
|
+
tools.register(submit);
|
|
3046
|
+
tools.register(test);
|
|
3047
|
+
}
|
|
3048
|
+
//#endregion
|
|
2660
3049
|
//#region src/index.ts
|
|
2661
3050
|
/** Stable Cordis plugin name. */
|
|
2662
3051
|
const name = "eval-guard";
|
|
@@ -2701,7 +3090,12 @@ function apply(ctx, config) {
|
|
|
2701
3090
|
on: (event, listener) => child.on(event, listener)
|
|
2702
3091
|
};
|
|
2703
3092
|
const guard = new Guard(ctx, services, config, TANK_DEFINITION);
|
|
2704
|
-
|
|
3093
|
+
const tasks = new TaskService(guard, services.workspaceRegistry);
|
|
3094
|
+
guard.onRunAssociated = (agent, run) => {
|
|
3095
|
+
registerAgentSubmissionTools(agent, guard, tasks);
|
|
3096
|
+
};
|
|
3097
|
+
guard.onAutoRun = (agent, cwd) => tasks.autoBind(agent, cwd);
|
|
3098
|
+
registerRoutes(ctx, guard, tasks);
|
|
2705
3099
|
child.inject(["settings"], (settingsCtx) => {
|
|
2706
3100
|
const settings = settingsCtx.get("settings");
|
|
2707
3101
|
settingsCtx.effect(() => settings.configure({ auto: false }, ctx.fiber), "eval-guard: settings page");
|