dsh-eval-guard 0.7.2 → 0.7.4

package/eval/neural-cpp.py

@@ -15,15 +15,12 @@ import sys
 import tempfile
 from pathlib import Path
 
-from common import EVAL_ROOT, clean_copy, emit, fail, parse_workspace
+from common import EVAL_ROOT, child_env, clean_copy, emit, fail, parse_workspace
 
 
 def run(cmd: list[str], cwd: Path, timeout: int = 600) -> subprocess.CompletedProcess[str]:
-    env = dict(os.environ)
-    env.pop('PYTHONPATH', None)
-    env.pop('PYTHONHOME', None)
     return subprocess.run(cmd, cwd=str(cwd), capture_output=True, text=True,
-                          encoding='utf-8', errors='replace', env=env, timeout=timeout)
+                          encoding='utf-8', errors='replace', env=child_env(), timeout=timeout)
 
 
 def main() -> int:
@@ -31,6 +28,9 @@ def main() -> int:
     hidden = EVAL_ROOT.parent / 'assets' / 'neural-cpp' / 'hidden' / 'test_neural.cpp'
     if not hidden.is_file():
         return fail({'task': 'neural-cpp', 'ok': False, 'error': 'hidden test missing'})
+    # Snapshot the hidden test BEFORE the candidate build runs: an agent-authored
+    # CMakeLists can otherwise overwrite the asset and substitute a passing test.
+    hidden_source = hidden.read_bytes()
 
     with tempfile.TemporaryDirectory(prefix='neural-cpp-eval-') as td:
         root = Path(td)
@@ -67,9 +67,12 @@ def main() -> int:
         if not lib.is_file():
             return fail({'task': 'neural-cpp', 'ok': False, 'error': 'build/libnn.a missing'})
 
+        # Compile the in-memory snapshot, not the on-disk asset the build could edit.
+        hidden_copy = root / 'test_neural.cpp'
+        hidden_copy.write_bytes(hidden_source)
         exe = root / 'test_neural'
         cc = run([compiler, '-std=c++17', '-O3', '-pthread',
-                  '-I', str(source / 'include'), str(hidden), str(lib),
+                  '-I', str(source / 'include'), str(hidden_copy), str(lib),
                   '-lm', '-o', str(exe)], root, 300)
         if cc.returncode != 0:
             return fail({'task': 'neural-cpp', 'ok': False, 'error': 'hidden test compile failed',

package/eval/neural-fp32.py

@@ -20,7 +20,7 @@ import numpy as np
 import torch
 import torch.nn.functional as F
 
-from common import EVAL_ROOT, clean_copy, emit, fail, parse_workspace
+from common import EVAL_ROOT, child_env, clean_copy, emit, fail, parse_workspace
 
 torch.set_num_threads(1)
 torch.set_float32_matmul_precision('highest')
@@ -29,6 +29,9 @@ torch.backends.mkldnn.deterministic = True
 torch.manual_seed(1234)
 
 SEEDS = (11, 22, 33)
+# A single CLI invocation budget and a total budget for the whole oracle run.
+INVOCATION_TIMEOUT_SECONDS = 60
+OPS_DEADLINE_SECONDS = 1800
 
 
 def write_tensor(path: Path, t: torch.Tensor) -> None:
@@ -65,11 +68,12 @@ def build_candidate(workspace: Path, root: Path) -> Path:
         shutil.rmtree(build, ignore_errors=True)
     build.mkdir(parents=True)
     cfg = subprocess.run(['cmake', '-S', '.', '-B', 'build', '-DCMAKE_BUILD_TYPE=Release'],
-                         cwd=str(source), capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=180)
+                         cwd=str(source), capture_output=True, text=True, encoding='utf-8', errors='replace',
+                         timeout=180, env=child_env())
     if cfg.returncode != 0:
         raise RuntimeError('cmake configure failed\n' + (cfg.stdout + cfg.stderr)[-4000:])
     made = subprocess.run(['cmake', '--build', 'build', '-j4'], cwd=str(source), capture_output=True,
-                          text=True, encoding='utf-8', errors='replace', timeout=1800)
+                          text=True, encoding='utf-8', errors='replace', timeout=1800, env=child_env())
     if made.returncode != 0:
         raise RuntimeError('build failed\n' + (made.stdout + made.stderr)[-4000:])
     binary = build / 'nn_task'
@@ -87,7 +91,8 @@ class Case:
         self.flops = flops
 
 
-def run_operation(name: str, threshold: float, cases: list[Case], binary: Path, root: Path) -> dict:
+def run_operation(name: str, threshold: float, cases: list[Case], binary: Path, root: Path,
+                  invocation_timeout: float = INVOCATION_TIMEOUT_SECONDS) -> dict:
     op_dir = root / 'ops' / name
     op_dir.mkdir(parents=True, exist_ok=True)
     executable = True
@@ -111,7 +116,8 @@ def run_operation(name: str, threshold: float, cases: list[Case], binary: Path,
         started = time.perf_counter()
         try:
             proc = subprocess.run(cmd, cwd=str(case_dir), capture_output=True, text=True,
-                                  encoding='utf-8', errors='replace', timeout=300)
+                                  encoding='utf-8', errors='replace', timeout=invocation_timeout,
+                                  env=child_env())
         except subprocess.TimeoutExpired:
             executable = False
             first_error = first_error or 'timeout'
@@ -459,7 +465,13 @@ def main() -> int:
         except Exception as exc:
             return fail({'task': 'neural-fp32', 'ok': False, 'error': str(exc)})
         rows = []
+        deadline = time.monotonic() + OPS_DEADLINE_SECONDS
         for name, threshold, builder in OPERATIONS:
+            if time.monotonic() > deadline:
+                rows.append({'name': name, 'executable': False, 'passed': False,
+                             'weighted_mean_relative_error': None, 'threshold': threshold,
+                             'speed_gflops': None, 'error': 'evaluation deadline exceeded'})
+                continue
             try:
                 cases = builder()
                 row = run_operation(name, threshold, cases, binary, root)

package/eval/swe-bench-pro.py

@@ -3,9 +3,12 @@
 from __future__ import annotations
 import os, tempfile
 from pathlib import Path
-from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, run_legacy
+from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, restore_baseline, run_legacy, verify_venv
 
 workspace = parse_workspace()
+problem = verify_venv(workspace)
+if problem is not None:
+    raise SystemExit(fail({'task': 'swe-bench-pro', 'ok': False, 'error': problem}))
 with tempfile.TemporaryDirectory(prefix='swe-pro-eval-') as td:
     temp = Path(td)
     candidate = temp / 'candidate' / 'workspace'
@@ -13,13 +16,14 @@ with tempfile.TemporaryDirectory(prefix='swe-pro-eval-') as td:
     clean_copy(workspace, candidate, ignore_venv=True)
     venv = workspace / '.venv'
     if venv.is_dir():
-        os.symlink(venv, candidate / '.venv')
+        try:
+            os.symlink(venv, candidate / '.venv')
+        except FileExistsError:
+            pass
+    restore_baseline(candidate)
     result = run_legacy(EVAL_ROOT / 'legacy' / 'swe_bench_pro.py', ['--submissions', str(temp)], temp / 'out')
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'swe-bench-pro', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    passed = row.get('passed', 0)
-    failed = row.get('failed', 0)
-    errors = row.get('errors', 0)
-    resolved = failed == 0 and errors == 0 and passed > 0
-    emit({'task': 'swe-bench-pro', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': passed, 'failed': failed, 'errors': errors, 'status': row.get('status')})
+    resolved = row.get('resolved') is True
+    emit({'task': 'swe-bench-pro', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': row.get('passed', 0), 'failed': row.get('failed', 0), 'errors': row.get('errors', 0), 'f2pPassed': row.get('f2p_passed', 0), 'f2pTotal': row.get('f2p_total', 0), 'status': row.get('status')})

package/eval/tank.py

@@ -16,5 +16,14 @@ with tempfile.TemporaryDirectory(prefix='tank-eval-') as td:
     result = run_legacy(EVAL_ROOT / 'legacy' / 'tank.py', ['--submissions', str(temp), '--arena', str(EVAL_ROOT / 'arena' / 'tank'), '--tier', 'quick'], temp / 'out')
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'tank', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
-    row = candidate_row(result['summary'])
-    emit({'task': 'tank', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})
+    summary = result['summary']
+    row = candidate_row(summary)
+    if not row:
+        raise SystemExit(fail({'task': 'tank', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    wins = draws = losses = 0
+    for pair in summary.get('pairwise') or []:
+        if pair.get('a') == 'candidate':
+            wins += int(pair.get('wins', 0)); draws += int(pair.get('draws', 0)); losses += int(pair.get('losses', 0))
+        elif pair.get('b') == 'candidate':
+            wins += int(pair.get('losses', 0)); draws += int(pair.get('draws', 0)); losses += int(pair.get('wins', 0))
+    emit({'task': 'tank', 'ok': True, 'score': row.get('rating', 0), 'rating': row.get('rating', 0), 'wins': wins, 'losses': losses, 'draws': draws, 'games': row.get('games', 0)})

package/eval/warehouse.py

@@ -17,4 +17,20 @@ with tempfile.TemporaryDirectory(prefix='warehouse-eval-') as td:
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'warehouse', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    emit({'task': 'warehouse', 'ok': True, 'score': row.get('speedup', 0), 'speedup': row.get('speedup', 0), 'correct': row.get('correct'), 'status': row.get('status')})
+    if not row:
+        raise SystemExit(fail({'task': 'warehouse', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    compile_error = row.get('compileError') or ''
+    correct = row.get('correct', 0)
+    total = row.get('total', 0)
+    status = 'compile-failed' if compile_error else ('ok' if correct == total and total else 'incorrect')
+    emit({
+        'task': 'warehouse',
+        'ok': True,
+        'score': row.get('score', 0),
+        'speedup': row.get('geomeanSpeedup', 0),
+        'geomeanSpeedup': row.get('geomeanSpeedup', 0),
+        'correct': correct,
+        'total': total,
+        'status': status,
+        'error': compile_error,
+    })

package/lib/client/index.js

@@ -285,7 +285,6 @@ window.__ModuleLoader__.load({
 			};
 			const selectTask = (next) => {
 				setTaskId(next);
-				save({ taskId: next });
 				try {
 					localStorage.setItem("evalGuard.taskId", next);
 				} catch {}
@@ -296,21 +295,24 @@ window.__ModuleLoader__.load({
 				setJobNotice("");
 				if (workspaceId !== "") loadManual(workspaceId, next);
 				const task = state?.tasks.find((item) => item.id === next);
-				if (task === void 0) return;
+				if (task === void 0) {
+					save({ taskId: next });
+					return;
+				}
 				const remotes = task.remotes ?? [];
 				const cloneSource = draft.cloneSource === "custom" || remotes.some((remote) => remote.id === draft.cloneSource) ? draft.cloneSource : remotes[0]?.id ?? "github";
-				setDraft((current) => ({
-					...current,
+				const patch = {
+					taskId: next,
 					timeLimitMinutes: task.timeLimitMinutes,
 					timeLimitEnabled: task.timeLimitEnabled,
 					cloneSource
+				};
+				setDraft((current) => ({
+					...current,
+					...patch
 				}));
 				setNotice(t("settings.taskDefaults").replace("{minutes}", String(task.timeLimitMinutes)).replace("{state}", task.timeLimitEnabled ? t("settings.on") : t("settings.off")));
-				save({
-					timeLimitMinutes: task.timeLimitMinutes,
-					timeLimitEnabled: task.timeLimitEnabled,
-					cloneSource
-				});
+				save(patch);
 			};
 			const toggleEnabled = (next) => {
 				if (!next) {
@@ -472,22 +474,43 @@ window.__ModuleLoader__.load({
 				}),
 				/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("section", {
 					style: box,
-					children: [/* @__PURE__ */ (0, react_jsx_runtime.jsx)("div", {
-						style: row,
-						children: /* @__PURE__ */ (0, react_jsx_runtime.jsxs)("label", { children: [
-							/* @__PURE__ */ (0, react_jsx_runtime.jsx)("input", {
-								type: "checkbox",
-								checked: draft.enabled,
-								disabled: !ready,
-								onChange: (event) => toggleEnabled(event.target.checked)
-							}),
-							" ",
-							t("settings.enabled")
-						] })
-					}), /* @__PURE__ */ (0, react_jsx_runtime.jsx)("p", {
-						style: note,
-						children: t("settings.enabledHint")
-					})]
+					children: [
+						/* @__PURE__ */ (0, react_jsx_runtime.jsx)("div", {
+							style: row,
+							children: /* @__PURE__ */ (0, react_jsx_runtime.jsxs)("label", { children: [
+								/* @__PURE__ */ (0, react_jsx_runtime.jsx)("input", {
+									type: "checkbox",
+									checked: draft.enabled,
+									disabled: !ready,
+									onChange: (event) => toggleEnabled(event.target.checked)
+								}),
+								" ",
+								t("settings.enabled")
+							] })
+						}),
+						/* @__PURE__ */ (0, react_jsx_runtime.jsx)("p", {
+							style: note,
+							children: t("settings.enabledHint")
+						}),
+						state?.error !== void 0 && /* @__PURE__ */ (0, react_jsx_runtime.jsxs)("p", {
+							style: {
+								...note,
+								color: "#d99"
+							},
+							children: [
+								t("settings.enableError"),
+								": ",
+								state.error
+							]
+						}),
+						draft.enabled && state !== null && !state.enabled && /* @__PURE__ */ (0, react_jsx_runtime.jsx)("p", {
+							style: {
+								...note,
+								color: "#d99"
+							},
+							children: t("settings.enableInactive")
+						})
+					]
 				}),
 				/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("section", {
 					style: box,
@@ -1023,7 +1046,7 @@ window.__ModuleLoader__.load({
 			"settings.nav": "评测守护",
 			"settings.title": "评测守护",
 			"settings.enabled": "启用评测守护",
-			"settings.enabledHint": "启用后，所有用户主对话只能读写自己的工作区，禁止联网、子代理与跨 agent 通信，并按任务时限强制终止。默认关闭。",
+			"settings.enabledHint": "启用后，用户主对话的文件工具被限制在会话工作区（系统目录只读可见，/tmp 为工作区私有临时目录）；禁止联网、子代理与跨 agent 通信；并按任务时限强制终止。默认关闭。",
 			"settings.runningWarning": "当前有正在运行的对话。启用评测守护会立即约束它们，是否强行开启？",
 			"settings.runningCancel": "放弃",
 			"settings.runningForce": "强行开启",
@@ -1033,7 +1056,7 @@ window.__ModuleLoader__.load({
 			"settings.parallelEvalHint": "启用后，安装任务时会在提示词中提醒 agent：运行环境可能因其他 agent 的行为而波动，需要自行适应。",
 			"settings.minutes": "分钟",
 			"settings.forbidden": "额外禁止路径",
-			"settings.forbiddenHint": "每行一个绝对路径；# 开头为注释；! 开头表示在禁止范围内放开该子路径；按行顺序生效。默认空。",
+			"settings.forbiddenHint": "每行一个绝对路径；# 开头为注释；! 开头表示在禁止范围内放开该子路径；按行顺序生效；含 * 或 ? 的行按首个通配符之前的前缀生效。默认空。",
 			"settings.blacklist": "工具黑名单",
 			"settings.blacklistHint": "每行一个工具名；默认内容按当前已安装工具生成，不存在的工具会被忽略。",
 			"settings.reset": "恢复默认黑名单",
@@ -1074,6 +1097,8 @@ window.__ModuleLoader__.load({
 			"settings.recent": "最近事件",
 			"settings.none": "暂无。",
 			"settings.warningTitle": "评测守护警告",
+			"settings.enableError": "启用失败",
+			"settings.enableInactive": "开关已保存，但守护未生效：请查看上方错误信息并修复依赖后重试。",
 			"settings.cifarProxy": "CIFAR 下载代理",
 			"settings.cifarMirror": "CIFAR 下载镜像",
 			"settings.close": "关闭"
@@ -1082,7 +1107,7 @@ window.__ModuleLoader__.load({
 			"settings.nav": "Eval Guard",
 			"settings.title": "Eval Guard",
 			"settings.enabled": "Enable evaluation guard",
-			"settings.enabledHint": "When on, every user main conversation may only read/write its own workspace, with network, subagents and cross-agent channels blocked, and a wall-clock limit. Off by default.",
+			"settings.enabledHint": "When on, user main conversations keep file tools inside their session workspace (system directories stay read-only visible; /tmp is a private workspace temp dir), network/subagents/cross-agent channels are blocked, and a wall-clock limit applies. Off by default.",
 			"settings.runningWarning": "Conversations are running right now. Enabling the guard will constrain them immediately. Force enable?",
 			"settings.runningCancel": "Cancel",
 			"settings.runningForce": "Force enable",
@@ -1092,7 +1117,7 @@ window.__ModuleLoader__.load({
 			"settings.parallelEvalHint": "When enabled, install prompts warn the agent that the runtime may fluctuate because other agents share the machine; it must adapt.",
 			"settings.minutes": "minutes",
 			"settings.forbidden": "Extra forbidden paths",
-			"settings.forbiddenHint": "One absolute path per line; # comments; ! re-allows a subpath; later lines win. Empty by default.",
+			"settings.forbiddenHint": "One absolute path per line; # comments; ! re-allows a subpath; later lines win; a line with * or ? applies to the prefix before the first wildcard. Empty by default.",
 			"settings.blacklist": "Tool blacklist",
 			"settings.blacklistHint": "One tool name per line; the default is generated from the tools installed now and missing tools are ignored.",
 			"settings.reset": "Reset default blacklist",
@@ -1135,6 +1160,8 @@ window.__ModuleLoader__.load({
 			"settings.recent": "Recent events",
 			"settings.none": "None.",
 			"settings.warningTitle": "Eval Guard warning",
+			"settings.enableError": "Enable failed",
+			"settings.enableInactive": "The switch was saved, but the guard is not active. See the error above, fix the dependency, then retry.",
 			"settings.close": "Close"
 		};
 		//#endregion