@uinaf/skillcheck 1.5.3 → 1.6.0

package/dist/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
+import { encodeRunNamePart, generateRun, isIsolated, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
 import { lintSkills } from "./lint.js";
 import { execFileSync, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
@@ -110,7 +110,8 @@ function runConfigOf(opts) {
 		judge_model: opts.judgeModel,
 		judge_effort: opts.judgeEffort ?? null,
 		trials: opts.trials ?? 1,
-		agent_access: "online"
+		agent_access: "online",
+		isolated: isIsolated()
 	};
 }
 function configKey(c) {
@@ -120,12 +121,13 @@ function configKey(c) {
 		c.judge_model,
 		c.judge_effort ?? null,
 		c.trials ?? 1,
-		c.agent_access ?? "offline"
+		c.agent_access ?? "offline",
+		c.isolated ?? false
 	]);
 }
 function describeConfig(c) {
 	const effort = (e) => e ? `@${e}` : "";
-	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}, ${c.agent_access ?? "offline"}`;
+	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}, ${c.agent_access ?? "offline"}, ${c.isolated ? "isolated" : "host"}`;
 }
 function assertUniformConfig(entries, allowMixed) {
 	if (allowMixed) return;
@@ -335,7 +337,8 @@ function resultRunConfig(raw, meta, harness) {
 		judge_model: m.judge_model,
 		judge_effort: m.judge_effort ?? null,
 		trials: m.trials ?? 1,
-		agent_access: m.agent_access === "online" ? "online" : "offline"
+		agent_access: m.agent_access === "online" ? "online" : "offline",
+		isolated: m.isolated === true
 	};
 	const r = raw;
 	const agent = r?.config?.providers?.[0]?.config;
@@ -348,7 +351,8 @@ function resultRunConfig(raw, meta, harness) {
 		judge_model: judgeName(judge),
 		judge_effort: typeof judgeEffort === "string" ? judgeEffort : null,
 		trials: r?.results?.results?.length ?? 1,
-		agent_access: "offline"
+		agent_access: "offline",
+		isolated: false
 	};
 }
 function readJson(file) {
@@ -390,11 +394,16 @@ function discoverScenarios(root) {
 	return found.sort();
 }
 const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N] [--control]";
+function warnHostRun() {
+	if (isIsolated()) return;
+	console.error("warning: host run; the agent can read this machine's installed skills, plugins, and files, so scores are not isolated. Run in the isolated image: docs/usage.md#isolated-runs");
+}
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length !== 1) fail(`usage: skillcheck run <scenario-dir> ${RUN_FLAGS}`);
 	const opts = runOptions(flags);
 	ensureEvalPackages(opts);
+	warnHostRun();
 	const o = runScenario(positional[0], opts, resolveRoot(flags));
 	if (o.stats === void 0) {
 		console.error(`ERROR ${o.name}: ${o.error ?? "no usable result"} (promptfoo rc=${o.rc})`);
@@ -409,6 +418,7 @@ function cmdSweep(argv) {
 	const root = resolveRoot(flags);
 	const opts = runOptions(flags);
 	ensureEvalPackages(opts);
+	warnHostRun();
 	const all = flags.get("--all") === true;
 	const resultsDir = stateDirs(root).results;
 	const wanted = configKey(runConfigOf(opts));

package/dist/scenario.js

@@ -75,6 +75,10 @@ function stripHiddenFlag(skillMd) {
 	if (!range) return skillMd;
 	return skillMd.split("\n").filter((l, i) => !(i >= range[0] && i < range[1] && l.startsWith("disable-model-invocation:"))).join("\n");
 }
+function isIsolated() {
+	return process.env.SKILLCHECK_ISOLATED === "1" && (fs.existsSync("/.dockerenv") || fs.existsSync("/run/.containerenv")) && process.getuid?.() === 0 && Number.isInteger(Number(process.env.SKILLCHECK_AGENT_UID)) && fs.existsSync(AGENT_WRAPPER);
+}
+const AGENT_WRAPPER = "/usr/local/libexec/skillcheck/agent-wrapper.sh";
 const MARKDOWN_LINK = /\]\(\s*<([^>]+)>|\]\(\s*([^)\s]+)|^ {0,3}\[[^\]]+\]:\s*<?([^\s>]+)/gm;
 function linkedSiblings(skillDir) {
 	const skillsRoot = path.dirname(skillDir);
@@ -167,7 +171,7 @@ function materialize(s, runDir, harness, control = false) {
 	};
 	walk(workdir);
 	const manifestPath = path.join(runDir, "manifest.json");
-	fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+	fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), { mode: 384 });
 	return {
 		workdir,
 		manifestPath
@@ -189,7 +193,7 @@ function agentProvider(opts, workdir, skill, paths) {
 			working_dir: workdir,
 			skip_git_repo_check: true,
 			enable_streaming: true,
-			sandbox_mode: "workspace-write",
+			sandbox_mode: isIsolated() ? "danger-full-access" : "workspace-write",
 			network_access_enabled: true,
 			web_search_enabled: true,
 			cli_config: { features: { plugins: false } },
@@ -358,6 +362,24 @@ function privateHome(dir) {
 		fs.symlinkSync(path.join(source, name), path.join(dir, name));
 	}
 }
+function handOverToAgent(runDir, trials) {
+	const uid = Number(process.env.SKILLCHECK_AGENT_UID);
+	const gid = Number(process.env.SKILLCHECK_AGENT_GID ?? process.env.SKILLCHECK_AGENT_UID);
+	if (!Number.isInteger(uid) || !Number.isInteger(gid)) throw new Error("isolated run without SKILLCHECK_AGENT_UID: refusing to run the agent as root");
+	const own = (p) => {
+		fs.lchownSync(p, uid, gid);
+		if (fs.lstatSync(p).isDirectory()) for (const e of fs.readdirSync(p)) own(path.join(p, e));
+	};
+	for (const dir of [path.dirname(runDir), runDir]) fs.chmodSync(dir, 457);
+	for (const t of trials) {
+		fs.chmodSync(path.dirname(t.workdir), 457);
+		own(t.workdir);
+	}
+	for (const home of ["codex-home", "home"]) {
+		const p = path.join(runDir, home);
+		if (fs.existsSync(p)) own(p);
+	}
+}
 function generateRun(scenarioDir, opts, paths) {
 	const s = loadScenario(scenarioDir);
 	const name = runNameFor(scenarioDir, opts.harness, opts.control);
@@ -382,7 +404,8 @@ function generateRun(scenarioDir, opts, paths) {
 	}
 	const config = buildConfig(s, trials, opts, paths);
 	const configPath = path.join(runDir, "promptfooconfig.json");
-	fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+	fs.writeFileSync(configPath, JSON.stringify(config, null, 2), { mode: 384 });
+	if (isIsolated()) handOverToAgent(runDir, trials);
 	return {
 		name,
 		configPath,
@@ -391,4 +414,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { DEFAULT_CLAUDE_AGENT, FILE_BLOCK, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, linkedSiblings, loadScenario, materialize, privateCodexHome, privateHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };
+export { DEFAULT_CLAUDE_AGENT, FILE_BLOCK, buildConfig, encodeRunNamePart, generateRun, handOverToAgent, isHiddenSkill, isIsolated, linkedSiblings, loadScenario, materialize, privateCodexHome, privateHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/docs/usage.md

@@ -57,6 +57,45 @@ and a Claude agent limit of 50 turns. `--max-turns` changes that limit only for
 Claude; passing it with `codex` or `grok` fails before the eval starts. On
 those harnesses, omitting `--agent` leaves the model to that CLI's own default.
 
+### Isolated runs
+
+Run evals inside the image in [container/](../container/Dockerfile), which
+follows promptfoo's guidance for
+[coding agents](https://www.promptfoo.dev/docs/guides/evaluate-coding-agents/):
+a workspace is not a sandbox, so agents run in an ephemeral container with a
+read-only mount of the repository, writes on a separate volume, a
+project-local Codex home, and only the credentials they need. On the host the
+agent shares the operator's machine: it can read installed skills, enabled
+plugins, global guidance, and any file by absolute path. `run` and `sweep` warn
+when they are not isolated.
+
+```sh
+docker build -t skillcheck:<version> --build-arg SKILLCHECK_VERSION=<version> container
+docker run --rm --env-file <owner-only env file> \
+  -v "$PWD:/srv/work:ro" -v "$PWD/.skillcheck:/srv/work/.skillcheck" \
+  -v <codex config.toml>:/etc/skillcheck/codex/config.toml:ro \
+  skillcheck:<version> run --root /srv/work skills/<skill>/evals/<scenario> --trials 3
+```
+
+skillcheck runs as root and reads the repository from `/srv`, which only root
+can enter. Each agent binary starts through a wrapper that drops to the
+unprivileged `agent` user, which owns only its workdir and the run's homes; the
+run's promptfoo config and manifests, which carry the criteria, and promptfoo's
+own state stay root-only. Start one container per `run` or `sweep`
+invocation, as above: agents in one container share the agent user, so a
+control must never share a container with a skill run. The image's home is
+empty. Pass Claude gateway auth
+through the env file ([auth](#auth)) and mount a Codex `config.toml` written for
+the run that names only the model provider and its auth, never the operator's
+own; a token its auth command reads must be readable by the agent user. Codex's
+namespace sandbox cannot start inside a container, so skillcheck runs it with
+full access there; the container and the user drop are the boundary. A run
+counts as isolated only inside a container, with the image's marker variable,
+skillcheck running as root, an agent user configured, and the wrapper
+installed; results record `isolated` in their run
+configuration, so `summarize` refuses to mix isolated and host rows and `sweep`
+reruns host results.
+
 ### Trials
 
 One trial is one sample of a noisy process: the same scenario and skill can
@@ -221,6 +260,7 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
   "judge_effort": null,
   "trials": 3,
   "agent_access": "online",
+  "isolated": true,
   "aggregate": { "pass": false, "pass_rate": 0.6667, "score": 0.81, "...": "..." },
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"
@@ -228,8 +268,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```
 
 `summarize` reads those sidecars and refuses to mix skills-tree revisions or
-run configurations (agent model and effort, judge model and effort, trials, and
-agent access) in
+run configurations (agent model and effort, judge model and effort, trials,
+agent access, and isolation) in
 one scorecard, including retained rows from partial reruns, unless
 `--allow-mixed`. Configurations are compared within a harness, since harnesses
 differ by design. Sidecars written before run configurations were recorded fall

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.5.3",
+  "version": "1.6.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {