oh-my-promptfoo 1.23.0 → 1.23.1

package/dist/delegate-runner.js

@@ -6,7 +6,7 @@ import { a as nativeMcpAttributes, c as graderConfig, d as object, f as validate
 import { builtinModules } from "node:module";
 import fs, { lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
 import { access, lstat, mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises";
-import path, { basename, dirname, isAbsolute, join, resolve } from "node:path";
+import path, { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import assert from "node:assert";
 import process$1 from "node:process";
@@ -14,6 +14,7 @@ import v8 from "node:v8";
 import { format, inspect } from "node:util";
 import "node:child_process";
 import { tmpdir } from "node:os";
+import { createHash } from "node:crypto";
 // node_modules/.pnpm/@opentelemetry+context-async-hooks@2.12.0_@opentelemetry+api@1.9.1/node_modules/@opentelemetry/context-async-hooks/build/src/AbstractAsyncHooksContextManager.js
 var require_AbstractAsyncHooksContextManager = /* @__PURE__ */ __commonJSMin(((exports) => {
 	Object.defineProperty(exports, "__esModule", { value: true });
@@ -8808,6 +8809,134 @@ var NativeToolInventory = class {
 	}
 };
 // packages/oh-my-promptfoo/src/skill-evidence.ts
+function fields(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function literalWords(command, powershell) {
+	if (/[\r\n\0$`;&|<>()[\]{}*?#!,]/.test(command) || !powershell && command.includes("\\")) return;
+	const words = [];
+	let rest = command.trim();
+	while (rest) {
+		const match = /^(?:"([^"]*)"|'([^']*)'|([^\s'"]+))(?=[ \t]|$)/.exec(rest);
+		if (!match) return;
+		words.push(match[1] ?? match[2] ?? match[3]);
+		if (words.length > 8) return;
+		rest = rest.slice(match[0].length).replace(/^[ \t]+/, "");
+	}
+	return words;
+}
+function literalPath(path, powershell) {
+	const native = powershell ? path.replace(/\\/g, sep) : path;
+	if (!native || native.startsWith("-") || native.startsWith("~") || native.split(/[\\/]/).includes("..")) return;
+	if (native.replace(process.platform === "win32" ? /^[A-Za-z]:/ : /^$/, "").includes(":")) return;
+	return native;
+}
+function containedShellDirectory(cwd, workingDir) {
+	let cursor = cwd;
+	const root = resolve(workingDir);
+	while (cursor !== root) {
+		if (!lstatSync(cursor).isDirectory()) return false;
+		const parent = dirname(cursor);
+		if (parent === cursor) return false;
+		cursor = parent;
+	}
+	return true;
+}
+/** Capture bounded local evidence at tool start; nothing is credited until completion. */
+function observedShellSkillRead(toolName, args, workingDir) {
+	if (toolName !== "bash" && toolName !== "powershell") return;
+	const input = fields(args);
+	if (!input || typeof input.command !== "string" || input.command.length > 8192) return;
+	const powershell = toolName === "powershell";
+	const parts = powershell ? [input.command] : input.command.split(";");
+	if (parts.length > 2) return;
+	const words = literalWords(parts[0], powershell);
+	if (!words) return;
+	let path;
+	if (!powershell) {
+		if (words[0] !== "cat") return;
+		if (words[1] === "--") words.splice(1, 1);
+		if (words.length !== 2) return;
+		path = words[1];
+	} else {
+		if (!/^get-content[ \t]/i.test(parts[0].trim())) return;
+		if (words.shift()?.toLowerCase() !== "get-content") return;
+		let raw = false;
+		let pathOption = false;
+		for (const word of words) {
+			const option = word.toLowerCase();
+			if (option === "-raw" && !raw) raw = true;
+			else if ((option === "-path" || option === "-literalpath") && !pathOption && !path) pathOption = true;
+			else if (!word.startsWith("-") && !path) path = word;
+			else return;
+		}
+	}
+	const nativePath = path && literalPath(path, powershell);
+	if (!nativePath) return;
+	if (input.workdir !== void 0 || input.workingDirectory !== void 0 || input.working_dir !== void 0) return;
+	const requestedCwd = input.cwd === void 0 ? "." : input.cwd;
+	if (typeof requestedCwd !== "string") return;
+	const nativeCwd = literalPath(requestedCwd, powershell);
+	if (!nativeCwd) return;
+	const cwd = resolve(workingDir, nativeCwd);
+	const candidate = resolve(cwd, nativePath);
+	const skill = observedSkillRead("read", { path: candidate }, workingDir);
+	if (!skill) return;
+	try {
+		if (!containedShellDirectory(cwd, workingDir)) return;
+		if (parts.length === 2) {
+			const listing = literalWords(parts[1], false);
+			const directory = listing?.[1] && literalPath(listing[1], false);
+			if (listing?.length !== 2 || listing[0] !== "ls" || !directory || resolve(cwd, directory) !== dirname(candidate)) return;
+		}
+		if (statSync(candidate).size > 1048576) return;
+		const body = readFileSync(candidate, "utf8").replace(/\r\n/g, "\n");
+		if (!body.trim()) return;
+		return {
+			skill,
+			cwd,
+			bodyHash: createHash("sha256").update(body).digest("hex")
+		};
+	} catch {
+		return;
+	}
+}
+/** Only model-facing output containing the complete unchanged file proves a read. */
+function completedShellSkillRead(read, completion, workingDir) {
+	const data = fields(completion);
+	const result = fields(data?.result);
+	if (data?.success !== true || data.error || typeof result?.content !== "string") return;
+	const execution = fields(data.shellExecution);
+	if (data.shellExecution !== void 0 && execution?.exitCode !== 0) return;
+	if (result.contents !== void 0 && !Array.isArray(result.contents)) return;
+	let exited = execution?.exitCode === 0;
+	let terminal = false;
+	for (const value of result.contents ?? []) {
+		const block = fields(value);
+		if (block?.type !== "shell_exit" && block?.type !== "terminal") continue;
+		terminal = true;
+		if (block.exitCode !== void 0 && block.exitCode !== 0) return;
+		if (block.type === "shell_exit" && block.exitCode !== 0) return;
+		exited ||= block.exitCode === 0;
+		if (block.cwd !== void 0) {
+			const cwd = typeof block.cwd === "string" && literalPath(block.cwd, true);
+			if (!cwd || resolve(read.cwd, cwd) !== read.cwd) return;
+		}
+	}
+	if (terminal && !exited) return;
+	const local = observedSkillRead("read", { path: read.skill.path }, workingDir);
+	if (!local) return;
+	try {
+		if (!containedShellDirectory(read.cwd, workingDir)) return;
+		if (statSync(local.path).size > 1048576) return;
+		const body = readFileSync(local.path, "utf8").replace(/\r\n/g, "\n");
+		if (createHash("sha256").update(body).digest("hex") !== read.bodyHash) return;
+		if (!result.content.replace(/\r\n/g, "\n").includes(body)) return;
+		return read.skill;
+	} catch {
+		return;
+	}
+}
 const skillParentDirectories = [
 	".agents",
 	".claude",
@@ -9542,6 +9671,7 @@ async function runCopilot(frame, signal) {
 	const cleanupErrors = [];
 	const tools = /* @__PURE__ */ new Set();
 	const skillReads = /* @__PURE__ */ new Map();
+	const shellSkillReads = /* @__PURE__ */ new Map();
 	const skillCalls = [];
 	const inventory = new ToolInventoryBuilder(sensitive);
 	const subagents = /* @__PURE__ */ new Map();
@@ -9688,6 +9818,8 @@ async function runCopilot(frame, signal) {
 					if (!data.mcpServerName && !data.mcpConfigServerName) {
 						const skillRead = observedSkillRead(data.toolName, data.arguments, frame.workingDir);
 						if (skillRead) skillReads.set(data.toolCallId, skillRead);
+						const shellRead = observedShellSkillRead(data.toolName, data.arguments, frame.workingDir);
+						if (shellRead) shellSkillReads.set(data.toolCallId, shellRead);
 					}
 					const attributes = JSON.parse(redact(jsonSafe(mcpAttributes), sensitive));
 					try {
@@ -9708,6 +9840,10 @@ async function runCopilot(frame, signal) {
 					const skillRead = skillReads.get(data.toolCallId);
 					skillReads.delete(data.toolCallId);
 					if (skillRead && data.success === true) skillCalls.push(skillRead);
+					const shellRead = shellSkillReads.get(data.toolCallId);
+					shellSkillReads.delete(data.toolCallId);
+					const completedRead = shellRead && completedShellSkillRead(shellRead, data, frame.workingDir);
+					if (completedRead) skillCalls.push(completedRead);
 					emit({
 						version: 1,
 						type: "tool",