oh-my-promptfoo 1.26.0-next.145 → 1.26.0

package/dist/assertions.cjs

@@ -6,6 +6,7 @@ let node_fs = require("node:fs");
 let node_fs_promises = require("node:fs/promises");
 let node_os = require("node:os");
 let node_module = require("node:module");
+let node_util = require("node:util");
 // packages/oh-my-promptfoo/src/rubric-contract.ts
 const PROMPTFOO_TEMPLATE_SLOT = /\{\{-?\s*(output|rubric)\b/;
 let templateEngine;
@@ -1342,6 +1343,152 @@ function skillNotLoaded(_output, context) {
 		return error(cause instanceof Error ? cause.message : String(cause));
 	}
 }
+// packages/oh-my-promptfoo/src/workspace-file.ts
+function criterion(input) {
+	const value = require_fs.closed(input, [
+		"path",
+		"equals",
+		"contains",
+		"regex",
+		"isJson",
+		"jsonPath",
+		"absent",
+		"byteLength",
+		"encoding"
+	], "workspace-file criterion");
+	workspacePath(value.path);
+	for (const key of ["equals", "contains"]) if (value[key] !== void 0 && typeof value[key] !== "string") throw new Error(`${key} must be a string`);
+	for (const key of ["isJson", "absent"]) if (value[key] !== void 0 && typeof value[key] !== "boolean") throw new Error(`${key} must be boolean`);
+	if (value.byteLength !== void 0 && (!Number.isSafeInteger(value.byteLength) || value.byteLength < 0)) throw new Error("byteLength must be a nonnegative integer");
+	if (value.encoding !== void 0 && ![
+		"utf8",
+		"utf16le",
+		"ascii",
+		"latin1"
+	].includes(String(value.encoding))) throw new Error("Unsupported workspace-file encoding");
+	if (value.absent === true && Object.keys(value).some((key) => key !== "path" && key !== "absent")) throw new Error("absent: true cannot be combined with content clauses");
+	if (value.jsonPath !== void 0) segments((typeof value.jsonPath === "string" ? { path: value.jsonPath } : require_fs.closed(value.jsonPath, ["path", "equals"], "jsonPath")).path);
+	if (value.regex !== void 0) pattern(value.regex);
+	return value;
+}
+function segments(path) {
+	if (typeof path !== "string") throw new Error("jsonPath requires a string path");
+	if (path === "$" || path === "") return [];
+	if (path.startsWith("/")) {
+		if (/~(?![01])/.test(path)) throw new Error("Invalid JSON Pointer escape");
+		return path.slice(1).split("/").map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~"));
+	}
+	const parts = path.replace(/^\$\./, "").split(".");
+	if (parts.some((part) => !part || /[[\]*]/.test(part))) throw new Error("jsonPath accepts JSON Pointer or dotted properties, not expressions");
+	return parts;
+}
+function pattern(value) {
+	if (typeof value === "string") return new RegExp(value);
+	const spec = require_fs.closed(value, ["regex", "flags"], "regex");
+	if (typeof spec.regex !== "string" || spec.flags !== void 0 && typeof spec.flags !== "string") throw new Error("regex requires a pattern and optional string flags");
+	return new RegExp(spec.regex, spec.flags);
+}
+function check(file, bytes) {
+	if (file.absent === true) return bytes === void 0 ? [] : ["File exists"];
+	if (bytes === void 0) throw new Error(`Workspace file ${file.path} is absent`);
+	const failures = [];
+	if (file.byteLength !== void 0 && bytes.length !== file.byteLength) failures.push(`Expected ${file.byteLength} bytes, got ${bytes.length}`);
+	if (file.encoding === void 0 && file.equals === void 0 && file.contains === void 0 && file.regex === void 0 && file.isJson === void 0 && file.jsonPath === void 0) return failures;
+	const encoding = file.encoding ?? "utf8";
+	let text;
+	try {
+		if (encoding === "ascii" && bytes.some((byte) => byte > 127)) throw new Error("non-ASCII bytes");
+		text = encoding === "latin1" || encoding === "ascii" ? bytes.toString(encoding) : new TextDecoder(encoding === "utf16le" ? "utf-16le" : "utf-8", {
+			fatal: true,
+			ignoreBOM: true
+		}).decode(bytes);
+	} catch {
+		return [...failures, `File is not valid ${encoding}`];
+	}
+	let selected = text;
+	if (file.isJson !== void 0 || file.jsonPath !== void 0) {
+		let parsed;
+		let validJson = true;
+		try {
+			parsed = JSON.parse(text);
+		} catch {
+			validJson = false;
+			if (file.isJson !== false || file.jsonPath !== void 0) return [...failures, "File is not valid JSON"];
+		}
+		if (file.isJson === false && validJson) failures.push("File is valid JSON");
+		if (file.jsonPath !== void 0) {
+			const spec = typeof file.jsonPath === "string" ? { path: file.jsonPath } : file.jsonPath;
+			selected = parsed;
+			for (const part of segments(spec.path)) {
+				if (selected === null || typeof selected !== "object" || !Object.hasOwn(selected, part)) return [...failures, `JSON field ${spec.path} is absent`];
+				selected = selected[part];
+			}
+			if ("equals" in spec && !(0, node_util.isDeepStrictEqual)(selected, spec.equals)) failures.push(`JSON field ${spec.path} does not equal the expected value`);
+		}
+	}
+	if (file.equals !== void 0 && selected !== file.equals) failures.push("Content does not equal the expected string");
+	if (file.contains !== void 0 && (typeof selected !== "string" || !selected.includes(file.contains))) failures.push("Content does not contain the expected string");
+	if (file.regex !== void 0 && (typeof selected !== "string" || !pattern(file.regex).test(selected))) failures.push("Content does not match the regex");
+	return failures;
+}
+/** Deterministic file criteria, all ANDed; evidence errors never certify a pass. */
+async function workspaceFile(_output, context) {
+	const componentResults = [];
+	try {
+		const config = context?.config;
+		let files;
+		if (config && typeof config === "object" && "files" in config) {
+			const spec = require_fs.closed(config, ["files"], "workspace-file config");
+			if (!Array.isArray(spec.files) || !spec.files.length) throw new Error("files must be a nonempty list");
+			files = spec.files.map(criterion);
+		} else files = [criterion(config)];
+		if (new Set(files.map((file) => file.path)).size !== files.length) throw new Error("files must have unique paths");
+		const workspace = new CapturedWorkspace(context);
+		for (const file of files) try {
+			const failures = check(file, await workspace.read(file.path));
+			const pass = failures.length === 0;
+			componentResults.push({
+				pass,
+				score: pass ? 1 : 0,
+				reason: failures.join("; ") || "File criteria passed",
+				assertion: {
+					type: "javascript",
+					metric: file.path
+				},
+				metadata: {}
+			});
+		} catch (cause) {
+			componentResults.push({
+				pass: false,
+				score: 0,
+				reason: cause instanceof Error ? cause.message : String(cause),
+				assertion: {
+					type: "javascript",
+					metric: file.path
+				},
+				metadata: { graderError: true }
+			});
+		}
+		const pass = componentResults.every((result) => result.pass);
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: pass ? "All workspace file criteria passed" : componentResults.filter((result) => !result.pass).map((result) => `${result.assertion.metric}: ${result.reason}`).join("; "),
+			namedScores: Object.fromEntries(componentResults.map((result) => [result.assertion.metric, result.score])),
+			componentResults,
+			metadata: componentResults.some((result) => result.metadata.graderError) ? { graderError: true } : {}
+		};
+	} catch (cause) {
+		return {
+			pass: false,
+			score: 0,
+			reason: cause instanceof Error ? cause.message : String(cause),
+			namedScores: {},
+			componentResults,
+			metadata: { graderError: true }
+		};
+	}
+}
 // packages/oh-my-promptfoo/src/assertions.ts
 const SELF = /^package:oh-my-promptfoo\/assertions:llmAssert$/;
 const MISPLACED_PROVIDER_ERROR = "llmAssert does not see provider: beside config: (Promptfoo does not pass it to package assertions): move it into this assertion's config.provider, or set the judge for the suite in defaultTest.options.provider, for a test in its options.provider, or for the run with --grader";
@@ -1428,5 +1575,6 @@ exports["skill-not-loaded"] = skillNotLoaded;
 exports["tool-called"] = toolCalled;
 exports["tool-not-called"] = toolNotCalled;
 exports.withAssertions = withAssertions;
+exports["workspace-file"] = workspaceFile;
 
 //# sourceMappingURL=assertions.cjs.map