oh-my-promptfoo 1.25.0 → 1.26.0-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assertions.cjs +239 -8
- package/dist/assertions.cjs.map +1 -1
- package/dist/assertions.d.cts +28 -1
- package/dist/assertions.d.ts +28 -1
- package/dist/assertions.js +239 -8
- package/dist/assertions.js.map +1 -1
- package/package.json +1 -1
package/dist/assertions.cjs
CHANGED
|
@@ -988,6 +988,204 @@ function withAssertions(configuration) {
|
|
|
988
988
|
}
|
|
989
989
|
return result;
|
|
990
990
|
}
|
|
991
|
+
// packages/oh-my-promptfoo/src/shell-selector.ts
|
|
992
|
+
function tokens(source) {
|
|
993
|
+
const result = [];
|
|
994
|
+
let word = "", active = false, quote = "", unquoted = true, assignment = false;
|
|
995
|
+
const flush = () => {
|
|
996
|
+
if (active) result.push({
|
|
997
|
+
word,
|
|
998
|
+
assignment
|
|
999
|
+
});
|
|
1000
|
+
word = "";
|
|
1001
|
+
active = false;
|
|
1002
|
+
unquoted = true;
|
|
1003
|
+
assignment = false;
|
|
1004
|
+
};
|
|
1005
|
+
for (let i = 0; i < source.length; i++) {
|
|
1006
|
+
const c = source[i];
|
|
1007
|
+
if (quote === "'") {
|
|
1008
|
+
if (c === "'") quote = "";
|
|
1009
|
+
else word += c;
|
|
1010
|
+
continue;
|
|
1011
|
+
}
|
|
1012
|
+
if (c === "\\") {
|
|
1013
|
+
const next = source[++i];
|
|
1014
|
+
if (next === void 0) throw new Error("Unfinished shell escape");
|
|
1015
|
+
if (next !== "\n") {
|
|
1016
|
+
unquoted = false;
|
|
1017
|
+
if (quote === "\"" && ![
|
|
1018
|
+
"$",
|
|
1019
|
+
"`",
|
|
1020
|
+
"\"",
|
|
1021
|
+
"\\"
|
|
1022
|
+
].includes(next)) word += "\\";
|
|
1023
|
+
word += next;
|
|
1024
|
+
active = true;
|
|
1025
|
+
}
|
|
1026
|
+
continue;
|
|
1027
|
+
}
|
|
1028
|
+
if (c === "$" || c === "`") throw new Error("Shell expansion/substitution is not statically observable");
|
|
1029
|
+
if (quote === "\"") {
|
|
1030
|
+
if (c === "\"") quote = "";
|
|
1031
|
+
else word += c;
|
|
1032
|
+
continue;
|
|
1033
|
+
}
|
|
1034
|
+
if (c === "'" || c === "\"") {
|
|
1035
|
+
quote = c;
|
|
1036
|
+
unquoted = false;
|
|
1037
|
+
active = true;
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
if (c === "#" && !active) {
|
|
1041
|
+
while (i < source.length && source[i] !== "\n") i++;
|
|
1042
|
+
flush();
|
|
1043
|
+
result.push({ op: ";" });
|
|
1044
|
+
continue;
|
|
1045
|
+
}
|
|
1046
|
+
if ("(){}".includes(c)) throw new Error("Shell groups and substitutions are not supported");
|
|
1047
|
+
if (c === "\n" || c === ";" || c === "|" || c === "&" || c === ">" || c === "<") {
|
|
1048
|
+
let op = c === "\n" ? ";" : c;
|
|
1049
|
+
if (source[i + 1] === c && [
|
|
1050
|
+
"|",
|
|
1051
|
+
"&",
|
|
1052
|
+
">",
|
|
1053
|
+
"<"
|
|
1054
|
+
].includes(c)) {
|
|
1055
|
+
op += c;
|
|
1056
|
+
i++;
|
|
1057
|
+
}
|
|
1058
|
+
if (op === "&" || op === "<<" || source[i + 1] === "&" && (op === ">" || op === "<")) throw new Error("Background commands, heredocs and descriptor duplication are not supported");
|
|
1059
|
+
if ((op === ">" || op === ">>" || op === "<") && unquoted && /^\d+$/.test(word)) {
|
|
1060
|
+
word = "";
|
|
1061
|
+
active = false;
|
|
1062
|
+
}
|
|
1063
|
+
flush();
|
|
1064
|
+
result.push({ op });
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
if (/\s/.test(c)) {
|
|
1068
|
+
flush();
|
|
1069
|
+
continue;
|
|
1070
|
+
}
|
|
1071
|
+
if ("*?[~".includes(c)) throw new Error("Shell glob and tilde expansion are not statically observable");
|
|
1072
|
+
if (c === "=" && unquoted && /^[A-Za-z_][A-Za-z0-9_]*$/.test(word)) assignment = true;
|
|
1073
|
+
active = true;
|
|
1074
|
+
word += c;
|
|
1075
|
+
}
|
|
1076
|
+
if (quote) throw new Error("Unterminated shell quote");
|
|
1077
|
+
flush();
|
|
1078
|
+
return result;
|
|
1079
|
+
}
|
|
1080
|
+
const reserved = /^(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|function|!|select|time|coproc)$/;
|
|
1081
|
+
const executionBuiltins = /^(?:eval|source|\.|builtin|trap|alias|unalias|enable|hash|bind|fc)$/;
|
|
1082
|
+
/**
|
|
1083
|
+
* Static simple-command subset, not a shell interpreter. Supports quotes/escapes,
|
|
1084
|
+
* leading assignments, > / >> / < redirects, pipes/lists, command [-p] [--] and
|
|
1085
|
+
* exec [--]. Expansions, control groups, backgrounding, heredocs, descriptor
|
|
1086
|
+
* duplication and unmodeled execution/resolution-changing builtins fail closed.
|
|
1087
|
+
* External wrappers, interpreters and scripts are not recursively interpreted.
|
|
1088
|
+
*/
|
|
1089
|
+
function parseShell(source) {
|
|
1090
|
+
if (source.includes("\0")) throw new Error("NUL in shell command");
|
|
1091
|
+
const commands = [];
|
|
1092
|
+
let words = [], conditional = false, requiresCommand = false;
|
|
1093
|
+
const flush = () => {
|
|
1094
|
+
let offset = 0;
|
|
1095
|
+
while (words[offset]?.assignment) offset++;
|
|
1096
|
+
const hadCommand = offset < words.length;
|
|
1097
|
+
if (hadCommand) {
|
|
1098
|
+
if (reserved.test(words[offset].word)) throw new Error(`Unsupported shell control syntax: ${words[offset].word}`);
|
|
1099
|
+
let external = false;
|
|
1100
|
+
while (words[offset].word === "command" || words[offset].word === "exec") {
|
|
1101
|
+
const builtin = words[offset++].word;
|
|
1102
|
+
if (builtin === "command" && words[offset]?.word === "-p") offset++;
|
|
1103
|
+
if (words[offset]?.word === "--") offset++;
|
|
1104
|
+
else if (words[offset]?.word.startsWith("-")) throw new Error(`Unsupported ${builtin} builtin options`);
|
|
1105
|
+
if (!words[offset]) throw new Error(`${builtin} builtin requires an executable`);
|
|
1106
|
+
if (builtin === "exec") {
|
|
1107
|
+
external = true;
|
|
1108
|
+
break;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
const program = words[offset].word;
|
|
1112
|
+
if (!external && executionBuiltins.test(program)) throw new Error(`Unsupported shell execution builtin: ${program}`);
|
|
1113
|
+
commands.push({
|
|
1114
|
+
program: program.split("/").at(-1) ?? program,
|
|
1115
|
+
args: words.slice(offset + 1).map((token) => token.word),
|
|
1116
|
+
conditional
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1119
|
+
words = [];
|
|
1120
|
+
return hadCommand;
|
|
1121
|
+
};
|
|
1122
|
+
const input = tokens(source);
|
|
1123
|
+
for (let i = 0; i < input.length; i++) {
|
|
1124
|
+
const token = input[i];
|
|
1125
|
+
if ("word" in token) {
|
|
1126
|
+
words.push(token);
|
|
1127
|
+
requiresCommand = false;
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
if ([
|
|
1131
|
+
">",
|
|
1132
|
+
">>",
|
|
1133
|
+
"<"
|
|
1134
|
+
].includes(token.op)) {
|
|
1135
|
+
if (!input[i + 1] || !("word" in input[i + 1])) throw new Error("Shell redirection requires a target");
|
|
1136
|
+
i++;
|
|
1137
|
+
continue;
|
|
1138
|
+
}
|
|
1139
|
+
const hadCommand = flush();
|
|
1140
|
+
if (token.op !== ";" && !hadCommand) throw new Error("Shell operator requires a command");
|
|
1141
|
+
if (token.op === ";") {
|
|
1142
|
+
if (requiresCommand) throw new Error("Shell operator requires a following command");
|
|
1143
|
+
conditional = false;
|
|
1144
|
+
} else {
|
|
1145
|
+
if (token.op === "&&" || token.op === "||") conditional = true;
|
|
1146
|
+
requiresCommand = true;
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
flush();
|
|
1150
|
+
if (requiresCommand) throw new Error("Shell operator requires a following command");
|
|
1151
|
+
return commands;
|
|
1152
|
+
}
|
|
1153
|
+
function matcher(value, literal) {
|
|
1154
|
+
if (typeof value === "string" && literal) return (text) => text === value;
|
|
1155
|
+
const spec = typeof value === "string" ? { regex: value } : require_fs.closed(value, ["regex", "flags"], "shell regex");
|
|
1156
|
+
if (typeof spec.regex !== "string" || spec.flags !== void 0 && typeof spec.flags !== "string") throw new Error("Shell regex requires a string pattern and optional flags");
|
|
1157
|
+
const regex = new RegExp(spec.regex, spec.flags);
|
|
1158
|
+
return (text) => {
|
|
1159
|
+
regex.lastIndex = 0;
|
|
1160
|
+
return regex.test(text);
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
1163
|
+
function normalize(args) {
|
|
1164
|
+
return args.flatMap((arg) => {
|
|
1165
|
+
const framework = /^(?:--framework|-f|-p:TargetFramework)(?:=(.*))?$/.exec(arg);
|
|
1166
|
+
if (framework) return framework[1] === void 0 ? ["--framework"] : ["--framework", framework[1]];
|
|
1167
|
+
const option = /^(--[^=]+)=(.*)$/.exec(arg);
|
|
1168
|
+
return option ? [option[1], option[2]] : [arg];
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
function shellMatcher(value) {
|
|
1172
|
+
const spec = require_fs.closed(value, [
|
|
1173
|
+
"program",
|
|
1174
|
+
"args",
|
|
1175
|
+
"regex"
|
|
1176
|
+
], "shell command selector");
|
|
1177
|
+
if (Object.keys(spec).length === 0) throw new Error("Shell command selector must have program, args or regex");
|
|
1178
|
+
const program = spec.program === void 0 ? void 0 : matcher(spec.program, true);
|
|
1179
|
+
const regex = spec.regex === void 0 ? void 0 : matcher(spec.regex, false);
|
|
1180
|
+
if (spec.args !== void 0 && (!Array.isArray(spec.args) || !spec.args.length)) throw new Error("Shell command args must be a nonempty token selector list");
|
|
1181
|
+
const args = spec.args === void 0 ? void 0 : spec.args.flatMap((entry) => typeof entry === "string" ? normalize([entry]).map((token) => matcher(token, true)) : [matcher(entry, true)]);
|
|
1182
|
+
return (command) => {
|
|
1183
|
+
if (program && !program(command.program)) return false;
|
|
1184
|
+
const actual = normalize(command.args);
|
|
1185
|
+
if (args && !actual.some((_, offset) => args.every((match, index) => actual[offset + index] !== void 0 && match(actual[offset + index])))) return false;
|
|
1186
|
+
return !regex || regex([command.program, ...command.args].join(" "));
|
|
1187
|
+
};
|
|
1188
|
+
}
|
|
991
1189
|
// packages/oh-my-promptfoo/src/tool-assertions.ts
|
|
992
1190
|
function error(reason) {
|
|
993
1191
|
return {
|
|
@@ -1022,7 +1220,7 @@ function pathSegments(path) {
|
|
|
1022
1220
|
if (parts.some((part) => !part)) throw new Error("argumentsMatch.path requires JSON Pointer or dotted property names");
|
|
1023
1221
|
return parts;
|
|
1024
1222
|
}
|
|
1025
|
-
function selector(value) {
|
|
1223
|
+
function selector(value, parsedCommands) {
|
|
1026
1224
|
const spec = require_fs.closed(value, [
|
|
1027
1225
|
"name",
|
|
1028
1226
|
"mcpServer",
|
|
@@ -1031,23 +1229,52 @@ function selector(value) {
|
|
|
1031
1229
|
const matchesName = spec.name === void 0 ? void 0 : nameMatcher(spec.name);
|
|
1032
1230
|
if (spec.mcpServer !== void 0 && (typeof spec.mcpServer !== "string" || !spec.mcpServer)) throw new Error("mcpServer must be a nonempty string");
|
|
1033
1231
|
let argumentPattern;
|
|
1232
|
+
let commandMatcher;
|
|
1233
|
+
let includeConditional = false;
|
|
1034
1234
|
let path = [];
|
|
1035
1235
|
if (spec.argumentsMatch !== void 0) {
|
|
1036
|
-
const match = require_fs.closed(spec.argumentsMatch, [
|
|
1236
|
+
const match = require_fs.closed(spec.argumentsMatch, [
|
|
1237
|
+
"path",
|
|
1238
|
+
"regex",
|
|
1239
|
+
"shell",
|
|
1240
|
+
"command",
|
|
1241
|
+
"includeConditional"
|
|
1242
|
+
], "argumentsMatch");
|
|
1037
1243
|
if (typeof match.path !== "string") throw new Error("argumentsMatch.path must be a string");
|
|
1038
1244
|
path = pathSegments(match.path);
|
|
1039
|
-
|
|
1245
|
+
if (match.shell === true) {
|
|
1246
|
+
if (match.regex !== void 0) throw new Error("Shell selectors put regex inside command");
|
|
1247
|
+
if (match.includeConditional !== void 0 && typeof match.includeConditional !== "boolean") throw new Error("includeConditional must be boolean");
|
|
1248
|
+
includeConditional = match.includeConditional === true;
|
|
1249
|
+
commandMatcher = shellMatcher(match.command);
|
|
1250
|
+
} else {
|
|
1251
|
+
if (match.shell !== void 0 || match.command !== void 0 || match.includeConditional !== void 0) throw new Error("Shell selector requires shell: true");
|
|
1252
|
+
argumentPattern = regex(match.regex);
|
|
1253
|
+
}
|
|
1040
1254
|
}
|
|
1041
1255
|
return (call) => {
|
|
1042
1256
|
if (matchesName && !matchesName(call.name)) return false;
|
|
1043
1257
|
if (spec.mcpServer !== void 0 && call.mcpServer !== spec.mcpServer) return false;
|
|
1044
|
-
if (!argumentPattern) return true;
|
|
1258
|
+
if (!argumentPattern && !commandMatcher) return true;
|
|
1045
1259
|
let value = call.arguments;
|
|
1046
1260
|
for (const part of path) {
|
|
1047
|
-
if (!value || typeof value !== "object" || !Object.hasOwn(value, part))
|
|
1261
|
+
if (!value || typeof value !== "object" || !Object.hasOwn(value, part)) {
|
|
1262
|
+
if (commandMatcher) throw new Error(`Shell command argument path is missing in call ${call.id}`);
|
|
1263
|
+
return false;
|
|
1264
|
+
}
|
|
1048
1265
|
value = value[part];
|
|
1049
1266
|
}
|
|
1267
|
+
if (commandMatcher) {
|
|
1268
|
+
if (typeof value !== "string") throw new Error(`Shell command argument in call ${call.id} must be a string`);
|
|
1269
|
+
const commands = parseShell(value);
|
|
1270
|
+
for (const command of commands) parsedCommands.push({
|
|
1271
|
+
callId: call.id,
|
|
1272
|
+
...command
|
|
1273
|
+
});
|
|
1274
|
+
return commands.some((command) => (includeConditional || !command.conditional) && commandMatcher(command));
|
|
1275
|
+
}
|
|
1050
1276
|
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
1277
|
+
if (!argumentPattern) return false;
|
|
1051
1278
|
argumentPattern.lastIndex = 0;
|
|
1052
1279
|
return typeof text === "string" && argumentPattern.test(text);
|
|
1053
1280
|
};
|
|
@@ -1067,10 +1294,11 @@ function toolAssertion(wanted, context) {
|
|
|
1067
1294
|
"argumentsMatch",
|
|
1068
1295
|
"before"
|
|
1069
1296
|
], "tool assertion config");
|
|
1070
|
-
const
|
|
1297
|
+
const parsedCommands = [];
|
|
1298
|
+
const matches = selector(spec, parsedCommands);
|
|
1071
1299
|
let boundary = inventory.calls.length;
|
|
1072
1300
|
if (before !== void 0) {
|
|
1073
|
-
const reference = selector(before);
|
|
1301
|
+
const reference = selector(before, parsedCommands);
|
|
1074
1302
|
boundary = inventory.calls.findIndex(reference);
|
|
1075
1303
|
if (boundary < 0) return error("The before selector has no observed invocation; chronology cannot be evaluated");
|
|
1076
1304
|
}
|
|
@@ -1080,7 +1308,10 @@ function toolAssertion(wanted, context) {
|
|
|
1080
1308
|
pass,
|
|
1081
1309
|
score: pass ? 1 : 0,
|
|
1082
1310
|
reason: `${matched.length} matching tool invocation(s)${before === void 0 ? "" : " before the first reference invocation"}`,
|
|
1083
|
-
metadata: {
|
|
1311
|
+
metadata: {
|
|
1312
|
+
matchedCallIds: matched.map((call) => call.id),
|
|
1313
|
+
...parsedCommands.length ? { parsedCommands } : {}
|
|
1314
|
+
}
|
|
1084
1315
|
};
|
|
1085
1316
|
} catch (cause) {
|
|
1086
1317
|
return error(cause instanceof Error ? cause.message : String(cause));
|