oh-my-promptfoo 1.25.0 → 1.26.0-next.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -988,6 +988,204 @@ function withAssertions(configuration) {
988
988
  }
989
989
  return result;
990
990
  }
991
+ // packages/oh-my-promptfoo/src/shell-selector.ts
992
+ function tokens(source) {
993
+ const result = [];
994
+ let word = "", active = false, quote = "", unquoted = true, assignment = false;
995
+ const flush = () => {
996
+ if (active) result.push({
997
+ word,
998
+ assignment
999
+ });
1000
+ word = "";
1001
+ active = false;
1002
+ unquoted = true;
1003
+ assignment = false;
1004
+ };
1005
+ for (let i = 0; i < source.length; i++) {
1006
+ const c = source[i];
1007
+ if (quote === "'") {
1008
+ if (c === "'") quote = "";
1009
+ else word += c;
1010
+ continue;
1011
+ }
1012
+ if (c === "\\") {
1013
+ const next = source[++i];
1014
+ if (next === void 0) throw new Error("Unfinished shell escape");
1015
+ if (next !== "\n") {
1016
+ unquoted = false;
1017
+ if (quote === "\"" && ![
1018
+ "$",
1019
+ "`",
1020
+ "\"",
1021
+ "\\"
1022
+ ].includes(next)) word += "\\";
1023
+ word += next;
1024
+ active = true;
1025
+ }
1026
+ continue;
1027
+ }
1028
+ if (c === "$" || c === "`") throw new Error("Shell expansion/substitution is not statically observable");
1029
+ if (quote === "\"") {
1030
+ if (c === "\"") quote = "";
1031
+ else word += c;
1032
+ continue;
1033
+ }
1034
+ if (c === "'" || c === "\"") {
1035
+ quote = c;
1036
+ unquoted = false;
1037
+ active = true;
1038
+ continue;
1039
+ }
1040
+ if (c === "#" && !active) {
1041
+ while (i < source.length && source[i] !== "\n") i++;
1042
+ flush();
1043
+ result.push({ op: ";" });
1044
+ continue;
1045
+ }
1046
+ if ("(){}".includes(c)) throw new Error("Shell groups and substitutions are not supported");
1047
+ if (c === "\n" || c === ";" || c === "|" || c === "&" || c === ">" || c === "<") {
1048
+ let op = c === "\n" ? ";" : c;
1049
+ if (source[i + 1] === c && [
1050
+ "|",
1051
+ "&",
1052
+ ">",
1053
+ "<"
1054
+ ].includes(c)) {
1055
+ op += c;
1056
+ i++;
1057
+ }
1058
+ if (op === "&" || op === "<<" || source[i + 1] === "&" && (op === ">" || op === "<")) throw new Error("Background commands, heredocs and descriptor duplication are not supported");
1059
+ if ((op === ">" || op === ">>" || op === "<") && unquoted && /^\d+$/.test(word)) {
1060
+ word = "";
1061
+ active = false;
1062
+ }
1063
+ flush();
1064
+ result.push({ op });
1065
+ continue;
1066
+ }
1067
+ if (/\s/.test(c)) {
1068
+ flush();
1069
+ continue;
1070
+ }
1071
+ if ("*?[~".includes(c)) throw new Error("Shell glob and tilde expansion are not statically observable");
1072
+ if (c === "=" && unquoted && /^[A-Za-z_][A-Za-z0-9_]*$/.test(word)) assignment = true;
1073
+ active = true;
1074
+ word += c;
1075
+ }
1076
+ if (quote) throw new Error("Unterminated shell quote");
1077
+ flush();
1078
+ return result;
1079
+ }
1080
+ const reserved = /^(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|function|!|select|time|coproc)$/;
1081
+ const executionBuiltins = /^(?:eval|source|\.|builtin|trap|alias|unalias|enable|hash|bind|fc)$/;
1082
+ /**
1083
+ * Static simple-command subset, not a shell interpreter. Supports quotes/escapes,
1084
+ * leading assignments, > / >> / < redirects, pipes/lists, command [-p] [--] and
1085
+ * exec [--]. Expansions, control groups, backgrounding, heredocs, descriptor
1086
+ * duplication and unmodeled execution/resolution-changing builtins fail closed.
1087
+ * External wrappers, interpreters and scripts are not recursively interpreted.
1088
+ */
1089
+ function parseShell(source) {
1090
+ if (source.includes("\0")) throw new Error("NUL in shell command");
1091
+ const commands = [];
1092
+ let words = [], conditional = false, requiresCommand = false;
1093
+ const flush = () => {
1094
+ let offset = 0;
1095
+ while (words[offset]?.assignment) offset++;
1096
+ const hadCommand = offset < words.length;
1097
+ if (hadCommand) {
1098
+ if (reserved.test(words[offset].word)) throw new Error(`Unsupported shell control syntax: ${words[offset].word}`);
1099
+ let external = false;
1100
+ while (words[offset].word === "command" || words[offset].word === "exec") {
1101
+ const builtin = words[offset++].word;
1102
+ if (builtin === "command" && words[offset]?.word === "-p") offset++;
1103
+ if (words[offset]?.word === "--") offset++;
1104
+ else if (words[offset]?.word.startsWith("-")) throw new Error(`Unsupported ${builtin} builtin options`);
1105
+ if (!words[offset]) throw new Error(`${builtin} builtin requires an executable`);
1106
+ if (builtin === "exec") {
1107
+ external = true;
1108
+ break;
1109
+ }
1110
+ }
1111
+ const program = words[offset].word;
1112
+ if (!external && executionBuiltins.test(program)) throw new Error(`Unsupported shell execution builtin: ${program}`);
1113
+ commands.push({
1114
+ program: program.split("/").at(-1) ?? program,
1115
+ args: words.slice(offset + 1).map((token) => token.word),
1116
+ conditional
1117
+ });
1118
+ }
1119
+ words = [];
1120
+ return hadCommand;
1121
+ };
1122
+ const input = tokens(source);
1123
+ for (let i = 0; i < input.length; i++) {
1124
+ const token = input[i];
1125
+ if ("word" in token) {
1126
+ words.push(token);
1127
+ requiresCommand = false;
1128
+ continue;
1129
+ }
1130
+ if ([
1131
+ ">",
1132
+ ">>",
1133
+ "<"
1134
+ ].includes(token.op)) {
1135
+ if (!input[i + 1] || !("word" in input[i + 1])) throw new Error("Shell redirection requires a target");
1136
+ i++;
1137
+ continue;
1138
+ }
1139
+ const hadCommand = flush();
1140
+ if (token.op !== ";" && !hadCommand) throw new Error("Shell operator requires a command");
1141
+ if (token.op === ";") {
1142
+ if (requiresCommand) throw new Error("Shell operator requires a following command");
1143
+ conditional = false;
1144
+ } else {
1145
+ if (token.op === "&&" || token.op === "||") conditional = true;
1146
+ requiresCommand = true;
1147
+ }
1148
+ }
1149
+ flush();
1150
+ if (requiresCommand) throw new Error("Shell operator requires a following command");
1151
+ return commands;
1152
+ }
1153
+ function matcher(value, literal) {
1154
+ if (typeof value === "string" && literal) return (text) => text === value;
1155
+ const spec = typeof value === "string" ? { regex: value } : require_fs.closed(value, ["regex", "flags"], "shell regex");
1156
+ if (typeof spec.regex !== "string" || spec.flags !== void 0 && typeof spec.flags !== "string") throw new Error("Shell regex requires a string pattern and optional flags");
1157
+ const regex = new RegExp(spec.regex, spec.flags);
1158
+ return (text) => {
1159
+ regex.lastIndex = 0;
1160
+ return regex.test(text);
1161
+ };
1162
+ }
1163
+ function normalize(args) {
1164
+ return args.flatMap((arg) => {
1165
+ const framework = /^(?:--framework|-f|-p:TargetFramework)(?:=(.*))?$/.exec(arg);
1166
+ if (framework) return framework[1] === void 0 ? ["--framework"] : ["--framework", framework[1]];
1167
+ const option = /^(--[^=]+)=(.*)$/.exec(arg);
1168
+ return option ? [option[1], option[2]] : [arg];
1169
+ });
1170
+ }
1171
+ function shellMatcher(value) {
1172
+ const spec = require_fs.closed(value, [
1173
+ "program",
1174
+ "args",
1175
+ "regex"
1176
+ ], "shell command selector");
1177
+ if (Object.keys(spec).length === 0) throw new Error("Shell command selector must have program, args or regex");
1178
+ const program = spec.program === void 0 ? void 0 : matcher(spec.program, true);
1179
+ const regex = spec.regex === void 0 ? void 0 : matcher(spec.regex, false);
1180
+ if (spec.args !== void 0 && (!Array.isArray(spec.args) || !spec.args.length)) throw new Error("Shell command args must be a nonempty token selector list");
1181
+ const args = spec.args === void 0 ? void 0 : spec.args.flatMap((entry) => typeof entry === "string" ? normalize([entry]).map((token) => matcher(token, true)) : [matcher(entry, true)]);
1182
+ return (command) => {
1183
+ if (program && !program(command.program)) return false;
1184
+ const actual = normalize(command.args);
1185
+ if (args && !actual.some((_, offset) => args.every((match, index) => actual[offset + index] !== void 0 && match(actual[offset + index])))) return false;
1186
+ return !regex || regex([command.program, ...command.args].join(" "));
1187
+ };
1188
+ }
991
1189
  // packages/oh-my-promptfoo/src/tool-assertions.ts
992
1190
  function error(reason) {
993
1191
  return {
@@ -1022,7 +1220,7 @@ function pathSegments(path) {
1022
1220
  if (parts.some((part) => !part)) throw new Error("argumentsMatch.path requires JSON Pointer or dotted property names");
1023
1221
  return parts;
1024
1222
  }
1025
- function selector(value) {
1223
+ function selector(value, parsedCommands) {
1026
1224
  const spec = require_fs.closed(value, [
1027
1225
  "name",
1028
1226
  "mcpServer",
@@ -1031,23 +1229,52 @@ function selector(value) {
1031
1229
  const matchesName = spec.name === void 0 ? void 0 : nameMatcher(spec.name);
1032
1230
  if (spec.mcpServer !== void 0 && (typeof spec.mcpServer !== "string" || !spec.mcpServer)) throw new Error("mcpServer must be a nonempty string");
1033
1231
  let argumentPattern;
1232
+ let commandMatcher;
1233
+ let includeConditional = false;
1034
1234
  let path = [];
1035
1235
  if (spec.argumentsMatch !== void 0) {
1036
- const match = require_fs.closed(spec.argumentsMatch, ["path", "regex"], "argumentsMatch");
1236
+ const match = require_fs.closed(spec.argumentsMatch, [
1237
+ "path",
1238
+ "regex",
1239
+ "shell",
1240
+ "command",
1241
+ "includeConditional"
1242
+ ], "argumentsMatch");
1037
1243
  if (typeof match.path !== "string") throw new Error("argumentsMatch.path must be a string");
1038
1244
  path = pathSegments(match.path);
1039
- argumentPattern = regex(match.regex);
1245
+ if (match.shell === true) {
1246
+ if (match.regex !== void 0) throw new Error("Shell selectors put regex inside command");
1247
+ if (match.includeConditional !== void 0 && typeof match.includeConditional !== "boolean") throw new Error("includeConditional must be boolean");
1248
+ includeConditional = match.includeConditional === true;
1249
+ commandMatcher = shellMatcher(match.command);
1250
+ } else {
1251
+ if (match.shell !== void 0 || match.command !== void 0 || match.includeConditional !== void 0) throw new Error("Shell selector requires shell: true");
1252
+ argumentPattern = regex(match.regex);
1253
+ }
1040
1254
  }
1041
1255
  return (call) => {
1042
1256
  if (matchesName && !matchesName(call.name)) return false;
1043
1257
  if (spec.mcpServer !== void 0 && call.mcpServer !== spec.mcpServer) return false;
1044
- if (!argumentPattern) return true;
1258
+ if (!argumentPattern && !commandMatcher) return true;
1045
1259
  let value = call.arguments;
1046
1260
  for (const part of path) {
1047
- if (!value || typeof value !== "object" || !Object.hasOwn(value, part)) return false;
1261
+ if (!value || typeof value !== "object" || !Object.hasOwn(value, part)) {
1262
+ if (commandMatcher) throw new Error(`Shell command argument path is missing in call ${call.id}`);
1263
+ return false;
1264
+ }
1048
1265
  value = value[part];
1049
1266
  }
1267
+ if (commandMatcher) {
1268
+ if (typeof value !== "string") throw new Error(`Shell command argument in call ${call.id} must be a string`);
1269
+ const commands = parseShell(value);
1270
+ for (const command of commands) parsedCommands.push({
1271
+ callId: call.id,
1272
+ ...command
1273
+ });
1274
+ return commands.some((command) => (includeConditional || !command.conditional) && commandMatcher(command));
1275
+ }
1050
1276
  const text = typeof value === "string" ? value : JSON.stringify(value);
1277
+ if (!argumentPattern) return false;
1051
1278
  argumentPattern.lastIndex = 0;
1052
1279
  return typeof text === "string" && argumentPattern.test(text);
1053
1280
  };
@@ -1067,10 +1294,11 @@ function toolAssertion(wanted, context) {
1067
1294
  "argumentsMatch",
1068
1295
  "before"
1069
1296
  ], "tool assertion config");
1070
- const matches = selector(spec);
1297
+ const parsedCommands = [];
1298
+ const matches = selector(spec, parsedCommands);
1071
1299
  let boundary = inventory.calls.length;
1072
1300
  if (before !== void 0) {
1073
- const reference = selector(before);
1301
+ const reference = selector(before, parsedCommands);
1074
1302
  boundary = inventory.calls.findIndex(reference);
1075
1303
  if (boundary < 0) return error("The before selector has no observed invocation; chronology cannot be evaluated");
1076
1304
  }
@@ -1080,7 +1308,10 @@ function toolAssertion(wanted, context) {
1080
1308
  pass,
1081
1309
  score: pass ? 1 : 0,
1082
1310
  reason: `${matched.length} matching tool invocation(s)${before === void 0 ? "" : " before the first reference invocation"}`,
1083
- metadata: { matchedCallIds: matched.map((call) => call.id) }
1311
+ metadata: {
1312
+ matchedCallIds: matched.map((call) => call.id),
1313
+ ...parsedCommands.length ? { parsedCommands } : {}
1314
+ }
1084
1315
  };
1085
1316
  } catch (cause) {
1086
1317
  return error(cause instanceof Error ? cause.message : String(cause));