oh-my-promptfoo 1.19.0 → 1.20.0

package/README.md

@@ -75,7 +75,8 @@ in `defaultTest.options.provider` as above, or with
 `OPENAI_BASE_URL` to the full `https://<resource>.openai.azure.com/openai/v1`
 address and use `openai:chat:<deployment>`. The
 [grading guide](https://github.com/allagentsdev/oh-my-promptfoo/blob/main/docs/grading-guide.md#choosing-llmasserts-judge)
-covers per-test judges and falling back to another judge during an outage.
+covers per-test and per-assertion judges and falling back to another judge
+during an outage.
 
 The [examples](https://github.com/allagentsdev/oh-my-promptfoo/tree/main/examples) include configs for Codex, Claude Code, Copilot,
 pinned Git and OCI sources, and larger runs.

package/dist/assertions.cjs

@@ -2975,6 +2975,12 @@ async function loadJudge(spec, context2 = {}) {
     `A judge must be a provider id, a provider object with an id, or a { text: \u2026 } map; received ${JSON.stringify(spec) ?? String(spec)}`
   );
 }
+function judgeName(spec, judge) {
+  if (judge.label) return judge.label;
+  if (typeof spec === "object" && spec !== null && !isJudge(spec) && "label" in spec && typeof spec.label === "string" && spec.label)
+    return spec.label;
+  return judge.id();
+}
 
 // packages/oh-my-promptfoo/src/rubric-contract.ts
 var import_node_module2 = require("module");
@@ -3598,7 +3604,7 @@ ${rubric.user}${fileChanges === void 0 ? "" : "\n\nUntrusted evidence (JSON):\n"
 
 // packages/oh-my-promptfoo/src/assertions.ts
 var SELF = /^package:oh-my-promptfoo\/assertions:llmAssert$/;
-var PER_ASSERTION_JUDGE_ERROR = "llmAssert takes no per-assertion judge: remove provider from this assertion and set the judge in defaultTest.options.provider for the suite, in a test's options.provider, or with --grader for the run";
+var MISPLACED_PROVIDER_ERROR = "llmAssert does not see provider: beside config: (Promptfoo does not pass it to package assertions): move it into this assertion's config.provider, or set the judge for the suite in defaultTest.options.provider, for a test in its options.provider, or for the run with --grader";
 function misplacedProvider(assertions) {
   if (!Array.isArray(assertions)) return false;
   return assertions.some(
@@ -3606,8 +3612,9 @@ function misplacedProvider(assertions) {
   );
 }
 function judgeSpec(context2) {
-  if (context2?.config && Object.hasOwn(context2.config, "provider") || misplacedProvider(context2?.test?.assert))
-    throw new Error(PER_ASSERTION_JUDGE_ERROR);
+  if (misplacedProvider(context2?.test?.assert)) throw new Error(MISPLACED_PROVIDER_ERROR);
+  const own3 = context2?.config?.provider;
+  if (own3) return own3;
   const spec = context2?.test?.options?.provider;
   if (spec === void 0 || spec === null || spec === "")
     throw new Error(
@@ -3618,12 +3625,13 @@ function judgeSpec(context2) {
 }
 async function llmAssert(output, context2) {
   const rubric = prepareRubric(output, context2);
-  const judge = await loadJudge(judgeSpec(context2));
+  const spec = judgeSpec(context2);
+  const judge = await loadJudge(spec);
   const prompt = JSON.stringify([
     { role: "system", content: rubric.system },
     { role: "user", content: rubric.user }
   ]);
-  return gradeRubric(rubric, prompt, async (attempt) => {
+  const result = await gradeRubric(rubric, prompt, async (attempt) => {
     const response = await judge.callApi(prompt, {
       prompt: { raw: prompt, label: "llmAssert" },
       vars: context2?.vars ?? {},
@@ -3638,6 +3646,8 @@ async function llmAssert(output, context2) {
       metadata: response.metadata
     };
   });
+  result.metadata.judge = judgeName(spec, judge);
+  return result;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {