oh-my-promptfoo 1.26.0 → 1.27.0

package/README.md

@@ -94,7 +94,8 @@ pinned Git and OCI sources, and larger runs.
   assertion works with all four agents, with no tracing setup.
 - **Grade many criteria in one judge call.** `llmAssert` works like
   `llm-rubric`, but scores a whole list of named criteria at once, and each
-  one shows up as its own score.
+  one shows up as its own score. Both rubric assertions default to binary
+  scoring; set `config.scoring: graded` for partial credit or thresholds below 1.
 - **Grade what the agent did, not what it says it did.** `agentAssert` sends a
   read-only agent into the workspace to check the changes. Turn on
   `fileChanges` to read them in your own JavaScript assertions.

package/dist/assertions.cjs

@@ -45,11 +45,11 @@ function render(source, vars, label) {
 function prepareRubric(output, context, workspace = false) {
 	const config = context?.config;
 	const components = config?.components;
-	const scoring = config?.scoring ?? "graded";
+	const scoring = config?.scoring ?? "binary";
 	if (config && Object.hasOwn(config, "rubric")) throw new Error("LLM assertion config.rubric was renamed to config.rubricPrompt");
 	if (config && Object.hasOwn(config, "candidate")) throw new Error("LLM assertion config.candidate was removed; state the grading rule in config.rubricPrompt");
 	if (config?.rubricPrompt !== void 0 && typeof config.rubricPrompt !== "string") throw new Error("LLM assertion rubricPrompt must be a string, not an object or array (including parsed JSON/YAML file contents): use inline text or a text-file string. llmAssert keeps its own system prompt and reply format; chat templates cannot replace them");
-	if (!config || !Array.isArray(components) || components.length === 0 || !components.every((component) => component && typeof component.metric === "string" && component.metric.trim() && typeof component.value === "string" && component.value.trim() && (component.weight === void 0 || typeof component.weight === "number" && Number.isFinite(component.weight) && component.weight > 0)) || new Set(components.map(({ metric }) => metric)).size !== components.length || ![config.threshold, ...components.map(({ threshold }) => threshold)].every((threshold) => threshold === void 0 || typeof threshold === "number" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1 && (scoring !== "binary" || threshold === 1)) || config.rubricPrompt !== void 0 && !config.rubricPrompt.trim() || config.scoring !== void 0 && config.scoring !== "graded" && config.scoring !== "binary") throw new Error("LLM assertion requires unique named components with nonempty criteria, positive weights, a config.threshold and component thresholds between 0 and 1, a nonempty string rubricPrompt, and scoring of 'graded' or 'binary' without a conflicting config.threshold or component threshold");
+	if (!config || !Array.isArray(components) || components.length === 0 || !components.every((component) => component && typeof component.metric === "string" && component.metric.trim() && typeof component.value === "string" && component.value.trim() && (component.weight === void 0 || typeof component.weight === "number" && Number.isFinite(component.weight) && component.weight > 0)) || new Set(components.map(({ metric }) => metric)).size !== components.length || ![config.threshold, ...components.map(({ threshold }) => threshold)].every((threshold) => threshold === void 0 || typeof threshold === "number" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1 && (scoring !== "binary" || threshold === 1)) || config.rubricPrompt !== void 0 && !config.rubricPrompt.trim() || config.scoring !== void 0 && config.scoring !== "graded" && config.scoring !== "binary") throw new Error("LLM assertion requires unique named components with nonempty criteria, positive weights, a config.threshold and component thresholds between 0 and 1, a nonempty string rubricPrompt, and scoring of 'graded' or 'binary' without a conflicting config.threshold or component threshold; scoring defaults to 'binary', so set scoring: 'graded' for thresholds other than 1");
 	if (components.some(({ metric }) => Object.hasOwn(Object.prototype, metric))) throw new Error("LLM assertion requires no reserved component metric names");
 	const totalWeight = components.reduce((sum, component) => sum + (component.weight ?? 1), 0);
 	if (!Number.isFinite(totalWeight)) throw new Error("LLM assertion component weights exceed the supported range");