oh-my-promptfoo 1.26.0 → 1.28.0

package/README.md

@@ -94,7 +94,8 @@ pinned Git and OCI sources, and larger runs.
   assertion works with all four agents, with no tracing setup.
 - **Grade many criteria in one judge call.** `llmAssert` works like
   `llm-rubric`, but scores a whole list of named criteria at once, and each
-  one shows up as its own score.
+  one shows up as its own score. Both rubric assertions default to binary
+  scoring; set `config.scoring: graded` for partial credit or thresholds below 1.
 - **Grade what the agent did, not what it says it did.** `agentAssert` sends a
   read-only agent into the workspace to check the changes. Turn on
   `fileChanges` to read them in your own JavaScript assertions.
@@ -120,6 +121,39 @@ providers here and dropped their `Provider` suffix; the ids of earlier versions
 no longer resolve (see the
 [changelog](https://github.com/allagentsdev/oh-my-promptfoo/blob/main/CHANGELOG.md)).
 
+## Copilot conversations
+
+Use [Promptfoo's ordinary chat templates](https://www.promptfoo.dev/docs/configuration/chat/)
+with either `CopilotSdk` or `Agent` delegating to `copilot-sdk`. JSON and YAML
+prompt files, simplified chat markup expanded by Nunjucks, shared fixed history,
+`_conversation` replay, `metadata.conversationId`, `storeOutputAs` and output
+transforms belong to Promptfoo, not a JavaScript conversation adapter.
+Normal assertions grade each new reply; the provider reads only the rendered prompt.
+
+Copilot's optional `inputFormat` is `auto` (default), `chat` or `text`. Set it in
+direct provider config or the Agent's `delegate.config`; per-test
+`options.delegate.config.inputFormat` overrides either. `auto` recognizes only
+complete chat-shaped JSON arrays; business records with other roles or incomplete
+records remain literal. Chat-shaped business data is inherently ambiguous: use
+`text` to bypass provider parsing and preserve the supplied/rendered string exactly.
+It does not bypass Promptfoo's earlier Nunjucks rendering or variable expansion.
+YAML **file** loading is Promptfoo's job; raw YAML text is not provider chat input.
+
+All system messages, even later ones, are consolidated in original system order
+and appended to Copilot's core instructions through the public SDK. Other
+messages preserve string names, tool IDs, order and content in a delimited
+**user-channel JSON transcript**, not native assistant/tool roles. Delimiters
+are not a security boundary. Each row uses a fresh session and, with `Agent`, a
+fresh workspace. History is supplied input, not tool evidence or filesystem state.
+Returned errors, including settled aborts, retain published workspaces until
+cleanup; thrown failures release unpublished workspaces.
+
+The [two-provider examples](examples/copilot-conversations/promptfooconfig.yaml)
+include three generated turns, interleaved IDs, JSON/YAML and simplified fixed
+history, embedded/literal JSON, full-prompt replay and transformed-output storage.
+See the [feature/provenance and compatibility matrices](docs/provider-guide.md#feature-and-verification-provenance)
+for engine-owned behavior, SDK limits and verification status.
+
 ## Local MCP fixtures
 
 Copilot and Claude candidates can call suite-authored Node MCP servers staged in

package/dist/assertions.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_fs = require("./fs-D9YrrmPG.cjs");
+const require_fs = require("./fs-CNr5ZxiH.cjs");
 let node_path = require("node:path");
 let node_url = require("node:url");
 let node_fs = require("node:fs");
@@ -45,11 +45,11 @@ function render(source, vars, label) {
 function prepareRubric(output, context, workspace = false) {
 	const config = context?.config;
 	const components = config?.components;
-	const scoring = config?.scoring ?? "graded";
+	const scoring = config?.scoring ?? "binary";
 	if (config && Object.hasOwn(config, "rubric")) throw new Error("LLM assertion config.rubric was renamed to config.rubricPrompt");
 	if (config && Object.hasOwn(config, "candidate")) throw new Error("LLM assertion config.candidate was removed; state the grading rule in config.rubricPrompt");
 	if (config?.rubricPrompt !== void 0 && typeof config.rubricPrompt !== "string") throw new Error("LLM assertion rubricPrompt must be a string, not an object or array (including parsed JSON/YAML file contents): use inline text or a text-file string. llmAssert keeps its own system prompt and reply format; chat templates cannot replace them");
-	if (!config || !Array.isArray(components) || components.length === 0 || !components.every((component) => component && typeof component.metric === "string" && component.metric.trim() && typeof component.value === "string" && component.value.trim() && (component.weight === void 0 || typeof component.weight === "number" && Number.isFinite(component.weight) && component.weight > 0)) || new Set(components.map(({ metric }) => metric)).size !== components.length || ![config.threshold, ...components.map(({ threshold }) => threshold)].every((threshold) => threshold === void 0 || typeof threshold === "number" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1 && (scoring !== "binary" || threshold === 1)) || config.rubricPrompt !== void 0 && !config.rubricPrompt.trim() || config.scoring !== void 0 && config.scoring !== "graded" && config.scoring !== "binary") throw new Error("LLM assertion requires unique named components with nonempty criteria, positive weights, a config.threshold and component thresholds between 0 and 1, a nonempty string rubricPrompt, and scoring of 'graded' or 'binary' without a conflicting config.threshold or component threshold");
+	if (!config || !Array.isArray(components) || components.length === 0 || !components.every((component) => component && typeof component.metric === "string" && component.metric.trim() && typeof component.value === "string" && component.value.trim() && (component.weight === void 0 || typeof component.weight === "number" && Number.isFinite(component.weight) && component.weight > 0)) || new Set(components.map(({ metric }) => metric)).size !== components.length || ![config.threshold, ...components.map(({ threshold }) => threshold)].every((threshold) => threshold === void 0 || typeof threshold === "number" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1 && (scoring !== "binary" || threshold === 1)) || config.rubricPrompt !== void 0 && !config.rubricPrompt.trim() || config.scoring !== void 0 && config.scoring !== "graded" && config.scoring !== "binary") throw new Error("LLM assertion requires unique named components with nonempty criteria, positive weights, a config.threshold and component thresholds between 0 and 1, a nonempty string rubricPrompt, and scoring of 'graded' or 'binary' without a conflicting config.threshold or component threshold; scoring defaults to 'binary', so set scoring: 'graded' for thresholds other than 1");
 	if (components.some(({ metric }) => Object.hasOwn(Object.prototype, metric))) throw new Error("LLM assertion requires no reserved component metric names");
 	const totalWeight = components.reduce((sum, component) => sum + (component.weight ?? 1), 0);
 	if (!Number.isFinite(totalWeight)) throw new Error("LLM assertion component weights exceed the supported range");