oh-my-promptfoo 1.22.1 → 1.22.2-next.124

package/README.md

@@ -135,6 +135,7 @@ providers:
     config:
       model: gpt-5.6-sol           # required, exactly as Copilot lists it
       reasoning_effort: medium     # optional: low, medium or high
+      max_tokens: 32000            # optional; defaults to the model's output limit
 defaultTest:
   options:
     provider:
@@ -145,7 +146,7 @@ defaultTest:
         response_format: { type: json_object }   # optional; the only accepted format
 ```
 
-Those three keys are the whole config. The endpoint and credentials always come
+Those four keys are the whole config. The endpoint and credentials always come
 from Copilot: a private Copilot SDK process signs in with `GH_TOKEN` (never your
 stored logins or `OPENAI_*`, `ANTHROPIC_*` or `AZURE_*` settings), asks Copilot for
 the model's endpoint and short-lived credentials, and refreshes them before they
@@ -169,6 +170,21 @@ API in its own shape:
 - `response_format: { type: json_object }` works on Responses and Chat
   Completions. The Anthropic Messages API has no JSON-object mode, so a Claude
   model with `response_format` fails and names the model.
+- `max_tokens` caps each reply's output tokens: the Responses
+  `max_output_tokens`, the Chat Completions `max_tokens` (or
+  `max_completion_tokens` for models Promptfoo treats as reasoning models) and
+  the Messages `max_tokens`. Without it, the cap is the model's
+  `capabilities.limits.max_output_tokens` from Copilot's model list, so long
+  answers are not cut off at Promptfoo's default of 1024 (2048 for Claude
+  thinking models). When the list reports no limit, Promptfoo's default
+  applies. The cap in force is in `metadata.copilotMaxOutputTokens`.
+  `OPENAI_MAX_TOKENS`, `OPENAI_MAX_COMPLETION_TOKENS` and
+  `ANTHROPIC_MAX_TOKENS` never override it.
+- No request carries a `temperature`, so each model uses its own default.
+  Promptfoo's default of 0 and `OPENAI_TEMPERATURE` or `ANTHROPIC_TEMPERATURE`
+  are not sent; some models, such as Claude Opus 4.7 and later, reject them.
+- Messages requests stream, because the Anthropic SDK refuses a
+  non-streaming request with a large `max_tokens`.
 - Claude replies contain only the answer, not its thinking.
 
 Every instance in a run shares that one SDK process. The provider id is

package/dist/providers.cjs

@@ -5659,6 +5659,7 @@ const CONFIG_KEYS = {
 	model: true,
 	reasoning_effort: true,
 	response_format: true,
+	max_tokens: true,
 	basePath: true
 };
 /** Validate authored config; anything that could redirect endpoint or authentication is rejected. */
@@ -5667,17 +5668,30 @@ function copilotModelConfig(input) {
 	const config = input;
 	const unsupported = Object.keys(config).filter((key) => !Object.hasOwn(CONFIG_KEYS, key));
 	if (unsupported.length) throw new Error(`CopilotModel does not accept config.${unsupported.join(", config.")}; the endpoint and credentials come from Copilot discovery.`);
-	const { model, reasoning_effort, response_format } = config;
+	const { model, reasoning_effort, response_format, max_tokens } = config;
 	if (typeof model !== "string" || !model.trim() || model !== model.trim()) throw new Error("CopilotModel requires an explicit config.model.");
 	if (reasoning_effort !== void 0 && !EFFORTS.includes(reasoning_effort)) throw new Error(`CopilotModel config.reasoning_effort must be one of: ${EFFORTS.join(", ")}.`);
 	if (response_format !== void 0 && (typeof response_format !== "object" || response_format === null || Object.keys(response_format).join() !== "type" || !("type" in response_format) || response_format.type !== "json_object")) throw new Error("CopilotModel config.response_format must be { type: \"json_object\" }.");
+	if (max_tokens !== void 0 && !positiveInteger(max_tokens)) throw new Error("CopilotModel config.max_tokens must be a positive integer.");
 	return Object.freeze({
 		model,
 		...reasoning_effort === void 0 ? {} : { reasoning_effort },
-		...response_format === void 0 ? {} : { response_format: Object.freeze({ type: "json_object" }) }
+		...response_format === void 0 ? {} : { response_format: Object.freeze({ type: "json_object" }) },
+		...max_tokens === void 0 ? {} : { max_tokens }
 	});
 }
 const describe = (value) => typeof value === "string" ? JSON.stringify(value) : "none";
+const positiveInteger = (value) => typeof value === "number" && Number.isSafeInteger(value) && value > 0;
+/**
+* The model's maximum output tokens from Copilot's model list
+* (`capabilities.limits.max_output_tokens`), when reported.
+*/
+function modelOutputLimit(models, model) {
+	if (!Array.isArray(models)) return void 0;
+	const limits = models.find((candidate) => candidate?.id === model)?.capabilities?.limits;
+	const limit = limits && typeof limits === "object" ? limits.max_output_tokens : void 0;
+	return positiveInteger(limit) ? limit : void 0;
+}
 /** The Copilot API a discovery snapshot describes; anything but the three HTTP APIs is rejected. */
 function endpointOf(snapshot, model) {
 	const { type, wireApi, transport } = snapshot ?? {};
@@ -5751,8 +5765,15 @@ function anthropicReasoning(models, model, effort) {
 		effort
 	};
 }
-/** The Promptfoo provider id and options that send `config` to the discovered endpoint. */
-function nativeProvider(config, snapshot, { endpoint, headers }, models) {
+/**
+* The Promptfoo provider id and options that send `config` to the discovered endpoint.
+* `maxOutputTokens` replaces Promptfoo's default output cap and ambient `*_MAX_TOKENS`;
+* without it Promptfoo's default stands. Temperature is never sent: Promptfoo defaults it
+* to 0 (or ambient `*_TEMPERATURE`), which some models reject, so an `undefined` override
+* that is spread last drops the key from the request body.
+*/
+function nativeProvider(config, snapshot, { endpoint, headers }, models, maxOutputTokens) {
+	const noTemperature = { temperature: void 0 };
 	const { model, reasoning_effort, response_format } = config;
 	if (endpoint === "messages") {
 		if (response_format !== void 0) throw new Error(`CopilotModel config.response_format is not supported for ${model}: Copilot serves it over the Anthropic Messages API, which has no JSON-object mode.`);
@@ -5762,6 +5783,9 @@ function nativeProvider(config, snapshot, { endpoint, headers }, models) {
 			options: {
 				config: {
 					...reasoning_effort === void 0 ? {} : anthropicReasoning(models, model, reasoning_effort),
+					...maxOutputTokens === void 0 ? {} : { max_tokens: maxOutputTokens },
+					stream: true,
+					extra_body: noTemperature,
 					showThinking: false,
 					apiBaseUrl: snapshot.baseUrl,
 					apiKey: snapshot.apiKey,
@@ -5785,9 +5809,14 @@ function nativeProvider(config, snapshot, { endpoint, headers }, models) {
 	if (endpoint === "chat/completions") return {
 		id: `openai:chat:${model}`,
 		options: { config: {
-			...reasoning_effort === void 0 ? {} : {
-				reasoning_effort,
-				passthrough: { reasoning_effort }
+			...reasoning_effort === void 0 ? {} : { reasoning_effort },
+			...maxOutputTokens === void 0 ? {} : {
+				max_tokens: maxOutputTokens,
+				max_completion_tokens: maxOutputTokens
+			},
+			passthrough: {
+				...reasoning_effort === void 0 ? {} : { reasoning_effort },
+				...noTemperature
 			},
 			...openai
 		} }
@@ -5796,6 +5825,8 @@ function nativeProvider(config, snapshot, { endpoint, headers }, models) {
 		id: `openai:responses:${model}`,
 		options: { config: {
 			...reasoning_effort === void 0 ? {} : { reasoning_effort },
+			...maxOutputTokens === void 0 ? {} : { max_output_tokens: maxOutputTokens },
+			passthrough: noTemperature,
 			store: false,
 			...openai
 		} }
@@ -5857,7 +5888,8 @@ function copilotReply(response, generation) {
 		metadata: {
 			...reply.metadata,
 			copilotEndpoint: generation.endpoint,
-			...generation.prices ? { copilotTokenPrices: generation.prices } : {}
+			...generation.prices ? { copilotTokenPrices: generation.prices } : {},
+			...generation.maxOutputTokens === void 0 ? {} : { copilotMaxOutputTokens: generation.maxOutputTokens }
 		}
 	}, generation.endpoint);
 }
@@ -5897,7 +5929,8 @@ function createCopilotModel({ createEndpointClient = () => createCopilotEndpoint
 		try {
 			models = await owner.models();
 		} catch {}
-		const { id, options } = nativeProvider(config, snapshot, route, models);
+		const maxOutputTokens = config.max_tokens ?? modelOutputLimit(models, config.model);
+		const { id, options } = nativeProvider(config, snapshot, route, models, maxOutputTokens);
 		const { endpoint, refreshAt } = route;
 		const prices = tokenPrices(models, config.model);
 		const provider = await loadProvider(id, { options });
@@ -5907,6 +5940,7 @@ function createCopilotModel({ createEndpointClient = () => createCopilotEndpoint
 			provider,
 			endpoint,
 			prices,
+			maxOutputTokens,
 			refreshAt,
 			active: 0,
 			retired: false