@copilotkit/aimock 1.44.0 → 1.45.1

package/dist/bedrock-converse.js

@@ -1,6 +1,7 @@
-import { extractOverrides, flattenHeaders, generateToolUseId, getContext, getTestId, isContentWithToolCallsResponse, isErrorResponse, isJsonObject, isTextResponse, isToolCallResponse, resolveFixtureBlockOutcome, resolveReasoningForModel, resolveResponse, resolveStrictMode, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField } from "./helpers.js";
+import { InvalidToolArgumentsError, estimatePromptTokens, estimateTokens, extractOverrides, flattenHeaders, generateToolUseId, getContext, getTestId, isCombinedFixtureResponse, isErrorResponse, isJsonObject, isTextResponse, isToolCallResponse, requireEmittedFunctionToolCalls, resolveFixtureBlockOutcome, resolveReasoningForModel, resolveResponse, resolveStrictMode, servedToolArgs, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField, toolArgsForWire } from "./helpers.js";
 import { writeErrorResponse } from "./sse-writer.js";
 import { applyChaosAsync } from "./chaos.js";
+import { planMisbehavior, recordMisbehaviorOutcome } from "./misbehavior.js";
 import { matchFixtureDiagnostic, recordMatchOptions } from "./router.js";
 import { proxyAndRecord } from "./recorder.js";
 import { createInterruptionSignal } from "./interruption.js";
@@ -37,13 +38,7 @@ function converseUsage(overrides) {
 	};
 }
 function parseConverseToolArgumentsForStream(toolCall, logger) {
-	try {
-		const parsed = JSON.parse(toolCall.arguments || "{}");
-		return JSON.stringify(parsed);
-	} catch {
-		logger.warn(`Malformed JSON in fixture tool call arguments for "${toolCall.name}": ${toolCall.arguments}`);
-		return "{}";
-	}
+	return servedToolArgs(toolCall, "string", logger).text;
 }
 function buildBedrockStreamTextEvents(content, chunkSize, reasoning, overrides) {
 	const events = [{
@@ -430,17 +425,11 @@ function buildConverseToolCallResponse(toolCalls, logger, reasoning, overrides)
 	const contentBlocks = [];
 	if (reasoning) contentBlocks.push({ reasoningContent: { reasoningText: { text: reasoning } } });
 	for (const tc of toolCalls) {
-		let argsObj;
-		try {
-			argsObj = JSON.parse(tc.arguments || "{}");
-		} catch {
-			logger.warn(`Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`);
-			argsObj = {};
-		}
+		const args = servedToolArgs(tc, "object", logger);
 		contentBlocks.push({ toolUse: {
 			toolUseId: tc.id || generateToolUseId(),
 			name: tc.name,
-			input: argsObj
+			input: args.value
 		} });
 	}
 	return {
@@ -457,17 +446,11 @@ function buildConverseContentWithToolCallsResponse(content, toolCalls, logger, r
 	const contentBlocks = [];
 	if (reasoning) contentBlocks.push({ reasoningContent: { reasoningText: { text: reasoning } } });
 	const toolUseBlock = (tc) => {
-		let argsObj;
-		try {
-			argsObj = JSON.parse(tc.arguments || "{}");
-		} catch {
-			logger.warn(`Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`);
-			argsObj = {};
-		}
+		const args = servedToolArgs(tc, "object", logger);
 		return { toolUse: {
 			toolUseId: tc.id || generateToolUseId(),
 			name: tc.name,
-			input: argsObj
+			input: args.value
 		} };
 	};
 	let blockHasTools;
@@ -495,6 +478,160 @@ function buildConverseContentWithToolCallsResponse(content, toolCalls, logger, r
 		metrics: { latencyMs: 0 }
 	};
 }
+/** Prepare actual emitted calls and identities before journaling or writing bytes. */
+function prepareConverseMisbehavior(plan, request, stream, defaults, headers) {
+	const response = plan.response;
+	const combined = isCombinedFixtureResponse(response);
+	let blocks = (combined && response.blocks?.length ? resolveFixtureBlockOutcome(response.blocks) : void 0)?.ordered.map((block) => ({ ...block })) ?? [..."content" in response && response.content ? [{
+		type: "text",
+		text: response.content
+	}] : [], ...(combined || isToolCallResponse(response) ? response.toolCalls ?? [] : []).map((call) => ({
+		type: "toolCall",
+		...call
+	}))];
+	const calls = blocks.filter((block) => block.type === "toolCall");
+	const preparedCalls = calls.map((call, index) => {
+		const args = toolArgsForWire(call);
+		return {
+			...call,
+			arguments: args.kind === "parsed" ? args.text : args.raw,
+			...index === plan.duplicateId?.destinationIndex ? {} : { id: call.id || generateToolUseId() }
+		};
+	});
+	if (plan.duplicateId) preparedCalls[plan.duplicateId.destinationIndex].id = preparedCalls[plan.duplicateId.sourceIndex].id;
+	let callIndex = 0;
+	blocks = blocks.map((block) => block.type === "toolCall" ? preparedCalls[callIndex++] : block);
+	let stopReason = converseStopReason("finishReason" in response ? response.finishReason : void 0, calls.length ? "tool_use" : "end_turn");
+	if (plan.stop === "stop") stopReason = "end_turn";
+	if (plan.stop === "length") stopReason = "max_tokens";
+	if (plan.stop === "content_filter") stopReason = "content_filtered";
+	if (!stream && plan.summary.fault === "tool-args-invalid-json") {
+		blocks = blocks.filter((block) => block.type !== "toolCall");
+		stopReason = "malformed_tool_use";
+	}
+	if (!stream && plan.summary.fault === "stop-length-mid-tool") {
+		let index = 0;
+		blocks = blocks.map((block) => {
+			if (block.type !== "toolCall") return block;
+			return index++ === plan.target?.index ? {
+				...block,
+				arguments: "{}"
+			} : block;
+		});
+	}
+	const reasoning = plan.reasoning ?? resolveReasoningForModel("reasoning" in response ? response.reasoning : void 0, request.model, resolveStrictMode(defaults.strict, headers), defaults.logger) ?? "";
+	const servedToolCalls = blocks.flatMap((block) => block.type === "toolCall" ? [{
+		name: block.name,
+		arguments: block.arguments,
+		id: block.id
+	}] : []);
+	const inputTokens = estimatePromptTokens(request.messages);
+	const outputTokens = estimateTokens(reasoning + blocks.map((block) => block.type === "text" ? block.text : block.name + block.arguments).join(""));
+	return {
+		plan: {
+			...plan,
+			summary: {
+				...plan.summary,
+				servedToolCalls
+			}
+		},
+		blocks,
+		reasoning,
+		stopReason,
+		usage: {
+			inputTokens,
+			outputTokens,
+			totalTokens: inputTokens + outputTokens
+		}
+	};
+}
+function buildConverseMisbehaviorResponse(prepared) {
+	const content = prepared.reasoning || prepared.plan.summary.fault === "reasoning-only" ? [{ reasoningContent: { reasoningText: { text: prepared.reasoning } } }] : [];
+	for (const block of prepared.blocks) if (block.type === "text") content.push({ text: block.text });
+	else {
+		const args = toolArgsForWire(block);
+		if (args.kind === "verbatim") throw new InvalidToolArgumentsError(block);
+		content.push({ toolUse: {
+			toolUseId: block.id,
+			name: block.name,
+			input: args.value
+		} });
+	}
+	return {
+		output: { message: {
+			role: "assistant",
+			content
+		} },
+		stopReason: prepared.stopReason,
+		usage: prepared.usage,
+		metrics: { latencyMs: 0 }
+	};
+}
+function buildConverseMisbehaviorEvents(prepared, chunkSize) {
+	const events = [{
+		eventType: "messageStart",
+		payload: { role: "assistant" }
+	}];
+	let index = 0;
+	if (prepared.reasoning || prepared.plan.summary.fault === "reasoning-only") {
+		events.push({
+			eventType: "contentBlockStart",
+			payload: {
+				contentBlockIndex: index,
+				start: { reasoningContent: {} }
+			}
+		});
+		for (let i = 0; i < prepared.reasoning.length; i += chunkSize) events.push({
+			eventType: "contentBlockDelta",
+			payload: {
+				contentBlockIndex: index,
+				delta: { reasoningContent: { text: prepared.reasoning.slice(i, i + chunkSize) } }
+			}
+		});
+		events.push({
+			eventType: "contentBlockStop",
+			payload: { contentBlockIndex: index++ }
+		});
+	}
+	for (const block of prepared.blocks) {
+		events.push({
+			eventType: "contentBlockStart",
+			payload: {
+				contentBlockIndex: index,
+				start: block.type === "toolCall" ? { toolUse: {
+					toolUseId: block.id,
+					name: block.name
+				} } : {}
+			}
+		});
+		const value = block.type === "text" ? block.text : block.arguments;
+		for (let i = 0; i < value.length; i += chunkSize) {
+			const part = value.slice(i, i + chunkSize);
+			events.push({
+				eventType: "contentBlockDelta",
+				payload: {
+					contentBlockIndex: index,
+					delta: block.type === "text" ? { text: part } : { toolUse: { input: part } }
+				}
+			});
+		}
+		events.push({
+			eventType: "contentBlockStop",
+			payload: { contentBlockIndex: index++ }
+		});
+	}
+	events.push({
+		eventType: "messageStop",
+		payload: { stopReason: prepared.stopReason }
+	}, {
+		eventType: "metadata",
+		payload: {
+			usage: prepared.usage,
+			metrics: { latencyMs: 0 }
+		}
+	});
+	return events;
+}
 async function handleConverse(req, res, raw, modelId, fixtures, journal, defaults, setCorsHeaders) {
 	const { logger } = defaults;
 	setCorsHeaders(res);
@@ -627,9 +764,70 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 		return;
 	}
 	const response = await resolveResponse(fixture, completionReq);
+	let misbehavior = planMisbehavior({
+		wire: "bedrock-converse",
+		emitsToolCallIds: true,
+		fixture,
+		response,
+		request: completionReq,
+		stream: false,
+		defaults,
+		rawHeaders: req.headers,
+		url: req.url
+	});
+	const addResponseEntry = (input) => {
+		const entry = journal.add(input);
+		recordMisbehaviorOutcome({
+			entry,
+			summary: misbehavior.summary,
+			defaults,
+			testId
+		});
+		return entry;
+	};
+	if (misbehavior.kind === "error") {
+		if (!misbehavior.summary?.evaluations.length) logger.error(misbehavior.message);
+		addResponseEntry({
+			method: req.method ?? "POST",
+			path: urlPath,
+			headers: flattenHeaders(req.headers),
+			body: completionReq,
+			response: {
+				status: misbehavior.status,
+				fixture,
+				error: misbehavior.message
+			}
+		});
+		const type = misbehavior.status === 400 ? "ValidationException" : "InternalServerException";
+		res.setHeader("x-amzn-errortype", type);
+		writeErrorResponse(res, misbehavior.status, JSON.stringify({
+			__type: type,
+			message: misbehavior.message,
+			code: misbehavior.code
+		}));
+		return;
+	}
+	if (misbehavior.kind === "applied") {
+		const prepared = prepareConverseMisbehavior(misbehavior, completionReq, false, defaults, req.headers);
+		misbehavior = prepared.plan;
+		const body = buildConverseMisbehaviorResponse(prepared);
+		addResponseEntry({
+			method: req.method ?? "POST",
+			path: urlPath,
+			headers: flattenHeaders(req.headers),
+			body: completionReq,
+			response: {
+				status: 200,
+				fixture
+			}
+		});
+		res.writeHead(200, { "Content-Type": "application/json" });
+		res.end(JSON.stringify(body));
+		return;
+	}
 	if (isErrorResponse(response)) {
 		const status = response.status ?? 500;
-		journal.add({
+		addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -649,11 +847,11 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 		writeErrorResponse(res, status, JSON.stringify(errBody), { retryAfter: response.retryAfter });
 		return;
 	}
-	if (isContentWithToolCallsResponse(response)) {
+	if (isCombinedFixtureResponse(response)) {
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		journal.add({
+		addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -663,7 +861,8 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 				fixture
 			}
 		});
-		const body = buildConverseContentWithToolCallsResponse(response.content ?? "", response.toolCalls ?? [], logger, effReasoning, overrides, response.blocks);
+		const functionToolCalls = requireEmittedFunctionToolCalls(response, "Bedrock Converse");
+		const body = buildConverseContentWithToolCallsResponse(response.content ?? "", functionToolCalls, logger, effReasoning, overrides, response.blocks);
 		res.writeHead(200, { "Content-Type": "application/json" });
 		res.end(JSON.stringify(body));
 		return;
@@ -672,7 +871,7 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		journal.add({
+		addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -691,7 +890,7 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		journal.add({
+		addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -701,12 +900,12 @@ async function handleConverse(req, res, raw, modelId, fixtures, journal, default
 				fixture
 			}
 		});
-		const body = buildConverseToolCallResponse(response.toolCalls, logger, effReasoning, overrides);
+		const body = buildConverseToolCallResponse(requireEmittedFunctionToolCalls(response, "Bedrock Converse"), logger, effReasoning, overrides);
 		res.writeHead(200, { "Content-Type": "application/json" });
 		res.end(JSON.stringify(body));
 		return;
 	}
-	journal.add({
+	addResponseEntry({
 		method: req.method ?? "POST",
 		path: urlPath,
 		headers: flattenHeaders(req.headers),
@@ -854,11 +1053,84 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 		return;
 	}
 	const response = await resolveResponse(fixture, completionReq);
+	let misbehavior = planMisbehavior({
+		wire: "bedrock-converse",
+		emitsToolCallIds: true,
+		fixture,
+		response,
+		request: completionReq,
+		stream: true,
+		defaults,
+		rawHeaders: req.headers,
+		url: req.url
+	});
+	const addResponseEntry = (input) => {
+		const entry = journal.add(input);
+		recordMisbehaviorOutcome({
+			entry,
+			summary: misbehavior.summary,
+			defaults,
+			testId
+		});
+		return entry;
+	};
+	if (misbehavior.kind === "error") {
+		if (!misbehavior.summary?.evaluations.length) logger.error(misbehavior.message);
+		addResponseEntry({
+			method: req.method ?? "POST",
+			path: urlPath,
+			headers: flattenHeaders(req.headers),
+			body: completionReq,
+			response: {
+				status: misbehavior.status,
+				fixture,
+				error: misbehavior.message
+			}
+		});
+		const type = misbehavior.status === 400 ? "ValidationException" : "InternalServerException";
+		res.setHeader("x-amzn-errortype", type);
+		writeErrorResponse(res, misbehavior.status, JSON.stringify({
+			__type: type,
+			message: misbehavior.message,
+			code: misbehavior.code
+		}));
+		return;
+	}
 	const latency = fixture.latency ?? defaults.latency;
 	const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize);
+	if (misbehavior.kind === "applied") {
+		const prepared = prepareConverseMisbehavior(misbehavior, completionReq, true, defaults, req.headers);
+		misbehavior = prepared.plan;
+		const events = buildConverseMisbehaviorEvents(prepared, chunkSize);
+		const entry = addResponseEntry({
+			method: req.method ?? "POST",
+			path: urlPath,
+			headers: flattenHeaders(req.headers),
+			body: completionReq,
+			response: {
+				status: 200,
+				fixture
+			}
+		});
+		const interruption = createInterruptionSignal(fixture);
+		if (!await writeEventStream(res, events, {
+			latency,
+			streamingProfile: fixture.streamingProfile,
+			recordedTimings: fixture.recordedTimings,
+			replaySpeed: fixture.replaySpeed ?? defaults.replaySpeed,
+			signal: interruption?.signal,
+			onChunkSent: interruption?.tick
+		})) {
+			if (!res.writableEnded) res.destroy();
+			entry.response.interrupted = true;
+			entry.response.interruptReason = interruption?.reason();
+		}
+		interruption?.cleanup();
+		return;
+	}
 	if (isErrorResponse(response)) {
 		const status = response.status ?? 500;
-		journal.add({
+		addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -878,11 +1150,11 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 		writeErrorResponse(res, status, JSON.stringify(errBody), { retryAfter: response.retryAfter });
 		return;
 	}
-	if (isContentWithToolCallsResponse(response)) {
+	if (isCombinedFixtureResponse(response)) {
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		const journalEntry = journal.add({
+		const journalEntry = addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -892,7 +1164,8 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 				fixture
 			}
 		});
-		const events = buildBedrockStreamContentWithToolCallsEvents(response.content ?? "", response.toolCalls ?? [], chunkSize, logger, effReasoning, overrides, response.blocks);
+		const functionToolCalls = requireEmittedFunctionToolCalls(response, "Bedrock Converse");
+		const events = buildBedrockStreamContentWithToolCallsEvents(response.content ?? "", functionToolCalls, chunkSize, logger, effReasoning, overrides, response.blocks);
 		const interruption = createInterruptionSignal(fixture);
 		if (!await writeEventStream(res, events, {
 			latency,
@@ -913,7 +1186,7 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		const journalEntry = journal.add({
+		const journalEntry = addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -944,7 +1217,7 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 		if (response.webSearches?.length) logger.warn("webSearches in fixture response are not supported for Bedrock Converse API — ignoring");
 		const overrides = extractOverrides(response);
 		const effReasoning = resolveReasoningForModel(response.reasoning, completionReq.model, resolveStrictMode(defaults.strict, req.headers), logger);
-		const journalEntry = journal.add({
+		const journalEntry = addResponseEntry({
 			method: req.method ?? "POST",
 			path: urlPath,
 			headers: flattenHeaders(req.headers),
@@ -954,7 +1227,7 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 				fixture
 			}
 		});
-		const events = buildBedrockStreamToolCallEvents(response.toolCalls, chunkSize, logger, effReasoning, overrides);
+		const events = buildBedrockStreamToolCallEvents(requireEmittedFunctionToolCalls(response, "Bedrock Converse"), chunkSize, logger, effReasoning, overrides);
 		const interruption = createInterruptionSignal(fixture);
 		if (!await writeEventStream(res, events, {
 			latency,
@@ -971,7 +1244,7 @@ async function handleConverseStream(req, res, raw, modelId, fixtures, journal, d
 		interruption?.cleanup();
 		return;
 	}
-	journal.add({
+	addResponseEntry({
 		method: req.method ?? "POST",
 		path: urlPath,
 		headers: flattenHeaders(req.headers),