copilot-relay 0.2.6 → 0.3.0

package/README.md

@@ -24,13 +24,25 @@ Public API:
 - `POST /v1/messages/count_tokens`
 - `GET /v1/models`
 - `GET /healthz`
+- `GET|HEAD /api/hello`
+
+`/api/hello` is a static reachability probe Claude Code sends on startup and
+around real traffic. Like `/healthz` it never contacts Copilot, so it proves the
+relay is listening and nothing more.
 
 Claude WebSearch is bridge-managed: when the model selects the WebSearch tool,
 the relay executes Copilot `/responses` with `web_search_preview`, then sends the
 retrieved context through a final model pass and returns Claude
-`server_tool_use` / `web_search_tool_result` blocks. Unknown API routes return
-`500` and log method, path, selected headers, and request payload to help
-implement compatible endpoints later.
+`server_tool_use` / `web_search_tool_result` blocks. The final pass keeps your
+other tools available, so the model can act on what it found in the same turn.
+
+Advertising WebSearch no longer costs streaming. The relay reads the model's
+response only as far as it takes to tell whether a search is coming, so a turn
+that never searches streams normally — which is most of them, since Claude Code
+offers the tool on every request.
+
+Unknown API routes return `500` and log method, path, selected headers, and
+request payload to help implement compatible endpoints later.
 
 Routing:
 

package/dist/main.js

@@ -1508,6 +1508,7 @@ function getFinishReason(response, hasFunctionCalls) {
 //#region src/copilot/chat.ts
 const usesMaxCompletionTokens = (modelId) => modelId.startsWith("gpt-5");
 const continuePrefillPrompt$1 = "Continue the assistant response from the previous assistant message.";
+const continueAfterNonUserPrompt = "Continue based on the context above.";
 const maxUserLength = 64;
 const sanitizeReasoningEffortForModel = (_modelId, reasoningEffort) => {
 	return reasoningEffort ?? void 0;
@@ -1536,7 +1537,14 @@ const buildRequestPayload = (payload) => {
 };
 const normalizeFinalAssistantPrefill$1 = (payload) => {
 	const lastMessage = payload.messages.at(-1);
-	if (lastMessage?.role !== "assistant") return payload;
+	if (!lastMessage || lastMessage.role === "user") return payload;
+	if (lastMessage.role !== "assistant") return {
+		...payload,
+		messages: [...payload.messages, {
+			role: "user",
+			content: continueAfterNonUserPrompt
+		}]
+	};
 	const messages = [...payload.messages];
 	if (typeof lastMessage.content !== "string") {
 		messages.pop();
@@ -2435,10 +2443,10 @@ const getWebSearchResultText = (search) => {
 	].join("\n");
 };
 const createWebSearchResultContextMessage = (search) => ({
-	role: "system",
+	role: "user",
 	content: [
 		"Trusted bridge retrieval context: the assistant selected web_search, and copilot-relay executed it.",
-		"Use this context for the final answer. Do not describe it as user-provided or injected. Do not call web_search again.",
+		"Use this context to complete the request. Do not describe it as user-provided or injected.",
 		"If the user requested a specific output format, answer using only matching information from this context.",
 		"If the user asked for a URL only, output only that URL with no surrounding text.",
 		"",
@@ -2447,24 +2455,26 @@ const createWebSearchResultContextMessage = (search) => ({
 		getWebSearchResultText(search)
 	].join("\n")
 });
-const createFinalWebSearchPayload = (payload, search) => {
-	const messages = payload.messages.flatMap((message) => {
-		if (message.role === "tool") return [{
-			role: "developer",
-			content: ["Prior local tool result from the conversation:", String(message.content ?? "")].join("\n")
-		}];
-		if (message.tool_calls && message.tool_calls.length > 0) return message.content ? [{
-			...message,
-			tool_calls: void 0
-		}] : [];
-		return [message];
-	});
+const isWebSearchTool = (tool, toolNameMapper) => isClaudeWebSearchToolName(toolNameMapper.toClaude(tool.function.name));
+const removeWebSearchTool = (tools, toolNameMapper) => {
+	if (!tools || tools.length === 0) return;
+	const remaining = tools.filter((tool) => !isWebSearchTool(tool, toolNameMapper));
+	return remaining.length > 0 ? remaining : void 0;
+};
+const normalizeFinalToolChoice = (toolChoice, tools, toolNameMapper) => {
+	if (!tools || tools.length === 0) return;
+	if (toolChoice === "required") return "auto";
+	if (toolChoice && typeof toolChoice === "object" && isClaudeWebSearchToolName(toolNameMapper.toClaude(toolChoice.function.name))) return "auto";
+	return toolChoice;
+};
+const createFinalWebSearchPayload = (payload, search, toolNameMapper) => {
+	const tools = removeWebSearchTool(payload.tools, toolNameMapper);
 	return {
 		...payload,
 		stream: false,
-		tools: void 0,
-		tool_choice: void 0,
-		messages: [...messages, createWebSearchResultContextMessage(search)]
+		tools,
+		tool_choice: normalizeFinalToolChoice(payload.tool_choice, tools, toolNameMapper),
+		messages: [...payload.messages, createWebSearchResultContextMessage(search)]
 	};
 };
 const mergeWebSearchAndFinalResponse = (searchResponse, finalResponse) => ({
@@ -2478,6 +2488,137 @@ const mergeWebSearchAndFinalResponse = (searchResponse, finalResponse) => ({
 	}
 });
 
+//#endregion
+//#region src/claude/web-search-stream.ts
+const isWebSearchName = (name, toolNameMapper, isWebSearchToolName) => !!name && isWebSearchToolName(toolNameMapper.toClaude(name));
+const chunkSettlesDecision = (chunk) => {
+	const choice = chunk.choices[0];
+	if (!choice) return false;
+	return choice.delta?.tool_calls?.some((call) => call.function?.name) === true || !!choice.finish_reason;
+};
+const chunkHasWebSearchCall = (chunk, toolNameMapper, isWebSearchToolName) => chunk.choices[0]?.delta?.tool_calls?.some((call) => isWebSearchName(call.function?.name, toolNameMapper, isWebSearchToolName)) ?? false;
+const accumulateChunks = (chunks) => {
+	const first = chunks[0];
+	const toolCalls = [];
+	let content = "";
+	let reasoning = "";
+	let finishReason = null;
+	let usage;
+	for (const chunk of chunks) {
+		if (chunk.usage) usage = chunk.usage;
+		const choice = chunk.choices[0];
+		if (!choice) continue;
+		if (typeof choice.delta?.content === "string") content += choice.delta.content;
+		const chunkReasoning = choice.delta?.reasoning_text ?? choice.delta?.reasoning_content;
+		if (typeof chunkReasoning === "string") reasoning += chunkReasoning;
+		if (choice.finish_reason) finishReason = choice.finish_reason;
+		for (const call of choice.delta?.tool_calls ?? []) {
+			const existing = toolCalls[call.index];
+			if (existing) {
+				existing.function.arguments += call.function?.arguments ?? "";
+				continue;
+			}
+			toolCalls[call.index] = {
+				id: call.id ?? `call_${call.index}`,
+				type: "function",
+				function: {
+					name: call.function?.name ?? "",
+					arguments: call.function?.arguments ?? ""
+				}
+			};
+		}
+	}
+	const collectedToolCalls = toolCalls.filter(Boolean);
+	return {
+		id: first?.id ?? "chat_stream_accumulated",
+		object: "chat.completion",
+		created: first?.created ?? 0,
+		model: first?.model ?? "",
+		choices: [{
+			index: 0,
+			message: {
+				role: "assistant",
+				content: content || null,
+				...reasoning ? { reasoning_text: reasoning } : {},
+				...collectedToolCalls.length > 0 ? { tool_calls: collectedToolCalls } : {}
+			},
+			finish_reason: finishReason ?? "stop"
+		}],
+		...usage ? { usage } : {}
+	};
+};
+const resolveWebSearchStreamDecision = async (stream, toolNameMapper, isWebSearchToolName, onChunk) => {
+	const buffered = [];
+	const iterator = stream[Symbol.asyncIterator]();
+	let sawWebSearch = false;
+	let settled = false;
+	let done = false;
+	let alreadyStreamed = false;
+	while (!settled) {
+		const next = await iterator.next();
+		if (next.done) {
+			done = true;
+			break;
+		}
+		const raw = next.value;
+		if (raw.data === "[DONE]") {
+			done = true;
+			break;
+		}
+		if (!raw.data) continue;
+		let chunk;
+		try {
+			chunk = JSON.parse(raw.data);
+		} catch {
+			continue;
+		}
+		buffered.push(chunk);
+		if (chunkHasWebSearchCall(chunk, toolNameMapper, isWebSearchToolName)) {
+			sawWebSearch = true;
+			settled = true;
+			break;
+		}
+		if (onChunk) {
+			await onChunk(chunk);
+			alreadyStreamed = true;
+		}
+		if (chunkSettlesDecision(chunk)) settled = true;
+	}
+	const rest = { async *[Symbol.asyncIterator]() {
+		if (done) return;
+		while (true) {
+			const next = await iterator.next();
+			if (next.done) return;
+			yield next.value;
+		}
+	} };
+	if (!sawWebSearch) return {
+		decision: {
+			kind: "streamed",
+			buffered: onChunk ? [] : buffered
+		},
+		rest,
+		alreadyStreamed
+	};
+	for await (const raw of rest) {
+		if (raw.data === "[DONE]") break;
+		if (!raw.data) continue;
+		try {
+			buffered.push(JSON.parse(raw.data));
+		} catch {
+			continue;
+		}
+	}
+	return {
+		decision: {
+			kind: "webSearch",
+			response: accumulateChunks(buffered)
+		},
+		rest: { async *[Symbol.asyncIterator]() {} },
+		alreadyStreamed
+	};
+};
+
 //#endregion
 //#region src/lib/tokenizer.ts
 const encodingMap = {
@@ -2706,6 +2847,65 @@ const eventsFromClaudeResponse = (response) => {
 	events$1.push({ type: "message_stop" });
 	return events$1;
 };
+const continuationEventsFromClaudeResponse = (response, state) => {
+	const events$1 = [];
+	if (state.contentBlockOpen) {
+		events$1.push({
+			type: "content_block_stop",
+			index: state.contentBlockIndex
+		});
+		state.contentBlockIndex += 1;
+		state.contentBlockOpen = false;
+		state.thinkingBlockOpen = false;
+	}
+	for (const block of response.content) {
+		const index = state.contentBlockIndex;
+		events$1.push({
+			type: "content_block_start",
+			index,
+			content_block: block
+		});
+		if (block.type === "text") events$1.push({
+			type: "content_block_delta",
+			index,
+			delta: {
+				type: "text_delta",
+				text: block.text
+			}
+		});
+		else if (block.type === "thinking") events$1.push({
+			type: "content_block_delta",
+			index,
+			delta: {
+				type: "thinking_delta",
+				thinking: block.thinking
+			}
+		});
+		else if (block.type === "tool_use" || block.type === "server_tool_use") events$1.push({
+			type: "content_block_delta",
+			index,
+			delta: {
+				type: "input_json_delta",
+				partial_json: JSON.stringify(block.input)
+			}
+		});
+		events$1.push({
+			type: "content_block_stop",
+			index
+		});
+		state.contentBlockIndex += 1;
+	}
+	events$1.push({
+		type: "message_delta",
+		delta: {
+			stop_reason: response.stop_reason,
+			stop_sequence: response.stop_sequence
+		},
+		usage: { output_tokens: response.usage.output_tokens }
+	});
+	events$1.push({ type: "message_stop" });
+	return events$1;
+};
 const createQueuedClaudeStreamWriter = (write) => {
 	let pending = Promise.resolve();
 	return (event) => {
@@ -2722,7 +2922,8 @@ const handleClaudeMessageRequest = async (config, claudePayload, requestSignal,
 	const decisionPayload = shouldLetModelDecideWebSearch ? prepareClaudeWebSearchDecisionPayload(claudePayload) : claudePayload;
 	const toolNameMapper = createClaudeToolNameMapper(decisionPayload.tools, { ...getToolNameMapperOptionsForModel(upstreamModel) });
 	const openAIPayload = translateToOpenAI(decisionPayload, void 0, toolNameMapper);
-	if (shouldLetModelDecideWebSearch) openAIPayload.stream = false;
+	const canStreamWebSearchDecision = shouldLetModelDecideWebSearch && !!writeEvent;
+	if (shouldLetModelDecideWebSearch && !canStreamWebSearchDecision) openAIPayload.stream = false;
 	const response = await createChatCompletions(config, openAIPayload, {
 		client: "claude",
 		requestedModel: claudePayload.model,
@@ -2732,8 +2933,31 @@ const handleClaudeMessageRequest = async (config, claudePayload, requestSignal,
 		signal: requestSignal,
 		timeoutMs: config.upstreamTimeoutMs
 	});
-	if (isNonStreamingResponse(response)) {
-		const webSearchToolCall = shouldLetModelDecideWebSearch ? getClaudeWebSearchToolCallFromChatResponse(response, toolNameMapper) : void 0;
+	let decidedResponse;
+	let bufferedChunks = [];
+	let remainingStream;
+	const streamState = {
+		messageStartSent: false,
+		contentBlockIndex: 0,
+		contentBlockOpen: false,
+		thinkingBlockOpen: false,
+		toolCalls: {}
+	};
+	let streamedBeforeDecision = false;
+	if (canStreamWebSearchDecision && !isNonStreamingResponse(response)) {
+		const { decision, rest, alreadyStreamed } = await resolveWebSearchStreamDecision(response, toolNameMapper, isClaudeWebSearchToolName, async (chunk) => {
+			await writeClaudeStreamEvents(translateChunkToClaudeEvents(chunk, streamState, toolNameMapper), writeEvent);
+		});
+		streamedBeforeDecision = alreadyStreamed;
+		if (decision.kind === "webSearch") decidedResponse = decision.response;
+		else {
+			bufferedChunks = decision.buffered;
+			remainingStream = rest;
+		}
+	}
+	const effectiveResponse = decidedResponse ?? response;
+	if (isNonStreamingResponse(effectiveResponse)) {
+		const webSearchToolCall = shouldLetModelDecideWebSearch ? getClaudeWebSearchToolCallFromChatResponse(effectiveResponse, toolNameMapper) : void 0;
 		let claudeResponse;
 		if (webSearchToolCall) {
 			const search = await createClaudeWebSearchExecution(config, claudePayload, webSearchToolCall.query, {
@@ -2744,7 +2968,7 @@ const handleClaudeMessageRequest = async (config, claudePayload, requestSignal,
 			const searchResponse = createClaudeWebSearchResponse(search);
 			if (search.results.length === 0) claudeResponse = searchResponse;
 			else {
-				const finalResponse = await createChatCompletions(config, createFinalWebSearchPayload(openAIPayload, search), {
+				const finalResponse = await createChatCompletions(config, createFinalWebSearchPayload(openAIPayload, search, toolNameMapper), {
 					client: "claude",
 					requestedModel: claudePayload.model,
 					requestedThinkEffort: getClaudeRequestedThinkEffort(claudePayload),
@@ -2759,14 +2983,14 @@ const handleClaudeMessageRequest = async (config, claudePayload, requestSignal,
 				}));
 				claudeResponse = mergeWebSearchAndFinalResponse(searchResponse, translateToClaude(finalResponse, toolNameMapper));
 			}
-		} else claudeResponse = translateToClaude(response, toolNameMapper);
+		} else claudeResponse = translateToClaude(effectiveResponse, toolNameMapper);
 		if (writeEvent) {
-			await writeClaudeStreamEvents(eventsFromClaudeResponse(claudeResponse), writeEvent);
+			await writeClaudeStreamEvents(streamedBeforeDecision ? continuationEventsFromClaudeResponse(claudeResponse, streamState) : eventsFromClaudeResponse(claudeResponse), writeEvent);
 			return;
 		}
 		return claudeResponse;
 	}
-	if (shouldLetModelDecideWebSearch) throw new HTTPError("Claude web search model-decision request unexpectedly streamed", new Response("Claude web search model-decision request unexpectedly streamed", {
+	if (shouldLetModelDecideWebSearch && !canStreamWebSearchDecision) throw new HTTPError("Claude web search model-decision request unexpectedly streamed", new Response("Claude web search model-decision request unexpectedly streamed", {
 		status: 502,
 		headers: { "content-type": "text/plain" }
 	}));
@@ -2774,14 +2998,9 @@ const handleClaudeMessageRequest = async (config, claudePayload, requestSignal,
 		status: 502,
 		headers: { "content-type": "text/plain" }
 	}));
-	const streamState = {
-		messageStartSent: false,
-		contentBlockIndex: 0,
-		contentBlockOpen: false,
-		thinkingBlockOpen: false,
-		toolCalls: {}
-	};
-	for await (const rawEvent of response) {
+	for (const chunk of bufferedChunks) await writeClaudeStreamEvents(translateChunkToClaudeEvents(chunk, streamState, toolNameMapper), writeEvent);
+	const upstreamStream = remainingStream ?? effectiveResponse;
+	for await (const rawEvent of upstreamStream) {
 		if (rawEvent.data === "[DONE]") break;
 		if (!rawEvent.data) continue;
 		let chunk;
@@ -2924,6 +3143,7 @@ const createServer = (config) => {
 		status: "ok"
 	}));
 	app.get("/healthz", (c) => c.json({ ok: true }));
+	app.on(["GET", "HEAD"], "/api/hello", (c) => c.body(null, 200));
 	app.route("/v1", claudeRoutes);
 	app.notFound(async (c) => {
 		const message = "Unsupported Claude API route";