copilot-relay 0.0.6 → 0.0.8

package/README.md

@@ -60,6 +60,7 @@ claudeSetup: true
 logLevel: info
 logRetentionDays: 3
 thinkEffort: xhigh
+upstreamTimeoutSeconds: 180
 webSearchBackend:
 ```
 
@@ -75,6 +76,10 @@ Any other `logLevel` value is invalid and stops startup.
 
 Valid `thinkEffort`: `none`, `low`, `medium`, `high`, `xhigh`.
 
+`upstreamTimeoutSeconds` controls the maximum time a single Claude request can
+spend waiting on upstream Copilot calls, including chat, Responses, preflight,
+and bridge-managed WebSearch calls. The default is `180`.
+
 `webSearchBackend` controls bridge-managed Claude WebSearch. Leave it empty to
 use `gptModel`, or set a Copilot Responses model ID such as `gpt-5.5`.
 

package/config.default.yaml

@@ -35,6 +35,9 @@ logRetentionDays: 3
 # Valid values: none, low, medium, high, xhigh.
 thinkEffort: xhigh
 
+# Max seconds to wait for one Claude request's upstream Copilot calls.
+upstreamTimeoutSeconds: 180
+
 # Copilot model used for bridge-managed Claude WebSearch.
 # Leave empty to use gptModel.
 webSearchBackend:

package/dist/main.js

@@ -372,6 +372,7 @@ const defaultConfig = {
 	opusModel: "claude-opus-4.8",
 	port: 4142,
 	thinkEffort: defaultReasoningEffort,
+	upstreamTimeoutSeconds: 180,
 	webSearchBackend: void 0
 };
 const isLogLevelName = (value) => typeof value === "string" && logLevels.includes(value.toLowerCase());
@@ -403,6 +404,12 @@ const normalizeThinkEffort = (value) => {
 	const normalized = value.toLowerCase() === "minimal" ? "low" : value.toLowerCase();
 	return isReasoningEffort(normalized) ? normalized : void 0;
 };
+const normalizeUpstreamTimeoutSeconds = (value) => {
+	if (value === void 0) return;
+	const timeout = normalizePositiveInteger(value);
+	if (timeout === void 0) throw new Error("Invalid upstreamTimeoutSeconds: expected a positive integer");
+	return timeout;
+};
 const readRawConfig = async () => {
 	try {
 		return parseConfigYaml(await fs.readFile(paths.configPath, "utf8"));
@@ -476,6 +483,10 @@ const parseConfigYaml = (content) => {
 			case "think_effort":
 				config.thinkEffort = unquoteYamlScalar(value);
 				break;
+			case "upstreamTimeoutSeconds":
+			case "upstream_timeout_seconds":
+				config.upstreamTimeoutSeconds = unquoteYamlScalar(value);
+				break;
 			case "webSearchBackend":
 			case "web_search_backend":
 				config.webSearchBackend = unquoteYamlScalar(value);
@@ -522,6 +533,9 @@ const serializeConfig = (config) => [
 	"# Default upstream thinking/reasoning effort: none, low, medium, high, xhigh.",
 	`thinkEffort: ${config.thinkEffort}`,
 	"",
+	"# Max seconds to wait for a single Claude request's upstream Copilot calls.",
+	`upstreamTimeoutSeconds: ${config.upstreamTimeoutSeconds}`,
+	"",
 	"# Copilot model used for bridge-managed Claude WebSearch. Empty uses gptModel.",
 	`webSearchBackend: ${config.webSearchBackend ?? ""}`,
 	"",
@@ -542,6 +556,7 @@ async function readAppConfig() {
 	const logRetentionDays$1 = normalizePositiveInteger(raw.logRetentionDays);
 	const port = normalizePort(raw.port);
 	const thinkEffort = normalizeThinkEffort(raw.thinkEffort);
+	const upstreamTimeoutSeconds = normalizeUpstreamTimeoutSeconds(raw.upstreamTimeoutSeconds);
 	const config = {
 		claudeSetup: claudeSetup ?? defaultConfig.claudeSetup,
 		copilotBaseUrl: normalizeString(raw.copilotBaseUrl) ?? defaultConfig.copilotBaseUrl,
@@ -552,6 +567,7 @@ async function readAppConfig() {
 		opusModel: normalizeString(raw.opusModel) ?? defaultConfig.opusModel,
 		port: port ?? defaultConfig.port,
 		thinkEffort: thinkEffort ?? defaultConfig.thinkEffort,
+		upstreamTimeoutSeconds: upstreamTimeoutSeconds ?? defaultConfig.upstreamTimeoutSeconds,
 		webSearchBackend: normalizeString(raw.webSearchBackend)
 	};
 	await writeConfig(config);
@@ -590,6 +606,7 @@ const readProxyConfig = (config) => ({
 	copilotToken: void 0,
 	host: config.host,
 	port: config.port,
+	upstreamTimeoutMs: config.upstreamTimeoutSeconds * 1e3,
 	vsCodeVersion: vscodeVersion,
 	webSearchBackend: config.webSearchBackend
 });
@@ -674,12 +691,50 @@ const editorPluginVersion = `copilot-chat/${copilotVersion}`;
 const userAgent = `GitHubCopilotChat/${copilotVersion}`;
 const apiVersion = "2025-04-01";
 const maxFetchAttempts = 2;
+const copilotRequestTimeoutMs = 18e4;
 const getCopilotProviderContext = (config) => ({
 	baseUrl: config.copilotBaseUrl,
 	token: config.copilotToken,
 	vsCodeVersion: config.vsCodeVersion
 });
 const shouldRetryResponse = (response) => response.status >= 500 && response.status <= 599;
+const createCopilotRequestSignal = (signal, timeoutMs = copilotRequestTimeoutMs) => {
+	const signals = [signal, timeoutMs > 0 ? AbortSignal.timeout(timeoutMs) : void 0].filter((value) => value !== void 0);
+	if (signals.length === 0) return;
+	return signals.length === 1 ? signals[0] : AbortSignal.any(signals);
+};
+const getAbortName = (value) => typeof value === "object" && value !== null && "name" in value && typeof value.name === "string" ? value.name : void 0;
+const isAbortLikeError = (error) => {
+	const name = getAbortName(error);
+	return name === "AbortError" || name === "TimeoutError";
+};
+const toCopilotAbortHTTPError = (error, signal, timeoutMs = copilotRequestTimeoutMs) => {
+	if (!signal?.aborted && !isAbortLikeError(error)) return;
+	const timedOut = getAbortName(signal?.reason) === "TimeoutError" || getAbortName(error) === "TimeoutError";
+	const message = timedOut ? `Copilot upstream request timed out after ${Math.round(timeoutMs / 1e3)}s.` : "Client request cancelled before Copilot upstream completed.";
+	const code = timedOut ? "upstream_timeout" : "request_cancelled";
+	return new HTTPError(message, new Response(JSON.stringify({ error: {
+		message,
+		code
+	} }), {
+		status: timedOut ? 504 : 499,
+		headers: { "content-type": "application/json" }
+	}), message);
+};
+const readCopilotJson = async (response, signal, timeoutMs = copilotRequestTimeoutMs) => {
+	try {
+		return await response.json();
+	} catch (error) {
+		throw toCopilotAbortHTTPError(error, signal, timeoutMs) ?? error;
+	}
+};
+const readCopilotText = async (response, signal, timeoutMs = copilotRequestTimeoutMs) => {
+	try {
+		return await response.text();
+	} catch (error) {
+		throw toCopilotAbortHTTPError(error, signal, timeoutMs) ?? error;
+	}
+};
 const buildHeaders = (provider, init, options) => {
 	const headers = new Headers(init.headers);
 	headers.set("authorization", `Bearer ${provider.token}`);
@@ -699,18 +754,23 @@ const buildHeaders = (provider, init, options) => {
 };
 const fetchCopilot = async (provider, path$1, init, options = {}) => {
 	if (!provider.token) throw new ProxyNotImplementedError("Copilot token is not configured for copilot-relay.");
+	const timeoutMs = options.timeoutMs ?? copilotRequestTimeoutMs;
+	const signal = createCopilotRequestSignal(options.signal, timeoutMs);
 	let lastError;
 	for (let attempt = 1; attempt <= maxFetchAttempts; attempt++) try {
 		const started = performance.now();
 		const response = await fetch(`${provider.baseUrl}${path$1}`, {
 			...init,
-			headers: buildHeaders(provider, init, options)
+			headers: buildHeaders(provider, init, options),
+			signal
 		});
 		const ms = Math.round(performance.now() - started);
 		log.debug(`Copilot ${init.method ?? "GET"} ${path$1} -> ${response.status} ${ms}ms (attempt ${attempt})`);
 		if (!shouldRetryResponse(response) || attempt === maxFetchAttempts) return response;
 		log.error(`Copilot ${path$1} returned ${response.status}; retrying (${attempt}/${maxFetchAttempts})`);
 	} catch (error) {
+		const abortError = toCopilotAbortHTTPError(error, signal, timeoutMs);
+		if (abortError) throw abortError;
 		lastError = error;
 		if (attempt === maxFetchAttempts) throw error;
 		log.error(`Copilot ${path$1} request failed; retrying (${attempt}/${maxFetchAttempts})`, error);
@@ -733,12 +793,21 @@ function buildResponsesRequestPayload(payload, reasoningEffort) {
 		temperature: payload.temperature,
 		top_p: payload.top_p,
 		user: sanitizeUserIdentifier(payload.user),
+		prompt_cache_key: deriveResponsesCacheKey(payload),
 		tools: translateTools(payload.tools),
 		tool_choice: translateToolChoice(payload.tool_choice),
 		reasoning: reasoningEffort ? { effort: reasoningEffort } : void 0,
 		text: payload.response_format?.type === "json_object" ? { format: { type: "json_object" } } : void 0
 	};
 }
+function deriveResponsesCacheKey(payload) {
+	const user = sanitizeUserIdentifier(payload.user);
+	if (user) return `cr-${createHash("sha256").update(user).digest("hex").slice(0, 32)}`;
+	const systemPrompt = payload.messages.find((message) => message.role === "system");
+	const systemText = typeof systemPrompt?.content === "string" ? systemPrompt.content : Array.isArray(systemPrompt?.content) ? systemPrompt.content.map((part) => part.type === "text" ? part.text : "").join("") : "";
+	if (!systemText) return;
+	return `cr-sys-${createHash("sha256").update(systemText).digest("hex").slice(0, 32)}`;
+}
 function translateResponsesToChatCompletion(response) {
 	const assistantMessages = response.output.filter((item) => item.type === "message");
 	const functionCalls = response.output.filter((item) => item.type === "function_call");
@@ -1077,6 +1146,7 @@ function getFinishReason(response, hasFunctionCalls) {
 //#endregion
 //#region src/copilot/chat.ts
 const usesMaxCompletionTokens = (modelId) => modelId.startsWith("gpt-5");
+const continuePrefillPrompt$1 = "Continue the assistant response from the previous assistant message.";
 const maxUserLength = 64;
 const sanitizeReasoningEffortForModel = (_modelId, reasoningEffort) => {
 	return reasoningEffort ?? void 0;
@@ -1103,6 +1173,38 @@ const buildRequestPayload = (payload) => {
 		user: sanitizeUserIdentifier(payload.user)
 	};
 };
+const normalizeFinalAssistantPrefill$1 = (payload) => {
+	const lastMessage = payload.messages.at(-1);
+	if (lastMessage?.role !== "assistant") return payload;
+	const messages = [...payload.messages];
+	if (typeof lastMessage.content !== "string") {
+		messages.pop();
+		return {
+			...payload,
+			messages
+		};
+	}
+	const content = lastMessage.content.trimEnd();
+	if (!content) {
+		messages.pop();
+		return {
+			...payload,
+			messages
+		};
+	}
+	messages[messages.length - 1] = {
+		...lastMessage,
+		content
+	};
+	messages.push({
+		role: "user",
+		content: continuePrefillPrompt$1
+	});
+	return {
+		...payload,
+		messages
+	};
+};
 const isAgentInitiator = (messages) => messages.some((msg) => msg.role === "assistant" || msg.role === "tool") ? "agent" : "user";
 const messagesIncludeImage = (messages) => messages.some((msg) => typeof msg.content !== "string" && msg.content?.some((part) => part.type === "image_url"));
 const createChatCompletions = async (config, payload, options = {}) => {
@@ -1110,53 +1212,62 @@ const createChatCompletions = async (config, payload, options = {}) => {
 	const requestedModel = options.requestedModel ?? payload.model;
 	const requestedThinkEffort = options.requestedThinkEffort ?? payload.reasoning_effort ?? "none";
 	const requestedThinking = options.requestedThinking ?? "none";
+	const signal = createCopilotRequestSignal(options.signal, options.timeoutMs);
 	const upstreamModelId = routeModelId(payload.model);
 	const upstreamPayload = upstreamModelId === payload.model ? payload : {
 		...payload,
 		model: upstreamModelId
 	};
+	const useResponsesApi = shouldUseResponsesApiForModel(upstreamPayload.model);
+	const compatiblePayload = useResponsesApi ? upstreamPayload : normalizeFinalAssistantPrefill$1(upstreamPayload);
 	const provider = getCopilotProviderContext(config);
-	const enableVision = messagesIncludeImage(upstreamPayload.messages);
-	const initiator = isAgentInitiator(upstreamPayload.messages);
-	const requestPayload = buildRequestPayload(upstreamPayload);
+	const enableVision = messagesIncludeImage(compatiblePayload.messages);
+	const initiator = isAgentInitiator(compatiblePayload.messages);
+	const requestPayload = buildRequestPayload(compatiblePayload);
 	log.debug([
 		"Model request",
 		`client=${client}`,
 		`requested_model=${requestedModel}`,
-		`upstream_model=${upstreamPayload.model}`,
+		`upstream_model=${compatiblePayload.model}`,
 		`requested_think_effort=${requestedThinkEffort}`,
 		`requested_thinking=${requestedThinking}`,
 		`effective_think_effort=${requestPayload.reasoning_effort ?? "none"}`
 	].join(" "));
 	log.debug("Full request payload", { payload: requestPayload });
-	if (shouldUseResponsesApiForModel(upstreamPayload.model)) return createResponses(provider, upstreamPayload, {
+	if (useResponsesApi) return createResponses(provider, compatiblePayload, {
 		vision: enableVision,
-		initiator
+		initiator,
+		signal,
+		timeoutMs: options.timeoutMs
 	});
 	const response = await fetchCopilot(provider, "/chat/completions", {
 		method: "POST",
 		headers: {
-			accept: upstreamPayload.stream ? "text/event-stream" : "application/json",
+			accept: compatiblePayload.stream ? "text/event-stream" : "application/json",
 			"content-type": "application/json"
 		},
 		body: JSON.stringify(requestPayload)
 	}, {
 		vision: enableVision,
-		initiator
+		initiator,
+		signal,
+		timeoutMs: options.timeoutMs
 	});
 	if (!response.ok) {
-		if (await shouldRetryWithResponses(response)) return createResponses(provider, upstreamPayload, {
+		if (await shouldRetryWithResponses(response, signal, options.timeoutMs)) return createResponses(provider, compatiblePayload, {
 			vision: enableVision,
-			initiator
+			initiator,
+			signal,
+			timeoutMs: options.timeoutMs
 		});
 		throw new HTTPError("Failed to create chat completions", response, await logUpstreamError("Failed to create chat completions", response, {
 			model: payload.model,
 			request: requestPayload,
 			route: "/chat/completions"
-		}));
+		}, signal, options.timeoutMs));
 	}
-	if (upstreamPayload.stream) return events(response);
-	return await response.json();
+	if (compatiblePayload.stream) return events(response);
+	return readCopilotJson(response, signal, options.timeoutMs);
 };
 async function createResponses(provider, payload, options) {
 	const requestPayload = buildResponsesRequestPayload(payload, getRequestedReasoningEffort(payload));
@@ -1169,18 +1280,20 @@ async function createResponses(provider, payload, options) {
 		body: JSON.stringify(requestPayload)
 	}, {
 		vision: options.vision,
-		initiator: options.initiator
+		initiator: options.initiator,
+		signal: options.signal,
+		timeoutMs: options.timeoutMs
 	});
 	if (!response.ok) throw new HTTPError("Failed to create responses", response, await logUpstreamError("Failed to create responses", response, {
 		model: payload.model,
 		request: requestPayload,
 		route: "/responses"
-	}));
+	}, options.signal, options.timeoutMs));
 	if (payload.stream) return translateResponsesStreamToChatCompletionStream(events(response));
-	return translateResponsesToChatCompletion(await response.json());
+	return translateResponsesToChatCompletion(await readCopilotJson(response, options.signal, options.timeoutMs));
 }
-async function logUpstreamError(message, response, context) {
-	const errorBody = await response.clone().text().catch(() => "");
+async function logUpstreamError(message, response, context, signal, timeoutMs) {
+	const errorBody = await readCopilotText(response.clone(), signal, timeoutMs).catch(() => "");
 	const detail = getUpstreamErrorDetail(response, errorBody);
 	log.error(`${message}: route=${context.route} model=${context.model} status=${response.status}`, {
 		message,
@@ -1206,9 +1319,9 @@ function getUpstreamErrorDetail(response, body) {
 		return body.slice(0, 240);
 	}
 }
-async function shouldRetryWithResponses(response) {
+async function shouldRetryWithResponses(response, signal, timeoutMs) {
 	try {
-		return (await response.clone().json()).error?.code === "unsupported_api_for_model";
+		return (await readCopilotJson(response.clone(), signal, timeoutMs)).error?.code === "unsupported_api_for_model";
 	} catch {
 		return false;
 	}
@@ -1217,12 +1330,17 @@ async function shouldRetryWithResponses(response) {
 //#endregion
 //#region src/lib/preflight.ts
 const getUpstreamModelIds = async (config) => {
-	const response = await fetchCopilot(getCopilotProviderContext(config), "/models", {
+	const provider = getCopilotProviderContext(config);
+	const signal = createCopilotRequestSignal(void 0, config.upstreamTimeoutMs);
+	const response = await fetchCopilot(provider, "/models", {
 		method: "GET",
 		headers: { accept: "application/json" }
+	}, {
+		signal,
+		timeoutMs: config.upstreamTimeoutMs
 	});
 	if (!response.ok) throw new HTTPError("Failed to validate upstream models", response);
-	const payload = await response.json();
+	const payload = await readCopilotJson(response, signal, config.upstreamTimeoutMs);
 	return new Set((payload.data ?? []).map((model) => model.id).filter((id) => typeof id === "string" && id.length > 0));
 };
 const ensureRequiredModels = async (config) => {
@@ -1245,7 +1363,8 @@ const validateModelRequest = async (config, model, thinkEffort) => {
 	try {
 		const response = await createChatCompletions(config, createProbePayload(model), {
 			client: "generic",
-			requestedModel: model
+			requestedModel: model,
+			timeoutMs: config.upstreamTimeoutMs
 		});
 		if (typeof response !== "object" || response === null || !("choices" in response)) throw new Error(`Preflight request for ${model} unexpectedly streamed`);
 		log.info(`Preflight OK: model=${model} think_effort=${thinkEffort}`);
@@ -1862,8 +1981,9 @@ const createFailedSearchExecution = (payload, requestedQuery, model, message) =>
 	results: [],
 	text: message
 });
-const createClaudeWebSearchExecution = async (config, payload, requestedQuery) => {
+const createClaudeWebSearchExecution = async (config, payload, requestedQuery, options = {}) => {
 	const backendModel = getWebSearchBackendModel(config);
+	const signal = createCopilotRequestSignal(options.signal, options.timeoutMs);
 	const response = await fetchCopilot(getCopilotProviderContext(config), "/responses", {
 		method: "POST",
 		headers: {
@@ -1871,12 +1991,16 @@ const createClaudeWebSearchExecution = async (config, payload, requestedQuery) =
 			"content-type": "application/json"
 		},
 		body: JSON.stringify(buildWebSearchRequestPayload(payload, requestedQuery, backendModel))
-	}, { initiator: "agent" });
+	}, {
+		initiator: "agent",
+		signal,
+		timeoutMs: options.timeoutMs
+	});
 	if (!response.ok) {
-		const detail = await response.text().catch(() => "");
+		const detail = await readCopilotText(response, signal, options.timeoutMs).catch(() => "");
 		return createFailedSearchExecution(payload, requestedQuery, backendModel, [`Copilot web search is not available for model ${backendModel}.`, detail ? `Upstream response: ${detail}` : ""].filter(Boolean).join("\n"));
 	}
-	const upstream = await response.json();
+	const upstream = await readCopilotJson(response, signal, options.timeoutMs);
 	const text = getResponseText(upstream);
 	const query = getSearchQuery(upstream, requestedQuery);
 	const results = parseSearchResults(text);
@@ -2224,6 +2348,7 @@ claudeRoutes.get("/models", (c) => c.json({
 claudeRoutes.post("/messages", async (c) => {
 	const config = c.get("config");
 	const claudePayload = await c.req.json();
+	const requestSignal = createCopilotRequestSignal(c.req.raw.signal, config.upstreamTimeoutMs);
 	log.debug("Full Claude request payload", { payload: claudePayload });
 	try {
 		const upstreamModel = translateModelName(claudePayload.model);
@@ -2236,13 +2361,18 @@ claudeRoutes.post("/messages", async (c) => {
 			client: "claude",
 			requestedModel: claudePayload.model,
 			requestedThinkEffort: getClaudeRequestedThinkEffort(claudePayload),
-			requestedThinking: getClaudeRequestedThinking(claudePayload)
+			requestedThinking: getClaudeRequestedThinking(claudePayload),
+			signal: requestSignal,
+			timeoutMs: config.upstreamTimeoutMs
 		});
 		if (isNonStreamingResponse(response)) {
 			const webSearchToolCall = shouldLetModelDecideWebSearch ? getClaudeWebSearchToolCallFromChatResponse(response, toolNameMapper) : void 0;
 			let claudeResponse;
 			if (webSearchToolCall) {
-				const search = await createClaudeWebSearchExecution(config, claudePayload, webSearchToolCall.query);
+				const search = await createClaudeWebSearchExecution(config, claudePayload, webSearchToolCall.query, {
+					signal: requestSignal,
+					timeoutMs: config.upstreamTimeoutMs
+				});
 				const searchResponse = createClaudeWebSearchResponse(search);
 				if (search.results.length === 0) claudeResponse = searchResponse;
 				else {
@@ -2250,7 +2380,9 @@ claudeRoutes.post("/messages", async (c) => {
 						client: "claude",
 						requestedModel: claudePayload.model,
 						requestedThinkEffort: getClaudeRequestedThinkEffort(claudePayload),
-						requestedThinking: getClaudeRequestedThinking(claudePayload)
+						requestedThinking: getClaudeRequestedThinking(claudePayload),
+						signal: requestSignal,
+						timeoutMs: config.upstreamTimeoutMs
 					});
 					if (!isNonStreamingResponse(finalResponse)) throw new HTTPError("Claude web search final answer request unexpectedly streamed", new Response("Claude web search final answer request unexpectedly streamed", {
 						status: 502,
@@ -2433,6 +2565,8 @@ const start = defineCommand({
 			config.copilotBaseUrl = nextConfig.copilotBaseUrl;
 			config.host = nextConfig.host;
 			config.port = nextConfig.port;
+			config.upstreamTimeoutMs = nextConfig.upstreamTimeoutSeconds * 1e3;
+			config.webSearchBackend = nextConfig.webSearchBackend;
 			runtimeState.modelRouting = {
 				gptModel: nextConfig.gptModel,
 				opusModel: nextConfig.opusModel
@@ -2441,6 +2575,7 @@ const start = defineCommand({
 		applyRuntimeConfig(appConfig);
 		log.info(`Log level: ${appConfig.logLevel}`);
 		log.info(`Think effort: ${appConfig.thinkEffort}`);
+		log.info(`Upstream timeout: ${appConfig.upstreamTimeoutSeconds}s`);
 		log.info(`Exposed models: ${getExposedModelIds().join(", ")}`);
 		const authSession = await setupProxyAuth(config);
 		try {
@@ -2470,7 +2605,7 @@ const start = defineCommand({
 		}
 		watchAppConfig((nextConfig) => {
 			applyRuntimeConfig(nextConfig);
-			log.info(`Config reloaded: logLevel=${nextConfig.logLevel} thinkEffort=${nextConfig.thinkEffort}`);
+			log.info(`Config reloaded: logLevel=${nextConfig.logLevel} thinkEffort=${nextConfig.thinkEffort} upstreamTimeoutSeconds=${nextConfig.upstreamTimeoutSeconds}`);
 		});
 		await new Promise((resolve$1, reject) => {
 			server.on("close", resolve$1);