@dianshuv/copilot-api 0.11.3 → 0.11.4

package/dist/main.mjs

@@ -1348,7 +1348,7 @@ const patchClaude = defineCommand({
 
 //#endregion
 //#region package.json
-var version = "0.11.3";
+var version = "0.11.4";
 
 //#endregion
 //#region src/lib/adaptive-rate-limiter.ts
@@ -7392,6 +7392,81 @@ function supportsDirectAnthropicApi(modelId) {
 	return resolveAnthropicModelForDirectPath(modelId) !== void 0;
 }
 
+//#endregion
+//#region src/lib/stream-keepalive.ts
+/** SSE comment line used as a keepalive heartbeat. */
+const SSE_PING = ": ping\n\n";
+/**
+* Grace period before opening a keepalive stream. Normal upstream responses
+* resolve sub-second (response headers arrive immediately, the body is not
+* buffered); only a request queued behind the rate limiter takes >=10s. 3s
+* cleanly separates the two — a request still pending after 3s is queued.
+*/
+const RATE_LIMIT_GRACE_MS = 3e3;
+/**
+* Interval between keepalive pings once the stream is open. Frequent enough to
+* prove liveness and defeat idle-connection drops, far below Claude Code's
+* default 10-minute request timeout.
+*/
+const KEEPALIVE_PING_INTERVAL_MS = 5e3;
+/**
+* Capture a promise's outcome as a value. The returned promise never rejects,
+* so it can be raced against a timer and awaited again later without producing
+* an unhandled rejection or a second pending chain on the original promise.
+*/
+function settle(promise) {
+	return promise.then((value) => ({
+		kind: "done",
+		value
+	}), (error) => ({
+		kind: "error",
+		error
+	}));
+}
+/**
+* Race an already-settled promise against a grace timer. Resolves with the
+* settled outcome if it arrives within `graceMs`, otherwise `{ kind: "timeout" }`.
+* The underlying work keeps running — await the SAME `settled` promise afterward
+* to obtain its eventual result.
+*/
+async function raceWithGrace(settled, graceMs) {
+	let timer;
+	const timeout = new Promise((resolve) => {
+		timer = setTimeout(() => resolve({ kind: "timeout" }), graceMs);
+	});
+	const result = await Promise.race([settled, timeout]);
+	if (timer !== void 0) clearTimeout(timer);
+	return result;
+}
+/**
+* Drive an open SSE stream while the upstream request is still queued: emit a
+* ping immediately, then every `pingIntervalMs`, until `settled` resolves —
+* then hand off to `onResponse` (success) or `onError` (failure). Pings stop on
+* client abort, on resolution, and never write to an aborted stream.
+*/
+async function runStreamWithKeepalive(opts) {
+	const { stream, settled, pingIntervalMs, onResponse, onError } = opts;
+	let pinging = true;
+	const sendPing = () => {
+		if (pinging && !stream.aborted) stream.write(SSE_PING);
+	};
+	sendPing();
+	const interval = setInterval(sendPing, pingIntervalMs);
+	stream.onAbort(() => {
+		pinging = false;
+		clearInterval(interval);
+	});
+	let result;
+	try {
+		result = await settled;
+	} finally {
+		pinging = false;
+		clearInterval(interval);
+	}
+	if (result.kind === "done") await onResponse(result.value);
+	else if (result.kind === "error") await onError(result.error);
+}
+
 //#endregion
 //#region src/routes/messages/message-utils.ts
 function convertAnthropicMessages(messages) {
@@ -8599,27 +8674,66 @@ async function handleDirectAnthropicCompletion(c, anthropicPayload, ctx, initiat
 	} else if (state.autoTruncate && !selectedModel) consola.debug(`[Anthropic] Model '${anthropicPayload.model}' not found, skipping auto-truncate`);
 	if (state.manualApprove) await awaitApproval();
 	const clientAnthropicBetaHeader = c.req.header("anthropic-beta");
+	const isStreaming = anthropicPayload.stream === true;
 	try {
-		const { result: response, queueWaitMs } = await executeWithAdaptiveRateLimit(() => createAnthropicMessages(effectivePayload, {
+		const settled = settle(executeWithAdaptiveRateLimit(() => createAnthropicMessages(effectivePayload, {
 			initiator: initiatorOverride,
 			injectContext1mBeta: needsContext1mBeta,
 			errorModelIdOverride: needsContext1mBeta ? originalModelId : void 0,
 			clientAnthropicBetaHeader
-		}));
-		ctx.queueWaitMs = queueWaitMs;
-		if (Symbol.asyncIterator in response) {
-			consola.debug("Streaming response from Copilot (direct Anthropic)");
-			updateTrackerStatus(ctx.trackingId, "streaming");
-			return streamSSE(c, async (stream) => {
-				await handleDirectAnthropicStreamingResponse({
-					stream,
-					response,
-					anthropicPayload: effectivePayload,
-					ctx
+		})));
+		const raced = isStreaming ? await raceWithGrace(settled, RATE_LIMIT_GRACE_MS) : await settled;
+		if (raced.kind === "error") throw raced.error;
+		if (raced.kind === "done") {
+			const { result: response, queueWaitMs } = raced.value;
+			ctx.queueWaitMs = queueWaitMs;
+			if (Symbol.asyncIterator in response) {
+				consola.debug("Streaming response from Copilot (direct Anthropic)");
+				updateTrackerStatus(ctx.trackingId, "streaming");
+				return streamSSE(c, async (stream) => {
+					await handleDirectAnthropicStreamingResponse({
+						stream,
+						response,
+						anthropicPayload: effectivePayload,
+						ctx
+					});
 				});
-			});
+			}
+			return handleDirectAnthropicNonStreamingResponse(c, recoverLeakedToolCallsInResponse(response, toolNameSet(effectivePayload.tools)), ctx, truncateResult, effectivePayload);
 		}
-		return handleDirectAnthropicNonStreamingResponse(c, recoverLeakedToolCallsInResponse(response, toolNameSet(effectivePayload.tools)), ctx, truncateResult, effectivePayload);
+		consola.debug("[RateLimiter] Request queued past grace; opening keepalive stream (direct Anthropic)");
+		updateTrackerStatus(ctx.trackingId, "streaming");
+		return streamSSE(c, async (stream) => {
+			await runStreamWithKeepalive({
+				stream,
+				settled,
+				pingIntervalMs: KEEPALIVE_PING_INTERVAL_MS,
+				onResponse: async ({ result: response, queueWaitMs }) => {
+					ctx.queueWaitMs = queueWaitMs;
+					await handleDirectAnthropicStreamingResponse({
+						stream,
+						response,
+						anthropicPayload: effectivePayload,
+						ctx
+					});
+				},
+				onError: async (error) => {
+					recordStreamError({
+						acc: createAnthropicStreamAccumulator(),
+						fallbackModel: anthropicPayload.model,
+						ctx,
+						error,
+						endpoint: "messages"
+					});
+					failTracking(ctx.trackingId, error);
+					const errorEvent = translateErrorToAnthropicErrorEvent(error);
+					await stream.writeSSE({
+						event: errorEvent.type,
+						data: JSON.stringify(errorEvent)
+					});
+				}
+			});
+		});
 	} catch (error) {
 		if (error instanceof HTTPError && error.status === 413) logPayloadSizeInfoAnthropic(effectivePayload, selectedModel);
 		recordErrorResponse(ctx, anthropicPayload.model, error, "messages", anthropicPayload.stream ?? false);
@@ -8864,30 +8978,71 @@ async function handleTranslatedCompletion(c, anthropicPayload, ctx, initiatorOve
 	if (autoTruncateConfig.tokenLimitCacheKeyOverride !== void 0) errorModelIdOverride = autoTruncateConfig.tokenLimitCacheKeyOverride;
 	else if (needsContext1mBeta && selectedModel) errorModelIdOverride = selectedModel.id;
 	else if (hasOneMillionSuffix) errorModelIdOverride = originalModelId;
+	const isStreaming = anthropicPayload.stream === true;
 	try {
-		const { result: response, queueWaitMs } = await executeWithAdaptiveRateLimit(() => createChatCompletions(openAIPayload, {
+		const settled = settle(executeWithAdaptiveRateLimit(() => createChatCompletions(openAIPayload, {
 			initiator: initiatorOverride,
 			resolvedModel: selectedModel,
 			anthropicBeta,
 			errorModelIdOverride
-		}));
-		ctx.queueWaitMs = queueWaitMs;
-		if (isNonStreaming(response)) return handleNonStreamingResponse({
-			c,
-			response,
-			toolNameMapping,
-			ctx,
-			anthropicPayload
-		});
-		consola.debug("Streaming response from Copilot");
+		})));
+		const raced = isStreaming ? await raceWithGrace(settled, RATE_LIMIT_GRACE_MS) : await settled;
+		if (raced.kind === "error") throw raced.error;
+		if (raced.kind === "done") {
+			const { result: response, queueWaitMs } = raced.value;
+			ctx.queueWaitMs = queueWaitMs;
+			if (isNonStreaming(response)) return handleNonStreamingResponse({
+				c,
+				response,
+				toolNameMapping,
+				ctx,
+				anthropicPayload
+			});
+			consola.debug("Streaming response from Copilot");
+			updateTrackerStatus(ctx.trackingId, "streaming");
+			return streamSSE(c, async (stream) => {
+				await handleStreamingResponse({
+					stream,
+					response,
+					toolNameMapping,
+					anthropicPayload,
+					ctx
+				});
+			});
+		}
+		consola.debug("[RateLimiter] Request queued past grace; opening keepalive stream (translated)");
 		updateTrackerStatus(ctx.trackingId, "streaming");
 		return streamSSE(c, async (stream) => {
-			await handleStreamingResponse({
+			await runStreamWithKeepalive({
 				stream,
-				response,
-				toolNameMapping,
-				anthropicPayload,
-				ctx
+				settled,
+				pingIntervalMs: KEEPALIVE_PING_INTERVAL_MS,
+				onResponse: async ({ result: response, queueWaitMs }) => {
+					ctx.queueWaitMs = queueWaitMs;
+					if (isNonStreaming(response)) return;
+					await handleStreamingResponse({
+						stream,
+						response,
+						toolNameMapping,
+						anthropicPayload,
+						ctx
+					});
+				},
+				onError: async (error) => {
+					recordStreamError({
+						acc: createAnthropicStreamAccumulator(),
+						fallbackModel: anthropicPayload.model,
+						ctx,
+						error,
+						endpoint: "messages"
+					});
+					failTracking(ctx.trackingId, error);
+					const errorEvent = translateErrorToAnthropicErrorEvent(error);
+					await stream.writeSSE({
+						event: errorEvent.type,
+						data: JSON.stringify(errorEvent)
+					});
+				}
 			});
 		});
 	} catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dianshuv/copilot-api",
-  "version": "0.11.3",
+  "version": "0.11.4",
   "description": "Turn GitHub Copilot into OpenAI/Anthropic API compatible server. Usable with Claude Code!",
   "author": "dianshuv",
   "type": "module",