@vscode/chat-lib 0.0.1

package/src/_internal/platform/openai/node/fetch.ts

@@ -0,0 +1,596 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ClientHttp2Stream } from 'http2';
+import type { CancellationToken } from 'vscode';
+import { createRequestHMAC } from '../../../util/common/crypto';
+import { generateUuid } from '../../../util/vs/base/common/uuid';
+import { IAuthenticationService } from '../../authentication/common/authentication';
+import { IChatQuotaService } from '../../chat/common/chatQuotaService';
+import { ChatLocation } from '../../chat/common/commonTypes';
+import { IInteractionService } from '../../chat/common/interactionService';
+import { ICAPIClientService } from '../../endpoint/common/capiClient';
+import { IDomainService } from '../../endpoint/common/domainService';
+import { IEnvService } from '../../env/common/envService';
+import { ILogService } from '../../log/common/logService';
+import { FinishedCallback, OptionalChatRequestParams, RequestId, getProcessingTime, getRequestId } from '../../networking/common/fetch';
+import { IFetcherService, Response } from '../../networking/common/fetcherService';
+import { IChatEndpoint, IEndpointBody, postRequest, stringifyUrlOrRequestMetadata } from '../../networking/common/networking';
+import { CAPIChatMessage, ChatCompletion } from '../../networking/common/openai';
+import { sendEngineMessagesTelemetry } from '../../networking/node/chatStream';
+import { sendCommunicationErrorTelemetry } from '../../networking/node/stream';
+import { ITelemetryService, TelemetryProperties } from '../../telemetry/common/telemetry';
+import { TelemetryData } from '../../telemetry/common/telemetryData';
+
+/** based on https://platform.openai.com/docs/api-reference/chat/create */
+interface RequiredChatRequestParams {
+	model: string;
+	messages: CAPIChatMessage[];
+}
+
+interface CopilotOnlyParams {
+
+	/** Copilot-only: names of experimental features to enable in the proxy. */
+	feature_flags?: string[];
+
+	/** Copilot-only: NWO of repository, if any */
+	nwo?: string;
+
+	copilot_thread_id?: string;
+}
+
+export interface ChatRequest extends
+	RequiredChatRequestParams, OptionalChatRequestParams, CopilotOnlyParams {
+}
+
+export enum FetchResponseKind {
+	Success = 'success',
+	Failed = 'failed',
+	Canceled = 'canceled',
+}
+
+export interface ChatResults {
+	type: FetchResponseKind.Success;
+	chatCompletions: AsyncIterable<ChatCompletion>;
+	getProcessingTime(): number;
+}
+
+export interface ChatRequestFailed {
+	type: FetchResponseKind.Failed;
+	modelRequestId: RequestId | undefined;
+	failKind: ChatFailKind;
+	reason: string;
+	data?: Record<string, any>;
+}
+
+export interface ChatRequestCanceled {
+	type: FetchResponseKind.Canceled;
+	reason: string;
+}
+
+export enum ChatFailKind {
+	OffTopic = 'offTopic',
+	TokenExpiredOrInvalid = 'tokenExpiredOrInvalid',
+	ServerCanceled = 'serverCanceled',
+	ClientNotSupported = 'clientNotSupported',
+	RateLimited = 'rateLimited',
+	QuotaExceeded = 'quotaExceeded',
+	ExtensionBlocked = 'extensionBlocked',
+	ServerError = 'serverError',
+	ContentFilter = 'contentFilter',
+	AgentUnauthorized = 'unauthorized',
+	AgentFailedDependency = 'failedDependency',
+	ValidationFailed = 'validationFailed',
+	InvalidPreviousResponseId = 'invalidPreviousResponseId',
+	NotFound = 'notFound',
+	Unknown = 'unknown',
+}
+
+/**
+ * A fetcher specialized to fetch ChatML completions. This differs from the standard fetcher in the form that ChatML
+ * requires a different datamodel. Details can be found here https://platform.openai.com/docs/guides/chat
+ *
+ * This fetcher was created because the standard fetcher is tightly coupled to the OpenAI API completion models and a major refactoring
+ * or rewrite is necessary to have a more generic fetcher that can be used for both completions and chat models.
+ */
+export async function fetchAndStreamChat(
+	logService: ILogService,
+	telemetryService: ITelemetryService,
+	fetcherService: IFetcherService,
+	envService: IEnvService,
+	chatQuotaService: IChatQuotaService,
+	domainService: IDomainService,
+	capiClientService: ICAPIClientService,
+	authenticationService: IAuthenticationService,
+	interactionService: IInteractionService,
+	chatEndpointInfo: IChatEndpoint,
+	request: IEndpointBody,
+	baseTelemetryData: TelemetryData,
+	finishedCb: FinishedCallback,
+	secretKey: string | undefined,
+	location: ChatLocation,
+	ourRequestId: string,
+	nChoices: number | undefined,
+	userInitiatedRequest?: boolean,
+	cancel?: CancellationToken | undefined,
+	telemetryProperties?: TelemetryProperties | undefined
+): Promise<ChatResults | ChatRequestFailed | ChatRequestCanceled> {
+	if (cancel?.isCancellationRequested) {
+		return { type: FetchResponseKind.Canceled, reason: 'before fetch request' };
+	}
+
+	logService.debug(`modelMaxPromptTokens ${chatEndpointInfo.modelMaxPromptTokens}`);
+	logService.debug(`modelMaxResponseTokens ${request.max_tokens ?? 2048}`);
+	logService.debug(`chat model ${chatEndpointInfo.model}`);
+
+	secretKey ??= (await authenticationService.getCopilotToken()).token;
+	if (!secretKey) {
+		// If no key is set we error
+		const urlOrRequestMetadata = stringifyUrlOrRequestMetadata(chatEndpointInfo.urlOrRequestMetadata);
+		logService.error(`Failed to send request to ${urlOrRequestMetadata} due to missing key`);
+		sendCommunicationErrorTelemetry(telemetryService, `Failed to send request to ${urlOrRequestMetadata} due to missing key`);
+		return {
+			type: FetchResponseKind.Failed,
+			modelRequestId: undefined,
+			failKind: ChatFailKind.TokenExpiredOrInvalid,
+			reason: 'key is missing'
+		};
+	}
+
+	// Generate unique ID to link input and output messages
+	const modelCallId = generateUuid();
+
+	const response = await fetchWithInstrumentation(
+		logService,
+		telemetryService,
+		fetcherService,
+		envService,
+		domainService,
+		capiClientService,
+		interactionService,
+		chatEndpointInfo,
+		ourRequestId,
+		request,
+		secretKey,
+		location,
+		userInitiatedRequest,
+		cancel,
+		{ ...telemetryProperties, modelCallId });
+
+	if (cancel?.isCancellationRequested) {
+		const body = await response!.body();
+		try {
+			// Destroy the stream so that the server is hopefully notified we don't want any more data
+			// and can cancel/forget about the request itself.
+			(body as ClientHttp2Stream).destroy();
+		} catch (e) {
+			logService.error(e, `Error destroying stream`);
+			telemetryService.sendGHTelemetryException(e, 'Error destroying stream');
+		}
+		return { type: FetchResponseKind.Canceled, reason: 'after fetch request' };
+	}
+
+	if (response.status === 200 && authenticationService.copilotToken?.isFreeUser && authenticationService.copilotToken?.isChatQuotaExceeded) {
+		authenticationService.resetCopilotToken();
+	}
+
+	if (response.status !== 200) {
+		const telemetryData = createTelemetryData(chatEndpointInfo, location, ourRequestId);
+		logService.info('Request ID for failed request: ' + ourRequestId);
+		return handleError(logService, telemetryService, authenticationService, telemetryData, response, ourRequestId);
+	}
+
+	// Extend baseTelemetryData with modelCallId for output messages
+	const extendedBaseTelemetryData = baseTelemetryData.extendedBy({ modelCallId });
+
+	const chatCompletions = await chatEndpointInfo.processResponseFromChatEndpoint(
+		telemetryService,
+		logService,
+		response,
+		nChoices ?? /* OpenAI's default */ 1,
+		finishedCb,
+		extendedBaseTelemetryData,
+		cancel
+	);
+
+	// CAPI will return us a Copilot Edits Session Header which is our token to using the speculative decoding endpoint
+	// We should store this in the auth service for easy use later
+	if (response.headers.get('Copilot-Edits-Session')) {
+		authenticationService.speculativeDecodingEndpointToken = response.headers.get('Copilot-Edits-Session') ?? undefined;
+	}
+
+	chatQuotaService.processQuotaHeaders(response.headers);
+
+	return {
+		type: FetchResponseKind.Success,
+		chatCompletions: chatCompletions,
+		getProcessingTime: () => getProcessingTime(response),
+	};
+}
+
+function createTelemetryData(chatEndpointInfo: IChatEndpoint, location: ChatLocation, headerRequestId: string) {
+	return TelemetryData.createAndMarkAsIssued({
+		endpoint: 'completions',
+		engineName: 'chat',
+		uiKind: ChatLocation.toString(location),
+		headerRequestId
+	});
+}
+
+async function handleError(
+	logService: ILogService,
+	telemetryService: ITelemetryService,
+	authenticationService: IAuthenticationService,
+	telemetryData: TelemetryData,
+	response: Response,
+	requestId: string,
+): Promise<ChatRequestFailed> {
+	const modelRequestIdObj = getRequestId(response, undefined);
+	requestId = modelRequestIdObj.headerRequestId || requestId;
+	modelRequestIdObj.headerRequestId = requestId;
+
+	telemetryData.properties.error = `Response status was ${response.status}`;
+	telemetryData.properties.status = String(response.status);
+	telemetryService.sendGHTelemetryEvent('request.shownWarning', telemetryData.properties, telemetryData.measurements);
+
+	const text = await response.text();
+	let jsonData: Record<string, any> | undefined;
+	try {
+		jsonData = JSON.parse(text);
+		jsonData = jsonData?.error ?? jsonData; // Extract nested error object if it exists
+	} catch {
+		// JSON parsing failed, it's not json content.
+	}
+
+	if (400 <= response.status && response.status < 500) {
+
+		if (response.status === 400 && text.includes('off_topic')) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.OffTopic,
+				reason: 'filtered as off_topic by intent classifier: message was not programming related',
+			};
+		}
+
+		if (response.status === 401 && text.includes('authorize_url') && jsonData?.authorize_url) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.AgentUnauthorized,
+				reason: response.statusText || response.statusText,
+				data: jsonData
+			};
+		}
+
+		if (response.status === 400 && jsonData?.code === 'previous_response_not_found') {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.InvalidPreviousResponseId,
+				reason: jsonData.message || 'Invalid previous response ID',
+				data: jsonData,
+			};
+		}
+
+		if (response.status === 401 || response.status === 403) {
+			// Token has expired or invalid, fetch a new one on next request
+			// TODO(drifkin): these actions should probably happen in vsc specific code
+			authenticationService.resetCopilotToken(response.status);
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.TokenExpiredOrInvalid,
+				reason: jsonData?.message || `token expired or invalid: ${response.status}`,
+			};
+		}
+
+		if (response.status === 402) {
+			// When we receive a 402, we have exceed a quota
+			// This is stored on the token so let's refresh it
+			authenticationService.resetCopilotToken(response.status);
+
+			const retryAfter = response.headers.get('retry-after');
+
+			const convertToDate = (retryAfterString: string | null): Date | undefined => {
+				if (!retryAfterString) {
+					return undefined;
+				}
+
+				// Try treating it as a date
+				const retryAfterDate = new Date(retryAfterString);
+				if (!isNaN(retryAfterDate.getDate())) {
+					return retryAfterDate;
+				}
+
+				// It is not a date, try treating it as a duration from the current date
+				const retryAfterDuration = parseInt(retryAfterString, 10);
+				if (isNaN(retryAfterDuration)) {
+					return undefined;
+				}
+
+				return new Date(Date.now() + retryAfterDuration * 1000);
+			};
+
+			const retryAfterDate = convertToDate(retryAfter);
+
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.QuotaExceeded,
+				reason: jsonData?.message ?? 'Free tier quota exceeded',
+				data: {
+					capiError: jsonData,
+					retryAfter: retryAfterDate
+				}
+			};
+		}
+
+		if (response.status === 404) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.NotFound,
+				reason: 'Resource not found'
+			};
+		}
+
+		if (response.status === 422) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.ContentFilter,
+				reason: 'Filtered by Responsible AI Service'
+			};
+		}
+
+		if (response.status === 424) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.AgentFailedDependency,
+				reason: text
+			};
+		}
+
+		if (response.status === 429) {
+			let rateLimitReason = text;
+			rateLimitReason = jsonData?.message ?? jsonData?.code;
+
+			if (text.includes('extension_blocked') && jsonData?.code === 'extension_blocked' && jsonData?.type === 'rate_limit_error') {
+				return {
+					type: FetchResponseKind.Failed,
+					modelRequestId: modelRequestIdObj,
+					failKind: ChatFailKind.ExtensionBlocked,
+					reason: 'Extension blocked',
+					data: {
+						...jsonData?.message,
+						retryAfter: response.headers.get('retry-after'),
+					}
+				};
+			}
+
+			// HTTP 429 Too Many Requests
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.RateLimited,
+				reason: rateLimitReason,
+				data: {
+					retryAfter: response.headers.get('retry-after'),
+					rateLimitKey: response.headers.get('x-ratelimit-exceeded'),
+					capiError: jsonData
+				}
+			};
+		}
+
+		if (response.status === 466) {
+			logService.info(text);
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.ClientNotSupported,
+				reason: `client not supported: ${text}`
+			};
+		}
+
+		if (response.status === 499) {
+			logService.info('Cancelled by server');
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.ServerCanceled,
+				reason: 'canceled by server'
+			};
+		}
+
+	} else if (500 <= response.status && response.status < 600) {
+
+		if (response.status === 503) {
+			return {
+				type: FetchResponseKind.Failed,
+				modelRequestId: modelRequestIdObj,
+				failKind: ChatFailKind.RateLimited,
+				reason: 'Upstream provider rate limit hit',
+				data: {
+					retryAfter: null,
+					rateLimitKey: null,
+					capiError: { code: 'upstream_provider_rate_limit', message: text }
+				}
+			};
+		}
+
+		const reasonNoText = `Server error: ${response.status}`;
+		const reason = `${reasonNoText} ${text}`;
+		logService.error(reason);
+		// HTTP 5xx Server Error
+		return {
+			type: FetchResponseKind.Failed,
+			modelRequestId: modelRequestIdObj,
+			failKind: ChatFailKind.ServerError,
+			reason: reasonNoText,
+		};
+	}
+
+	logService.error(`Request Failed: ${response.status} ${text}`);
+
+	sendCommunicationErrorTelemetry(telemetryService, 'Unhandled status from server: ' + response.status, text);
+
+	return {
+		type: FetchResponseKind.Failed,
+		modelRequestId: modelRequestIdObj,
+		failKind: ChatFailKind.Unknown,
+		reason: `Request Failed: ${response.status} ${text}`
+	};
+}
+
+async function fetchWithInstrumentation(
+	logService: ILogService,
+	telemetryService: ITelemetryService,
+	fetcherService: IFetcherService,
+	envService: IEnvService,
+	domainService: IDomainService,
+	capiClientService: ICAPIClientService,
+	interactionService: IInteractionService,
+	chatEndpoint: IChatEndpoint,
+	ourRequestId: string,
+	request: IEndpointBody,
+	secretKey: string,
+	location: ChatLocation,
+	userInitiatedRequest?: boolean,
+	cancel?: CancellationToken,
+	telemetryProperties?: TelemetryProperties
+): Promise<Response> {
+
+	// If request contains an image, we include this header.
+	const additionalHeaders: Record<string, string> = {
+		'X-Interaction-Id': interactionService.interactionId,
+		'X-Initiator': userInitiatedRequest ? 'user' : 'agent', // Agent = a system request / not the primary user query.
+	};
+	if (request.messages?.some((m: CAPIChatMessage) => Array.isArray(m.content) ? m.content.some(c => 'image_url' in c) : false) && chatEndpoint.supportsVision) {
+		additionalHeaders['Copilot-Vision-Request'] = 'true';
+	}
+	const telemetryData = TelemetryData.createAndMarkAsIssued({
+		endpoint: 'completions',
+		engineName: 'chat',
+		uiKind: ChatLocation.toString(location),
+		...telemetryProperties // This includes the modelCallId from fetchAndStreamChat
+	}, {
+		maxTokenWindow: chatEndpoint.modelMaxPromptTokens
+	});
+
+	for (const [key, value] of Object.entries(request)) {
+		if (key === 'messages') {
+			continue;
+		} // Skip messages (PII)
+		telemetryData.properties[`request.option.${key}`] = JSON.stringify(value) ?? 'undefined';
+	}
+
+	// The request ID we are passed in is sent in the request to the proxy, and included in our pre-request telemetry.
+	// We hope (but do not rely on) that the model will use the same ID in the response, allowing us to correlate
+	// the request and response.
+	telemetryData.properties['headerRequestId'] = ourRequestId;
+
+	telemetryService.sendGHTelemetryEvent('request.sent', telemetryData.properties, telemetryData.measurements);
+
+	const requestStart = Date.now();
+	const intent = locationToIntent(location);
+
+	// Wrap the Promise with success/error callbacks so we can log/measure it
+	return postRequest(
+		fetcherService,
+		envService,
+		telemetryService,
+		domainService,
+		capiClientService,
+		chatEndpoint,
+		secretKey,
+		await createRequestHMAC(process.env.HMAC_SECRET),
+		intent,
+		ourRequestId,
+		request,
+		additionalHeaders,
+		cancel
+	).then(response => {
+		const apim = response.headers.get('apim-request-id');
+		if (apim) {
+			logService.debug(`APIM request id: ${apim}`);
+		}
+		const ghRequestId = response.headers.get('x-github-request-id');
+		if (ghRequestId) {
+			logService.debug(`GH request id: ${ghRequestId}`);
+		}
+		// This ID is hopefully the one the same as ourRequestId, but it is not guaranteed.
+		// If they are different then we will override the original one we set in telemetryData above.
+		const modelRequestId = getRequestId(response, undefined);
+		telemetryData.extendWithRequestId(modelRequestId);
+
+		// TODO: Add response length (requires parsing)
+		const totalTimeMs = Date.now() - requestStart;
+		telemetryData.measurements.totalTimeMs = totalTimeMs;
+
+		logService.debug(`request.response: [${stringifyUrlOrRequestMetadata(chatEndpoint.urlOrRequestMetadata)}], took ${totalTimeMs} ms`);
+
+		if (request.messages) {
+			logService.debug(`messages: ${JSON.stringify(request.messages)}`);
+		} else if (request.input) {
+			logService.debug(`input: ${JSON.stringify(request.input)}`);
+		}
+
+		telemetryService.sendGHTelemetryEvent('request.response', telemetryData.properties, telemetryData.measurements);
+
+		return response;
+	})
+		.catch(error => {
+			if (fetcherService.isAbortError(error)) {
+				// If we cancelled a network request, we don't want to log a `request.error`
+				throw error;
+			}
+
+			const warningTelemetry = telemetryData.extendedBy({ error: 'Network exception' });
+			telemetryService.sendGHTelemetryEvent('request.shownWarning', warningTelemetry.properties, warningTelemetry.measurements);
+
+			telemetryData.properties.code = String(error.code ?? '');
+			telemetryData.properties.errno = String(error.errno ?? '');
+			telemetryData.properties.message = String(error.message ?? '');
+			telemetryData.properties.type = String(error.type ?? '');
+
+			const totalTimeMs = Date.now() - requestStart;
+			telemetryData.measurements.totalTimeMs = totalTimeMs;
+
+			logService.debug(`request.response: [${chatEndpoint.urlOrRequestMetadata}] took ${totalTimeMs} ms`);
+
+			telemetryService.sendGHTelemetryEvent('request.error', telemetryData.properties, telemetryData.measurements);
+
+			throw error;
+		})
+		.finally(() => {
+			sendEngineMessagesTelemetry(telemetryService, request.messages ?? [], telemetryData, false, logService);
+		});
+}
+
+/**
+ * WARNING: The value that is returned from this function drives the disablement of RAI for full-file rewrite requests
+ * in Copilot Edits, Copilot Chat, Agent Mode, and Inline Chat.
+ * If your chat location generates full-file rewrite requests and you are unsure if changing something here will cause problems, please talk to @roblourens
+ */
+function locationToIntent(location: ChatLocation): string {
+	switch (location) {
+		case ChatLocation.Panel:
+			return 'conversation-panel';
+		case ChatLocation.Editor:
+			return 'conversation-inline';
+		case ChatLocation.EditingSession:
+			return 'conversation-edits';
+		case ChatLocation.Notebook:
+			return 'conversation-notebook';
+		case ChatLocation.Terminal:
+			return 'conversation-terminal';
+		case ChatLocation.Other:
+			return 'conversation-other';
+		case ChatLocation.Agent:
+			return 'conversation-agent';
+	}
+}