@vscode/chat-lib 0.0.1

package/src/_internal/platform/nesFetch/common/completionsFetchService.ts

@@ -0,0 +1,118 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as l10n from '@vscode/l10n';
+import type { ChatErrorDetails } from 'vscode';
+import { Result } from '../../../util/common/result';
+import { createServiceIdentifier } from '../../../util/common/services';
+import { CancellationToken } from '../../../util/vs/base/common/cancellation';
+import { ResponseStream } from './responseStream';
+
+interface BaseCompletionsParams {
+	prompt: string;
+	stop?: string[];
+	top_p?: number;
+	best_of?: number;
+	max_tokens?: number;
+	temperature?: number;
+	presence_penalty?: number;
+	frequency_penalty?: number;
+	// required to access certain experimental models
+	model?: string;
+	logprobs?: number;
+	n?: number;
+	stream: true;
+}
+
+interface CodexV2Params {
+	suffix?: string;
+	extra?: { [key: string]: any };
+	code_annotations?: boolean;
+}
+
+export interface ModelParams extends BaseCompletionsParams, CodexV2Params { }
+
+export type FetchOptions = {
+	requestId: string;
+	headers: { [name: string]: string };
+	body: any;
+};
+
+export type CompletionsFetchFailure =
+	| { kind: 'cancelled' }
+	| { kind: 'model_overloaded' }
+	| { kind: 'model_error' }
+	| { kind: 'not-registered' }
+	| { kind: 'not-configured' }
+	| { kind: 'quota-exceeded' }
+	| {
+		kind: 'context-window-exceeded';
+		/** Message we get in response body. */
+		message: string;
+	}
+	| {
+		kind: 'invalid-api-key';
+		/** Message we get in response body. */
+		message: string;
+	}
+	| {
+		kind: 'exceeded-rate-limit';
+		/** Message we get in response body. */
+		message: string;
+	}
+	| {
+		kind: 'not-200-status';
+		status: number;
+		statusText: string;
+	}
+	;
+
+export type CompletionsFetchErrorType = 'stop_content_filter' | 'stop_length' | 'unknown';
+
+export class CompletionsFetchError extends Error {
+	constructor(
+		readonly type: CompletionsFetchErrorType,
+		readonly requestId: string,
+		message: string
+	) {
+		super(message);
+	}
+}
+
+export const ICompletionsFetchService = createServiceIdentifier<ICompletionsFetchService>('ICompletionsFetchService');
+
+/**
+ * OpenAI has completions and _chat_ completions endpoints. This's (non-chat) completions endpoint fetcher.
+ */
+export interface ICompletionsFetchService {
+	readonly _serviceBrand: undefined;
+
+	fetch(url: string, secretKey: string, params: ModelParams, requestId: string, ct: CancellationToken, headerOverrides?: Record<string, string>): Promise<Result<ResponseStream, CompletionsFetchFailure>>;
+}
+
+export function getErrorDetailsFromFetchError(requestId: string, error: CompletionsFetchFailure): ChatErrorDetails {
+	switch (error.kind) {
+		case 'cancelled':
+			return { message: 'Cancelled' };
+		case 'exceeded-rate-limit':
+			return {
+				message: l10n.t(`Sorry, your request was rate-limited. Please wait and try again.`),
+				responseIsFiltered: true,
+			};
+		case 'quota-exceeded':
+			return {
+				message: l10n.t(`You've reached your monthly chat messages limit. [Upgrade to Copilot Pro]({0}) (30-day Free Trial) or wait for your limit to reset.`, 'https://aka.ms/github-copilot-upgrade-plan'),
+			};
+		case 'model_overloaded':
+		case 'model_error':
+		case 'not-registered':
+		case 'not-200-status':
+		case 'context-window-exceeded':
+		case 'invalid-api-key':
+		case 'not-configured':
+		default:
+			return { message: l10n.t(`Sorry, your request failed. Please try again. Request id: {0}`, requestId) };
+	}
+}

package/src/_internal/platform/nesFetch/common/responseStream.ts

@@ -0,0 +1,151 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Result } from '../../../util/common/result';
+import { AsyncIterableObject, DeferredPromise } from '../../../util/vs/base/common/async';
+import { safeStringify } from '../../../util/vs/base/common/objects';
+import { assertType } from '../../../util/vs/base/common/types';
+import { Completion } from './completionsAPI';
+
+export class ResponseStream {
+	/**
+	 * A promise that resolves to the array of completions that were emitted by the stream.
+	 *
+	 * (it's expected to not throw)
+	 */
+	public readonly aggregatedStream: Promise<Result<Completion[], Error>>;
+
+	/**
+	 * A completion that aggregates completions stream.
+	 *
+	 * (it's expected to not throw)
+	 */
+	public readonly response: Promise<Result<Completion, Error>>;
+
+	/**
+	 * The stream of completions that were emitted by the response.
+	 *
+	 * @throws {Error} if the response stream throws an error.
+	 */
+	public readonly stream: AsyncIterableObject<Completion>;
+
+	constructor(stream: AsyncIterable<Completion>) {
+		const tokensDeferredPromise = new DeferredPromise<Result<Completion[], Error>>();
+		this.aggregatedStream = tokensDeferredPromise.p;
+		this.response = this.aggregatedStream.then((completions) => {
+			if (completions.isError()) {
+				return completions;
+			}
+			try {
+				return Result.ok(ResponseStream.aggregateCompletionsStream(completions.val));
+			} catch (err) {
+				return Result.error(err);
+			}
+		});
+
+		this.stream = new AsyncIterableObject(async (emitter) => {
+			const completions: Completion[] = [];
+			let error: Error | undefined;
+			try {
+				for await (const completion of stream) {
+					completions.push(completion);
+					emitter.emitOne(completion);
+				}
+			} catch (e) {
+				if (e instanceof Error) {
+					error = e;
+				} else {
+					error = new Error(safeStringify(e));
+				}
+				emitter.reject(e);
+			} finally {
+				tokensDeferredPromise.complete(
+					error ? Result.error(error) : Result.ok(completions)
+				);
+			}
+		});
+	}
+
+	private static aggregateCompletionsStream(stream: Completion[]): Completion {
+		let text = '';
+		let finishReason: Completion.FinishReason | null = null;
+		let aggregatedLogsProbs: Completion.LogProbs | null = null;
+		let aggregatedUsage: Completion.Usage | undefined = undefined;
+
+		for (const completion of stream) {
+			const choice = completion.choices[0]; // TODO@ulugbekna: we only support choice.index=0
+			text += choice.text ?? '';
+			if (choice.logprobs) {
+				if (aggregatedLogsProbs === null) {
+					aggregatedLogsProbs = {
+						tokens: [...choice.logprobs.tokens],
+						token_logprobs: [...choice.logprobs.token_logprobs],
+						text_offset: [...choice.logprobs.text_offset],
+						top_logprobs: [...choice.logprobs.top_logprobs],
+					};
+				} else {
+					aggregatedLogsProbs.tokens.push(...choice.logprobs.tokens);
+					aggregatedLogsProbs.token_logprobs.push(...choice.logprobs.token_logprobs);
+					aggregatedLogsProbs.text_offset.push(...choice.logprobs.text_offset);
+					aggregatedLogsProbs.top_logprobs.push(...choice.logprobs.top_logprobs);
+				}
+			}
+			if (completion.usage) {
+				if (aggregatedUsage === undefined) {
+					aggregatedUsage = {
+						completion_tokens: completion.usage.completion_tokens,
+						prompt_tokens: completion.usage.prompt_tokens,
+						total_tokens: completion.usage.total_tokens,
+						completion_tokens_details: {
+							audio_tokens: completion.usage.completion_tokens_details.audio_tokens,
+							reasoning_tokens: completion.usage.completion_tokens_details.reasoning_tokens,
+						},
+						prompt_tokens_details: {
+							audio_tokens: completion.usage.prompt_tokens_details.audio_tokens,
+							reasoning_tokens: completion.usage.prompt_tokens_details.reasoning_tokens,
+						}
+					};
+				} else {
+					aggregatedUsage.completion_tokens += completion.usage.completion_tokens;
+					aggregatedUsage.prompt_tokens += completion.usage.prompt_tokens;
+					aggregatedUsage.total_tokens += completion.usage.total_tokens;
+					aggregatedUsage.completion_tokens_details.audio_tokens += completion.usage.completion_tokens_details.audio_tokens;
+					aggregatedUsage.completion_tokens_details.reasoning_tokens += completion.usage.completion_tokens_details.reasoning_tokens;
+					aggregatedUsage.prompt_tokens_details.audio_tokens += completion.usage.prompt_tokens_details.audio_tokens;
+					aggregatedUsage.prompt_tokens_details.reasoning_tokens += completion.usage.prompt_tokens_details.reasoning_tokens;
+				}
+			}
+			if (choice.finish_reason) {
+				assertType(
+					finishReason === null,
+					'cannot already have finishReason if just seeing choice.finish_reason'
+				);
+				finishReason = choice.finish_reason;
+			}
+		}
+
+		if (stream.length === 0) {
+			throw new Error(`Response is empty!`);
+		}
+
+		const completion = stream[0];
+
+		const choice: Completion.Choice = {
+			index: 0,
+			finish_reason: finishReason,
+			logprobs: aggregatedLogsProbs,
+			text,
+		};
+
+		const aggregatedCompletion: Completion = {
+			choices: [choice],
+			system_fingerprint: completion.system_fingerprint,
+			object: completion.object,
+			usage: aggregatedUsage,
+		};
+
+		return aggregatedCompletion;
+	}
+}

package/src/_internal/platform/nesFetch/node/completionsFetchServiceImpl.ts

@@ -0,0 +1,236 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Readable } from 'stream';
+import * as stream_consumers from 'stream/consumers';
+import { Result } from '../../../util/common/result';
+import { AsyncIterableObject } from '../../../util/vs/base/common/async';
+import { CancellationToken } from '../../../util/vs/base/common/cancellation';
+import { safeStringify } from '../../../util/vs/base/common/objects';
+import { IAuthenticationService } from '../../authentication/common/authentication';
+import { IFetcherService, IHeaders } from '../../networking/common/fetcherService';
+import { CompletionsFetchFailure, FetchOptions, ICompletionsFetchService, ModelParams } from '../common/completionsFetchService';
+import { ResponseStream } from '../common/responseStream';
+import { jsonlStreamToCompletions, streamToLines } from './streamTransformer';
+
+export type FetchResponse = {
+	status: number;
+	statusText: string;
+	headers: { [name: string]: string };
+	body: AsyncIterableObject<string>;
+};
+
+export interface IFetchRequestParams extends ModelParams { }
+
+export class CompletionsFetchService implements ICompletionsFetchService {
+	readonly _serviceBrand: undefined;
+
+	constructor(
+		@IAuthenticationService private authService: IAuthenticationService,
+		@IFetcherService private fetcherService: IFetcherService,
+	) {
+	}
+
+	public async fetch(
+		url: string,
+		secretKey: string,
+		params: IFetchRequestParams,
+		requestId: string,
+		ct: CancellationToken,
+		headerOverrides?: Record<string, string>,
+	): Promise<Result<ResponseStream, CompletionsFetchFailure>> {
+
+		if (ct.isCancellationRequested) {
+			return Result.error({ kind: 'cancelled' });
+		}
+
+		const options = {
+			requestId,
+			headers: this.getHeaders(requestId, secretKey, headerOverrides),
+			body: {
+				...params,
+				stream: true,
+			}
+		};
+
+		const fetchResponse = await this._fetchFromUrl(url, options, ct);
+
+		if (fetchResponse.isError()) {
+			return fetchResponse;
+		}
+
+		if (fetchResponse.val.status === 200) {
+
+			const jsonlStream = streamToLines(fetchResponse.val.body);
+			const completionsStream = jsonlStreamToCompletions(jsonlStream);
+
+			const completions = completionsStream.map(completion => {
+				return {
+					...completion,
+					choices: completion.choices.filter(choice => choice.index === 0),
+				};
+			}).filter(c => {
+				return c.choices.length > 0;
+			}); // we only support `n=1`, so we only get choice.index = 0
+
+			const response = new ResponseStream(completions);
+
+			return Result.ok(response);
+
+		} else {
+
+			const body = await stream_consumers.text(fetchResponse.val.body);
+			if (body.match(/This model's maximum context length is /)) {
+				return Result.error({ kind: 'context-window-exceeded', message: body });
+			}
+			if (
+				body.match(
+					/Access denied due to invalid subscription key or wrong API endpoint/
+				) || fetchResponse.val.status === 401 || fetchResponse.val.status === 403
+			) {
+				return Result.error({ kind: 'invalid-api-key', message: body });
+			}
+			if (body.match(/exceeded call rate limit/)) {
+				return Result.error({ kind: 'exceeded-rate-limit', message: body });
+			}
+			const error: CompletionsFetchFailure = {
+				kind: 'not-200-status',
+				status: fetchResponse.val.status,
+				statusText: fetchResponse.val.statusText,
+			};
+			return Result.error(error);
+		}
+	}
+
+	protected async _fetchFromUrl(url: string, options: FetchOptions, ct: CancellationToken): Promise<Result<FetchResponse, CompletionsFetchFailure>> {
+
+		const fetchAbortCtl = this.fetcherService.makeAbortController();
+
+		const onCancellationDisposable = ct.onCancellationRequested(() => {
+			fetchAbortCtl.abort();
+		});
+
+		try {
+
+			const response = await this.fetcherService.fetch(url, {
+				headers: options.headers,
+				json: options.body,
+				signal: fetchAbortCtl.signal,
+				method: 'POST',
+			});
+
+			if (response.status === 200 && this.authService.copilotToken?.isFreeUser && this.authService.copilotToken?.isChatQuotaExceeded) {
+				this.authService.resetCopilotToken();
+			}
+
+			if (response.status !== 200) {
+				if (response.status === 402) {
+					// When we receive a 402, we have exceed the free tier quota
+					// This is stored on the token so let's refresh it
+					this.authService.resetCopilotToken(response.status);
+					return Result.error<CompletionsFetchFailure>({ kind: 'quota-exceeded' });
+				}
+
+				const error: CompletionsFetchFailure = {
+					kind: 'not-200-status',
+					status: response.status,
+					statusText: response.statusText,
+				};
+				return Result.error(error);
+			}
+
+			const responseBody = await response.body();
+
+			const body = (
+				responseBody instanceof Readable
+					? responseBody
+					: (
+						responseBody
+							? new Readable().wrap(responseBody as NodeJS.ReadableStream)
+							: new Readable()
+					)
+			);
+
+			body.setEncoding('utf8');
+
+			const responseStream = new AsyncIterableObject<string>(async (emitter) => {
+				try {
+					for await (const str of body) {
+						emitter.emitOne(str);
+					}
+				} catch (err: unknown) {
+					if (!(err instanceof Error)) {
+						throw new Error(safeStringify(err));
+					}
+
+					if (this.fetcherService.isAbortError(err) || err.name === 'AbortError') {
+						// stream aborted - ignore
+					} else if (
+						err.message === 'ERR_HTTP2_STREAM_ERROR' ||
+						(err as any).code === 'ERR_HTTP2_STREAM_ERROR'
+					) {
+						// stream closed - ignore
+					} else {
+						throw err;
+					}
+				} finally {
+					onCancellationDisposable.dispose();
+				}
+			});
+
+			return Result.ok({
+				status: response.status,
+				statusText: response.statusText,
+				headers: headersObjectToKv(response.headers),
+				body: responseStream,
+			});
+
+		} catch (reason: any) { // TODO: replace with unknown with proper error handling
+
+			onCancellationDisposable.dispose();
+
+			if (reason instanceof Error && reason.message === 'This operation was aborted') {
+				return Result.error({ kind: 'cancelled', errorMessage: reason.message });
+			}
+
+			if (
+				reason.code === 'ECONNRESET' ||
+				reason.code === 'ETIMEDOUT' ||
+				reason.code === 'ERR_HTTP2_INVALID_SESSION' ||
+				reason.message === 'ERR_HTTP2_GOAWAY_SESSION' ||
+				reason.code === '429'
+			) {
+				return Result.error({ kind: 'model_overloaded', errorMessage: reason.message });
+			} else {
+				return Result.error({ kind: 'model_error', errorMessage: reason.message });
+			}
+		}
+	}
+
+	private getHeaders(
+		requestId: string,
+		secretKey: string,
+		headerOverrides: Record<string, string> = {},
+	): Record<string, string> {
+		const headers: Record<string, string> = {
+			'Content-Type': 'application/json',
+			'x-policy-id': 'nil',
+			Authorization: 'Bearer ' + secretKey,
+			'X-Request-Id': requestId,
+			'X-GitHub-Api-Version': '2025-04-01',
+			...headerOverrides,
+		};
+
+		return headers;
+	}
+}
+
+function headersObjectToKv(headers: IHeaders): { [name: string]: string } {
+	const result: { [name: string]: string } = {};
+	for (const [name, value] of headers) {
+		result[name] = value;
+	}
+	return result;
+}

package/src/_internal/platform/nesFetch/node/streamTransformer.ts

@@ -0,0 +1,95 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { AsyncIterableObject } from '../../../util/vs/base/common/async';
+import { Completion } from '../common/completionsAPI';
+
+/**
+ * Transforms a stream of strings into a stream of lines.
+ *
+ * Listener should handle the errors coming from the input stream.
+ */
+export function streamToLines(stream: AsyncIterableObject<string>): AsyncIterableObject<string> {
+	return new AsyncIterableObject<string>(async (emitter) => {
+		let buffer = '';
+
+		for await (const str of stream) {
+			buffer += str;
+			do {
+				const newlineIndex = buffer.indexOf('\n');
+				if (newlineIndex === -1) {
+					break;
+				}
+
+				// take the first line
+				const line = buffer.substring(0, newlineIndex);
+				buffer = buffer.substring(newlineIndex + 1);
+
+				emitter.emitOne(line);
+			} while (true);
+		}
+
+		if (buffer.length > 0) {
+			// last line which doesn't end with \n
+			emitter.emitOne(buffer);
+		}
+	});
+}
+
+export function jsonlStreamToCompletions(jsonlStream: AsyncIterableObject<string>): AsyncIterableObject<Completion> {
+
+	return new AsyncIterableObject<Completion>(async (emitter) => {
+
+		for await (const line of jsonlStream) {
+
+			if (line.trim() === 'data: [DONE]') {
+				continue;
+			}
+
+			if (line.startsWith('data: ')) {
+				try {
+					const message: Completion & { error?: { message: string } } = JSON.parse(line.substring('data: '.length));
+
+					if (message.error) {
+						emitter.reject(new Error(message.error.message));
+						return;
+					}
+
+					emitter.emitOne(message);
+				} catch (err) {
+					emitter.reject(err);
+					return;
+				}
+			}
+		}
+	});
+}
+
+// function replaceBytes(s: string): string {
+// 	if (!s.startsWith('bytes:')) {
+// 		return s;
+// 	}
+// 	const bytes: number[] = [];
+// 	let i = 'bytes:'.length;
+// 	const textEncoder = new TextEncoder();
+// 	while (i < s.length) {
+// 		if (s.slice(i, i + 3) === '\\\\x') {
+// 			bytes.push(parseInt(s.slice(i + 3, i + 5), 16));
+// 			i += 5;
+// 		} else if (s.slice(i, i + 2) === '\\x') {
+// 			bytes.push(parseInt(s.slice(i + 2, i + 4), 16));
+// 			i += 4;
+// 		} else {
+// 			const encoded = textEncoder.encode(s.slice(i, i + 1));
+// 			for (const b of encoded) {
+// 				bytes.push(b);
+// 			}
+// 			i += 1;
+// 		}
+// 	}
+// 	return new TextDecoder('utf8', { fatal: false }).decode(
+// 		new Uint8Array(bytes)
+// 	);
+// }