@vscode/chat-lib 0.0.1

package/src/_internal/platform/notebook/common/alternativeContent.ts

@@ -0,0 +1,89 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { LanguageModelChat, NotebookDocument, Uri } from 'vscode';
+import { findCell } from '../../../util/common/notebooks';
+import { createServiceIdentifier } from '../../../util/common/services';
+import { Range } from '../../../util/common/test/shims/vscodeTypesShim';
+import { ConfigKey, IConfigurationService } from '../../configuration/common/configurationService';
+import { modelPrefersJsonNotebookRepresentation } from '../../endpoint/common/chatModelCapabilities';
+import { IChatEndpoint } from '../../networking/common/networking';
+import { IExperimentationService } from '../../telemetry/common/nullExperimentationService';
+import { BaseAlternativeNotebookContentProvider } from './alternativeContentProvider';
+import { AlternativeJsonNotebookContentProvider, isJsonContent } from './alternativeContentProvider.json';
+import { AlternativeTextNotebookContentProvider } from './alternativeContentProvider.text';
+import { AlternativeXmlNotebookContentProvider, isXmlContent } from './alternativeContentProvider.xml';
+
+export type AlternativeContentFormat = 'xml' | 'text' | 'json';
+
+export function getAlternativeNotebookDocumentProvider(kind: 'xml' | 'text' | 'json'): BaseAlternativeNotebookContentProvider {
+	switch (kind) {
+		case 'xml':
+			return new AlternativeXmlNotebookContentProvider();
+		case 'text':
+			return new AlternativeTextNotebookContentProvider();
+		case 'json':
+			return new AlternativeJsonNotebookContentProvider();
+		default:
+			throw new Error(`Unsupported kind '${kind}'`);
+	}
+}
+
+/**
+ * Given the content, determine the format of the content.
+ */
+export function inferAlternativeNotebookContentFormat(content: string): AlternativeContentFormat {
+	if (isXmlContent(content)) {
+		return 'xml';
+	}
+	if (isJsonContent(content)) {
+		return 'json';
+	}
+	return 'text';
+}
+
+
+export const IAlternativeNotebookContentService = createServiceIdentifier<IAlternativeNotebookContentService>('IAlternativeNotebookContentService');
+
+export interface IAlternativeNotebookContentService {
+	readonly _serviceBrand: undefined;
+	getFormat(options: LanguageModelChat | IChatEndpoint | undefined): AlternativeContentFormat;
+	create(format: AlternativeContentFormat): BaseAlternativeNotebookContentProvider;
+}
+
+export class AlternativeNotebookContentService implements IAlternativeNotebookContentService {
+	declare readonly _serviceBrand: undefined;
+	constructor(
+		@IConfigurationService private readonly configurationService: IConfigurationService,
+		@IExperimentationService private readonly experimentationService: IExperimentationService,
+	) {
+		//
+	}
+	getFormat(options: LanguageModelChat | IChatEndpoint | undefined): AlternativeContentFormat {
+		// GPT 4.1 supports apply_patch, such models work best with JSON format (doesn't have great support for XML yet, thats being worked on).
+		if (options && modelPrefersJsonNotebookRepresentation(options)) {
+			return 'json';
+		}
+
+		return this.configurationService.getExperimentBasedConfig(ConfigKey.Internal.NotebookAlternativeDocumentFormat, this.experimentationService);
+	}
+
+	create(format: AlternativeContentFormat): BaseAlternativeNotebookContentProvider {
+		return getAlternativeNotebookDocumentProvider(format);
+	}
+}
+
+export function getAltNotebookRange(range: Range, cellUri: Uri, notebook: NotebookDocument, format: AlternativeContentFormat) {
+	// If we have a range for cell, then translate that from notebook cell range to alternative range.
+	const cell = findCell(cellUri, notebook);
+	if (!cell) {
+		return undefined;
+	}
+	const doc = getAlternativeNotebookDocumentProvider(format).getAlternativeDocument(notebook);
+	return new Range(
+		doc.fromCellPosition(cell, range.start),
+		doc.fromCellPosition(cell, range.end),
+	);
+}

package/src/_internal/platform/notebook/common/alternativeContentFormat.ts

@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export enum AlternativeNotebookFormat {
+	json = 'json',
+	xml = 'xml',
+	text = 'text'
+}

package/src/_internal/platform/notebook/common/alternativeContentProvider.json.ts

@@ -0,0 +1,262 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import { visit } from 'jsonc-parser';
+import type { CancellationToken, NotebookCell, NotebookDocument, TextDocument } from 'vscode';
+import { AsyncIterableObject } from '../../../util/vs/base/common/async';
+import { URI } from '../../../util/vs/base/common/uri';
+import { NotebookCellKind, Position, Range, Uri } from '../../../util/common/test/shims/vscodeTypesShim';
+import { BaseAlternativeNotebookContentProvider } from './alternativeContentProvider';
+import { AlternativeNotebookDocument } from './alternativeNotebookDocument';
+import { EOL, getCellId, getCellIdMap, getDefaultLanguage, LineOfCellText, LineOfText, summarize } from './helpers';
+
+const IndentSize = 4;
+
+export function isJsonContent(text: string): boolean {
+	if (text.startsWith('{') || text.trim().startsWith('{')) {
+		return true;
+	}
+
+	if ((text.includes('{') || text.includes('}')) && text.includes('"source":') && text.includes('"cell_type":')) {
+		return true;
+	}
+	return false;
+}
+
+class AlternativeJsonDocument extends AlternativeNotebookDocument {
+	override fromCellPosition(cell: NotebookCell, position: Position): Position {
+		const cellId = getCellId(cell);
+
+		const alternativeContentText = this.getText();
+		const sourcePrefix = `                `; // we know we're indented by 4 spaces and source is 3 levels deep
+		const cellMarker = `"id": "${cellId}",`;
+		const positionOfSource = alternativeContentText.indexOf(`"source": [`, alternativeContentText.indexOf(cellMarker));
+
+		// Assume the text in the line is `print("Hello World")`
+		// & the position is the white space before `World`
+		// Position = line = n, character = 12
+		// In Json this would be `            "print(\"Hello World\")"`
+		// That would be translated as character position in translated = `            "print(\"Hello`.length;
+		const firstLineIndexOfCellSource = this.positionAt(positionOfSource).line + 1;
+		const leadingCharacters = cell.document.getText(new Range(position.line, 0, position.line, position.character));
+		// -1 to exclude to trailing `"`
+		const characterPositionInAltContent = `${sourcePrefix}${JSON.stringify(leadingCharacters).slice(0, -1)}`;
+		const linePositionInAltContent = position.line + firstLineIndexOfCellSource;
+		// -1 to exclude to trailing `"`
+		return new Position(linePositionInAltContent, characterPositionInAltContent.length);
+	}
+	override toCellPosition(position: Position): { cell: NotebookCell; position: Position } | undefined {
+		throw new Error('Method not implemented.');
+	}
+}
+
+export class AlternativeJsonNotebookContentProvider extends BaseAlternativeNotebookContentProvider {
+	constructor() {
+		super('json');
+	}
+
+	public stripCellMarkers(text: string): string {
+		return text;
+	}
+
+	public override parseAlternateContent(notebookOrUri: NotebookDocument | Uri, inputStream: AsyncIterable<LineOfText>, token: CancellationToken): AsyncIterable<LineOfCellText> {
+		return this.parseAlternateContentImpl(notebookOrUri, inputStream, token);
+	}
+
+	public override getAlternativeDocument(notebook: NotebookDocument, excludeMarkdownCells?: boolean): AlternativeNotebookDocument {
+		const cells = notebook.getCells().filter(cell => excludeMarkdownCells ? cell.kind !== NotebookCellKind.Markup : true).map(cell => {
+			const summary = summarize(cell);
+			const source = getCellCode(cell.document);
+
+			return {
+				cell_type: summary.cell_type,
+				id: summary.id,
+				metadata: {
+					language: summary.language
+				},
+				source,
+			} satisfies SummaryCell;
+		});
+
+		const json: Notebook = { cells };
+		const text = JSON.stringify(json, undefined, IndentSize);
+
+		return new AlternativeJsonDocument(text, notebook);
+	}
+
+	public override getSummaryOfStructure(notebook: NotebookDocument, cellsToInclude: NotebookCell[], existingCodeMarker: string): string {
+		const lines = ['{', '    "cells: ['];
+		const existingCodeMarkerWithComment = `// ${existingCodeMarker}`;
+		notebook.getCells().forEach((cell) => {
+			if (cellsToInclude.includes(cell)) {
+				const cellSummary = summarize(cell);
+				if (cellSummary.source.length && cellSummary.source[0].trim().length) {
+					cellSummary.source = [cellSummary.source[0], existingCodeMarkerWithComment];
+				} else if (cellSummary.source.length && cellSummary.source.some(line => line.trim().length)) {
+					cellSummary.source = [existingCodeMarkerWithComment, cellSummary.source.filter(line => line.trim().length)[0], existingCodeMarkerWithComment];
+				} else {
+					cellSummary.source = [existingCodeMarkerWithComment];
+				}
+				const summary = JSON.stringify(cellSummary, undefined, IndentSize).split(/\r?\n/).map(line => `    ${line}`);
+				lines.push(...summary);
+				lines.push(',');
+			} else if (!lines.length || lines[lines.length - 1] !== existingCodeMarkerWithComment) {
+				lines.push(existingCodeMarkerWithComment);
+			}
+		});
+		lines.push(`    ]`);
+		lines.push(`}`);
+		return lines.join(EOL);
+	}
+
+	private parseAlternateContentImpl(notebookOrUri: NotebookDocument | URI, inputStream: AsyncIterable<LineOfText>, token: CancellationToken): AsyncIterable<LineOfCellText> {
+		return new AsyncIterableObject<LineOfCellText>(async (emitter) => {
+			const cellIdMap = URI.isUri(notebookOrUri) ? new Map<string, NotebookCell>() : getCellIdMap(notebookOrUri);
+			const cellIdsSeen = new Set<string>();
+			let jsonText = '';
+			let lastSeenOffset = -1;
+			const cellInfo: { id?: string; index: number; kind: NotebookCellKind; source: string[]; uri?: Uri; language?: string; startOffset: number; endOffset: number } = {
+				index: -1,
+				startOffset: -1,
+				endOffset: -1,
+				kind: NotebookCellKind.Code,
+				source: [],
+			};
+			const defaultLanguage = URI.isUri(notebookOrUri) ? 'python' : getDefaultLanguage(notebookOrUri);
+
+			const emitCell = (endOffset: number) => {
+				// LLM can return duplicate cell with the same id.
+				cellInfo.language = cellInfo.language || defaultLanguage;
+				if (cellInfo.id && cellIdMap.get(cellInfo.id)?.document.languageId === cellInfo.language) {
+					if (cellIdsSeen.has(cellInfo.id)) {
+						cellInfo.id = '';
+					} else {
+						cellIdsSeen.add(cellInfo.id);
+					}
+				} else {
+					// Possible duplicate cell with the same id but different language.
+					// In such cases, treat them as new cells.
+					cellInfo.id = '';
+				}
+				const cell = cellIdMap.get(cellInfo.id);
+				cellInfo.uri = cell?.document.uri;
+				cellInfo.kind = cell?.kind || (cellInfo.language === 'markdown' ? NotebookCellKind.Markup : NotebookCellKind.Code);
+
+				emitter.emitOne({ index: cellInfo.index, type: 'start', kind: cellInfo.kind, language: cellInfo.language, uri: cellInfo.uri, id: cellInfo.id });
+				cellInfo.source.forEach(cellLine => emitter.emitOne({ index: cellInfo.index, type: 'line', line: cellLine }));
+				emitter.emitOne({ index: cellInfo.index, type: 'end' });
+			};
+
+			let finalOffset = 0;
+			for await (const lineOfText of inputStream) {
+				if (token.isCancellationRequested) {
+					break;
+				}
+				const line = lineOfText.value;
+
+
+				jsonText += line;
+
+				visit(jsonText, {
+					onObjectEnd(offset, _length, _startLine, _startCharacter) {
+						finalOffset = offset;
+					},
+					onLiteralValue: (value, offset, _length, _startLine, _startCharacter, pathSupplier) => {
+						if (lastSeenOffset >= offset) {
+							return;
+						}
+						const segments = pathSupplier();
+						if (segments.length < 2) {
+							return;
+						}
+						if (segments.shift() !== 'cells') {
+							return;
+						}
+						const cellIndex = segments.shift();
+						if (typeof cellIndex !== 'number') {
+							return;
+						}
+
+						const property = segments.shift()! as string;
+						lastSeenOffset = offset;
+
+						if (cellInfo.index !== -1 && cellInfo.index !== cellIndex) {
+							emitCell(offset);
+
+							cellInfo.startOffset = offset;
+							cellInfo.id = undefined;
+							cellInfo.kind = NotebookCellKind.Code;
+							cellInfo.source = [];
+							cellInfo.uri = undefined;
+							cellInfo.language = undefined;
+						}
+
+						cellInfo.index = cellIndex;
+
+						if (property === 'cell_type') {
+							cellInfo.kind = value === 'code' ? NotebookCellKind.Code : NotebookCellKind.Markup;
+							if (cellInfo.kind === NotebookCellKind.Markup) {
+								cellInfo.language = 'markdown';
+							}
+						} else if (property === 'id') {
+							// This is for scenarios when LLM sends the id as part of the cell instead of metdata.
+							cellInfo.id = value;
+						} else if (property === 'metadata' && segments[0] === 'id') {
+							cellInfo.id = value;
+						} else if (property === 'metadata' && segments[0] === 'language') {
+							cellInfo.language = value;
+							if (cellInfo.language === 'markdown') {
+								cellInfo.kind = NotebookCellKind.Markup;
+							}
+						} else if (property === 'source' && segments.length && typeof segments[0] === 'number') {
+							if (segments[0] === 0) {
+								cellInfo.startOffset = offset;
+							}
+							let code = typeof value === 'string' ? value : `${value || ''}`;
+							// Generally code in jupyter cells always end with `\n` when persisted in JSON file.
+							// However we do not want to deal with the \n as we're only interested in the lines of code.
+							// This is because we're going to edit a line at a time, new line means we have two lines to edit, but thats not possible.
+							// A line cannot contain new line in editor (then its just two lines in editor).
+							if (code.endsWith('\n')) {
+								code = code.substr(0, code.length - 1);
+							}
+							cellInfo.source.push(code);
+						}
+					}
+				});
+			}
+
+			if (cellInfo.index !== -1) {
+				emitCell(finalOffset);
+			}
+		});
+	}
+}
+
+function getCellCode(document: TextDocument): string[] {
+	if (document.lineCount === 0) {
+		return [];
+	}
+	if (document.lineCount === 1) {
+		return [document.lineAt(0).text];
+	}
+	const lineCount = document.lineCount;
+	return new Array(lineCount).fill('').map((_, i) => document.lineAt(i).text);
+}
+
+type Notebook = {
+	cells: SummaryCell[];
+};
+
+/**
+ * Shortend version of a Jupyter cell JSON.
+ */
+type SummaryCell = {
+	cell_type: 'code' | 'markdown';
+	source: string[];
+	id: string;
+	metadata: {
+		language: string;
+	};
+};

package/src/_internal/platform/notebook/common/alternativeContentProvider.text.ts

@@ -0,0 +1,243 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import type { CancellationToken, NotebookCell, NotebookDocument, Position, Uri } from 'vscode';
+import { getLanguage } from '../../../util/common/languages';
+import { isUri } from '../../../util/common/types';
+import { EndOfLine, NotebookCellKind } from '../../../util/common/test/shims/vscodeTypesShim';
+import { BaseAlternativeNotebookContentProvider } from './alternativeContentProvider';
+import { AlternativeNotebookDocument } from './alternativeNotebookDocument';
+import { EOL, getCellIdMap, getDefaultLanguage, LineOfCellText, LineOfText, summarize, SummaryCell } from './helpers';
+import { findLast } from '../../../util/vs/base/common/arraysFind';
+
+export function generateCellTextMarker(cell: SummaryCell, lineComment: string): string {
+	const cellIdStr = cell.id ? `[id=${cell.id}] ` : '';
+	return `${lineComment}%% vscode.cell ${cellIdStr}[language=${cell.language}]`;
+}
+
+export function lineMightHaveCellMarker(line: string) {
+	return line.toLowerCase().includes('vscode.cell');
+}
+
+class AlternativeTextDocument extends AlternativeNotebookDocument {
+	constructor(text: string, private readonly cellOffsetMap: { offset: number; sourceOffset: number; cell: NotebookCell }[], notebook: NotebookDocument) {
+		super(text, notebook);
+	}
+
+	override fromCellPosition(cell: NotebookCell, position: Position): Position {
+		const cellSummary = summarize(cell);
+		const lineCommentStart = getLineCommentStart(this.notebook);
+		const cellMarker = generateCellTextMarker(cellSummary, lineCommentStart);
+
+		const eolLength = cell.document.eol === EndOfLine.LF ? 1 : 2;
+		const blockComment = getBlockComment(this.notebook);
+		const alternativeContentText = this.getText();
+		const offsetInCell = cell.document.offsetAt(position);
+		const markdownOffset = cell.kind === NotebookCellKind.Markup ? blockComment[0].length + eolLength : 0;
+		const offset = alternativeContentText.indexOf(cellMarker) + cellMarker.length + eolLength + markdownOffset + offsetInCell;
+		return this.positionAt(offset);
+	}
+
+	override toCellPosition(position: Position): { cell: NotebookCell; position: Position } | undefined {
+		const offset = this.offsetAt(position);
+		const cell = findLast(this.cellOffsetMap, (cell) => cell.sourceOffset <= offset);
+		if (!cell) {
+			return undefined;
+		}
+		const cellPosition = cell.cell.document.positionAt(offset - cell.sourceOffset);
+		return { cell: cell.cell, position: cellPosition };
+	}
+}
+
+
+export class AlternativeTextNotebookContentProvider extends BaseAlternativeNotebookContentProvider {
+	constructor() {
+		super('text');
+	}
+
+	public stripCellMarkers(text: string): string {
+		const lines = text.split(EOL);
+		if (lines.length && lineMightHaveCellMarker(lines[0])) {
+			lines.shift();
+			return lines.join(EOL);
+		} else {
+			return text;
+		}
+	}
+
+	public override getSummaryOfStructure(notebook: NotebookDocument, cellsToInclude: NotebookCell[], existingCodeMarker: string): string {
+		const blockComment = getBlockComment(notebook);
+		const lineCommentStart = getLineCommentStart(notebook);
+		const existingCodeMarkerWithComment = `${lineCommentStart} ${existingCodeMarker}`;
+		const lines: string[] = [];
+		notebook.getCells().forEach((cell) => {
+			if (cellsToInclude.includes(cell)) {
+				const cellSummary = summarize(cell);
+				if (cellSummary.source.length && cellSummary.source[0].trim().length) {
+					cellSummary.source = [cellSummary.source[0], existingCodeMarkerWithComment];
+				} else if (cellSummary.source.length && cellSummary.source.some(line => line.trim().length)) {
+					cellSummary.source = [existingCodeMarkerWithComment, cellSummary.source.filter(line => line.trim().length)[0], existingCodeMarkerWithComment];
+				} else {
+					cellSummary.source = [existingCodeMarkerWithComment];
+				}
+				lines.push(generateAlternativeCellTextContent(cellSummary, lineCommentStart, blockComment).content);
+			} else if (!lines.length || lines[lines.length - 1] !== existingCodeMarkerWithComment) {
+				lines.push(existingCodeMarkerWithComment);
+			}
+		});
+		return lines.join(EOL);
+	}
+
+
+	public override async *parseAlternateContent(notebookOrUri: NotebookDocument | Uri, inputStream: AsyncIterable<LineOfText>, token: CancellationToken): AsyncIterable<LineOfCellText> {
+		const isNotebook = !isUri(notebookOrUri);
+		const cellIdMap = isNotebook ? getCellIdMap(notebookOrUri) : new Map<string, NotebookCell>();
+
+		let inMarkdownCell = false;
+		let isInTripleQuotes = false;
+		let pendingTripleQuotes = false;
+		let emittedStart = false;
+		let cellIndex = -1;
+
+		const lineCommentStart = getLineCommentStart(isNotebook ? notebookOrUri : undefined);
+		const blockComment = getBlockComment(isNotebook ? notebookOrUri : undefined);
+		const defaultLanguage = isNotebook ? getLanguage(getDefaultLanguage(notebookOrUri)).languageId : undefined;
+		const cellIdsSeen = new Set<string>();
+		for await (const lineOfText of inputStream) {
+			if (token.isCancellationRequested) {
+				break;
+			}
+			const line = lineOfText.value;
+
+			// Check for new cell delimiter
+			// Sometimes LLM returns cells without the `vscode.cell` marker such as .
+			const isLineCommentForEmptyCellWithoutCellMarker = line.startsWith(`${lineCommentStart}%% [`) && line.trimEnd().endsWith(']');
+			const isLineCommentWithCellMarker = line.startsWith(`${lineCommentStart}%% vscode.cell`);
+			// Attempt to extract only if we think we have a cell marker, else we end up doing this for every single line and thats expensive.
+			const cellParts = (isLineCommentWithCellMarker || isLineCommentForEmptyCellWithoutCellMarker) ? extractCellParts(line, defaultLanguage) : undefined;
+			if ((isLineCommentWithCellMarker || isLineCommentForEmptyCellWithoutCellMarker) && cellParts?.language) {
+				if (pendingTripleQuotes) {
+					pendingTripleQuotes = false;
+				}
+				const lineOfCellText: LineOfCellText & { emitted: Boolean } = { index: -1, uri: undefined, language: undefined, kind: NotebookCellKind.Code, emitted: false, type: 'start' };
+				lineOfCellText.index = cellIndex += 1;
+				lineOfCellText.emitted = false;
+				// LLM returns duplicate cell with the same id.
+				if (cellParts.id && cellIdMap.get(cellParts.id)?.document.languageId === cellParts.language) {
+					if (cellIdsSeen.has(cellParts.id)) {
+						cellParts.id = '';
+					} else {
+						cellIdsSeen.add(cellParts.id);
+					}
+				} else {
+					// Possible duplicate cell with the same id but different language.
+					// In such cases, treat them as new cells.
+					cellParts.id = '';
+				}
+
+				const cell = cellIdMap.get(cellParts.id);
+				lineOfCellText.id = cellParts.id;
+				lineOfCellText.language = cellParts.language;
+				lineOfCellText.uri = cell?.document.uri;
+				lineOfCellText.kind = cell?.kind || (lineOfCellText.language === 'markdown' ? NotebookCellKind.Markup : NotebookCellKind.Code);
+				inMarkdownCell = lineOfCellText.language === 'markdown';
+				isInTripleQuotes = false;
+
+				if (emittedStart) {
+					yield { index: cellIndex - 1, type: 'end' };
+				}
+
+				emittedStart = true;
+				yield lineOfCellText;
+				continue;
+			}
+
+			if (!emittedStart) {
+				continue;
+			}
+			if (inMarkdownCell) {
+				if (!isInTripleQuotes) {
+					// Look for the opening triple quotes
+					if (line === blockComment[0]) {
+						isInTripleQuotes = true;
+					} else {
+						// lineEmitted = true;
+						yield { index: cellIndex, line, type: 'line' };
+					}
+				} else {
+					// We are in triple quotes
+					if (line === blockComment[1]) {
+						// Closing triple quotes found
+						isInTripleQuotes = false;
+						pendingTripleQuotes = true;
+					} else {
+						yield { index: cellIndex, line, type: 'line' };
+					}
+				}
+			} else {
+				// Non-markdown cell or default
+				yield { index: cellIndex, line, type: 'line' };
+			}
+		}
+
+		if (emittedStart) {
+			yield { index: cellIndex, type: 'end' };
+		}
+	}
+
+	public override getAlternativeDocument(notebook: NotebookDocument, excludeMarkdownCells?: boolean): AlternativeNotebookDocument {
+		const cells = notebook.getCells().filter(cell => excludeMarkdownCells ? cell.kind !== NotebookCellKind.Markup : true).map(cell => summarize(cell));
+		const blockComment = getBlockComment(notebook);
+		const lineCommentStart = getLineCommentStart(notebook);
+		const cellContent = cells.map(cell => ({ ...generateAlternativeCellTextContent(cell, lineCommentStart, blockComment), cell: notebook.cellAt(cell.index) }));
+		const content = cellContent.map(cell => cell.content).join(EOL);
+		const cellOffsetMap = cellContent.map(cellContent => {
+			const offset = content.indexOf(cellContent.content);
+			const sourceOffset = offset + cellContent.prefix.length;
+			return { offset, sourceOffset, cell: notebook.cellAt(cellContent.cell.index) };
+		});
+
+		return new AlternativeTextDocument(content, cellOffsetMap, notebook);
+	}
+
+}
+
+function generateAlternativeCellTextContent(cell: SummaryCell, lineCommentStart: string, blockComment: [string, string]): { content: string; prefix: string } {
+	const cellMarker = generateCellTextMarker(cell, lineCommentStart);
+	const src = cell.source.join(EOL);
+	const prefix = cell.language === 'markdown' ? `${cellMarker}${EOL}${blockComment[0]}${EOL}` : `${cellMarker}${EOL}`;
+	const content = cell.language === 'markdown'
+		? `${prefix}${src}${EOL}${blockComment[1]}`
+		: `${prefix}${src}`;
+	return { content, prefix };
+}
+
+export function getBlockComment(notebook?: NotebookDocument): [string, string] {
+	if (!notebook) {
+		return ['"""', '"""'];
+	}
+	const language = getLanguage(getDefaultLanguage(notebook));
+	return language.blockComment ?? ['```', '```'];
+}
+
+export function getLineCommentStart(notebook?: NotebookDocument): string {
+	if (!notebook) {
+		return '#';
+	}
+	const language = getLanguage(getDefaultLanguage(notebook));
+	return language.lineComment.start || '#';
+}
+
+function extractCellParts(line: string, defaultLanguage: string | undefined): { id: string; language: string } | undefined {
+	const idMatch = line.match(/\[id=(.+?)\]/);
+	const languageMatch = line.match(/\[language=(.+?)\]/);
+	if (!languageMatch) {
+		if (lineMightHaveCellMarker(line) && typeof defaultLanguage === 'string') {
+			// If we have a cell marker but no language, we assume the default language.
+			return { id: idMatch ? idMatch[1].trim() : '', language: defaultLanguage };
+		}
+		return;
+	}
+	return { id: idMatch ? idMatch[1].trim() : '', language: languageMatch[1].trim() };
+}

package/src/_internal/platform/notebook/common/alternativeContentProvider.ts

@@ -0,0 +1,41 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { CancellationToken, NotebookCell, NotebookDocument } from 'vscode';
+import { Uri } from '../../../util/common/test/shims/vscodeTypesShim';
+import { AlternativeNotebookDocument } from './alternativeNotebookDocument';
+import { LineOfCellText, LineOfText } from './helpers';
+
+
+export abstract class BaseAlternativeNotebookContentProvider {
+	constructor(public readonly kind: 'xml' | 'text' | 'json') { }
+
+	/**
+	 * Give the code for a cell, strips the cell markers and returns the code.
+	 * XML and Jupytext formats have cell markers and this allows stripped those markers.
+	 */
+	public abstract stripCellMarkers(text: string): string;
+
+	/**
+	 * Generate the Document of the notebook document that is LLM friendly.
+	 */
+	public abstract getAlternativeDocument(notebook: NotebookDocument, excludeMarkdownCells?: boolean): AlternativeNotebookDocument;
+
+	/**
+	 * Generate the summary of the structure of the notebook document that is LLM friendly.
+	 * & includes just the cells that are passed in.
+	 * This is used to help the LLM understand the structure of the notebook, without including the entire code of all cells.
+	 */
+	public abstract getSummaryOfStructure(notebook: NotebookDocument, cellsToInclude: NotebookCell[], existingCodeMarker: string): string;
+
+	/**
+	 * Given the input stream of response parts, parse the response and return an async iterable of lines of text for a given cell.
+	 * We accept a NotebookDocument or a Uri.
+	 * This is because its possible the Notebook may not have been created/loaded as of yet.
+	 * I.e. for new Notebooks, we can emity the Insert Cell Edits without the notebook being created.
+	 */
+	public abstract parseAlternateContent(notebookOrUri: NotebookDocument | Uri, inputStream: AsyncIterable<LineOfText>, token: CancellationToken): AsyncIterable<LineOfCellText>;
+
+}