@zzzen/pyright-internal 1.2.0-dev.20251228 → 1.2.0-dev.20260111

package/dist/tests/harness/fourslash/fourSlashParser.js

@@ -14,6 +14,7 @@ const pathUtils_1 = require("../../../common/pathUtils");
 const uriUtils_1 = require("../../../common/uri/uriUtils");
 const factory_1 = require("../vfs/factory");
 const fourSlashTypes_1 = require("./fourSlashTypes");
+const fourSlashRawUtils_1 = require("./fourSlashRawUtils");
 /**
  * Parse given fourslash markup code and return content with markup/range data
  *
@@ -24,17 +25,21 @@ const fourSlashTypes_1 = require("./fourSlashTypes");
  */
 function parseTestData(basePath, contents, fileName) {
     const normalizedBasePath = (0, pathUtils_1.normalizeSlashes)(basePath);
-    // Regex for parsing options in the format "@Alpha: Value of any sort"
-    const optionRegex = /^\s*@(\w+):\s*(.*)\s*/;
+    // Historically, many fourslash strings ended with a trailing newline (often from a closing backtick on its own
+    // line). Some parsing logic assumes line feeds exist for line-to-offset mapping. Normalize to ensure the
+    // input always ends with an LF, so callers don't need to add a trailing empty line.
+    const rawText = contents.endsWith('\n') ? contents : `${contents}\n`;
+    const rawTokens = [];
     // List of all the subfiles we've parsed out
     const files = [];
     // Global options
     const globalOptions = {};
+    const globalOptionsRawData = {};
     // Marker positions
     // Split up the input file by line
     // Note: IE JS engine incorrectly handles consecutive delimiters here when using RegExp split, so
     // we have to string-based splitting instead and try to figure out the delimiting chars
-    const lines = contents.split('\n');
+    const lines = rawText.split('\n');
     let i = 0;
     const markerPositions = new Map();
     const markers = [];
@@ -43,6 +48,10 @@ function parseTestData(basePath, contents, fileName) {
     let currentFileContent;
     let currentFileName = (0, pathUtils_1.normalizeSlashes)(fileName);
     let currentFileOptions = {};
+    let currentFileOptionsRawData = {};
+    let currentFileTokenRanges = [];
+    let currentFileContentToRawSegments = [];
+    let lastFourSlashLineLfOffset;
     let normalizedProjectRoot = normalizedBasePath;
     function nextFile() {
         if (currentFileContent === undefined) {
@@ -55,38 +64,92 @@ function parseTestData(basePath, contents, fileName) {
             currentFileName = (0, pathUtils_1.normalizePath)((0, pathUtils_1.combinePaths)(factory_1.distlibFolder.getFilePath(), (0, pathUtils_1.getRelativePath)(currentFileName, normalizedBasePath)));
         }
         const ignoreCase = (0, core_1.toBoolean)(globalOptions["ignorecase" /* GlobalMetadataOptionNames.ignoreCase */]);
-        const file = parseFileContent(currentFileContent, currentFileName, ignoreCase, markerPositions, markers, ranges);
+        const file = parseFileContent(currentFileContent, currentFileContentToRawSegments, rawTokens, currentFileName, ignoreCase, markerPositions, markers, ranges);
         file.fileOptions = currentFileOptions;
+        const mappingRawData = file.rawData;
+        file.rawData = {
+            tokenRanges: currentFileTokenRanges,
+            fileOptionsRawData: currentFileOptionsRawData,
+            rawToContent: mappingRawData?.rawToContent,
+            contentToRaw: mappingRawData?.contentToRaw,
+        };
         // Store result file
         files.push(file);
         currentFileContent = undefined;
         currentFileOptions = {};
+        currentFileOptionsRawData = {};
+        currentFileTokenRanges = [];
+        currentFileContentToRawSegments = [];
+        lastFourSlashLineLfOffset = undefined;
         currentFileName = fileName;
     }
-    for (let line of lines) {
+    let rawOffset = 0;
+    for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+        const lineWithPotentialCr = lines[lineIndex];
         i++;
+        const lineStartRawOffset = rawOffset;
+        const hasLf = lineIndex < lines.length - 1;
+        const lineTokenStart = rawTokens.length;
+        const tokenizeResult = tokenizeRawLine(rawText, rawTokens, lineStartRawOffset, lineWithPotentialCr, hasLf);
+        const lineTokenEnd = rawTokens.length;
+        // Maintain legacy parsing behavior: treat CRLF as LF-delimited lines with a trailing '\r' in the line text.
+        let line = lineWithPotentialCr;
         if (line.length > 0 && line.charAt(line.length - 1) === '\r') {
             line = line.substr(0, line.length - 1);
         }
         if (line.substr(0, 4) === '////') {
             const text = line.substr(4);
-            currentFileContent = currentFileContent === undefined ? text : currentFileContent + '\n' + text;
+            currentFileTokenRanges.push({ startToken: lineTokenStart, endToken: lineTokenEnd });
+            if (currentFileContent === undefined) {
+                currentFileContent = text;
+                currentFileContentToRawSegments = [
+                    {
+                        contentStart: 0,
+                        contentEnd: text.length,
+                        rawStart: lineStartRawOffset + 4,
+                    },
+                ];
+            }
+            else {
+                const newlineContentOffset = currentFileContent.length;
+                currentFileContent = currentFileContent + '\n' + text;
+                if (lastFourSlashLineLfOffset === undefined) {
+                    throw new Error(`Missing line feed mapping for four-slash line ending at line ${i - 1}`);
+                }
+                currentFileContentToRawSegments.push({
+                    contentStart: newlineContentOffset,
+                    contentEnd: newlineContentOffset + 1,
+                    rawStart: lastFourSlashLineLfOffset,
+                });
+                const textContentStart = newlineContentOffset + 1;
+                currentFileContentToRawSegments.push({
+                    contentStart: textContentStart,
+                    contentEnd: textContentStart + text.length,
+                    rawStart: lineStartRawOffset + 4,
+                });
+            }
+            // Record the raw offset of the '\n' for this line (used if another four-slash line follows).
+            lastFourSlashLineLfOffset = hasLf ? lineStartRawOffset + lineWithPotentialCr.length : undefined;
         }
         else if (line.substr(0, 3) === '///' && currentFileContent !== undefined) {
             throw new Error(`Three-slash line in the middle of four-slash region at line ${i}`);
         }
         else if (line.substr(0, 2) === '//') {
             // Comment line, check for global/file @options and record them
-            const match = optionRegex.exec(line.substr(2));
-            if (match) {
-                const key = match[1].toLowerCase();
-                const value = match[2];
+            const directive = tryParseOptionDirective(line.substr(2));
+            if (directive) {
+                const key = directive.key.toLowerCase();
+                const value = directive.value;
+                const directiveRawData = tokenizeResult.directiveRawData;
                 if (!(0, collectionUtils_1.contains)(fourSlashTypes_1.fileMetadataNames, key)) {
                     // Check if the match is already existed in the global options
                     if (globalOptions[key] !== undefined) {
                         throw new Error(`Global option '${key}' already exists`);
                     }
                     globalOptions[key] = value;
+                    if (directiveRawData) {
+                        globalOptionsRawData[key] = directiveRawData;
+                    }
                     if (key === "projectroot" /* GlobalMetadataOptionNames.projectRoot */) {
                         normalizedProjectRoot = (0, pathUtils_1.combinePaths)(normalizedBasePath, value);
                     }
@@ -101,11 +164,17 @@ function parseTestData(basePath, contents, fileName) {
                                 ? normalizedPath
                                 : (0, pathUtils_1.combinePaths)(normalizedProjectRoot, normalizedPath);
                             currentFileOptions[key] = value;
+                            if (directiveRawData) {
+                                currentFileOptionsRawData[key] = directiveRawData;
+                            }
                             break;
                         }
                         default:
                             // Add other fileMetadata flag
                             currentFileOptions[key] = value;
+                            if (directiveRawData) {
+                                currentFileOptionsRawData[key] = directiveRawData;
+                            }
                     }
                 }
             }
@@ -117,14 +186,265 @@ function parseTestData(basePath, contents, fileName) {
             // Code line, terminate current subfile if there is one
             nextFile();
         }
+        rawOffset += lineWithPotentialCr.length + (hasLf ? 1 : 0);
     }
     return {
         markerPositions,
         markers,
         globalOptions,
+        globalOptionsRawData,
         files,
         ranges,
+        rawText,
+        rawTokens,
+    };
+}
+function tryParseOptionDirective(textAfterTwoSlash) {
+    // Matches the legacy behavior of: /^\s*@(\w+):\s*(.*)\s*/
+    let i = 0;
+    while (i < textAfterTwoSlash.length && /\s/.test(textAfterTwoSlash[i])) {
+        i++;
+    }
+    if (i >= textAfterTwoSlash.length || textAfterTwoSlash[i] !== '@') {
+        return undefined;
+    }
+    i++;
+    const nameStart = i;
+    while (i < textAfterTwoSlash.length && /\w/.test(textAfterTwoSlash[i])) {
+        i++;
+    }
+    if (i === nameStart) {
+        return undefined;
+    }
+    const key = textAfterTwoSlash.substring(nameStart, i);
+    while (i < textAfterTwoSlash.length && /\s/.test(textAfterTwoSlash[i])) {
+        i++;
+    }
+    if (i >= textAfterTwoSlash.length || textAfterTwoSlash[i] !== ':') {
+        return undefined;
+    }
+    i++;
+    while (i < textAfterTwoSlash.length && /\s/.test(textAfterTwoSlash[i])) {
+        i++;
+    }
+    const valueStart = i;
+    let valueEnd = textAfterTwoSlash.length;
+    while (valueEnd > valueStart && /\s/.test(textAfterTwoSlash[valueEnd - 1])) {
+        valueEnd--;
+    }
+    const value = textAfterTwoSlash.substring(valueStart, valueEnd);
+    return { key, value };
+}
+function tokenizeRawLine(rawText, rawTokens, lineStartRawOffset, lineTextIncludingOptionalCR, hasLf) {
+    const lineTokenStart = rawTokens.length;
+    const hasCr = lineTextIncludingOptionalCR.length > 0 && lineTextIncludingOptionalCR.endsWith('\r');
+    const lineBody = hasCr
+        ? lineTextIncludingOptionalCR.substring(0, lineTextIncludingOptionalCR.length - 1)
+        : lineTextIncludingOptionalCR;
+    const lineBodyStart = lineStartRawOffset;
+    const lineBodyEnd = lineStartRawOffset + lineBody.length;
+    let directiveRawData;
+    const push = (kind, start, end) => {
+        if (start < end) {
+            rawTokens.push({ kind, start, end });
+        }
     };
+    const tokenizePlain = (start, end) => {
+        let pos = start;
+        while (pos < end) {
+            const ch = rawText[pos];
+            if (ch === ' ' || ch === '\t') {
+                const wsStart = pos;
+                pos++;
+                while (pos < end && (rawText[pos] === ' ' || rawText[pos] === '\t')) {
+                    pos++;
+                }
+                push("whitespace" /* RawTokenKind.Whitespace */, wsStart, pos);
+            }
+            else {
+                const textStart = pos;
+                pos++;
+                while (pos < end && rawText[pos] !== ' ' && rawText[pos] !== '\t') {
+                    pos++;
+                }
+                push("text" /* RawTokenKind.Text */, textStart, pos);
+            }
+        }
+    };
+    if (lineBody.startsWith('////')) {
+        push("fourSlashPrefix" /* RawTokenKind.FourSlashPrefix */, lineBodyStart, lineBodyStart + 4);
+        tokenizeFourSlashRemainder(rawText, rawTokens, lineBodyStart + 4, lineBodyEnd);
+    }
+    else if (lineBody.startsWith('//')) {
+        const prefixStartToken = rawTokens.length;
+        push("twoSlashPrefix" /* RawTokenKind.TwoSlashPrefix */, lineBodyStart, lineBodyStart + 2);
+        const afterPrefixStart = lineBodyStart + 2;
+        const afterPrefixText = rawText.substring(afterPrefixStart, lineBodyEnd);
+        const directive = tryParseOptionDirective(afterPrefixText);
+        if (!directive) {
+            tokenizePlain(afterPrefixStart, lineBodyEnd);
+        }
+        else {
+            const isWhitespaceNotNewline = (ch) => /\s/.test(ch) && ch !== '\r' && ch !== '\n';
+            const consumeWhitespaceTokens = (pos, stopChar) => {
+                while (pos < lineBodyEnd &&
+                    isWhitespaceNotNewline(rawText[pos]) &&
+                    (stopChar === undefined || rawText[pos] !== stopChar)) {
+                    const wsStart = pos;
+                    pos++;
+                    while (pos < lineBodyEnd &&
+                        isWhitespaceNotNewline(rawText[pos]) &&
+                        (stopChar === undefined || rawText[pos] !== stopChar)) {
+                        pos++;
+                    }
+                    push("whitespace" /* RawTokenKind.Whitespace */, wsStart, pos);
+                }
+                return pos;
+            };
+            // Tokenize with directive structure. This keeps token spans aligned to the stored value.
+            let pos = afterPrefixStart;
+            // Leading whitespace.
+            pos = consumeWhitespaceTokens(pos);
+            const atStartToken = rawTokens.length;
+            push("directiveAt" /* RawTokenKind.DirectiveAt */, pos, pos + 1);
+            pos++;
+            const nameStart = pos;
+            while (pos < lineBodyEnd && /\w/.test(rawText[pos])) {
+                pos++;
+            }
+            const nameTokenIndex = rawTokens.length;
+            push("directiveName" /* RawTokenKind.DirectiveName */, nameStart, pos);
+            // Whitespace before ':'
+            pos = consumeWhitespaceTokens(pos, ':');
+            const colonTokenIndex = rawTokens.length;
+            push("directiveColon" /* RawTokenKind.DirectiveColon */, pos, pos + 1);
+            pos++;
+            // Whitespace after ':'
+            pos = consumeWhitespaceTokens(pos);
+            const valueStart = pos;
+            let valueEnd = lineBodyEnd;
+            while (valueEnd > valueStart && /\s/.test(rawText[valueEnd - 1])) {
+                valueEnd--;
+            }
+            const valueTokenStart = rawTokens.length;
+            push("directiveValue" /* RawTokenKind.DirectiveValue */, valueStart, valueEnd);
+            const valueTokenEnd = rawTokens.length;
+            // Trailing whitespace.
+            tokenizePlain(valueEnd, lineBodyEnd);
+            directiveRawData = {
+                directiveLine: { startToken: lineTokenStart, endToken: -1 },
+                prefix: { startToken: prefixStartToken, endToken: prefixStartToken + 1 },
+                name: { startToken: atStartToken, endToken: nameTokenIndex + 1 },
+                colon: { startToken: colonTokenIndex, endToken: colonTokenIndex + 1 },
+                value: { startToken: valueTokenStart, endToken: valueTokenEnd },
+            };
+        }
+    }
+    else {
+        tokenizePlain(lineBodyStart, lineBodyEnd);
+    }
+    // CR and LF are tokenized separately.
+    if (hasCr) {
+        push("newlineCR" /* RawTokenKind.NewLineCR */, lineBodyEnd, lineBodyEnd + 1);
+    }
+    if (hasLf) {
+        const lfStart = lineStartRawOffset + lineTextIncludingOptionalCR.length;
+        push("newlineLF" /* RawTokenKind.NewLineLF */, lfStart, lfStart + 1);
+    }
+    const lineTokenEnd = rawTokens.length;
+    if (directiveRawData) {
+        directiveRawData.directiveLine.endToken = lineTokenEnd;
+    }
+    return { directiveRawData };
+}
+function tokenizeFourSlashRemainder(rawText, rawTokens, start, end) {
+    const push = (kind, s, e) => {
+        if (s < e) {
+            rawTokens.push({ kind, start: s, end: e });
+        }
+    };
+    const validMarkerChars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz$1234567890_';
+    let pos = start;
+    while (pos < end) {
+        const ch = rawText[pos];
+        if (ch === ' ' || ch === '\t') {
+            const wsStart = pos;
+            pos++;
+            while (pos < end && (rawText[pos] === ' ' || rawText[pos] === '\t')) {
+                pos++;
+            }
+            push("whitespace" /* RawTokenKind.Whitespace */, wsStart, pos);
+            continue;
+        }
+        // Range delimiters.
+        if (pos + 1 < end && rawText[pos] === '[' && rawText[pos + 1] === '|') {
+            push("rangeStart" /* RawTokenKind.RangeStart */, pos, pos + 2);
+            pos += 2;
+            continue;
+        }
+        if (pos + 1 < end && rawText[pos] === '|' && rawText[pos + 1] === ']') {
+            push("rangeEnd" /* RawTokenKind.RangeEnd */, pos, pos + 2);
+            pos += 2;
+            continue;
+        }
+        // Object markers.
+        if (pos + 1 < end && rawText[pos] === '{' && rawText[pos + 1] === '|') {
+            const closeIndex = rawText.indexOf('|}', pos + 2);
+            if (closeIndex >= 0 && closeIndex + 2 <= end) {
+                push("objectMarkerStart" /* RawTokenKind.ObjectMarkerStart */, pos, pos + 2);
+                if (pos + 2 < closeIndex) {
+                    push("objectMarkerText" /* RawTokenKind.ObjectMarkerText */, pos + 2, closeIndex);
+                }
+                push("objectMarkerEnd" /* RawTokenKind.ObjectMarkerEnd */, closeIndex, closeIndex + 2);
+                pos = closeIndex + 2;
+                continue;
+            }
+        }
+        // Slash-star markers.
+        if (pos + 1 < end && rawText[pos] === '/' && rawText[pos + 1] === '*') {
+            const closeIndex = rawText.indexOf('*/', pos + 2);
+            if (closeIndex >= 0 && closeIndex + 2 <= end) {
+                let isValidMarker = true;
+                for (let j = pos + 2; j < closeIndex; j++) {
+                    if (validMarkerChars.indexOf(rawText[j]) < 0) {
+                        isValidMarker = false;
+                        break;
+                    }
+                }
+                if (isValidMarker) {
+                    push("markerStart" /* RawTokenKind.MarkerStart */, pos, pos + 2);
+                    if (pos + 2 < closeIndex) {
+                        push("markerName" /* RawTokenKind.MarkerName */, pos + 2, closeIndex);
+                    }
+                    push("markerEnd" /* RawTokenKind.MarkerEnd */, closeIndex, closeIndex + 2);
+                    pos = closeIndex + 2;
+                    continue;
+                }
+            }
+        }
+        // Plain text chunk until whitespace or a known delimiter.
+        const textStart = pos;
+        pos++;
+        while (pos < end) {
+            const c = rawText[pos];
+            if (c === ' ' || c === '\t') {
+                break;
+            }
+            if (pos + 1 < end) {
+                const c2 = rawText[pos + 1];
+                if ((c === '[' && c2 === '|') ||
+                    (c === '|' && c2 === ']') ||
+                    (c === '{' && c2 === '|') ||
+                    (c === '|' && c2 === '}') ||
+                    (c === '/' && c2 === '*') ||
+                    (c === '*' && c2 === '/')) {
+                    break;
+                }
+            }
+            pos++;
+        }
+        push("text" /* RawTokenKind.Text */, textStart, pos);
+    }
 }
 var State;
 (function (State) {
@@ -136,37 +456,43 @@ function reportError(fileName, line, col, message) {
     const errorMessage = `${fileName}(${line},${col}): ${message}`;
     throw new Error(errorMessage);
 }
-function recordObjectMarker(fileName, ignoreCase, location, text, markerMap, markers) {
+function recordObjectMarker(fileName, ignoreCase, location, text, markerMap, markers, rawData) {
     let markerValue;
     try {
         // Attempt to parse the marker value as JSON
         markerValue = JSON.parse('{ ' + text + ' }');
     }
     catch (e) {
-        reportError(fileName, location.sourceLine, location.sourceColumn, `Unable to parse marker text ${e.message}`);
+        const message = e instanceof Error ? e.message : String(e);
+        reportError(fileName, location.sourceLine, location.sourceColumn, `Unable to parse marker text ${message}`);
     }
-    if (markerValue === undefined) {
+    if (markerValue === undefined || markerValue === null || typeof markerValue !== 'object') {
         reportError(fileName, location.sourceLine, location.sourceColumn, 'Object markers can not be empty');
         return undefined;
     }
+    const markerData = markerValue;
     const marker = {
         fileName,
         fileUri: uriUtils_1.UriEx.file(fileName, !ignoreCase),
         position: location.position,
-        data: markerValue,
+        data: markerData,
+        rawData,
     };
     // Object markers can be anonymous
-    if (markerValue.name) {
-        markerMap.set(markerValue.name, marker);
+    const markerNameValue = markerValue.name;
+    if (markerNameValue) {
+        // Preserve legacy behavior: this may not be a string at runtime.
+        markerMap.set(markerNameValue, marker);
     }
     markers.push(marker);
     return marker;
 }
-function recordMarker(fileName, ignoreCase, location, name, markerMap, markers) {
+function recordMarker(fileName, ignoreCase, location, name, markerMap, markers, rawData) {
     const marker = {
         fileName,
         fileUri: uriUtils_1.UriEx.file(fileName, !ignoreCase),
         position: location.position,
+        rawData,
     };
     // Verify markers for uniqueness
     if (markerMap.has(name)) {
@@ -180,12 +506,14 @@ function recordMarker(fileName, ignoreCase, location, name, markerMap, markers)
         return marker;
     }
 }
-function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ranges) {
-    content = chompLeadingSpace(content);
+function parseFileContent(content, contentToRawSegments, rawTokens, fileName, ignoreCase, markerMap, markers, ranges) {
+    ({ content, segments: contentToRawSegments } = chompLeadingSpaceWithMapping(content, contentToRawSegments));
     // Any slash-star comment with a character not in this string is not a marker.
     const validMarkerChars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz$1234567890_';
     /// The file content (minus metacharacters) so far
     let output = '';
+    // Mapping segments for the final FourSlashFile.content -> rawText offsets.
+    const contentToRawOutputSegments = [];
     /// The current marker (or maybe multi-line comment?) we're parsing, possibly
     let openMarker;
     /// A stack of the open range markers that are still unclosed
@@ -202,9 +530,13 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
     let line = 1;
     let column = 1;
     const flush = (lastSafeCharIndex) => {
-        output =
-            output +
-                content.substr(lastNormalCharPosition, lastSafeCharIndex === undefined ? undefined : lastSafeCharIndex - lastNormalCharPosition);
+        const safeIndex = lastSafeCharIndex ?? content.length;
+        if (safeIndex <= lastNormalCharPosition) {
+            return;
+        }
+        const outputStart = output.length;
+        output = output + content.substring(lastNormalCharPosition, safeIndex);
+        appendOutputMappingSegments(contentToRawSegments, lastNormalCharPosition, safeIndex, outputStart, contentToRawOutputSegments);
     };
     if (content.length > 0) {
         let previousChar = content.charAt(0);
@@ -214,11 +546,13 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
                 case 0 /* State.none */:
                     if (previousChar === '[' && currentChar === '|') {
                         // found a range start
+                        const rawOpen = getRawSpanFromContentSpan(contentToRawSegments, i - 1, i + 1);
                         openRanges.push({
                             position: i - 1 - difference,
                             sourcePosition: i - 1,
                             sourceLine: line,
                             sourceColumn: column,
+                            rawOpen,
                         });
                         // copy all text up to marker position
                         flush(i - 1);
@@ -231,12 +565,23 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
                         if (!rangeStart) {
                             reportError(fileName, line, column, 'Found range end with no matching start.');
                         }
+                        const rawClose = getRawSpanFromContentSpan(contentToRawSegments, i - 1, i + 1);
+                        const rawSelectedStart = rangeStart.rawOpen?.rawEnd ?? rawClose.rawStart;
+                        const rawSelectedEnd = rawClose.rawStart;
+                        const rawFullStart = rangeStart.rawOpen?.rawStart ?? rawClose.rawStart;
+                        const rawFullEnd = rawClose.rawEnd;
                         const range = {
                             fileName,
                             fileUri: uriUtils_1.UriEx.file(fileName, !ignoreCase),
                             pos: rangeStart.position,
                             end: i - 1 - difference,
                             marker: rangeStart.marker,
+                            rawData: {
+                                full: getTokenRangeCoveringRawSpan(rawTokens, rawFullStart, rawFullEnd),
+                                open: getTokenRangeCoveringRawSpan(rawTokens, rangeStart.rawOpen?.rawStart ?? rawClose.rawStart, rangeStart.rawOpen?.rawEnd ?? rawClose.rawStart),
+                                selected: getTokenRangeCoveringRawSpan(rawTokens, rawSelectedStart, rawSelectedEnd),
+                                close: getTokenRangeCoveringRawSpan(rawTokens, rawClose.rawStart, rawClose.rawEnd),
+                            },
                         };
                         localRanges.push(range);
                         // copy all text up to range marker position
@@ -269,9 +614,19 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
                 case 2 /* State.inObjectMarker */:
                     // Object markers are only ever terminated by |} and have no content restrictions
                     if (previousChar === '|' && currentChar === '}') {
+                        const rawFull = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition, i + 1);
+                        const rawStart = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition, openMarker.sourcePosition + 2);
+                        const rawEnd = getRawSpanFromContentSpan(contentToRawSegments, i - 1, i + 1);
+                        const rawTextSpan = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition + 2, i - 1);
                         // Record the marker
                         const objectMarkerNameText = content.substring(openMarker.sourcePosition + 2, i - 1).trim();
-                        const marker = recordObjectMarker(fileName, ignoreCase, openMarker, objectMarkerNameText, markerMap, markers);
+                        const marker = recordObjectMarker(fileName, ignoreCase, openMarker, objectMarkerNameText, markerMap, markers, {
+                            kind: 'object',
+                            full: getTokenRangeCoveringRawSpan(rawTokens, rawFull.rawStart, rawFull.rawEnd),
+                            start: getTokenRangeCoveringRawSpan(rawTokens, rawStart.rawStart, rawStart.rawEnd),
+                            text: getTokenRangeCoveringRawSpan(rawTokens, rawTextSpan.rawStart, rawTextSpan.rawEnd),
+                            end: getTokenRangeCoveringRawSpan(rawTokens, rawEnd.rawStart, rawEnd.rawEnd),
+                        });
                         if (openRanges.length > 0) {
                             openRanges[openRanges.length - 1].marker = marker;
                         }
@@ -285,10 +640,20 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
                     break;
                 case 1 /* State.inSlashStarMarker */:
                     if (previousChar === '*' && currentChar === '/') {
+                        const rawFull = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition, i + 1);
+                        const rawStart = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition, openMarker.sourcePosition + 2);
+                        const rawEnd = getRawSpanFromContentSpan(contentToRawSegments, i - 1, i + 1);
+                        const rawNameSpan = getRawSpanFromContentSpan(contentToRawSegments, openMarker.sourcePosition + 2, i - 1);
                         // Record the marker
                         // start + 2 to ignore the */, -1 on the end to ignore the * (/ is next)
                         const markerNameText = content.substring(openMarker.sourcePosition + 2, i - 1).trim();
-                        const marker = recordMarker(fileName, ignoreCase, openMarker, markerNameText, markerMap, markers);
+                        const marker = recordMarker(fileName, ignoreCase, openMarker, markerNameText, markerMap, markers, {
+                            kind: 'slashStar',
+                            full: getTokenRangeCoveringRawSpan(rawTokens, rawFull.rawStart, rawFull.rawEnd),
+                            start: getTokenRangeCoveringRawSpan(rawTokens, rawStart.rawStart, rawStart.rawEnd),
+                            name: getTokenRangeCoveringRawSpan(rawTokens, rawNameSpan.rawStart, rawNameSpan.rawEnd),
+                            end: getTokenRangeCoveringRawSpan(rawTokens, rawEnd.rawStart, rawEnd.rawEnd),
+                        });
                         if (openRanges.length > 0) {
                             openRanges[openRanges.length - 1].marker = marker;
                         }
@@ -342,21 +707,113 @@ function parseFileContent(content, fileName, ignoreCase, markerMap, markers, ran
     localRanges.forEach((r) => {
         ranges.push(r);
     });
+    const contentToRaw = { segments: contentToRawOutputSegments };
+    const rawToContent = { segments: contentToRawOutputSegments };
     return {
         content: output,
         fileOptions: {},
         version: 0,
         fileName,
         fileUri: uriUtils_1.UriEx.file(fileName, !ignoreCase),
+        rawData: {
+            tokenRanges: [],
+            rawToContent,
+            contentToRaw,
+        },
     };
 }
-function chompLeadingSpace(content) {
+function getRawSpanFromContentSpan(segments, contentStart, contentEnd) {
+    if (contentStart === contentEnd) {
+        const rawOffset = tryGetRawOffsetFromContentIndex(segments, contentStart) ?? 0;
+        return { rawStart: rawOffset, rawEnd: rawOffset };
+    }
+    const rawStart = tryGetRawOffsetFromContentIndex(segments, contentStart) ?? 0;
+    const rawLast = tryGetRawOffsetFromContentIndex(segments, contentEnd - 1) ?? rawStart;
+    return { rawStart, rawEnd: rawLast + 1 };
+}
+function tryGetRawOffsetFromContentIndex(segments, contentIndex) {
+    const seg = (0, fourSlashRawUtils_1.findItemContainingOffset)(segments, contentIndex, (s) => s.contentStart, (s) => s.contentEnd);
+    if (!seg) {
+        return undefined;
+    }
+    return seg.rawStart + (contentIndex - seg.contentStart);
+}
+function getTokenRangeCoveringRawSpan(rawTokens, rawStart, rawEnd) {
+    if (rawStart === rawEnd) {
+        const tokenIndex = (0, fourSlashRawUtils_1.findTokenIndexAtOrAfter)(rawTokens, rawStart);
+        return { startToken: tokenIndex, endToken: tokenIndex };
+    }
+    const startToken = (0, fourSlashRawUtils_1.findTokenIndexAtOrAfter)(rawTokens, rawStart);
+    const endToken = (0, fourSlashRawUtils_1.findTokenIndexAtOrAfter)(rawTokens, rawEnd);
+    return { startToken, endToken };
+}
+function appendOutputMappingSegments(sourceSegments, sourceStart, sourceEnd, outputStart, out) {
+    for (const seg of sourceSegments) {
+        const overlapStart = Math.max(sourceStart, seg.contentStart);
+        const overlapEnd = Math.min(sourceEnd, seg.contentEnd);
+        if (overlapStart >= overlapEnd) {
+            continue;
+        }
+        const overlapLen = overlapEnd - overlapStart;
+        const rawStart = seg.rawStart + (overlapStart - seg.contentStart);
+        const contentStart = outputStart + (overlapStart - sourceStart);
+        out.push({
+            rawStart,
+            rawEnd: rawStart + overlapLen,
+            contentStart,
+            contentEnd: contentStart + overlapLen,
+        });
+    }
+}
+function chompLeadingSpaceWithMapping(content, contentToRawSegments) {
     const lines = content.split('\n');
     for (const line of lines) {
         if (line.length !== 0 && line.charAt(0) !== ' ') {
-            return content;
+            return { content, segments: contentToRawSegments };
+        }
+    }
+    // Remove one leading space from each line.
+    const newContent = lines.map((s) => s.substr(1)).join('\n');
+    // Rebuild mapping segments by walking line-by-line over the original content.
+    const newSegments = [];
+    let sourcePos = 0;
+    let outPos = 0;
+    for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+        const line = lines[lineIndex];
+        const lineStart = sourcePos;
+        const lineEnd = lineStart + line.length;
+        // Keep everything after the removed leading space.
+        if (line.length > 0) {
+            const keepStart = lineStart + 1;
+            const keepEnd = lineEnd;
+            appendChompedSegments(contentToRawSegments, keepStart, keepEnd, outPos, newSegments);
+            outPos += keepEnd - keepStart;
+        }
+        sourcePos = lineEnd;
+        // Keep newline except for the final line.
+        if (lineIndex < lines.length - 1) {
+            appendChompedSegments(contentToRawSegments, sourcePos, sourcePos + 1, outPos, newSegments);
+            sourcePos += 1;
+            outPos += 1;
+        }
+    }
+    return { content: newContent, segments: newSegments };
+}
+function appendChompedSegments(sourceSegments, sourceStart, sourceEnd, outputStart, out) {
+    for (const seg of sourceSegments) {
+        const overlapStart = Math.max(sourceStart, seg.contentStart);
+        const overlapEnd = Math.min(sourceEnd, seg.contentEnd);
+        if (overlapStart >= overlapEnd) {
+            continue;
         }
+        const overlapLen = overlapEnd - overlapStart;
+        const rawStart = seg.rawStart + (overlapStart - seg.contentStart);
+        const contentStart = outputStart + (overlapStart - sourceStart);
+        out.push({
+            contentStart,
+            contentEnd: contentStart + overlapLen,
+            rawStart,
+        });
     }
-    return lines.map((s) => s.substr(1)).join('\n');
 }
 //# sourceMappingURL=fourSlashParser.js.map