@typespec/compiler 0.52.0-dev.12 → 0.52.0-dev.13

package/dist/src/server/serverlib.js

@@ -1,67 +1,40 @@
 import { DiagnosticSeverity, DiagnosticTag, DocumentHighlightKind, MarkupKind, Range, SemanticTokensBuilder, TextDocumentSyncKind, TextEdit, Diagnostic as VSDiagnostic, } from "vscode-languageserver/node.js";
-import { defaultConfig, findTypeSpecConfigPath, loadTypeSpecConfigFile, } from "../config/config-loader.js";
-import { resolveOptionsFromConfig } from "../config/config-to-options.js";
 import { codePointBefore, isIdentifierContinue } from "../core/charcode.js";
-import { compilerAssert, createSourceFile, formatDiagnostic, getSourceLocation, } from "../core/diagnostics.js";
+import { compilerAssert, createSourceFile, getSourceLocation } from "../core/diagnostics.js";
 import { formatTypeSpec } from "../core/formatter.js";
 import { getTypeName } from "../core/helpers/type-name-utils.js";
 import { getNodeAtPosition, visitChildren } from "../core/parser.js";
-import { ensureTrailingDirectorySeparator, getDirectoryPath, joinPaths, } from "../core/path-utils.js";
-import { compile as compileProgram } from "../core/program.js";
-import { Token, TokenFlags, createScanner, isKeyword, isPunctuation, skipTrivia, skipWhiteSpace, } from "../core/scanner.js";
+import { ensureTrailingDirectorySeparator } from "../core/path-utils.js";
+import { skipTrivia, skipWhiteSpace } from "../core/scanner.js";
 import { SyntaxKind, } from "../core/types.js";
-import { doIO, getNormalizedRealPath, getSourceFileKindFromExt, loadFile, resolveTspMain, } from "../core/util.js";
+import { getNormalizedRealPath, getSourceFileKindFromExt } from "../core/util.js";
+import { getSemanticTokens } from "./classify.js";
+import { createCompileService } from "./compile-service.js";
 import { resolveCompletion } from "./completion.js";
+import { createFileService } from "./file-service.js";
+import { createFileSystemCache } from "./file-system-cache.js";
 import { getPositionBeforeTrivia } from "./server-utils.js";
 import { getSymbolStructure } from "./symbol-structure.js";
 import { getParameterDocumentation, getSymbolDetails, getTemplateParameterDocumentation, } from "./type-details.js";
-export var SemanticTokenKind;
-(function (SemanticTokenKind) {
-    SemanticTokenKind[SemanticTokenKind["Namespace"] = 0] = "Namespace";
-    SemanticTokenKind[SemanticTokenKind["Type"] = 1] = "Type";
-    SemanticTokenKind[SemanticTokenKind["Class"] = 2] = "Class";
-    SemanticTokenKind[SemanticTokenKind["Enum"] = 3] = "Enum";
-    SemanticTokenKind[SemanticTokenKind["Interface"] = 4] = "Interface";
-    SemanticTokenKind[SemanticTokenKind["Struct"] = 5] = "Struct";
-    SemanticTokenKind[SemanticTokenKind["TypeParameter"] = 6] = "TypeParameter";
-    SemanticTokenKind[SemanticTokenKind["Parameter"] = 7] = "Parameter";
-    SemanticTokenKind[SemanticTokenKind["Variable"] = 8] = "Variable";
-    SemanticTokenKind[SemanticTokenKind["Property"] = 9] = "Property";
-    SemanticTokenKind[SemanticTokenKind["EnumMember"] = 10] = "EnumMember";
-    SemanticTokenKind[SemanticTokenKind["Event"] = 11] = "Event";
-    SemanticTokenKind[SemanticTokenKind["Function"] = 12] = "Function";
-    SemanticTokenKind[SemanticTokenKind["Method"] = 13] = "Method";
-    SemanticTokenKind[SemanticTokenKind["Macro"] = 14] = "Macro";
-    SemanticTokenKind[SemanticTokenKind["Keyword"] = 15] = "Keyword";
-    SemanticTokenKind[SemanticTokenKind["Comment"] = 16] = "Comment";
-    SemanticTokenKind[SemanticTokenKind["String"] = 17] = "String";
-    SemanticTokenKind[SemanticTokenKind["Number"] = 18] = "Number";
-    SemanticTokenKind[SemanticTokenKind["Regexp"] = 19] = "Regexp";
-    SemanticTokenKind[SemanticTokenKind["Operator"] = 20] = "Operator";
-    SemanticTokenKind[SemanticTokenKind["DocCommentTag"] = 21] = "DocCommentTag";
-})(SemanticTokenKind || (SemanticTokenKind = {}));
-const serverOptions = {
-    noEmit: true,
-    designTimeBuild: true,
-    parseOptions: {
-        comments: true,
-        docs: true,
-    },
-};
+import { SemanticTokenKind, } from "./types.js";
 export function createServer(host) {
-    // Remember original URL when we convert it to a local path so that we can
-    // get it back. We can't convert it back because things like URL-encoding
-    // could give us back an equivalent but non-identical URL but the original
-    // URL is used as a key into the opened documents and so we must reproduce
-    // it exactly.
-    const pathToURLMap = new Map();
+    const fileService = createFileService({ serverHost: host });
     // Cache all file I/O. Only open documents are sent over the LSP pipe. When
     // the compiler reads a file that isn't open, we use this cache to avoid
     // hitting the disk. Entries are invalidated when LSP client notifies us of
     // a file change.
-    const fileSystemCache = createFileSystemCache();
+    const fileSystemCache = createFileSystemCache({
+        fileService,
+    });
     const compilerHost = createCompilerHost();
-    const oldPrograms = new Map();
+    const compileService = createCompileService({
+        fileService,
+        fileSystemCache,
+        compilerHost,
+        serverHost: host,
+        log,
+    });
+    compileService.on("compileEnd", (result) => reportDiagnostics(result));
     let workspaceFolders = [];
     let isInitialized = false;
     let pendingMessages = [];
@@ -72,7 +45,7 @@ export function createServer(host) {
         get workspaceFolders() {
             return workspaceFolders;
         },
-        compile,
+        compile: compileService.compile,
         initialize,
         initialized,
         workspaceFoldersChanged,
@@ -85,7 +58,7 @@ export function createServer(host) {
         findDocumentHighlight,
         prepareRename,
         rename,
-        getSemanticTokens,
+        getSemanticTokens: getSemanticTokensForDocument,
         buildSemanticTokens,
         checkChange,
         getFoldingRanges,
@@ -131,7 +104,7 @@ export function createServer(host) {
             for (const w of (_b = params.workspaceFolders) !== null && _b !== void 0 ? _b : []) {
                 workspaceFolders.push({
                     ...w,
-                    path: ensureTrailingDirectorySeparator(await fileURLToRealPath(w.uri)),
+                    path: ensureTrailingDirectorySeparator(await fileService.fileURLToRealPath(w.uri)),
                 });
             }
             capabilities.workspace = {
@@ -148,8 +121,9 @@ export function createServer(host) {
                     name: "<root>",
                     // eslint-disable-next-line deprecation/deprecation
                     uri: params.rootUri,
+                    path: ensureTrailingDirectorySeparator(
                     // eslint-disable-next-line deprecation/deprecation
-                    path: ensureTrailingDirectorySeparator(await fileURLToRealPath(params.rootUri)),
+                    await fileService.fileURLToRealPath(params.rootUri)),
                 },
             ];
             // eslint-disable-next-line deprecation/deprecation
@@ -182,7 +156,7 @@ export function createServer(host) {
         for (const folder of e.added) {
             map.set(folder.uri, {
                 ...folder,
-                path: ensureTrailingDirectorySeparator(await fileURLToRealPath(folder.uri)),
+                path: ensureTrailingDirectorySeparator(await fileService.fileURLToRealPath(folder.uri)),
             });
         }
         workspaceFolders = Array.from(map.values());
@@ -191,85 +165,8 @@ export function createServer(host) {
     function watchedFilesChanged(params) {
         fileSystemCache.notify(params.changes);
     }
-    async function compile(document, callback) {
-        const path = await getPath(document);
-        const mainFile = await getMainFileForDocument(path);
-        const config = await getConfig(mainFile);
-        const [optionsFromConfig, _] = resolveOptionsFromConfig(config, { cwd: path });
-        const options = {
-            ...optionsFromConfig,
-            ...serverOptions,
-        };
-        if (!upToDate(document)) {
-            return undefined;
-        }
-        let program;
-        try {
-            program = await compileProgram(compilerHost, mainFile, options, oldPrograms.get(mainFile));
-            oldPrograms.set(mainFile, program);
-            if (!upToDate(document)) {
-                return undefined;
-            }
-            if (mainFile !== path && !program.sourceFiles.has(path)) {
-                // If the file that changed wasn't imported by anything from the main
-                // file, retry using the file itself as the main file.
-                program = await compileProgram(compilerHost, path, options, oldPrograms.get(path));
-                oldPrograms.set(path, program);
-            }
-            if (!upToDate(document)) {
-                return undefined;
-            }
-            if (callback) {
-                const doc = "version" in document ? document : host.getOpenDocumentByURL(document.uri);
-                compilerAssert(doc, "Failed to get document.");
-                const path = await getPath(doc);
-                const script = program.sourceFiles.get(path);
-                compilerAssert(script, "Failed to get script.");
-                return await callback(program, doc, script);
-            }
-            return program;
-        }
-        catch (err) {
-            if (host.throwInternalErrors) {
-                throw err;
-            }
-            host.sendDiagnostics({
-                uri: document.uri,
-                diagnostics: [
-                    {
-                        severity: DiagnosticSeverity.Error,
-                        range: Range.create(0, 0, 0, 0),
-                        message: `Internal compiler error!\nFile issue at https://github.com/microsoft/typespec\n\n` +
-                            err.stack,
-                    },
-                ],
-            });
-            return undefined;
-        }
-    }
-    async function getConfig(mainFile) {
-        const entrypointStat = await host.compilerHost.stat(mainFile);
-        const lookupDir = entrypointStat.isDirectory() ? mainFile : getDirectoryPath(mainFile);
-        const configPath = await findTypeSpecConfigPath(compilerHost, lookupDir, true);
-        if (!configPath) {
-            return { ...defaultConfig, projectRoot: getDirectoryPath(mainFile) };
-        }
-        const cached = await fileSystemCache.get(configPath);
-        if (cached === null || cached === void 0 ? void 0 : cached.data) {
-            return cached.data;
-        }
-        const config = await loadTypeSpecConfigFile(compilerHost, configPath);
-        await fileSystemCache.setData(configPath, config);
-        return config;
-    }
-    async function getScript(document) {
-        var _a;
-        const file = await compilerHost.readFile(await getPath(document));
-        const cached = (_a = compilerHost.parseCache) === null || _a === void 0 ? void 0 : _a.get(file);
-        return cached !== null && cached !== void 0 ? cached : (await compile(document, (_, __, script) => script));
-    }
     async function getFoldingRanges(params) {
-        const ast = await getScript(params.textDocument);
+        const ast = await compileService.getScript(params.textDocument);
         if (!ast) {
             return [];
         }
@@ -323,29 +220,29 @@ export function createServer(host) {
         }
     }
     async function getDocumentSymbols(params) {
-        const ast = await getScript(params.textDocument);
+        const ast = await compileService.getScript(params.textDocument);
         if (!ast) {
             return [];
         }
         return getSymbolStructure(ast);
     }
     async function findDocumentHighlight(params) {
-        let highlights = [];
-        await compile(params.textDocument, (program, document, file) => {
-            const identifiers = findReferenceIdentifiers(program, file, document.offsetAt(params.position), [file]);
-            highlights = identifiers.map((identifier) => ({
-                range: getRange(identifier, file.file),
-                kind: DocumentHighlightKind.Read,
-            }));
-        });
-        return highlights;
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return [];
+        }
+        const { program, document, script } = result;
+        const identifiers = findReferenceIdentifiers(program, script, document.offsetAt(params.position), [script]);
+        return identifiers.map((identifier) => ({
+            range: getRange(identifier, script.file),
+            kind: DocumentHighlightKind.Read,
+        }));
     }
     async function checkChange(change) {
+        compileService.notifyChange(change.document);
+    }
+    async function reportDiagnostics({ program, document }) {
         var _a, _b, _c;
-        const program = await compile(change.document);
-        if (!program) {
-            return;
-        }
         // Group diagnostics by file.
         //
         // Initialize diagnostics for all source files in program to empty array
@@ -353,7 +250,7 @@ export function createServer(host) {
         // stale diagnostics from a previous run will stick around in the IDE.
         //
         const diagnosticMap = new Map();
-        diagnosticMap.set(change.document, []);
+        diagnosticMap.set(document, []);
         for (const each of program.sourceFiles.values()) {
             const document = (_a = each.file) === null || _a === void 0 ? void 0 : _a.document;
             if (document) {
@@ -361,10 +258,10 @@ export function createServer(host) {
             }
         }
         for (const each of program.diagnostics) {
-            let document;
+            let diagDocument;
             const location = getSourceLocation(each.target, { locateId: true });
             if (location === null || location === void 0 ? void 0 : location.file) {
-                document = location.file.document;
+                diagDocument = location.file.document;
             }
             else {
                 // https://github.com/microsoft/language-server-protocol/issues/256
@@ -372,20 +269,20 @@ export function createServer(host) {
                 // LSP does not currently allow sending a diagnostic with no location so
                 // we report diagnostics with no location on the document that changed to
                 // trigger.
-                document = change.document;
+                diagDocument = document;
             }
-            if (!document || !upToDate(document)) {
+            if (!diagDocument || !fileService.upToDate(diagDocument)) {
                 continue;
             }
-            const start = document.positionAt((_b = location === null || location === void 0 ? void 0 : location.pos) !== null && _b !== void 0 ? _b : 0);
-            const end = document.positionAt((_c = location === null || location === void 0 ? void 0 : location.end) !== null && _c !== void 0 ? _c : 0);
+            const start = diagDocument.positionAt((_b = location === null || location === void 0 ? void 0 : location.pos) !== null && _b !== void 0 ? _b : 0);
+            const end = diagDocument.positionAt((_c = location === null || location === void 0 ? void 0 : location.end) !== null && _c !== void 0 ? _c : 0);
             const range = Range.create(start, end);
             const severity = convertSeverity(each.severity);
             const diagnostic = VSDiagnostic.create(range, each.message, severity, each.code, "TypeSpec");
             if (each.code === "deprecated") {
                 diagnostic.tags = [DiagnosticTag.Deprecated];
             }
-            const diagnostics = diagnosticMap.get(document);
+            const diagnostics = diagnosticMap.get(diagDocument);
             compilerAssert(diagnostics, "Diagnostic reported against a source file that was not added to the program.");
             diagnostics.push(diagnostic);
         }
@@ -394,40 +291,42 @@ export function createServer(host) {
         }
     }
     async function getHover(params) {
-        const docString = await compile(params.textDocument, (program, document, file) => {
-            const id = getNodeAtPosition(file, document.offsetAt(params.position));
-            const sym = (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? program.checker.resolveIdentifier(id) : undefined;
-            if (sym) {
-                return getSymbolDetails(program, sym);
-            }
-            return undefined;
-        });
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return { contents: [] };
+        }
+        const { program, document, script } = result;
+        const id = getNodeAtPosition(script, document.offsetAt(params.position));
+        const sym = (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? program.checker.resolveIdentifier(id) : undefined;
         const markdown = {
             kind: MarkupKind.Markdown,
-            value: docString !== null && docString !== void 0 ? docString : "",
+            value: sym ? getSymbolDetails(program, sym) : "",
         };
         return {
             contents: markdown,
         };
     }
     async function getSignatureHelp(params) {
-        return await compile(params.textDocument, (program, document, file) => {
-            const data = getSignatureHelpNodeAtPosition(file, document.offsetAt(params.position));
-            if (data === undefined) {
-                return undefined;
-            }
-            const { node, argumentIndex } = data;
-            switch (node.kind) {
-                case SyntaxKind.TypeReference:
-                    return getSignatureHelpForTemplate(program, node, argumentIndex);
-                case SyntaxKind.DecoratorExpression:
-                case SyntaxKind.AugmentDecoratorStatement:
-                    return getSignatureHelpForDecorator(program, node, argumentIndex);
-                default:
-                    const _assertNever = node;
-                    compilerAssert(false, "Unreachable");
-            }
-        });
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return undefined;
+        }
+        const { script, document, program } = result;
+        const data = getSignatureHelpNodeAtPosition(script, document.offsetAt(params.position));
+        if (data === undefined) {
+            return undefined;
+        }
+        const { node, argumentIndex } = data;
+        switch (node.kind) {
+            case SyntaxKind.TypeReference:
+                return getSignatureHelpForTemplate(program, node, argumentIndex);
+            case SyntaxKind.DecoratorExpression:
+            case SyntaxKind.AugmentDecoratorStatement:
+                return getSignatureHelpForDecorator(program, node, argumentIndex);
+            default:
+                const _assertNever = node;
+                compilerAssert(false, "Unreachable");
+        }
     }
     function getSignatureHelpForTemplate(program, node, argumentIndex) {
         const sym = program.checker.resolveIdentifier(node.target.kind === SyntaxKind.MemberExpression ? node.target.id : node.target);
@@ -551,10 +450,12 @@ export function createServer(host) {
         return TextEdit.replace(Range.create(pos0, pos1), newText);
     }
     async function gotoDefinition(params) {
-        const sym = await compile(params.textDocument, (program, document, file) => {
-            const id = getNodeAtPosition(file, document.offsetAt(params.position));
-            return (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? program.checker.resolveIdentifier(id) : undefined;
-        });
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return [];
+        }
+        const id = getNodeAtPosition(result.script, result.document.offsetAt(params.position));
+        const sym = (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? result.program.checker.resolveIdentifier(id) : undefined;
         return getLocations(sym === null || sym === void 0 ? void 0 : sym.declarations);
     }
     async function complete(params) {
@@ -562,32 +463,41 @@ export function createServer(host) {
             isIncomplete: false,
             items: [],
         };
-        await compile(params.textDocument, async (program, document, file) => {
-            const node = getCompletionNodeAtPosition(file, document.offsetAt(params.position));
+        const result = await compileService.compile(params.textDocument);
+        if (result) {
+            const { script, document, program } = result;
+            const node = getCompletionNodeAtPosition(script, document.offsetAt(params.position));
             await resolveCompletion({
                 program,
-                file,
+                file: script,
                 completions,
                 params,
             }, node);
-        });
+        }
         return completions;
     }
     async function findReferences(params) {
-        const identifiers = await compile(params.textDocument, (program, document, file) => findReferenceIdentifiers(program, file, document.offsetAt(params.position)));
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return [];
+        }
+        const identifiers = findReferenceIdentifiers(result.program, result.script, result.document.offsetAt(params.position));
         return getLocations(identifiers);
     }
     async function prepareRename(params) {
-        return await compile(params.textDocument, (_, document, file) => {
-            var _a;
-            const id = getNodeAtPosition(file, document.offsetAt(params.position));
-            return (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? (_a = getLocation(id)) === null || _a === void 0 ? void 0 : _a.range : undefined;
-        });
+        var _a;
+        const result = await compileService.compile(params.textDocument);
+        if (result === undefined) {
+            return undefined;
+        }
+        const id = getNodeAtPosition(result.script, result.document.offsetAt(params.position));
+        return (id === null || id === void 0 ? void 0 : id.kind) === SyntaxKind.Identifier ? (_a = getLocation(id)) === null || _a === void 0 ? void 0 : _a.range : undefined;
     }
     async function rename(params) {
         const changes = {};
-        await compile(params.textDocument, (program, document, file) => {
-            const identifiers = findReferenceIdentifiers(program, file, document.offsetAt(params.position));
+        const result = await compileService.compile(params.textDocument);
+        if (result) {
+            const identifiers = findReferenceIdentifiers(result.program, result.script, result.document.offsetAt(params.position));
             for (const id of identifiers) {
                 const location = getLocation(id);
                 if (!location) {
@@ -601,7 +511,7 @@ export function createServer(host) {
                     changes[location.uri] = [change];
                 }
             }
-        });
+        }
         return { changes };
     }
     function findReferenceIdentifiers(program, file, pos, searchFiles = program.sourceFiles.values()) {
@@ -627,317 +537,17 @@ export function createServer(host) {
         }
         return references;
     }
-    async function getSemanticTokens(params) {
-        const ignore = -1;
-        const defer = -2;
-        const ast = await getScript(params.textDocument);
+    async function getSemanticTokensForDocument(params) {
+        const ast = await compileService.getScript(params.textDocument);
         if (!ast) {
             return [];
         }
-        const file = ast.file;
-        const tokens = mapTokens();
-        classifyNode(ast);
-        return Array.from(tokens.values()).filter((t) => t.kind !== undefined);
-        function mapTokens() {
-            const tokens = new Map();
-            const scanner = createScanner(file, () => { });
-            const templateStack = [];
-            while (scanner.scan() !== Token.EndOfFile) {
-                if (scanner.tokenFlags & TokenFlags.DocComment) {
-                    classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position });
-                }
-                else {
-                    handleToken(scanner.token, scanner.tokenFlags, {
-                        pos: scanner.tokenPosition,
-                        end: scanner.position,
-                    });
-                }
-            }
-            return tokens;
-            function classifyDocComment(range) {
-                scanner.scanRange(range, () => {
-                    while (scanner.scanDoc() !== Token.EndOfFile) {
-                        const kind = classifyDocToken(scanner.token);
-                        if (kind === ignore) {
-                            continue;
-                        }
-                        tokens.set(scanner.tokenPosition, {
-                            kind: kind === defer ? undefined : kind,
-                            pos: scanner.tokenPosition,
-                            end: scanner.position,
-                        });
-                    }
-                });
-            }
-            function handleToken(token, tokenFlags, range) {
-                switch (token) {
-                    case Token.StringTemplateHead:
-                        templateStack.push([token, tokenFlags]);
-                        classifyStringTemplate(token, range);
-                        break;
-                    case Token.OpenBrace:
-                        // If we don't have anything on the template stack,
-                        // then we aren't trying to keep track of a previously scanned template head.
-                        if (templateStack.length > 0) {
-                            templateStack.push([token, tokenFlags]);
-                        }
-                        handleSimpleToken(token, range);
-                        break;
-                    case Token.CloseBrace:
-                        // If we don't have anything on the template stack,
-                        // then we aren't trying to keep track of a previously scanned template head.
-                        if (templateStack.length > 0) {
-                            const [lastToken, lastTokenFlags] = templateStack[templateStack.length - 1];
-                            if (lastToken === Token.StringTemplateHead) {
-                                token = scanner.reScanStringTemplate(lastTokenFlags);
-                                // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us.
-                                if (token === Token.StringTemplateTail) {
-                                    templateStack.pop();
-                                    classifyStringTemplate(token, {
-                                        pos: scanner.tokenPosition,
-                                        end: scanner.position,
-                                    });
-                                }
-                                else {
-                                    compilerAssert(token === Token.StringTemplateMiddle, "Should have been a template middle.");
-                                    classifyStringTemplate(token, {
-                                        pos: scanner.tokenPosition,
-                                        end: scanner.position,
-                                    });
-                                }
-                            }
-                            else {
-                                compilerAssert(lastToken === Token.OpenBrace, "Should have been an open brace");
-                                templateStack.pop();
-                            }
-                            break;
-                        }
-                        handleSimpleToken(token, range);
-                        break;
-                    default:
-                        handleSimpleToken(token, range);
-                }
-            }
-            function handleSimpleToken(token, range) {
-                const kind = classifyToken(scanner.token);
-                if (kind === ignore) {
-                    return;
-                }
-                tokens.set(range.pos, {
-                    kind: kind === defer ? undefined : kind,
-                    ...range,
-                });
-            }
-            function classifyStringTemplate(token, range) {
-                const stringStart = token === Token.StringTemplateHead ? range.pos : range.pos + 1;
-                const stringEnd = token === Token.StringTemplateTail ? range.end : range.end - 2;
-                if (stringStart !== range.pos) {
-                    tokens.set(range.pos, {
-                        kind: SemanticTokenKind.Operator,
-                        pos: range.pos,
-                        end: stringStart,
-                    });
-                }
-                tokens.set(stringStart, {
-                    kind: SemanticTokenKind.String,
-                    pos: stringStart,
-                    end: stringEnd,
-                });
-                if (stringEnd !== range.end) {
-                    tokens.set(stringEnd, {
-                        kind: SemanticTokenKind.Operator,
-                        pos: stringEnd,
-                        end: range.end,
-                    });
-                }
-            }
-        }
-        function classifyToken(token) {
-            switch (token) {
-                case Token.Identifier:
-                    return defer;
-                case Token.StringLiteral:
-                    return SemanticTokenKind.String;
-                case Token.NumericLiteral:
-                    return SemanticTokenKind.Number;
-                case Token.MultiLineComment:
-                case Token.SingleLineComment:
-                    return SemanticTokenKind.Comment;
-                default:
-                    if (isKeyword(token)) {
-                        return SemanticTokenKind.Keyword;
-                    }
-                    if (isPunctuation(token)) {
-                        return SemanticTokenKind.Operator;
-                    }
-                    return ignore;
-            }
-        }
-        /** Classify tokens when scanning doc comment. */
-        function classifyDocToken(token) {
-            switch (token) {
-                case Token.NewLine:
-                case Token.Whitespace:
-                    return ignore;
-                case Token.DocText:
-                case Token.Star:
-                case Token.Identifier:
-                    return SemanticTokenKind.Comment;
-                case Token.At:
-                    return defer;
-                default:
-                    return ignore;
-            }
-        }
-        function classifyNode(node) {
-            switch (node.kind) {
-                case SyntaxKind.DirectiveExpression:
-                    classify(node.target, SemanticTokenKind.Keyword);
-                    break;
-                case SyntaxKind.TemplateParameterDeclaration:
-                    classify(node.id, SemanticTokenKind.TypeParameter);
-                    break;
-                case SyntaxKind.ModelProperty:
-                case SyntaxKind.UnionVariant:
-                    if (node.id) {
-                        classify(node.id, SemanticTokenKind.Property);
-                    }
-                    break;
-                case SyntaxKind.AliasStatement:
-                    classify(node.id, SemanticTokenKind.Struct);
-                    break;
-                case SyntaxKind.ModelStatement:
-                    classify(node.id, SemanticTokenKind.Struct);
-                    break;
-                case SyntaxKind.ScalarStatement:
-                    classify(node.id, SemanticTokenKind.Type);
-                    break;
-                case SyntaxKind.EnumStatement:
-                    classify(node.id, SemanticTokenKind.Enum);
-                    break;
-                case SyntaxKind.EnumMember:
-                    classify(node.id, SemanticTokenKind.EnumMember);
-                    break;
-                case SyntaxKind.NamespaceStatement:
-                    classify(node.id, SemanticTokenKind.Namespace);
-                    break;
-                case SyntaxKind.InterfaceStatement:
-                    classify(node.id, SemanticTokenKind.Interface);
-                    break;
-                case SyntaxKind.OperationStatement:
-                    classify(node.id, SemanticTokenKind.Function);
-                    break;
-                case SyntaxKind.DecoratorDeclarationStatement:
-                    classify(node.id, SemanticTokenKind.Function);
-                    break;
-                case SyntaxKind.FunctionDeclarationStatement:
-                    classify(node.id, SemanticTokenKind.Function);
-                    break;
-                case SyntaxKind.FunctionParameter:
-                    classify(node.id, SemanticTokenKind.Parameter);
-                    break;
-                case SyntaxKind.AugmentDecoratorStatement:
-                    classifyReference(node.targetType, SemanticTokenKind.Type);
-                    classifyReference(node.target, SemanticTokenKind.Macro);
-                    break;
-                case SyntaxKind.DecoratorExpression:
-                    classifyReference(node.target, SemanticTokenKind.Macro);
-                    break;
-                case SyntaxKind.TypeReference:
-                    classifyReference(node.target);
-                    break;
-                case SyntaxKind.MemberExpression:
-                    classifyReference(node);
-                    break;
-                case SyntaxKind.ProjectionStatement:
-                    classifyReference(node.selector);
-                    classify(node.id, SemanticTokenKind.Variable);
-                    break;
-                case SyntaxKind.Projection:
-                    classify(node.directionId, SemanticTokenKind.Keyword);
-                    for (const modifierId of node.modifierIds) {
-                        classify(modifierId, SemanticTokenKind.Keyword);
-                    }
-                    break;
-                case SyntaxKind.ProjectionParameterDeclaration:
-                    classifyReference(node.id, SemanticTokenKind.Parameter);
-                    break;
-                case SyntaxKind.ProjectionCallExpression:
-                    classifyReference(node.target, SemanticTokenKind.Function);
-                    for (const arg of node.arguments) {
-                        classifyReference(arg);
-                    }
-                    break;
-                case SyntaxKind.ProjectionMemberExpression:
-                    classifyReference(node.id);
-                    break;
-                case SyntaxKind.DocParamTag:
-                case SyntaxKind.DocTemplateTag:
-                    classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag);
-                    classifyOverride(node.paramName, SemanticTokenKind.Variable);
-                    break;
-                case SyntaxKind.DocReturnsTag:
-                    classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag);
-                    break;
-                case SyntaxKind.DocUnknownTag:
-                    classifyDocTag(node.tagName, SemanticTokenKind.Macro);
-                    break;
-                case SyntaxKind.TemplateArgument:
-                    if (node.name)
-                        classify(node.name, SemanticTokenKind.TypeParameter);
-                    break;
-                default:
-                    break;
-            }
-            visitChildren(node, classifyNode);
-        }
-        function classifyDocTag(node, kind) {
-            classifyOverride(node, kind);
-            const token = tokens.get(node.pos - 1); // Get the `@` token
-            if (token) {
-                token.kind = kind;
-            }
-        }
-        function classify(node, kind) {
-            const token = tokens.get(node.pos);
-            if (token && token.kind === undefined) {
-                token.kind = kind;
-            }
-        }
-        function classifyOverride(node, kind) {
-            const token = tokens.get(node.pos);
-            if (token) {
-                token.kind = kind;
-            }
-        }
-        function classifyReference(node, kind = SemanticTokenKind.Type) {
-            switch (node.kind) {
-                case SyntaxKind.MemberExpression:
-                    classifyIdentifier(node.base, SemanticTokenKind.Namespace);
-                    classifyIdentifier(node.id, kind);
-                    break;
-                case SyntaxKind.ProjectionMemberExpression:
-                    classifyReference(node.base, SemanticTokenKind.Namespace);
-                    classifyIdentifier(node.id, kind);
-                    break;
-                case SyntaxKind.TypeReference:
-                    classifyIdentifier(node.target, kind);
-                    break;
-                case SyntaxKind.Identifier:
-                    classify(node, kind);
-                    break;
-            }
-        }
-        function classifyIdentifier(node, kind) {
-            if (node.kind === SyntaxKind.Identifier) {
-                classify(node, kind);
-            }
-        }
+        return getSemanticTokens(ast);
     }
     async function buildSemanticTokens(params) {
         const builder = new SemanticTokensBuilder();
-        const tokens = await getSemanticTokens(params);
-        const file = await compilerHost.readFile(await getPath(params.textDocument));
+        const tokens = await getSemanticTokensForDocument(params);
+        const file = await compilerHost.readFile(await fileService.getPath(params.textDocument));
         const starts = file.getLineStarts();
         for (const token of tokens) {
             const start = file.getLineAndCharacterOfPosition(token.pos);
@@ -965,7 +575,7 @@ export function createServer(host) {
             return undefined;
         }
         return {
-            uri: getURL(location.file.path),
+            uri: fileService.getURL(location.file.path),
             range: getRange(location, location.file),
         };
     }
@@ -1004,123 +614,6 @@ export function createServer(host) {
             diagnostics,
         });
     }
-    /**
-     * Determine if the given document is the latest version.
-     *
-     * A document can become out-of-date if a change comes in during an async
-     * operation.
-     */
-    function upToDate(document) {
-        var _a;
-        if (!("version" in document)) {
-            return true;
-        }
-        return document.version === ((_a = host.getOpenDocumentByURL(document.uri)) === null || _a === void 0 ? void 0 : _a.version);
-    }
-    /**
-     * Infer the appropriate entry point (a.k.a. "main file") for analyzing a
-     * change to the file at the given path. This is necessary because different
-     * results can be obtained from compiling the same file with different entry
-     * points.
-     *
-     * Walk directory structure upwards looking for package.json with tspMain or
-     * main.tsp file. Stop search when reaching a workspace root. If a root is
-     * reached without finding an entry point, use the given path as its own
-     * entry point.
-     *
-     * Untitled documents are always treated as their own entry points as they
-     * do not exist in a directory that could pull them in via another entry
-     * point.
-     */
-    async function getMainFileForDocument(path) {
-        if (path.startsWith("untitled:")) {
-            return path;
-        }
-        let dir = getDirectoryPath(path);
-        const options = { allowFileNotFound: true };
-        while (true) {
-            let mainFile = "main.tsp";
-            let pkg;
-            const pkgPath = joinPaths(dir, "package.json");
-            const cached = await fileSystemCache.get(pkgPath);
-            if (cached === null || cached === void 0 ? void 0 : cached.data) {
-                pkg = cached.data;
-            }
-            else {
-                [pkg] = await loadFile(compilerHost, pkgPath, JSON.parse, logMainFileSearchDiagnostic, options);
-                await fileSystemCache.setData(pkgPath, pkg !== null && pkg !== void 0 ? pkg : {});
-            }
-            const tspMain = resolveTspMain(pkg);
-            if (typeof tspMain === "string") {
-                mainFile = tspMain;
-            }
-            const candidate = joinPaths(dir, mainFile);
-            const stat = await doIO(() => compilerHost.stat(candidate), candidate, logMainFileSearchDiagnostic, options);
-            if (stat === null || stat === void 0 ? void 0 : stat.isFile()) {
-                return candidate;
-            }
-            const parentDir = getDirectoryPath(dir);
-            if (parentDir === dir) {
-                break;
-            }
-            dir = parentDir;
-        }
-        return path;
-        function logMainFileSearchDiagnostic(diagnostic) {
-            log(`Unexpected diagnostic while looking for main file of ${path}`, formatDiagnostic(diagnostic));
-        }
-    }
-    async function getPath(document) {
-        if (isUntitled(document.uri)) {
-            return document.uri;
-        }
-        const path = await fileURLToRealPath(document.uri);
-        pathToURLMap.set(path, document.uri);
-        return path;
-    }
-    function getURL(path) {
-        var _a;
-        if (isUntitled(path)) {
-            return path;
-        }
-        return (_a = pathToURLMap.get(path)) !== null && _a !== void 0 ? _a : compilerHost.pathToFileURL(path);
-    }
-    function isUntitled(pathOrUrl) {
-        return pathOrUrl.startsWith("untitled:");
-    }
-    function getOpenDocument(path) {
-        const url = getURL(path);
-        return url ? host.getOpenDocumentByURL(url) : undefined;
-    }
-    async function fileURLToRealPath(url) {
-        return getNormalizedRealPath(compilerHost, compilerHost.fileURLToPath(url));
-    }
-    function createFileSystemCache() {
-        const cache = new Map();
-        let changes = [];
-        return {
-            async get(path) {
-                for (const change of changes) {
-                    const path = await fileURLToRealPath(change.uri);
-                    cache.delete(path);
-                }
-                changes = [];
-                return cache.get(path);
-            },
-            set(path, entry) {
-                cache.set(path, entry);
-            },
-            async setData(path, data) {
-                const entry = await this.get(path);
-                if (entry) {
-                    entry.data = data;
-                }
-            },
-            notify(changes) {
-                changes.push(...changes);
-            },
-        };
-    }
     function createCompilerHost() {
         const base = host.compilerHost;
         return {
@@ -1131,7 +624,7 @@ export function createServer(host) {
             getSourceFileKind,
         };
         async function readFile(path) {
-            const document = getOpenDocument(path);
+            const document = fileService.getOpenDocument(path);
             const cached = await fileSystemCache.get(path);
             // Try cache
             if (cached && (!document || document.version === cached.version)) {
@@ -1165,7 +658,7 @@ export function createServer(host) {
             var _a;
             // if we have an open document for the path or a cache entry, then we know
             // it's a file and not a directory and needn't hit the disk.
-            if (getOpenDocument(path) || ((_a = (await fileSystemCache.get(path))) === null || _a === void 0 ? void 0 : _a.type) === "file") {
+            if (fileService.getOpenDocument(path) || ((_a = (await fileSystemCache.get(path))) === null || _a === void 0 ? void 0 : _a.type) === "file") {
                 return {
                     isFile() {
                         return true;
@@ -1178,7 +671,7 @@ export function createServer(host) {
             return await base.stat(path);
         }
         function getSourceFileKind(path) {
-            const document = getOpenDocument(path);
+            const document = fileService.getOpenDocument(path);
             if ((document === null || document === void 0 ? void 0 : document.languageId) === "typespec") {
                 return "typespec";
             }