@staticbolt/lsp 1.0.0-beta.26 → 1.0.0-beta.27

package/src/modes/embedded-support.ts

@@ -1,40 +1,41 @@
 import vscodeHtml from "vscode-html-languageservice";
 import { TextDocument } from "vscode-languageserver-textdocument";
 
-import type { LanguageService, Range } from "vscode-html-languageservice";
+import { blankRegions, findMarkdownNonHtmlRegions } from "./markdown-regions.ts";
 
-export interface LanguageRange extends Range {
-  languageId: string | undefined;
-  attributeValue?: boolean;
-}
+import type { LanguageService } from "vscode-html-languageservice";
 
 export interface HTMLDocumentRegions {
-  getEmbeddedDocument(languageId: string, shouldIgnoreAttributeValues?: boolean): TextDocument;
-  getLanguageRanges(range: Range): LanguageRange[];
+  /** The document to serve HTML features from: itself for an HTML file, a copy declared as HTML for a markdown one. */
+  getHtmlDocument(): TextDocument;
   getLanguageAtPosition(position: Position): string | undefined;
   getLanguagesInDocument(): string[];
-  getImportedScripts(): string[];
 }
 
-const CSS_STYLE_RULE = "__";
 const TokenType = vscodeHtml.TokenType;
-const Position = vscodeHtml.Position;
 type Position = vscodeHtml.Position;
 
 interface EmbeddedRegion {
   languageId: string | undefined;
   start: number;
   end: number;
-  attributeValue?: boolean;
 }
 
 export function getDocumentRegions(languageService: LanguageService, document: TextDocument): HTMLDocumentRegions {
+  const source = document.getText();
+  const isMarkdown = document.languageId === "markdown";
+
+  const markdownRegions: EmbeddedRegion[] = isMarkdown
+    ? findMarkdownNonHtmlRegions(source).map(region => ({ ...region, languageId: undefined }))
+    : [];
+
+  const htmlText = isMarkdown ? blankRegions(source, markdownRegions) : source;
+
   const regions: EmbeddedRegion[] = [];
-  const scanner = languageService.createScanner(document.getText());
+  const scanner = languageService.createScanner(htmlText);
   let lastTagName: string = "";
   let lastAttributeName: string | null = null;
   let languageIdFromType: string | undefined;
-  const importedScripts: string[] = [];
 
   let token = scanner.scan();
 
@@ -59,13 +60,7 @@ export function getDocumentRegions(languageService: LanguageService, document: T
         break;
       }
       case TokenType.AttributeValue: {
-        if (lastAttributeName === "src" && lastTagName.toLowerCase() === "script") {
-          let value = scanner.getTokenText();
-          if (value[0] === "'" || value[0] === '"') {
-            value = value.slice(1, -1);
-          }
-          importedScripts.push(value);
-        } else if (lastAttributeName === "type" && lastTagName.toLowerCase() === "script") {
+        if (lastAttributeName === "type" && lastTagName.toLowerCase() === "script") {
           const token = scanner.getTokenText();
           if (/["'](module|(text|application)\/(java|ecma)script|text\/babel)["']/.test(token) || token === "module") {
             languageIdFromType = "javascript";
@@ -79,12 +74,12 @@ export function getDocumentRegions(languageService: LanguageService, document: T
           if (attributeLanguageId) {
             let start = scanner.getTokenOffset();
             let end = scanner.getTokenEnd();
-            const firstChar = document.getText()[start];
+            const firstChar = htmlText[start];
             if (firstChar === "'" || firstChar === '"') {
               start++;
               end--;
             }
-            regions.push({ languageId: attributeLanguageId, start, end, attributeValue: true });
+            regions.push({ languageId: attributeLanguageId, start, end });
           }
         }
         lastAttributeName = null;
@@ -93,76 +88,43 @@ export function getDocumentRegions(languageService: LanguageService, document: T
     }
     token = scanner.scan();
   }
+
+  const allRegions = mergeRegions(regions, markdownRegions);
+  const htmlDocument = isMarkdown ? TextDocument.create(document.uri, "html", document.version, htmlText) : document;
+
   return {
-    getLanguageRanges: (range: Range) => getLanguageRanges(document, regions, range),
-    getEmbeddedDocument: (languageId: string, shouldIgnoreAttributeValues: boolean) =>
-      getEmbeddedDocument(document, regions, languageId, shouldIgnoreAttributeValues),
-    getLanguageAtPosition: (position: Position) => getLanguageAtPosition(document, regions, position),
-    getLanguagesInDocument: () => getLanguagesInDocument(document, regions),
-    getImportedScripts: () => importedScripts,
+    getLanguageAtPosition: (position: Position) => getLanguageAtPosition(document, allRegions, position),
+    getLanguagesInDocument: () => getLanguagesInDocument(allRegions),
+    getHtmlDocument: () => htmlDocument,
   };
 }
 
-function getLanguageRanges(document: TextDocument, regions: EmbeddedRegion[], range: Range): LanguageRange[] {
-  const result: LanguageRange[] = [];
-  let currentPos = range ? range.start : Position.create(0, 0);
-  let currentOffset = range ? document.offsetAt(range.start) : 0;
-  const endOffset = range ? document.offsetAt(range.end) : document.getText().length;
-  for (const region of regions) {
-    if (!(region.end > currentOffset && region.start < endOffset)) {
-      continue;
-    }
+/** The lookups below read the regions in order, so an overlap is dropped: a scanned region always wins over a markdown one. */
+function mergeRegions(scanned: EmbeddedRegion[], markdown: EmbeddedRegion[]): EmbeddedRegion[] {
+  if (markdown.length === 0) return scanned;
 
-    const start = Math.max(region.start, currentOffset);
-    const startPos = document.positionAt(start);
-    if (currentOffset < region.start) {
-      result.push({
-        start: currentPos,
-        end: startPos,
-        languageId: "html",
-      });
-    }
-    const end = Math.min(region.end, endOffset);
-    const endPos = document.positionAt(end);
-    if (end > region.start) {
-      result.push({
-        start: startPos,
-        end: endPos,
-        languageId: region.languageId,
-        attributeValue: region.attributeValue,
-      });
-    }
-    currentOffset = end;
-    currentPos = endPos;
-  }
-  if (currentOffset < endOffset) {
-    const endPos = range ? range.end : document.positionAt(endOffset);
-    result.push({
-      start: currentPos,
-      end: endPos,
-      languageId: "html",
-    });
+  const merged = [...scanned, ...markdown].toSorted((a, b) => a.start - b.start || b.end - a.end);
+  const result: EmbeddedRegion[] = [];
+
+  for (const region of merged) {
+    const previous = result.at(-1);
+    if (previous && region.start < previous.end) continue;
+    result.push(region);
   }
+
   return result;
 }
 
-function getLanguagesInDocument(_document: TextDocument, regions: EmbeddedRegion[]): string[] {
-  const result: string[] = [];
+function getLanguagesInDocument(regions: EmbeddedRegion[]): string[] {
+  const languages = new Set(["html"]);
 
   for (const region of regions) {
-    if (!(region.languageId && !result.includes(region.languageId))) {
-      continue;
-    }
-
-    result.push(region.languageId);
-    if (result.length === 3) {
-      return result;
+    if (region.languageId) {
+      languages.add(region.languageId);
     }
   }
 
-  result.push("html");
-
-  return result;
+  return [...languages];
 }
 
 function getLanguageAtPosition(document: TextDocument, regions: EmbeddedRegion[], position: Position): string | undefined {
@@ -179,100 +141,6 @@ function getLanguageAtPosition(document: TextDocument, regions: EmbeddedRegion[]
   return "html";
 }
 
-function getEmbeddedDocument(
-  document: TextDocument,
-  contents: EmbeddedRegion[],
-  languageId: string,
-  shouldIgnoreAttributeValues: boolean
-): TextDocument {
-  let currentPos = 0;
-  const oldContent = document.getText();
-  let result = "";
-  let lastSuffix = "";
-  for (const c of contents) {
-    if (!(c.languageId === languageId && (!shouldIgnoreAttributeValues || !c.attributeValue))) {
-      continue;
-    }
-
-    result = substituteWithWhitespace(result, currentPos, c.start, oldContent, lastSuffix, getPrefix(c));
-    result += updateContent(c, oldContent.slice(c.start, c.end));
-    currentPos = c.end;
-    lastSuffix = getSuffix(c);
-  }
-  result = substituteWithWhitespace(result, currentPos, oldContent.length, oldContent, lastSuffix, "");
-  return TextDocument.create(document.uri, languageId, document.version, result);
-}
-
-function getPrefix(c: EmbeddedRegion) {
-  if (c.attributeValue) {
-    switch (c.languageId) {
-      case "css": {
-        return CSS_STYLE_RULE + "{";
-      }
-    }
-  }
-  return "";
-}
-function getSuffix(c: EmbeddedRegion) {
-  if (c.attributeValue) {
-    switch (c.languageId) {
-      case "css": {
-        return "}";
-      }
-      case "javascript": {
-        return ";";
-      }
-    }
-  }
-  return "";
-}
-function updateContent(c: EmbeddedRegion, content: string): string {
-  if (!c.attributeValue && c.languageId === "javascript") {
-    return content.replace(`<!--`, `/* `).replace(`-->`, ` */`);
-  }
-  if (c.languageId === "css") {
-    const quoteEscape = /(&quot;|&#34;)/g;
-    return content.replace(quoteEscape, (match, _, offset: number) => {
-      const spaces = " ".repeat(match.length - 1);
-      const afterChar = content[offset + match.length];
-      if (!afterChar || afterChar.includes(" ")) {
-        return `${spaces}"`;
-      }
-      return `"${spaces}`;
-    });
-  }
-  return content;
-}
-
-function substituteWithWhitespace(result: string, start: number, end: number, oldContent: string, before: string, after: string) {
-  result += before;
-  let accumulatedWS = -before.length; // start with a negative value to account for the before string
-  for (let index = start; index < end; index++) {
-    const ch = oldContent[index];
-    if (ch === "\n" || ch === "\r") {
-      // only write new lines, skip the whitespace
-      accumulatedWS = 0;
-      result += ch;
-    } else {
-      accumulatedWS++;
-    }
-  }
-  result = append(result, " ", accumulatedWS - after.length);
-  result += after;
-  return result;
-}
-
-function append(result: string, string_: string, n: number): string {
-  while (n > 0) {
-    if (n & 1) {
-      result += string_;
-    }
-    n >>= 1;
-    string_ += string_;
-  }
-  return result;
-}
-
 function getAttributeLanguage(attributeName: string): string | null {
   const match = attributeName.match(/^(style)$|^(on\w+)$/i);
   if (!match) {

package/src/modes/language-modes.ts

@@ -7,44 +7,10 @@ import { getHTMLMode } from "./html-mode.ts";
 import type { LanguageModelCache } from "../language-model-cache.ts";
 import type { FileSystemProvider } from "../requests.ts";
 import type { HTMLDocumentRegions } from "./embedded-support.ts";
-import type {
-  ClientCapabilities,
-  CompletionConfiguration,
-  DocumentContext,
-  HoverSettings,
-  HTMLDataV1,
-  IHTMLDataProvider,
-} from "vscode-html-languageservice";
-import type {
-  CompletionItem,
-  CompletionList,
-  DocumentLink,
-  Hover,
-  Position,
-  Range,
-  WorkspaceFolder,
-} from "vscode-languageserver";
+import type { ClientCapabilities, DocumentContext, HTMLDataV1, IHTMLDataProvider } from "vscode-html-languageservice";
+import type { CompletionItem, CompletionList, DocumentLink, Hover, Position } from "vscode-languageserver";
 import type { TextDocument } from "vscode-languageserver-textdocument";
 
-export interface Settings {
-  hover?: HoverSettings;
-  autoClosingTags?: boolean;
-  completion?: CompletionConfiguration;
-  suggest?: CompletionConfiguration;
-}
-
-export interface Workspace {
-  readonly settings: Settings;
-  readonly folders: WorkspaceFolder[];
-}
-
-export interface SemanticTokenData {
-  start: Position;
-  length: number;
-  typeIdx: number;
-  modifierSet: number;
-}
-
 export type CompletionItemData = {
   languageId: string;
   uri: string;
@@ -72,20 +38,15 @@ export interface LanguageMode {
 
 export interface LanguageModes {
   updateDataProviders(dataProviders: IHTMLDataProvider[]): void;
+  /** The document to hand to the HTML mode: itself for an HTML file, a copy holding only the raw HTML for a markdown one. */
+  getHtmlDocument(document: TextDocument): TextDocument;
   getModeAtPosition(document: TextDocument, position: Position): LanguageMode | undefined;
-  getModesInRange(document: TextDocument, range: Range): LanguageModeRange[];
-  getAllModes(): LanguageMode[];
   getAllModesInDocument(document: TextDocument): LanguageMode[];
   getMode(languageId: string): LanguageMode | undefined;
   onDocumentRemoved(document: TextDocument): void;
   dispose(): void;
 }
 
-export interface LanguageModeRange extends Range {
-  mode: LanguageMode | undefined;
-  attributeValue?: boolean;
-}
-
 export const FILE_PROTOCOL = "staticbolt-server";
 
 export function getLanguageModes(clientCapabilities: ClientCapabilities, requestService: FileSystemProvider): LanguageModes {
@@ -140,6 +101,10 @@ export function getLanguageModes(clientCapabilities: ClientCapabilities, request
       htmlLanguageService.setDataProviders(true, dataProviders);
     },
 
+    getHtmlDocument(document: TextDocument): TextDocument {
+      return documentRegions.get(document).getHtmlDocument();
+    },
+
     getModeAtPosition(document: TextDocument, position: Position): LanguageMode | undefined {
       const languageId = documentRegions.get(document).getLanguageAtPosition(position);
       if (languageId) {
@@ -149,20 +114,6 @@ export function getLanguageModes(clientCapabilities: ClientCapabilities, request
       return undefined;
     },
 
-    getModesInRange(document: TextDocument, range: Range): LanguageModeRange[] {
-      return documentRegions
-        .get(document)
-        .getLanguageRanges(range)
-        .map((r): LanguageModeRange => {
-          return {
-            start: r.start,
-            end: r.end,
-            mode: r.languageId ? modes[r.languageId] : undefined,
-            attributeValue: r.attributeValue,
-          };
-        });
-    },
-
     getAllModesInDocument(document: TextDocument): LanguageMode[] {
       const result: LanguageMode[] = [];
       for (const languageId of documentRegions.get(document).getLanguagesInDocument()) {
@@ -175,18 +126,6 @@ export function getLanguageModes(clientCapabilities: ClientCapabilities, request
       return result;
     },
 
-    getAllModes(): LanguageMode[] {
-      const result = [];
-      for (const languageId in modes) {
-        const mode = modes[languageId];
-        if (mode) {
-          result.push(mode);
-        }
-      }
-
-      return result;
-    },
-
     getMode(languageId: string): LanguageMode {
       return modes[languageId];
     },

package/src/modes/markdown-regions.ts

@@ -0,0 +1,92 @@
+/**
+ * Markdown allows raw HTML anywhere, so a ".md" file is served as HTML with the parts that can never be HTML — front matter, code
+ * blocks and code spans — blanked out first. Blanking keeps every offset, so positions still point at the same place in the
+ * file.
+ */
+import { markdownToMdast } from "satteri";
+
+import type { MdastNode } from "satteri";
+
+export interface TextRegion {
+  start: number;
+  end: number;
+}
+
+/** None of these nest, so the regions they produce never overlap. */
+const NON_HTML_NODES = new Set(["code", "inlineCode", "yaml", "toml"]);
+
+/** The regions of a markdown document that must not be treated as HTML, sorted by start offset. */
+export function findMarkdownNonHtmlRegions(text: string): TextRegion[] {
+  let tree: MdastNode;
+
+  try {
+    tree = markdownToMdast(text);
+  } catch (error) {
+    console.warn("[staticbolt] could not parse markdown, its whole content is handled as HTML:", error);
+    return [];
+  }
+
+  const regions: TextRegion[] = [];
+  const pending: MdastNode[] = [tree];
+
+  while (pending.length > 0) {
+    const node = pending.pop()!;
+
+    if (NON_HTML_NODES.has(node.type)) {
+      const start = node.position?.start.offset;
+      const end = node.position?.end.offset;
+
+      if (start !== undefined && end !== undefined) {
+        regions.push({ start, end });
+      }
+
+      continue;
+    }
+
+    if ("children" in node) {
+      pending.push(...node.children);
+    }
+  }
+
+  const sorted = regions.toSorted((a, b) => a.start - b.start);
+
+  return toUtf16Offsets(text, sorted);
+}
+
+/**
+ * The parser counts code points, the editor counts UTF-16 code units. The two only drift apart once a character outside the basic
+ * plane, an emoji most of the time, sits before a region.
+ */
+function toUtf16Offsets(text: string, regions: TextRegion[]): TextRegion[] {
+  if (!/[\uD800-\uDBFF]/.test(text)) return regions;
+
+  const astral: number[] = [];
+  let codePoint = 0;
+
+  for (const character of text) {
+    if (character.length === 2) {
+      astral.push(codePoint);
+    }
+
+    codePoint++;
+  }
+
+  const toUtf16 = (offset: number) => offset + astral.filter(position => position < offset).length;
+
+  return regions.map(region => ({ start: toUtf16(region.start), end: toUtf16(region.end) }));
+}
+
+/** Replaces every region with spaces, keeping the length of the text and its line breaks. Regions must be sorted and disjoint. */
+export function blankRegions(text: string, regions: readonly TextRegion[]): string {
+  if (regions.length === 0) return text;
+
+  let result = "";
+  let cursor = 0;
+
+  for (const region of regions) {
+    result += text.slice(cursor, region.start) + text.slice(region.start, region.end).replaceAll(/[^\n\r]/g, " ");
+    cursor = region.end;
+  }
+
+  return result + text.slice(cursor);
+}