@adecore/editor 0.22.1 → 0.23.0

package/src/shiki-document.ts

@@ -0,0 +1,143 @@
+import type { GrammarState, Highlighter, ShikiTransformer, ThemedToken } from 'shiki';
+import { documentLanguageOf, loadGrammar } from './shiki.ts';
+
+export interface ShikiDocumentTokenOptions {
+    readonly reset?: ReadonlySet<number>;
+    readonly tokenizeMaxLineLength?: number;
+    readonly tokenizeTimeLimit?: number;
+}
+
+type TokenOptions = Parameters<Highlighter['codeToTokensBase']>[1];
+
+export function shikiDocumentHighlighter(
+    highlighter: () => Promise<Highlighter>,
+    blocks: readonly string[],
+    language: string,
+    theme: string,
+    transformers: readonly ShikiTransformer[] = []
+): (index: number) => Promise<string> {
+    return documentHighlighter(highlighter, blocks, language, theme, (loaded, code, grammar, theme, state) => {
+        let root: Parameters<NonNullable<ShikiTransformer['root']>>[0] | undefined;
+        const value = loaded.codeToHtml(code, {
+            lang: grammar,
+            theme,
+            grammarState: state,
+            transformers: [
+                ...transformers,
+                {
+                    root(node) {
+                        root = node;
+                    }
+                }
+            ]
+        });
+        return { value, state: root === undefined ? undefined : loaded.getLastGrammarState(root) };
+    });
+}
+
+export function shikiDocumentTokens(
+    highlighter: () => Promise<Highlighter>,
+    blocks: readonly string[],
+    language: string,
+    theme: string,
+    options: ShikiDocumentTokenOptions = {}
+): (index: number) => Promise<ThemedToken[][]> {
+    return documentHighlighter(
+        highlighter,
+        blocks,
+        language,
+        theme,
+        (loaded, code, grammar, theme, state) => {
+            // Shiki 4.4 caches the active theme by argument identity, even after a same-name replacement.
+            const value = loaded.codeToTokensBase(code, {
+                lang: grammar,
+                theme: loaded.getTheme(theme),
+                grammarState: state,
+                tokenizeMaxLineLength: options.tokenizeMaxLineLength,
+                tokenizeTimeLimit: options.tokenizeTimeLimit
+            } as TokenOptions);
+            return { value, state: loaded.getLastGrammarState(value) };
+        },
+        options.reset
+    );
+}
+
+function documentHighlighter<Value>(
+    highlighter: () => Promise<Highlighter>,
+    blocks: readonly string[],
+    language: string,
+    theme: string,
+    render: (
+        highlighter: Highlighter,
+        code: string,
+        language: string,
+        theme: string,
+        state: GrammarState | undefined
+    ) => { value: Value; state: GrammarState | undefined },
+    reset: ReadonlySet<number> = new Set()
+): (index: number) => Promise<Value> {
+    const resets = new Set(reset);
+    // A CR before the removed LF belongs to the block separator, not the last source line.
+    const code = blocks.map((block, index) => (index < blocks.length - 1 ? block.replace(/\r$/, '') : block));
+    let prepared: Promise<{ highlighter: Highlighter; language: string }> | undefined;
+    let pending = Promise.resolve();
+    let grammar: unknown;
+    let themeRegistration: unknown;
+    let state: GrammarState | undefined;
+    let values: Value[] = [];
+
+    const prepare = async (): Promise<{ highlighter: Highlighter; language: string }> => {
+        const loaded = await highlighter();
+        if (!loaded.getLoadedThemes().includes(theme)) {
+            await loaded.loadTheme(theme as Parameters<Highlighter['loadTheme']>[0]);
+        }
+        const documentLanguage = documentLanguageOf(language);
+        const known =
+            loaded.getLoadedLanguages().includes(documentLanguage) || documentLanguage in loaded.getBundledLanguages() || documentLanguage !== language;
+        const grammar = known && (await loadGrammar(loaded, documentLanguage, code.join('\n'))) ? documentLanguage : 'text';
+        return { highlighter: loaded, language: grammar };
+    };
+
+    return (index) => {
+        if (!Number.isInteger(index) || index < 0 || index >= code.length) {
+            return Promise.reject(new RangeError('The highlighted block index is outside the document.'));
+        }
+        const result = pending.then(async () => {
+            prepared ??= prepare().catch((error: unknown) => {
+                prepared = undefined;
+                throw error;
+            });
+            const { highlighter: loaded, language: grammarId } = await prepared;
+            const refresh = (): void => {
+                const currentGrammar = grammarId === 'text' ? undefined : loaded.getLanguage(grammarId);
+                const currentTheme = loaded.getTheme(theme);
+                // Loading an embedded language rebuilds Shiki's grammar and invalidates its old states.
+                if (grammar !== currentGrammar || themeRegistration !== currentTheme) {
+                    grammar = currentGrammar;
+                    themeRegistration = currentTheme;
+                    state = undefined;
+                    values = [];
+                }
+            };
+            refresh();
+            while (values.length <= index) {
+                const block = values.length;
+                const rendered = render(loaded, code[block]!, grammarId, theme, state);
+                state = resets.has(block + 1) ? undefined : rendered.state;
+                values.push(rendered.value);
+                if (block < index) {
+                    // A jump deep into a file must leave room for input and painting between blocks.
+                    await new Promise<void>((resolve) => setTimeout(resolve, 0));
+                    refresh();
+                }
+            }
+            return values[index]!;
+        });
+        // A failed theme or grammar load can be retried without rejecting every later request.
+        pending = result.then(
+            () => undefined,
+            () => undefined
+        );
+        return result;
+    };
+}

package/src/shiki-worker.ts

@@ -0,0 +1,344 @@
+import type { Highlighter, ThemeRegistration, ThemedToken } from 'shiki';
+import { shikiDocumentTokens } from './shiki-document.ts';
+import { documentLanguageOf } from './shiki.ts';
+import type { LineToken } from './types.ts';
+
+export interface DiffSyntaxDocument {
+    readonly language: string;
+    readonly theme: string;
+    readonly blocks: readonly string[];
+    readonly resets: readonly number[];
+}
+
+export type DiffSyntaxInput =
+    | { readonly type: 'start'; readonly themes: readonly ThemeRegistration[]; readonly maxLines: number }
+    | { readonly type: 'request'; readonly generation: number; readonly index: number; readonly document?: DiffSyntaxDocument }
+    | { readonly type: 'cancel'; readonly generation: number }
+    | { readonly type: 'dispose'; readonly generation: number };
+
+export interface DiffSyntaxTokens {
+    readonly type: 'tokens';
+    readonly generation: number;
+    readonly index: number;
+    readonly colors: readonly string[];
+    readonly foreground: string;
+    readonly background: string;
+    readonly lines: Uint32Array;
+    readonly tokens: Uint32Array;
+}
+
+export type DiffSyntaxOutput =
+    | DiffSyntaxTokens
+    | { readonly type: 'ready' }
+    | { readonly type: 'error'; readonly message: string }
+    | { readonly type: 'failed'; readonly generation: number; readonly index: number; readonly reason: 'missing' | 'highlight'; readonly message: string };
+
+export interface DiffSyntaxService {
+    handle(message: DiffSyntaxInput): Promise<void>;
+    dispose(): void;
+}
+
+export interface DiffSyntaxServiceOptions {
+    readonly highlighter: () => Promise<Highlighter>;
+    readonly send: (message: DiffSyntaxOutput, transfer?: Transferable[]) => void;
+    readonly yield?: () => Promise<void>;
+}
+
+interface Entry {
+    document: DiffSyntaxDocument;
+    tokens: (index: number) => Promise<ThemedToken[][]>;
+    count: number;
+    done: number;
+    grammar?: unknown;
+    theme?: unknown;
+}
+
+interface Request {
+    generation: number;
+    index: number;
+    resolve: Array<() => void>;
+}
+
+function yieldToHost(): Promise<void> {
+    return new Promise((resolve) => {
+        const channel = new MessageChannel();
+        channel.port1.onmessage = () => {
+            channel.port1.close();
+            channel.port2.close();
+            resolve();
+        };
+        channel.port2.postMessage(null);
+    });
+}
+
+function packedTokens(
+    generation: number,
+    index: number,
+    code: string,
+    tokens: readonly ThemedToken[][],
+    appearance: { foreground: string; background: string }
+): DiffSyntaxTokens {
+    const colors: string[] = [''];
+    const colorIds = new Map([['', 0]]);
+    const starts = [0];
+    for (const match of code.matchAll(/\r\n|\n/g)) {
+        starts.push(match.index + match[0].length);
+    }
+    const lines = new Uint32Array(tokens.length + 1);
+    const packed = new Uint32Array(tokens.reduce((count, line) => count + line.length, 0) * 3);
+    let cursor = 0;
+    for (const [index, line] of tokens.entries()) {
+        lines[index] = cursor / 3;
+        for (const token of line) {
+            const color = token.color ?? '';
+            let colorId = colorIds.get(color);
+            if (colorId === undefined) {
+                colorId = colors.length;
+                colors.push(color);
+                colorIds.set(color, colorId);
+            }
+            packed[cursor++] = token.offset - starts[index]!;
+            packed[cursor++] = colorId;
+            packed[cursor++] = token.fontStyle ?? 0;
+        }
+    }
+    lines[tokens.length] = cursor / 3;
+    return { type: 'tokens', generation, index, colors, ...appearance, lines, tokens: packed };
+}
+
+/* The packet omits text and token lengths; its source line supplies the final token's end. */
+export function lineTokensOf(packet: DiffSyntaxTokens, line: number, text: string): LineToken[] {
+    if (!Number.isInteger(line) || line < 0 || line + 1 >= packet.lines.length) {
+        throw new RangeError('Unknown syntax line');
+    }
+    const result: LineToken[] = [];
+    const end = packet.lines[line + 1]!;
+    for (let index = packet.lines[line]!; index < end; index++) {
+        const from = packet.tokens[index * 3]!;
+        const to = index + 1 < end ? packet.tokens[(index + 1) * 3]! : text.length;
+        if (to > from) {
+            result.push({ length: to - from, color: packet.colors[packet.tokens[index * 3 + 1]!] ?? '', fontStyle: packet.tokens[index * 3 + 2]! });
+        }
+    }
+    return result;
+}
+
+export function createSyntaxService(options: DiffSyntaxServiceOptions): DiffSyntaxService {
+    const entries = new Map<number, Entry>();
+    const requests = new Map<string, Request>();
+    const pause = options.yield ?? yieldToHost;
+    let maxLines = 100000;
+    let count = 0;
+    let ready: Promise<Highlighter> | undefined;
+    let running = false;
+    let stopped = false;
+    let active: Entry | undefined;
+    const key = (generation: number, index: number): string => `${generation}:${index}`;
+    const cancel = (generation: number): void => {
+        for (const [id, request] of requests) {
+            if (request.generation === generation) {
+                requests.delete(id);
+                for (const resolve of request.resolve) {
+                    resolve();
+                }
+            }
+        }
+    };
+    const trim = (): void => {
+        for (const [generation, entry] of entries) {
+            if (count <= maxLines) {
+                break;
+            }
+            if (entry !== active) {
+                entries.delete(generation);
+                count -= entry.count;
+            }
+        }
+    };
+    const finish = (request: Request): void => {
+        const id = key(request.generation, request.index);
+        if (requests.get(id) === request) {
+            requests.delete(id);
+        }
+        for (const resolve of request.resolve) {
+            resolve();
+        }
+    };
+    const failed = (request: Request, reason: 'missing' | 'highlight', message: string): void => {
+        options.send({ type: 'failed', generation: request.generation, index: request.index, reason, message });
+        finish(request);
+    };
+    const grammarOf = (entry: Entry, loaded: Highlighter): unknown => {
+        const language = documentLanguageOf(entry.document.language);
+        return loaded.getLoadedLanguages().includes(language) ? loaded.getLanguage(language) : undefined;
+    };
+    const pump = async (): Promise<void> => {
+        if (running) {
+            return;
+        }
+        running = true;
+        try {
+            while (!stopped && requests.size > 0) {
+                const request = [...requests.values()].at(-1)!;
+                const id = key(request.generation, request.index);
+                const entry = entries.get(request.generation);
+                if (entry === undefined) {
+                    failed(request, 'missing', 'The syntax document was evicted; resend its definition.');
+                    continue;
+                }
+                active = entry;
+                entries.delete(request.generation);
+                entries.set(request.generation, entry);
+                try {
+                    const loaded = await (ready ??= options.highlighter());
+                    if (requests.get(id) !== request || entries.get(request.generation) !== entry || stopped) {
+                        continue;
+                    }
+                    if (entry.done >= 0 && (entry.grammar !== grammarOf(entry, loaded) || entry.theme !== loaded.getTheme(entry.document.theme))) {
+                        entry.done = -1;
+                    }
+                    if (request.index >= entry.document.blocks.length) {
+                        throw new RangeError('The syntax block index is outside the document.');
+                    }
+                    if ([...requests.values()].at(-1) !== request) {
+                        continue;
+                    }
+                    const index = Math.min(entry.done + 1, request.index);
+                    const tokens = await entry.tokens(index);
+                    entry.done = Math.max(entry.done, index);
+                    entry.grammar = grammarOf(entry, loaded);
+                    const theme = loaded.getTheme(entry.document.theme);
+                    entry.theme = theme;
+                    if (requests.get(id) === request && entries.get(request.generation) === entry && !stopped && index === request.index) {
+                        const raw = entry.document.blocks[index]!;
+                        const code = index < entry.document.blocks.length - 1 ? raw.replace(/\r$/, '') : raw;
+                        const packed = packedTokens(request.generation, index, code, tokens, { foreground: theme.fg, background: theme.bg });
+                        options.send(packed, [packed.lines.buffer, packed.tokens.buffer]);
+                        finish(request);
+                    }
+                } catch (error) {
+                    if (requests.get(id) === request && !stopped) {
+                        failed(request, 'highlight', error instanceof Error ? error.message : String(error));
+                    }
+                }
+                trim();
+                if (requests.size > 0 && !stopped) {
+                    await pause();
+                }
+            }
+        } finally {
+            active = undefined;
+            running = false;
+            trim();
+        }
+    };
+    const dispose = (): void => {
+        stopped = true;
+        for (const generation of entries.keys()) {
+            cancel(generation);
+        }
+        for (const request of requests.values()) {
+            finish(request);
+        }
+        entries.clear();
+        count = 0;
+    };
+    return {
+        dispose,
+        handle(message) {
+            if (stopped) {
+                return Promise.resolve();
+            }
+            if (message.type === 'start') {
+                if (!Number.isSafeInteger(message.maxLines) || message.maxLines < 1) {
+                    return Promise.reject(new RangeError('The syntax cache limit must be a positive safe integer.'));
+                }
+                maxLines = message.maxLines;
+                for (const generation of entries.keys()) {
+                    cancel(generation);
+                }
+                entries.clear();
+                count = 0;
+                ready = options.highlighter().then(async (loaded) => {
+                    const known = new Set(loaded.getLoadedThemes());
+                    const missing = message.themes.filter((theme) => theme.name === undefined || !known.has(theme.name));
+                    if (missing.length > 0) {
+                        await loaded.loadTheme(...missing);
+                    }
+                    return loaded;
+                });
+                return ready.then(
+                    () => options.send({ type: 'ready' }),
+                    (error: unknown) => options.send({ type: 'error', message: error instanceof Error ? error.message : String(error) })
+                );
+            }
+            if (message.type === 'cancel' || message.type === 'dispose') {
+                cancel(message.generation);
+                if (message.type === 'dispose') {
+                    const entry = entries.get(message.generation);
+                    if (entry !== undefined) {
+                        entries.delete(message.generation);
+                        count -= entry.count;
+                    }
+                }
+                return Promise.resolve();
+            }
+            if (!Number.isSafeInteger(message.generation) || message.generation < 1 || !Number.isSafeInteger(message.index) || message.index < 0) {
+                return Promise.reject(new RangeError('Syntax generations must be positive integers and block indexes nonnegative integers.'));
+            }
+            if (message.document !== undefined && !entries.has(message.generation)) {
+                const document = { ...message.document, blocks: [...message.document.blocks], resets: [...message.document.resets] };
+                const entry: Entry = {
+                    document,
+                    tokens: shikiDocumentTokens(() => (ready ??= options.highlighter()), document.blocks, document.language, document.theme, {
+                        reset: new Set(document.resets),
+                        // Shiki skips at the cutoff; include source lines of exactly 10,000 code units.
+                        tokenizeMaxLineLength: 10001,
+                        tokenizeTimeLimit: 20
+                    }),
+                    count: document.blocks.reduce((count, block) => count + block.split(/\r\n|\n/).length, 0),
+                    done: -1
+                };
+                count += entry.count;
+                entries.set(message.generation, entry);
+            }
+            return new Promise((resolve) => {
+                const id = key(message.generation, message.index);
+                const existing = requests.get(id);
+                const request: Request = existing ?? { generation: message.generation, index: message.index, resolve: [] };
+                request.resolve.push(resolve);
+                requests.delete(id);
+                requests.set(id, request);
+                void pump();
+            });
+        }
+    };
+}
+
+export interface DiffSyntaxWorkerEndpoint {
+    addEventListener(type: 'message', listener: (event: MessageEvent<DiffSyntaxInput>) => void): void;
+    removeEventListener(type: 'message', listener: (event: MessageEvent<DiffSyntaxInput>) => void): void;
+    postMessage(message: DiffSyntaxOutput, options?: { transfer: Transferable[] }): void;
+}
+
+export function serveDiffSyntax(endpoint: DiffSyntaxWorkerEndpoint): () => void {
+    let highlighter: Promise<Highlighter> | undefined;
+    const service = createSyntaxService({
+        highlighter: () => (highlighter ??= import('shiki').then(({ createHighlighter }) => createHighlighter({ langs: [], themes: [] }))),
+        send: (message, transfer) => endpoint.postMessage(message, { transfer: transfer ?? [] })
+    });
+    const listener = (event: MessageEvent<DiffSyntaxInput>): void => {
+        void service.handle(event.data).catch((error: unknown) => {
+            endpoint.postMessage({ type: 'error', message: error instanceof Error ? error.message : String(error) });
+        });
+    };
+    endpoint.addEventListener('message', listener);
+    return () => {
+        endpoint.removeEventListener('message', listener);
+        service.dispose();
+        void highlighter?.then(
+            (loaded) => loaded.dispose(),
+            () => undefined
+        );
+    };
+}

package/src/shiki.ts

@@ -1,7 +1,7 @@
 import type { GrammarState, Highlighter } from 'shiki';
 import { documentLanguageOf, loadDocumentLanguage } from './php-html.ts';
 import { scopeColorsOf, type ThemeRule } from './theme-scopes.ts';
-import type { LineToken, LineTokenizer, ScopeColorSource, TokenizerSource } from './types.ts';
+import type { LineToken, LineTokenKind, LineTokenizer, ScopeColorSource, TokenizerSource } from './types.ts';
 
 type LanguageId = Parameters<Highlighter['loadLanguage']>[0];
 type ThemeId = Parameters<Highlighter['loadTheme']>[0];
@@ -9,12 +9,16 @@ type TokenOptions = Parameters<Highlighter['codeToTokensBase']>[1];
 
 export { documentLanguageOf };
 
+/* The standard token types of TextMate: 1 comment, 2 string, 3 regex. */
+const KINDS: Record<number, LineTokenKind | undefined> = { 1: 'comment', 2: 'string', 3: 'regex' };
+
 const PLAIN_TEXT = new Set(['', 'text', 'txt', 'plain', 'plaintext']);
 
 /* Colors one line at a time and hands the grammar's state from one line to the next. A grammar Shiki rebuilt does not know the states of the old one. */
-function lineTokenizer(highlighter: Highlighter, language: string, theme: string): LineTokenizer {
+function lineTokenizer(highlighter: Highlighter, language: string, theme: string, embedded: EmbeddedIndex): LineTokenizer {
     let grammar = highlighter.getLanguage(language);
     return {
+        prepare: embedded.matcher === null ? undefined : (text) => loadEmbedded(highlighter, embedded, text),
         stale() {
             const current = highlighter.getLanguage(language);
             const changed = current !== grammar;
@@ -22,12 +26,18 @@ function lineTokenizer(highlighter: Highlighter, language: string, theme: string
             return changed;
         },
         tokenizeLine(text, state) {
-            const options = { lang: language, theme, grammarState: (state ?? undefined) as GrammarState | undefined } as TokenOptions;
+            const options = {
+                lang: language,
+                theme,
+                grammarState: (state ?? undefined) as GrammarState | undefined,
+                includeExplanation: 'tokenType'
+            } as TokenOptions;
             const lines = highlighter.codeToTokensBase(text, options);
             const tokens: LineToken[] = (lines[0] ?? []).map((token) => ({
                 length: token.content.length,
                 color: token.color ?? '',
-                fontStyle: token.fontStyle ?? 0
+                fontStyle: token.fontStyle ?? 0,
+                kind: KINDS[token.type ?? 0]
             }));
             return { tokens, state: highlighter.getLastGrammarState(lines) };
         },
@@ -42,41 +52,99 @@ function lineTokenizer(highlighter: Highlighter, language: string, theme: string
     };
 }
 
-/* The words of a document that name a language, an alias or the id itself, and not part of a longer word. */
-function mentions(text: string, names: readonly string[]): boolean {
-    return names.some((name) => new RegExp(`(?<![\\w-])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`, 'i').test(text));
+interface EmbeddedIndex {
+    readonly names: ReadonlyMap<string, string>;
+    readonly matcher: RegExp | null;
 }
 
-/*
- * A grammar embeds some languages only on demand (Vue's `lang="scss"` styles, the fences of Markdown),
- * and Shiki builds the host grammar anew each time one of them loads later, which drops the states
- * of every line colored before. So the ones a document names are loaded together with the host.
- */
-async function loadEmbedded(highlighter: Highlighter, grammar: string, text: string): Promise<void> {
-    const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
-    const bundled = bundledLanguages[grammar as keyof typeof bundledLanguages];
-    if (bundled === undefined) {
-        return;
+const languageLoads = new WeakMap<Highlighter, Map<string, Promise<void>>>();
+const embeddedIndexes = new Map<string, Promise<EmbeddedIndex>>();
+
+/* Concurrent documents await one load, so each learns the grammar was rebuilt. */
+function loadOnce(highlighter: Highlighter, id: string): Promise<void> {
+    if (highlighter.getLoadedLanguages().includes(id)) {
+        return Promise.resolve();
+    }
+    let loads = languageLoads.get(highlighter);
+    if (loads === undefined) {
+        loads = new Map();
+        languageLoads.set(highlighter, loads);
+    }
+    const pending = loads.get(id);
+    if (pending !== undefined) {
+        return pending;
     }
-    const registrations = (await bundled()).default as readonly { readonly name: string; readonly embeddedLangsLazy?: readonly string[] }[];
-    const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
-    const named = lazy.filter((id) => {
-        const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
-        return !highlighter.getLoadedLanguages().includes(id) && mentions(text, [id, ...aliases]);
+    const load = (async () => {
+        await loadDocumentLanguage(highlighter, id);
+        if (!highlighter.getLoadedLanguages().includes(id)) {
+            await highlighter.loadLanguage(id as LanguageId);
+        }
+    })();
+    loads.set(id, load);
+    void load.catch(() => {
+        if (loads.get(id) === load) {
+            loads.delete(id);
+        }
     });
-    await Promise.allSettled(named.map((id) => highlighter.loadLanguage(id as LanguageId)));
+    return load;
 }
 
-/*
- * Loads the grammar of a language with what a document in it embeds, and says whether Shiki has it.
- * `text` is the document, which decides the embedded languages that load only when named.
- */
-export async function loadGrammar(highlighter: Highlighter, grammar: string, text = ''): Promise<boolean> {
-    await loadDocumentLanguage(highlighter, grammar);
-    if (!highlighter.getLoadedLanguages().includes(grammar)) {
-        await highlighter.loadLanguage(grammar as LanguageId);
+function embedIndex(grammar: string): Promise<EmbeddedIndex> {
+    const cached = embeddedIndexes.get(grammar);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const pending = (async (): Promise<EmbeddedIndex> => {
+        const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
+        const bundled = bundledLanguages[grammar as keyof typeof bundledLanguages];
+        const registrations = bundled === undefined ? [] : (await bundled()).default;
+        const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
+        const names = new Map<string, string>();
+        for (const id of lazy) {
+            const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
+            for (const name of [id, ...aliases]) {
+                names.set(name.toLowerCase(), id);
+            }
+        }
+        if (names.size === 0) {
+            return { names, matcher: null };
+        }
+        let matcher: RegExp;
+        if (grammar === 'markdown' || grammar === 'mdx') {
+            matcher = /^\s*(?:`{3,}|~{3,})[ \t]*([\w#+.-]+)/gm;
+        } else if (grammar === 'vue' || grammar === 'svelte' || grammar === 'astro') {
+            matcher = /\blang\s*=\s*["']([\w#+.-]+)["']/gi;
+        } else {
+            const alternatives = [...names.keys()].map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
+            matcher = new RegExp(`(?<![\\w-])(${alternatives})(?![\\w-])`, 'gi');
+        }
+        return { names, matcher };
+    })();
+    embeddedIndexes.set(grammar, pending);
+    void pending.catch(() => embeddedIndexes.delete(grammar));
+    return pending;
+}
+
+async function loadEmbedded(highlighter: Highlighter, embedded: EmbeddedIndex, text: string): Promise<boolean> {
+    if (embedded.matcher === null) {
+        return false;
     }
-    await loadEmbedded(highlighter, grammar, text);
+    const loaded = new Set(highlighter.getLoadedLanguages());
+    const named = new Set<string>();
+    for (const match of text.matchAll(embedded.matcher)) {
+        const id = embedded.names.get(match[1]!.toLowerCase());
+        if (id !== undefined && !loaded.has(id)) {
+            named.add(id);
+        }
+    }
+    const results = await Promise.allSettled([...named].map((id) => loadOnce(highlighter, id)));
+    return results.some((result) => result.status === 'fulfilled');
+}
+
+/* Text names the optional embedded grammars that must load before the host hands out states. */
+export async function loadGrammar(highlighter: Highlighter, grammar: string, text = ''): Promise<boolean> {
+    await loadOnce(highlighter, grammar);
+    await loadEmbedded(highlighter, await embedIndex(grammar), text);
     return highlighter.getLoadedLanguages().includes(grammar);
 }
 
@@ -95,7 +163,7 @@ export function shikiTokenizers(highlighter: () => Promise<Highlighter>): Tokeni
             if (!shiki.getLoadedThemes().includes(theme)) {
                 await shiki.loadTheme(theme as ThemeId);
             }
-            return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme) : null;
+            return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme, await embedIndex(grammar)) : null;
         } catch {
             return null;
         }
@@ -116,3 +184,5 @@ export function shikiScopeColors(highlighter: () => Promise<Highlighter>): Scope
         }
     };
 }
+
+export { shikiDocumentHighlighter, shikiDocumentTokens, type ShikiDocumentTokenOptions } from './shiki-document.ts';