@adecore/editor 0.39.0 → 0.40.0

package/src/semantic.ts

@@ -17,27 +17,35 @@ export interface SemanticSpan {
  */
 export function overlayTokens(base: readonly LineToken[], spans: readonly SemanticSpan[]): LineToken[] {
     const result: LineToken[] = [];
+
     const push = (length: number, color: string, fontStyle: number): void => {
         if (length <= 0) {
             return;
         }
+
         const last = result[result.length - 1];
+
         if (last !== undefined && last.color === color && last.fontStyle === fontStyle) {
             result[result.length - 1] = { length: last.length + length, color, fontStyle };
         } else {
             result.push({ length, color, fontStyle });
         }
     };
+
     let span = 0;
     let start = 0;
+
     for (const token of base) {
         const end = start + token.length;
         let at = start;
+
         while (at < end) {
             while (span < spans.length && spans[span]!.to <= at) {
                 span++;
             }
+
             const current = spans[span];
+
             if (current === undefined || current.from >= end) {
                 push(end - at, token.color, token.fontStyle);
                 at = end;
@@ -46,15 +54,19 @@ export function overlayTokens(base: readonly LineToken[], spans: readonly Semant
                 at = current.from;
             } else {
                 const stop = Math.min(end, current.to);
+
                 if (current.within === undefined || token.kind === current.within) {
                     push(stop - at, current.color, current.fontStyle);
                 } else {
                     push(stop - at, token.color, token.fontStyle);
                 }
+
                 at = stop;
             }
         }
+
         start = end;
     }
+
     return result;
 }

package/src/shiki-document.ts

@@ -18,6 +18,7 @@ export function shikiDocumentHighlighter(
 ): (index: number) => Promise<string> {
     return documentHighlighter(highlighter, blocks, language, theme, (loaded, code, grammar, theme, state) => {
         let root: Parameters<NonNullable<ShikiTransformer['root']>>[0] | undefined;
+
         const value = loaded.codeToHtml(code, {
             lang: grammar,
             theme,
@@ -31,6 +32,7 @@ export function shikiDocumentHighlighter(
                 }
             ]
         });
+
         return { value, state: root === undefined ? undefined : loaded.getLastGrammarState(root) };
     });
 }
@@ -56,6 +58,7 @@ export function shikiDocumentTokens(
                 tokenizeMaxLineLength: options.tokenizeMaxLineLength,
                 tokenizeTimeLimit: options.tokenizeTimeLimit
             } as TokenOptions);
+
             return { value, state: loaded.getLastGrammarState(value) };
         },
         options.reset
@@ -88,13 +91,18 @@ function documentHighlighter<Value>(
 
     const prepare = async (): Promise<{ highlighter: Highlighter; language: string }> => {
         const loaded = await highlighter();
+
         if (!loaded.getLoadedThemes().includes(theme)) {
             await loaded.loadTheme(theme as Parameters<Highlighter['loadTheme']>[0]);
         }
+
         const documentLanguage = documentLanguageOf(language);
+
         const known =
             loaded.getLoadedLanguages().includes(documentLanguage) || documentLanguage in loaded.getBundledLanguages() || documentLanguage !== language;
+
         const grammar = known && (await loadGrammar(loaded, documentLanguage, code.join('\n'))) ? documentLanguage : 'text';
+
         return { highlighter: loaded, language: grammar };
     };
 
@@ -102,15 +110,20 @@ function documentHighlighter<Value>(
         if (!Number.isInteger(index) || index < 0 || index >= code.length) {
             return Promise.reject(new RangeError('The highlighted block index is outside the document.'));
         }
+
         const result = pending.then(async () => {
             prepared ??= prepare().catch((error: unknown) => {
                 prepared = undefined;
+
                 throw error;
             });
+
             const { highlighter: loaded, language: grammarId } = await prepared;
+
             const refresh = (): void => {
                 const currentGrammar = grammarId === 'text' ? undefined : loaded.getLanguage(grammarId);
                 const currentTheme = loaded.getTheme(theme);
+
                 // Loading an embedded language rebuilds Shiki's grammar and invalidates its old states.
                 if (grammar !== currentGrammar || themeRegistration !== currentTheme) {
                     grammar = currentGrammar;
@@ -119,25 +132,31 @@ function documentHighlighter<Value>(
                     values = [];
                 }
             };
+
             refresh();
+
             while (values.length <= index) {
                 const block = values.length;
                 const rendered = render(loaded, code[block]!, grammarId, theme, state);
                 state = resets.has(block + 1) ? undefined : rendered.state;
                 values.push(rendered.value);
+
                 if (block < index) {
                     // A jump deep into a file must leave room for input and painting between blocks.
                     await new Promise<void>((resolve) => setTimeout(resolve, 0));
                     refresh();
                 }
             }
+
             return values[index]!;
         });
+
         // A failed theme or grammar load can be retried without rejecting every later request.
         pending = result.then(
             () => undefined,
             () => undefined
         );
+
         return result;
     };
 }

package/src/shiki-worker.ts

@@ -63,11 +63,13 @@ interface Request {
 function yieldToHost(): Promise<void> {
     return new Promise((resolve) => {
         const channel = new MessageChannel();
+
         channel.port1.onmessage = () => {
             channel.port1.close();
             channel.port2.close();
             resolve();
         };
+
         channel.port2.postMessage(null);
     });
 }
@@ -82,28 +84,36 @@ function packedTokens(
     const colors: string[] = [''];
     const colorIds = new Map([['', 0]]);
     const starts = [0];
+
     for (const match of code.matchAll(/\r\n|\n/g)) {
         starts.push(match.index + match[0].length);
     }
+
     const lines = new Uint32Array(tokens.length + 1);
     const packed = new Uint32Array(tokens.reduce((count, line) => count + line.length, 0) * 3);
     let cursor = 0;
+
     for (const [index, line] of tokens.entries()) {
         lines[index] = cursor / 3;
+
         for (const token of line) {
             const color = token.color ?? '';
             let colorId = colorIds.get(color);
+
             if (colorId === undefined) {
                 colorId = colors.length;
                 colors.push(color);
                 colorIds.set(color, colorId);
             }
+
             packed[cursor++] = token.offset - starts[index]!;
             packed[cursor++] = colorId;
             packed[cursor++] = token.fontStyle ?? 0;
         }
     }
+
     lines[tokens.length] = cursor / 3;
+
     return { type: 'tokens', generation, index, colors, ...appearance, lines, tokens: packed };
 }
 
@@ -111,9 +121,11 @@ function packedTokens(
 async function loadMissingThemes(loaded: Highlighter, themes: readonly ThemeRegistration[]): Promise<Highlighter> {
     const known = new Set(loaded.getLoadedThemes());
     const missing = themes.filter((theme) => theme.name === undefined || !known.has(theme.name));
+
     if (missing.length > 0) {
         await loaded.loadTheme(...missing);
     }
+
     return loaded;
 }
 
@@ -122,15 +134,19 @@ export function lineTokensOf(packet: DiffSyntaxTokens, line: number, text: strin
     if (!Number.isInteger(line) || line < 0 || line + 1 >= packet.lines.length) {
         throw new RangeError('Unknown syntax line');
     }
+
     const result: LineToken[] = [];
     const end = packet.lines[line + 1]!;
+
     for (let index = packet.lines[line]!; index < end; index++) {
         const from = packet.tokens[index * 3]!;
         const to = index + 1 < end ? packet.tokens[(index + 1) * 3]! : text.length;
+
         if (to > from) {
             result.push({ length: to - from, color: packet.colors[packet.tokens[index * 3 + 1]!] ?? '', fontStyle: packet.tokens[index * 3 + 2]! });
         }
     }
+
     return result;
 }
 
@@ -144,82 +160,106 @@ export function createSyntaxService(options: DiffSyntaxServiceOptions): DiffSynt
     let running = false;
     let stopped = false;
     let active: Entry | undefined;
+
     const key = (generation: number, index: number): string => `${generation}:${index}`;
+
     const cancel = (generation: number): void => {
         for (const [id, request] of requests) {
             if (request.generation === generation) {
                 requests.delete(id);
+
                 for (const resolve of request.resolve) {
                     resolve();
                 }
             }
         }
     };
+
     const trim = (): void => {
         for (const [generation, entry] of entries) {
             if (count <= maxLines) {
                 break;
             }
+
             if (entry !== active) {
                 entries.delete(generation);
                 count -= entry.count;
             }
         }
     };
+
     const finish = (request: Request): void => {
         const id = key(request.generation, request.index);
+
         if (requests.get(id) === request) {
             requests.delete(id);
         }
+
         for (const resolve of request.resolve) {
             resolve();
         }
     };
+
     const failed = (request: Request, reason: 'missing' | 'highlight', message: string): void => {
         options.send({ type: 'failed', generation: request.generation, index: request.index, reason, message });
         finish(request);
     };
+
     const grammarOf = (entry: Entry, loaded: Highlighter): unknown => {
         const language = documentLanguageOf(entry.document.language);
+
         return loaded.getLoadedLanguages().includes(language) ? loaded.getLanguage(language) : undefined;
     };
+
     const pump = async (): Promise<void> => {
         if (running) {
             return;
         }
+
         running = true;
+
         try {
             while (!stopped && requests.size > 0) {
                 const request = [...requests.values()].at(-1)!;
                 const id = key(request.generation, request.index);
                 const entry = entries.get(request.generation);
+
                 if (entry === undefined) {
                     failed(request, 'missing', 'The syntax document was evicted; resend its definition.');
+
                     continue;
                 }
+
                 active = entry;
                 entries.delete(request.generation);
                 entries.set(request.generation, entry);
+
                 try {
                     const loaded = await (ready ??= options.highlighter());
+
                     if (requests.get(id) !== request || entries.get(request.generation) !== entry || stopped) {
                         continue;
                     }
+
                     if (entry.done >= 0 && (entry.grammar !== grammarOf(entry, loaded) || entry.theme !== loaded.getTheme(entry.document.theme))) {
                         entry.done = -1;
                     }
+
                     if (request.index >= entry.document.blocks.length) {
                         throw new RangeError('The syntax block index is outside the document.');
                     }
+
                     if ([...requests.values()].at(-1) !== request) {
                         continue;
                     }
+
                     const index = Math.min(entry.done + 1, request.index);
                     const tokens = await entry.tokens(index);
                     entry.done = Math.max(entry.done, index);
                     entry.grammar = grammarOf(entry, loaded);
                     const theme = loaded.getTheme(entry.document.theme);
                     entry.theme = theme;
+
                     if (requests.get(id) === request && entries.get(request.generation) === entry && !stopped && index === request.index) {
                         const raw = entry.document.blocks[index]!;
                         const code = index < entry.document.blocks.length - 1 ? raw.replace(/\r$/, '') : raw;
@@ -232,7 +272,9 @@ export function createSyntaxService(options: DiffSyntaxServiceOptions): DiffSynt
                         failed(request, 'highlight', error instanceof Error ? error.message : String(error));
                     }
                 }
+
                 trim();
+
                 if (requests.size > 0 && !stopped) {
                     await pause();
                 }
@@ -243,67 +285,86 @@ export function createSyntaxService(options: DiffSyntaxServiceOptions): DiffSynt
             trim();
         }
     };
+
     const dispose = (): void => {
         stopped = true;
+
         for (const generation of entries.keys()) {
             cancel(generation);
         }
+
         for (const request of requests.values()) {
             finish(request);
         }
+
         entries.clear();
         count = 0;
     };
+
     return {
         dispose,
         handle(message) {
             if (stopped) {
                 return Promise.resolve();
             }
+
             if (message.type === 'start') {
                 if (!Number.isSafeInteger(message.maxLines) || message.maxLines < 1) {
                     return Promise.reject(new RangeError('The syntax cache limit must be a positive safe integer.'));
                 }
+
                 maxLines = message.maxLines;
+
                 for (const generation of entries.keys()) {
                     cancel(generation);
                 }
+
                 entries.clear();
                 count = 0;
                 ready = options.highlighter().then((loaded) => loadMissingThemes(loaded, message.themes));
+
                 return ready.then(
                     () => options.send({ type: 'ready' }),
                     (error: unknown) => options.send({ type: 'error', message: error instanceof Error ? error.message : String(error) })
                 );
             }
+
             if (message.type === 'themes') {
                 // The queue awaits `ready` anew for every request, so a request sent after this one draws in these themes.
                 ready = (ready ?? options.highlighter()).then((loaded) =>
                     // A theme that fails to load fails only the requests drawn in it, never the ones in themes already in.
                     loadMissingThemes(loaded, message.themes).catch(() => loaded)
                 );
+
                 // A failed start has already reported its error.
                 return ready.then(
                     () => undefined,
                     () => undefined
                 );
             }
+
             if (message.type === 'cancel' || message.type === 'dispose') {
                 cancel(message.generation);
+
                 if (message.type === 'dispose') {
                     const entry = entries.get(message.generation);
+
                     if (entry !== undefined) {
                         entries.delete(message.generation);
                         count -= entry.count;
                     }
                 }
+
                 return Promise.resolve();
             }
+
             if (!Number.isSafeInteger(message.generation) || message.generation < 1 || !Number.isSafeInteger(message.index) || message.index < 0) {
                 return Promise.reject(new RangeError('Syntax generations must be positive integers and block indexes nonnegative integers.'));
             }
+
             if (message.document !== undefined && !entries.has(message.generation)) {
                 const document = { ...message.document, blocks: [...message.document.blocks], resets: [...message.document.resets] };
+
                 const entry: Entry = {
                     document,
                     tokens: shikiDocumentTokens(() => (ready ??= options.highlighter()), document.blocks, document.language, document.theme, {
@@ -315,9 +376,11 @@ export function createSyntaxService(options: DiffSyntaxServiceOptions): DiffSynt
                     count: document.blocks.reduce((count, block) => count + block.split(/\r\n|\n/).length, 0),
                     done: -1
                 };
+
                 count += entry.count;
                 entries.set(message.generation, entry);
             }
+
             return new Promise((resolve) => {
                 const id = key(message.generation, message.index);
                 const existing = requests.get(id);
@@ -339,19 +402,24 @@ export interface DiffSyntaxWorkerEndpoint {
 
 export function serveDiffSyntax(endpoint: DiffSyntaxWorkerEndpoint): () => void {
     let highlighter: Promise<Highlighter> | undefined;
+
     const service = createSyntaxService({
         highlighter: () => (highlighter ??= import('shiki').then(({ createHighlighter }) => createHighlighter({ langs: [], themes: [] }))),
         send: (message, transfer) => endpoint.postMessage(message, { transfer: transfer ?? [] })
     });
+
     const listener = (event: MessageEvent<DiffSyntaxInput>): void => {
         void service.handle(event.data).catch((error: unknown) => {
             endpoint.postMessage({ type: 'error', message: error instanceof Error ? error.message : String(error) });
         });
     };
+
     endpoint.addEventListener('message', listener);
+
     return () => {
         endpoint.removeEventListener('message', listener);
         service.dispose();
+
         void highlighter?.then(
             (loaded) => loaded.dispose(),
             () => undefined

package/src/shiki.ts

@@ -17,12 +17,14 @@ const PLAIN_TEXT = new Set(['', 'text', 'txt', 'plain', 'plaintext']);
 /* Colors one line at a time and hands the grammar's state from one line to the next. A grammar Shiki rebuilt does not know the states of the old one. */
 function lineTokenizer(highlighter: Highlighter, language: string, theme: string, embedded: EmbeddedIndex): LineTokenizer {
     let grammar = highlighter.getLanguage(language);
+
     return {
         prepare: embedded.matcher === null ? undefined : (text) => loadEmbedded(highlighter, embedded, text),
         stale() {
             const current = highlighter.getLanguage(language);
             const changed = current !== grammar;
             grammar = current;
+
             return changed;
         },
         tokenizeLine(text, state) {
@@ -32,21 +34,26 @@ function lineTokenizer(highlighter: Highlighter, language: string, theme: string
                 grammarState: (state ?? undefined) as GrammarState | undefined,
                 includeExplanation: 'tokenType'
             } as TokenOptions;
+
             const lines = highlighter.codeToTokensBase(text, options);
+
             const tokens: LineToken[] = (lines[0] ?? []).map((token) => ({
                 length: token.content.length,
                 color: token.color ?? '',
                 fontStyle: token.fontStyle ?? 0,
                 kind: KINDS[token.type ?? 0]
             }));
+
             return { tokens, state: highlighter.getLastGrammarState(lines) };
         },
         sameState(a, b) {
             if (a === b) {
                 return true;
             }
+
             const left = (a as GrammarState | null)?.getInternalStack(theme);
             const right = (b as GrammarState | null)?.getInternalStack(theme);
+
             return left !== undefined && right !== undefined && left.equals(right);
         }
     };
@@ -65,51 +72,67 @@ function loadOnce(highlighter: Highlighter, id: string): Promise<void> {
     if (highlighter.getLoadedLanguages().includes(id)) {
         return Promise.resolve();
     }
+
     let loads = languageLoads.get(highlighter);
+
     if (loads === undefined) {
         loads = new Map();
         languageLoads.set(highlighter, loads);
     }
+
     const pending = loads.get(id);
+
     if (pending !== undefined) {
         return pending;
     }
+
     const load = (async () => {
         await loadDocumentLanguage(highlighter, id);
+
         if (!highlighter.getLoadedLanguages().includes(id)) {
             await highlighter.loadLanguage(id as LanguageId);
         }
     })();
+
     loads.set(id, load);
+
     void load.catch(() => {
         if (loads.get(id) === load) {
             loads.delete(id);
         }
     });
+
     return load;
 }
 
 function embedIndex(grammar: string): Promise<EmbeddedIndex> {
     const cached = embeddedIndexes.get(grammar);
+
     if (cached !== undefined) {
         return cached;
     }
+
     const pending = (async (): Promise<EmbeddedIndex> => {
         const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
         const bundled = bundledLanguages[grammar as keyof typeof bundledLanguages];
         const registrations = bundled === undefined ? [] : (await bundled()).default;
         const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
         const names = new Map<string, string>();
+
         for (const id of lazy) {
             const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
+
             for (const name of [id, ...aliases]) {
                 names.set(name.toLowerCase(), id);
             }
         }
+
         if (names.size === 0) {
             return { names, matcher: null };
         }
+
         let matcher: RegExp;
+
         if (grammar === 'markdown' || grammar === 'mdx') {
             matcher = /^\s*(?:`{3,}|~{3,})[ \t]*([\w#+.-]+)/gm;
         } else if (grammar === 'vue' || grammar === 'svelte' || grammar === 'astro') {
@@ -118,10 +141,13 @@ function embedIndex(grammar: string): Promise<EmbeddedIndex> {
             const alternatives = [...names.keys()].map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
             matcher = new RegExp(`(?<![\\w-])(${alternatives})(?![\\w-])`, 'gi');
         }
+
         return { names, matcher };
     })();
+
     embeddedIndexes.set(grammar, pending);
     void pending.catch(() => embeddedIndexes.delete(grammar));
+
     return pending;
 }
 
@@ -129,15 +155,20 @@ async function loadEmbedded(highlighter: Highlighter, embedded: EmbeddedIndex, t
     if (embedded.matcher === null) {
         return false;
     }
+
     const loaded = new Set(highlighter.getLoadedLanguages());
     const named = new Set<string>();
+
     for (const match of text.matchAll(embedded.matcher)) {
         const id = embedded.names.get(match[1]!.toLowerCase());
+
         if (id !== undefined && !loaded.has(id)) {
             named.add(id);
         }
     }
+
     const results = await Promise.allSettled([...named].map((id) => loadOnce(highlighter, id)));
+
     return results.some((result) => result.status === 'fulfilled');
 }
 
@@ -145,6 +176,7 @@ async function loadEmbedded(highlighter: Highlighter, embedded: EmbeddedIndex, t
 export async function loadGrammar(highlighter: Highlighter, grammar: string, text = ''): Promise<boolean> {
     await loadOnce(highlighter, grammar);
     await loadEmbedded(highlighter, await embedIndex(grammar), text);
+
     return highlighter.getLoadedLanguages().includes(grammar);
 }
 
@@ -157,12 +189,15 @@ export function shikiTokenizers(highlighter: () => Promise<Highlighter>): Tokeni
         if (language === undefined || PLAIN_TEXT.has(language)) {
             return null;
         }
+
         const grammar = documentLanguageOf(language);
         const shiki = await highlighter();
+
         try {
             if (!shiki.getLoadedThemes().includes(theme)) {
                 await shiki.loadTheme(theme as ThemeId);
             }
+
             return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme, await embedIndex(grammar)) : null;
         } catch {
             return null;
@@ -174,10 +209,12 @@ export function shikiTokenizers(highlighter: () => Promise<Highlighter>): Tokeni
 export function shikiScopeColors(highlighter: () => Promise<Highlighter>): ScopeColorSource {
     return async (theme) => {
         const shiki = await highlighter();
+
         try {
             if (!shiki.getLoadedThemes().includes(theme)) {
                 await shiki.loadTheme(theme as ThemeId);
             }
+
             return scopeColorsOf(shiki.getTheme(theme).settings as readonly ThemeRule[]);
         } catch {
             return null;