@adecore/editor 0.22.0 → 0.23.0

package/dist/shiki-worker.js

@@ -0,0 +1,265 @@
+import { shikiDocumentTokens } from './shiki-document.js';
+import { documentLanguageOf } from './shiki.js';
+function yieldToHost() {
+    return new Promise((resolve) => {
+        const channel = new MessageChannel();
+        channel.port1.onmessage = () => {
+            channel.port1.close();
+            channel.port2.close();
+            resolve();
+        };
+        channel.port2.postMessage(null);
+    });
+}
+function packedTokens(generation, index, code, tokens, appearance) {
+    const colors = [''];
+    const colorIds = new Map([['', 0]]);
+    const starts = [0];
+    for (const match of code.matchAll(/\r\n|\n/g)) {
+        starts.push(match.index + match[0].length);
+    }
+    const lines = new Uint32Array(tokens.length + 1);
+    const packed = new Uint32Array(tokens.reduce((count, line) => count + line.length, 0) * 3);
+    let cursor = 0;
+    for (const [index, line] of tokens.entries()) {
+        lines[index] = cursor / 3;
+        for (const token of line) {
+            const color = token.color ?? '';
+            let colorId = colorIds.get(color);
+            if (colorId === undefined) {
+                colorId = colors.length;
+                colors.push(color);
+                colorIds.set(color, colorId);
+            }
+            packed[cursor++] = token.offset - starts[index];
+            packed[cursor++] = colorId;
+            packed[cursor++] = token.fontStyle ?? 0;
+        }
+    }
+    lines[tokens.length] = cursor / 3;
+    return { type: 'tokens', generation, index, colors, ...appearance, lines, tokens: packed };
+}
+/* The packet omits text and token lengths; its source line supplies the final token's end. */
+export function lineTokensOf(packet, line, text) {
+    if (!Number.isInteger(line) || line < 0 || line + 1 >= packet.lines.length) {
+        throw new RangeError('Unknown syntax line');
+    }
+    const result = [];
+    const end = packet.lines[line + 1];
+    for (let index = packet.lines[line]; index < end; index++) {
+        const from = packet.tokens[index * 3];
+        const to = index + 1 < end ? packet.tokens[(index + 1) * 3] : text.length;
+        if (to > from) {
+            result.push({ length: to - from, color: packet.colors[packet.tokens[index * 3 + 1]] ?? '', fontStyle: packet.tokens[index * 3 + 2] });
+        }
+    }
+    return result;
+}
+export function createSyntaxService(options) {
+    const entries = new Map();
+    const requests = new Map();
+    const pause = options.yield ?? yieldToHost;
+    let maxLines = 100000;
+    let count = 0;
+    let ready;
+    let running = false;
+    let stopped = false;
+    let active;
+    const key = (generation, index) => `${generation}:${index}`;
+    const cancel = (generation) => {
+        for (const [id, request] of requests) {
+            if (request.generation === generation) {
+                requests.delete(id);
+                for (const resolve of request.resolve) {
+                    resolve();
+                }
+            }
+        }
+    };
+    const trim = () => {
+        for (const [generation, entry] of entries) {
+            if (count <= maxLines) {
+                break;
+            }
+            if (entry !== active) {
+                entries.delete(generation);
+                count -= entry.count;
+            }
+        }
+    };
+    const finish = (request) => {
+        const id = key(request.generation, request.index);
+        if (requests.get(id) === request) {
+            requests.delete(id);
+        }
+        for (const resolve of request.resolve) {
+            resolve();
+        }
+    };
+    const failed = (request, reason, message) => {
+        options.send({ type: 'failed', generation: request.generation, index: request.index, reason, message });
+        finish(request);
+    };
+    const grammarOf = (entry, loaded) => {
+        const language = documentLanguageOf(entry.document.language);
+        return loaded.getLoadedLanguages().includes(language) ? loaded.getLanguage(language) : undefined;
+    };
+    const pump = async () => {
+        if (running) {
+            return;
+        }
+        running = true;
+        try {
+            while (!stopped && requests.size > 0) {
+                const request = [...requests.values()].at(-1);
+                const id = key(request.generation, request.index);
+                const entry = entries.get(request.generation);
+                if (entry === undefined) {
+                    failed(request, 'missing', 'The syntax document was evicted; resend its definition.');
+                    continue;
+                }
+                active = entry;
+                entries.delete(request.generation);
+                entries.set(request.generation, entry);
+                try {
+                    const loaded = await (ready ??= options.highlighter());
+                    if (requests.get(id) !== request || entries.get(request.generation) !== entry || stopped) {
+                        continue;
+                    }
+                    if (entry.done >= 0 && (entry.grammar !== grammarOf(entry, loaded) || entry.theme !== loaded.getTheme(entry.document.theme))) {
+                        entry.done = -1;
+                    }
+                    if (request.index >= entry.document.blocks.length) {
+                        throw new RangeError('The syntax block index is outside the document.');
+                    }
+                    if ([...requests.values()].at(-1) !== request) {
+                        continue;
+                    }
+                    const index = Math.min(entry.done + 1, request.index);
+                    const tokens = await entry.tokens(index);
+                    entry.done = Math.max(entry.done, index);
+                    entry.grammar = grammarOf(entry, loaded);
+                    const theme = loaded.getTheme(entry.document.theme);
+                    entry.theme = theme;
+                    if (requests.get(id) === request && entries.get(request.generation) === entry && !stopped && index === request.index) {
+                        const raw = entry.document.blocks[index];
+                        const code = index < entry.document.blocks.length - 1 ? raw.replace(/\r$/, '') : raw;
+                        const packed = packedTokens(request.generation, index, code, tokens, { foreground: theme.fg, background: theme.bg });
+                        options.send(packed, [packed.lines.buffer, packed.tokens.buffer]);
+                        finish(request);
+                    }
+                }
+                catch (error) {
+                    if (requests.get(id) === request && !stopped) {
+                        failed(request, 'highlight', error instanceof Error ? error.message : String(error));
+                    }
+                }
+                trim();
+                if (requests.size > 0 && !stopped) {
+                    await pause();
+                }
+            }
+        }
+        finally {
+            active = undefined;
+            running = false;
+            trim();
+        }
+    };
+    const dispose = () => {
+        stopped = true;
+        for (const generation of entries.keys()) {
+            cancel(generation);
+        }
+        for (const request of requests.values()) {
+            finish(request);
+        }
+        entries.clear();
+        count = 0;
+    };
+    return {
+        dispose,
+        handle(message) {
+            if (stopped) {
+                return Promise.resolve();
+            }
+            if (message.type === 'start') {
+                if (!Number.isSafeInteger(message.maxLines) || message.maxLines < 1) {
+                    return Promise.reject(new RangeError('The syntax cache limit must be a positive safe integer.'));
+                }
+                maxLines = message.maxLines;
+                for (const generation of entries.keys()) {
+                    cancel(generation);
+                }
+                entries.clear();
+                count = 0;
+                ready = options.highlighter().then(async (loaded) => {
+                    const known = new Set(loaded.getLoadedThemes());
+                    const missing = message.themes.filter((theme) => theme.name === undefined || !known.has(theme.name));
+                    if (missing.length > 0) {
+                        await loaded.loadTheme(...missing);
+                    }
+                    return loaded;
+                });
+                return ready.then(() => options.send({ type: 'ready' }), (error) => options.send({ type: 'error', message: error instanceof Error ? error.message : String(error) }));
+            }
+            if (message.type === 'cancel' || message.type === 'dispose') {
+                cancel(message.generation);
+                if (message.type === 'dispose') {
+                    const entry = entries.get(message.generation);
+                    if (entry !== undefined) {
+                        entries.delete(message.generation);
+                        count -= entry.count;
+                    }
+                }
+                return Promise.resolve();
+            }
+            if (!Number.isSafeInteger(message.generation) || message.generation < 1 || !Number.isSafeInteger(message.index) || message.index < 0) {
+                return Promise.reject(new RangeError('Syntax generations must be positive integers and block indexes nonnegative integers.'));
+            }
+            if (message.document !== undefined && !entries.has(message.generation)) {
+                const document = { ...message.document, blocks: [...message.document.blocks], resets: [...message.document.resets] };
+                const entry = {
+                    document,
+                    tokens: shikiDocumentTokens(() => (ready ??= options.highlighter()), document.blocks, document.language, document.theme, {
+                        reset: new Set(document.resets),
+                        // Shiki skips at the cutoff; include source lines of exactly 10,000 code units.
+                        tokenizeMaxLineLength: 10001,
+                        tokenizeTimeLimit: 20
+                    }),
+                    count: document.blocks.reduce((count, block) => count + block.split(/\r\n|\n/).length, 0),
+                    done: -1
+                };
+                count += entry.count;
+                entries.set(message.generation, entry);
+            }
+            return new Promise((resolve) => {
+                const id = key(message.generation, message.index);
+                const existing = requests.get(id);
+                const request = existing ?? { generation: message.generation, index: message.index, resolve: [] };
+                request.resolve.push(resolve);
+                requests.delete(id);
+                requests.set(id, request);
+                void pump();
+            });
+        }
+    };
+}
+export function serveDiffSyntax(endpoint) {
+    let highlighter;
+    const service = createSyntaxService({
+        highlighter: () => (highlighter ??= import('shiki').then(({ createHighlighter }) => createHighlighter({ langs: [], themes: [] }))),
+        send: (message, transfer) => endpoint.postMessage(message, { transfer: transfer ?? [] })
+    });
+    const listener = (event) => {
+        void service.handle(event.data).catch((error) => {
+            endpoint.postMessage({ type: 'error', message: error instanceof Error ? error.message : String(error) });
+        });
+    };
+    endpoint.addEventListener('message', listener);
+    return () => {
+        endpoint.removeEventListener('message', listener);
+        service.dispose();
+        void highlighter?.then((loaded) => loaded.dispose(), () => undefined);
+    };
+}

package/dist/shiki.d.ts

@@ -5,3 +5,4 @@ export { documentLanguageOf };
 export declare function loadGrammar(highlighter: Highlighter, grammar: string, text?: string): Promise<boolean>;
 export declare function shikiTokenizers(highlighter: () => Promise<Highlighter>): TokenizerSource;
 export declare function shikiScopeColors(highlighter: () => Promise<Highlighter>): ScopeColorSource;
+export { shikiDocumentHighlighter, shikiDocumentTokens, type ShikiDocumentTokenOptions } from './shiki-document.ts';

package/dist/shiki.js

@@ -1,11 +1,14 @@
 import { documentLanguageOf, loadDocumentLanguage } from './php-html.js';
 import { scopeColorsOf } from './theme-scopes.js';
 export { documentLanguageOf };
+/* The standard token types of TextMate: 1 comment, 2 string, 3 regex. */
+const KINDS = { 1: 'comment', 2: 'string', 3: 'regex' };
 const PLAIN_TEXT = new Set(['', 'text', 'txt', 'plain', 'plaintext']);
 /* Colors one line at a time and hands the grammar's state from one line to the next. A grammar Shiki rebuilt does not know the states of the old one. */
-function lineTokenizer(highlighter, language, theme) {
+function lineTokenizer(highlighter, language, theme, embedded) {
     let grammar = highlighter.getLanguage(language);
     return {
+        prepare: embedded.matcher === null ? undefined : (text) => loadEmbedded(highlighter, embedded, text),
         stale() {
             const current = highlighter.getLanguage(language);
             const changed = current !== grammar;
@@ -13,12 +16,18 @@ function lineTokenizer(highlighter, language, theme) {
             return changed;
         },
         tokenizeLine(text, state) {
-            const options = { lang: language, theme, grammarState: (state ?? undefined) };
+            const options = {
+                lang: language,
+                theme,
+                grammarState: (state ?? undefined),
+                includeExplanation: 'tokenType'
+            };
             const lines = highlighter.codeToTokensBase(text, options);
             const tokens = (lines[0] ?? []).map((token) => ({
                 length: token.content.length,
                 color: token.color ?? '',
-                fontStyle: token.fontStyle ?? 0
+                fontStyle: token.fontStyle ?? 0,
+                kind: KINDS[token.type ?? 0]
             }));
             return { tokens, state: highlighter.getLastGrammarState(lines) };
         },
@@ -32,39 +41,92 @@ function lineTokenizer(highlighter, language, theme) {
         }
     };
 }
-/* The words of a document that name a language, an alias or the id itself, and not part of a longer word. */
-function mentions(text, names) {
-    return names.some((name) => new RegExp(`(?<![\\w-])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`, 'i').test(text));
-}
-/*
- * A grammar embeds some languages only on demand (Vue's `lang="scss"` styles, the fences of Markdown),
- * and Shiki builds the host grammar anew each time one of them loads later, which drops the states
- * of every line colored before. So the ones a document names are loaded together with the host.
- */
-async function loadEmbedded(highlighter, grammar, text) {
-    const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
-    const bundled = bundledLanguages[grammar];
-    if (bundled === undefined) {
-        return;
+const languageLoads = new WeakMap();
+const embeddedIndexes = new Map();
+/* Concurrent documents await one load, so each learns the grammar was rebuilt. */
+function loadOnce(highlighter, id) {
+    if (highlighter.getLoadedLanguages().includes(id)) {
+        return Promise.resolve();
     }
-    const registrations = (await bundled()).default;
-    const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
-    const named = lazy.filter((id) => {
-        const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
-        return !highlighter.getLoadedLanguages().includes(id) && mentions(text, [id, ...aliases]);
+    let loads = languageLoads.get(highlighter);
+    if (loads === undefined) {
+        loads = new Map();
+        languageLoads.set(highlighter, loads);
+    }
+    const pending = loads.get(id);
+    if (pending !== undefined) {
+        return pending;
+    }
+    const load = (async () => {
+        await loadDocumentLanguage(highlighter, id);
+        if (!highlighter.getLoadedLanguages().includes(id)) {
+            await highlighter.loadLanguage(id);
+        }
+    })();
+    loads.set(id, load);
+    void load.catch(() => {
+        if (loads.get(id) === load) {
+            loads.delete(id);
+        }
     });
-    await Promise.allSettled(named.map((id) => highlighter.loadLanguage(id)));
+    return load;
 }
-/*
- * Loads the grammar of a language with what a document in it embeds, and says whether Shiki has it.
- * `text` is the document, which decides the embedded languages that load only when named.
- */
-export async function loadGrammar(highlighter, grammar, text = '') {
-    await loadDocumentLanguage(highlighter, grammar);
-    if (!highlighter.getLoadedLanguages().includes(grammar)) {
-        await highlighter.loadLanguage(grammar);
+function embedIndex(grammar) {
+    const cached = embeddedIndexes.get(grammar);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const pending = (async () => {
+        const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
+        const bundled = bundledLanguages[grammar];
+        const registrations = bundled === undefined ? [] : (await bundled()).default;
+        const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
+        const names = new Map();
+        for (const id of lazy) {
+            const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
+            for (const name of [id, ...aliases]) {
+                names.set(name.toLowerCase(), id);
+            }
+        }
+        if (names.size === 0) {
+            return { names, matcher: null };
+        }
+        let matcher;
+        if (grammar === 'markdown' || grammar === 'mdx') {
+            matcher = /^\s*(?:`{3,}|~{3,})[ \t]*([\w#+.-]+)/gm;
+        }
+        else if (grammar === 'vue' || grammar === 'svelte' || grammar === 'astro') {
+            matcher = /\blang\s*=\s*["']([\w#+.-]+)["']/gi;
+        }
+        else {
+            const alternatives = [...names.keys()].map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
+            matcher = new RegExp(`(?<![\\w-])(${alternatives})(?![\\w-])`, 'gi');
+        }
+        return { names, matcher };
+    })();
+    embeddedIndexes.set(grammar, pending);
+    void pending.catch(() => embeddedIndexes.delete(grammar));
+    return pending;
+}
+async function loadEmbedded(highlighter, embedded, text) {
+    if (embedded.matcher === null) {
+        return false;
+    }
+    const loaded = new Set(highlighter.getLoadedLanguages());
+    const named = new Set();
+    for (const match of text.matchAll(embedded.matcher)) {
+        const id = embedded.names.get(match[1].toLowerCase());
+        if (id !== undefined && !loaded.has(id)) {
+            named.add(id);
+        }
     }
-    await loadEmbedded(highlighter, grammar, text);
+    const results = await Promise.allSettled([...named].map((id) => loadOnce(highlighter, id)));
+    return results.some((result) => result.status === 'fulfilled');
+}
+/* Text names the optional embedded grammars that must load before the host hands out states. */
+export async function loadGrammar(highlighter, grammar, text = '') {
+    await loadOnce(highlighter, grammar);
+    await loadEmbedded(highlighter, await embedIndex(grammar), text);
     return highlighter.getLoadedLanguages().includes(grammar);
 }
 /*
@@ -82,7 +144,7 @@ export function shikiTokenizers(highlighter) {
             if (!shiki.getLoadedThemes().includes(theme)) {
                 await shiki.loadTheme(theme);
             }
-            return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme) : null;
+            return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme, await embedIndex(grammar)) : null;
         }
         catch {
             return null;
@@ -104,3 +166,4 @@ export function shikiScopeColors(highlighter) {
         }
     };
 }
+export { shikiDocumentHighlighter, shikiDocumentTokens } from './shiki-document.js';

package/dist/testing.d.ts

@@ -9,6 +9,7 @@ export interface KeyOptions {
     metaKey?: boolean;
     altKey?: boolean;
     shiftKey?: boolean;
+    pointerType?: string;
 }
 export declare function press(window: Window, target: Element, key: string, options?: KeyOptions): boolean;
 export declare function release(window: Window, target: Element, key: string, options?: KeyOptions): void;

package/dist/tokens.d.ts

@@ -8,6 +8,7 @@ export declare class TokenCache {
     private tokenizer;
     private entries;
     private frontier;
+    private staleFrom;
     private reusableFrom;
     private pivotLine;
     private pivotState;
@@ -20,6 +21,7 @@ export declare class TokenCache {
     tokenizeText(lines: readonly string[]): (readonly LineToken[])[] | null;
     tokensOf(line: number): readonly LineToken[] | null;
     edited(batches: readonly (readonly DocumentChange[])[], lineOf: (offset: number) => number): void;
+    revalidate(): boolean;
     advance(until: number, budget: number, now?: () => number): {
         from: number;
         to: number;

package/dist/tokens.js

@@ -12,6 +12,8 @@ export class TokenCache {
     entries = [];
     /* Every line above it is colored for the text as it is now. */
     frontier = 0;
+    /* Colors survive a rebuild, but the states below this boundary cannot settle recoloring. */
+    staleFrom = Number.POSITIVE_INFINITY;
     /* The last line the edit touched; lines below it may be told right by their old state. */
     reusableFrom = Number.POSITIVE_INFINITY;
     /* The state the last line of the edited stretch ended in before the edit; the same state now means the lines below are right. */
@@ -31,6 +33,7 @@ export class TokenCache {
         this.tokenizer = tokenizer;
         this.lines = this.source.getLineCount();
         this.entries = [];
+        this.staleFrom = Number.POSITIVE_INFINITY;
         this.frontier = 0;
         this.forgetPivot();
     }
@@ -69,6 +72,7 @@ export class TokenCache {
         const first = batches[0]?.[0];
         if (first === undefined) {
             this.entries = [];
+            this.staleFrom = Number.POSITIVE_INFINITY;
             this.frontier = 0;
             this.forgetPivot();
             return;
@@ -83,8 +87,22 @@ export class TokenCache {
         const lastNew = last === undefined ? -1 : lineOf(last.from + shift + last.insertedLength);
         const lastOld = lastNew - delta;
         this.frontier = Math.min(this.frontier, firstLine);
+        if (last === undefined && Number.isFinite(this.staleFrom)) {
+            // A history batch has no single mapping for the old grammar boundary.
+            this.staleFrom = 0;
+            this.frontier = 0;
+        }
+        else if (this.staleFrom > lastOld) {
+            this.staleFrom += delta;
+        }
+        else if (this.staleFrom > firstLine) {
+            this.staleFrom = firstLine;
+        }
         if (last === undefined || lastOld < firstLine || this.entries.length <= firstLine) {
             this.entries.length = Math.min(this.entries.length, firstLine);
+            if (this.staleFrom >= this.entries.length) {
+                this.staleFrom = Number.POSITIVE_INFINITY;
+            }
             this.forgetPivot();
             return;
         }
@@ -94,6 +112,16 @@ export class TokenCache {
         this.entries.splice(firstLine, lastOld - firstLine + 1, ...new Array(lastNew - firstLine + 1));
         this.reusableFrom = lastNew + 1;
     }
+    /* Old colors stay visible until the rebuilt grammar reaches each line. */
+    revalidate() {
+        if (this.tokenizer?.stale?.() !== true) {
+            return false;
+        }
+        this.frontier = 0;
+        this.staleFrom = 0;
+        this.forgetPivot();
+        return true;
+    }
     /*
      * Colors lines from the frontier until `until` is reached, the colors are right for the rest of
      * the document, or `budget` milliseconds are spent. Returns the first line that changed, so a
@@ -104,10 +132,7 @@ export class TokenCache {
         if (tokenizer === null) {
             return null;
         }
-        if (tokenizer.stale?.() === true) {
-            this.frontier = 0;
-            this.forgetPivot();
-        }
+        this.revalidate();
         const count = this.source.getLineCount();
         const limit = Math.min(until, count - 1);
         const deadline = now() + budget;
@@ -127,10 +152,15 @@ export class TokenCache {
             }
             this.entries[line] = next;
             this.frontier = line + 1;
-            const settled = (line === this.pivotLine && tokenizer.sameState(this.pivotState, next.state)) ||
-                (previous !== undefined && line >= this.reusableFrom && tokenizer.sameState(previous.state, next.state));
+            const fresh = line < this.staleFrom;
+            if (line === this.staleFrom) {
+                this.staleFrom = line + 1 >= this.entries.length ? Number.POSITIVE_INFINITY : line + 1;
+            }
+            const settled = fresh &&
+                ((line === this.pivotLine && tokenizer.sameState(this.pivotState, next.state)) ||
+                    (previous !== undefined && line >= this.reusableFrom && tokenizer.sameState(previous.state, next.state)));
             if (settled) {
-                this.frontier = this.entries.length;
+                this.frontier = Math.min(this.entries.length, this.staleFrom);
                 this.forgetPivot();
                 break;
             }

package/dist/touch.d.ts

@@ -0,0 +1,27 @@
+import type { DocumentModel } from '@adecore/editor-core';
+export interface TouchHost {
+    readonly model: DocumentModel;
+    offsetAt(clientX: number, clientY: number): number;
+    camelHumps(): boolean;
+    lineAt(offset: number): {
+        from: number;
+        to: number;
+    };
+    focus(): void;
+}
+export declare class TouchSelection {
+    private contact;
+    private timer;
+    private lastTap;
+    private readonly host;
+    constructor(host: TouchHost);
+    get pressed(): boolean;
+    start(event: PointerEvent, area?: 'text' | 'gutter', taken?: () => boolean): void;
+    move(event: PointerEvent): void;
+    end(event: PointerEvent): void;
+    dispose(): void;
+    private tap;
+    private press;
+    private clearTimer;
+    private cancel;
+}