@adecore/editor 0.0.1 → 0.17.0-beta.1

package/src/shiki.ts

@@ -0,0 +1,118 @@
+import type { GrammarState, Highlighter } from 'shiki';
+import { documentLanguageOf, loadDocumentLanguage } from './php-html.ts';
+import { scopeColorsOf, type ThemeRule } from './theme-scopes.ts';
+import type { LineToken, LineTokenizer, ScopeColorSource, TokenizerSource } from './types.ts';
+
+type LanguageId = Parameters<Highlighter['loadLanguage']>[0];
+type ThemeId = Parameters<Highlighter['loadTheme']>[0];
+type TokenOptions = Parameters<Highlighter['codeToTokensBase']>[1];
+
+export { documentLanguageOf };
+
+const PLAIN_TEXT = new Set(['', 'text', 'txt', 'plain', 'plaintext']);
+
+/* Colors one line at a time and hands the grammar's state from one line to the next. A grammar Shiki rebuilt does not know the states of the old one. */
+function lineTokenizer(highlighter: Highlighter, language: string, theme: string): LineTokenizer {
+    let grammar = highlighter.getLanguage(language);
+    return {
+        stale() {
+            const current = highlighter.getLanguage(language);
+            const changed = current !== grammar;
+            grammar = current;
+            return changed;
+        },
+        tokenizeLine(text, state) {
+            const options = { lang: language, theme, grammarState: (state ?? undefined) as GrammarState | undefined } as TokenOptions;
+            const lines = highlighter.codeToTokensBase(text, options);
+            const tokens: LineToken[] = (lines[0] ?? []).map((token) => ({
+                length: token.content.length,
+                color: token.color ?? '',
+                fontStyle: token.fontStyle ?? 0
+            }));
+            return { tokens, state: highlighter.getLastGrammarState(lines) };
+        },
+        sameState(a, b) {
+            if (a === b) {
+                return true;
+            }
+            const left = (a as GrammarState | null)?.getInternalStack(theme);
+            const right = (b as GrammarState | null)?.getInternalStack(theme);
+            return left !== undefined && right !== undefined && left.equals(right);
+        }
+    };
+}
+
+/* The words of a document that name a language, an alias or the id itself, and not part of a longer word. */
+function mentions(text: string, names: readonly string[]): boolean {
+    return names.some((name) => new RegExp(`(?<![\\w-])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`, 'i').test(text));
+}
+
+/*
+ * A grammar embeds some languages only on demand (Vue's `lang="scss"` styles, the fences of Markdown),
+ * and Shiki builds the host grammar anew each time one of them loads later, which drops the states
+ * of every line colored before. So the ones a document names are loaded together with the host.
+ */
+async function loadEmbedded(highlighter: Highlighter, grammar: string, text: string): Promise<void> {
+    const { bundledLanguages, bundledLanguagesInfo } = await import('shiki');
+    const bundled = bundledLanguages[grammar as keyof typeof bundledLanguages];
+    if (bundled === undefined) {
+        return;
+    }
+    const registrations = (await bundled()).default as readonly { readonly name: string; readonly embeddedLangsLazy?: readonly string[] }[];
+    const lazy = registrations.find((registration) => registration.name === grammar)?.embeddedLangsLazy ?? [];
+    const named = lazy.filter((id) => {
+        const aliases = bundledLanguagesInfo.find((info) => info.id === id)?.aliases ?? [];
+        return !highlighter.getLoadedLanguages().includes(id) && mentions(text, [id, ...aliases]);
+    });
+    await Promise.allSettled(named.map((id) => highlighter.loadLanguage(id as LanguageId)));
+}
+
+/*
+ * Loads the grammar of a language with what a document in it embeds, and says whether Shiki has it.
+ * `text` is the document, which decides the embedded languages that load only when named.
+ */
+export async function loadGrammar(highlighter: Highlighter, grammar: string, text = ''): Promise<boolean> {
+    await loadDocumentLanguage(highlighter, grammar);
+    if (!highlighter.getLoadedLanguages().includes(grammar)) {
+        await highlighter.loadLanguage(grammar as LanguageId);
+    }
+    await loadEmbedded(highlighter, grammar, text);
+    return highlighter.getLoadedLanguages().includes(grammar);
+}
+
+/*
+ * Colors with the viewer's highlighter, so a grammar loads once for both. A language Shiki has no
+ * grammar for, or a theme it cannot load, is plain text.
+ */
+export function shikiTokenizers(highlighter: () => Promise<Highlighter>): TokenizerSource {
+    return async (language, theme, text) => {
+        if (language === undefined || PLAIN_TEXT.has(language)) {
+            return null;
+        }
+        const grammar = documentLanguageOf(language);
+        const shiki = await highlighter();
+        try {
+            if (!shiki.getLoadedThemes().includes(theme)) {
+                await shiki.loadTheme(theme as ThemeId);
+            }
+            return (await loadGrammar(shiki, grammar, text)) ? lineTokenizer(shiki, grammar, theme) : null;
+        } catch {
+            return null;
+        }
+    };
+}
+
+/* The colors of a theme's own rules by scope, so what a language server classifies is drawn in the theme's colors. */
+export function shikiScopeColors(highlighter: () => Promise<Highlighter>): ScopeColorSource {
+    return async (theme) => {
+        const shiki = await highlighter();
+        try {
+            if (!shiki.getLoadedThemes().includes(theme)) {
+                await shiki.loadTheme(theme as ThemeId);
+            }
+            return scopeColorsOf(shiki.getTheme(theme).settings as readonly ThemeRule[]);
+        } catch {
+            return null;
+        }
+    };
+}

package/src/smart-keys.ts

@@ -0,0 +1,17 @@
+import type { EditorSmartKeys } from './types.ts';
+
+export const DEFAULT_SMART_KEYS: EditorSmartKeys = {
+    autoPairBrackets: true,
+    autoPairQuotes: true,
+    surroundSelection: true,
+    tabOutOfClosers: true,
+    smartIndentOnEnter: true,
+    indentOnPaste: true,
+    smartSemicolon: true,
+    camelHumps: false
+};
+
+/* The keys with what a host left out filled in from the defaults. */
+export function resolveSmartKeys(keys: Partial<EditorSmartKeys> | undefined): EditorSmartKeys {
+    return { ...DEFAULT_SMART_KEYS, ...keys };
+}

package/src/testing.ts

@@ -0,0 +1,157 @@
+import { parseHTML } from 'linkedom';
+import { createSmartEditorEngine } from './engine.ts';
+import type { Editor, EditorOptions, SmartEditorEngineOptions } from './types.ts';
+
+/*
+ * A page for tests: linkedom has no layout, so every box is the editor's fallback size, and its
+ * textarea has no selection of its own, which the editor's input sink reads and writes.
+ */
+export function createPage(): { document: Document; window: Window & typeof globalThis; host: HTMLElement } {
+    const { document, window } = parseHTML('<html><body><div id="host"></div></body></html>');
+    const prototype = (window as unknown as { HTMLTextAreaElement: { prototype: Record<string, unknown> } }).HTMLTextAreaElement.prototype;
+    const selections = new WeakMap<object, { start: number; end: number; direction: string }>();
+    const read = (element: object): { start: number; end: number; direction: string } => selections.get(element) ?? { start: 0, end: 0, direction: 'none' };
+    Object.defineProperty(prototype, 'selectionStart', {
+        get: function (this: object) {
+            return read(this).start;
+        },
+        configurable: true
+    });
+    Object.defineProperty(prototype, 'selectionEnd', {
+        get: function (this: object) {
+            return read(this).end;
+        },
+        configurable: true
+    });
+    Object.defineProperty(prototype, 'selectionDirection', {
+        get: function (this: object) {
+            return read(this).direction;
+        },
+        configurable: true
+    });
+    prototype.setSelectionRange = function (this: object, start: number, end: number, direction = 'none'): void {
+        selections.set(this, { start, end, direction });
+    };
+    const element = (window as unknown as { HTMLElement: { prototype: object } }).HTMLElement.prototype;
+    for (const name of ['scrollTop', 'scrollLeft']) {
+        Object.defineProperty(element, name, { value: 0, writable: true, configurable: true });
+    }
+    return {
+        document: document as unknown as Document,
+        window: window as unknown as Window & typeof globalThis,
+        host: document.getElementById('host') as unknown as HTMLElement
+    };
+}
+
+export interface KeyOptions {
+    ctrlKey?: boolean;
+    metaKey?: boolean;
+    altKey?: boolean;
+    shiftKey?: boolean;
+}
+
+/* Fires a keydown at the element and says whether the editor took the key. */
+export function press(window: Window, target: Element, key: string, options: KeyOptions = {}): boolean {
+    const event = new (window as unknown as { Event: typeof Event }).Event('keydown', { bubbles: true, cancelable: true });
+    Object.assign(event, { key, ctrlKey: false, metaKey: false, altKey: false, shiftKey: false, isComposing: false, ...options });
+    target.dispatchEvent(event);
+    return event.defaultPrevented;
+}
+
+/* Fires a keyup at the element, the other half of `press`. */
+export function release(window: Window, target: Element, key: string, options: KeyOptions = {}): void {
+    const event = new (window as unknown as { Event: typeof Event }).Event('keyup', { bubbles: true, cancelable: true });
+    Object.assign(event, { key, ctrlKey: false, metaKey: false, altKey: false, shiftKey: false, isComposing: false, ...options });
+    target.dispatchEvent(event);
+}
+
+/* What a browser sends before it puts text in a textarea. */
+export function typeInto(window: Window, target: Element, text: string): void {
+    const event = new (window as unknown as { Event: typeof Event }).Event('beforeinput', { bubbles: true, cancelable: true });
+    Object.assign(event, { inputType: 'insertText', data: text, isComposing: false });
+    target.dispatchEvent(event);
+}
+
+/* A clipboard event with the data it carries; what a copy or cut sets is read back through `data`. */
+export function clipboard(
+    window: Window,
+    target: Element,
+    type: 'copy' | 'cut' | 'paste',
+    data: { text: string } = { text: '' }
+): { prevented: boolean; text: string } {
+    const event = new (window as unknown as { Event: typeof Event }).Event(type, { bubbles: true, cancelable: true });
+    Object.assign(event, {
+        clipboardData: {
+            getData: () => data.text,
+            setData: (_format: string, value: string) => {
+                data.text = value;
+            }
+        }
+    });
+    target.dispatchEvent(event);
+    return { prevented: event.defaultPrevented, text: data.text };
+}
+
+/* A pointer event at a point of the screen; the page has no scroll or scale, so it is also the point of the content. */
+export function pointer(
+    window: Window,
+    target: Element,
+    type: 'pointerdown' | 'pointermove' | 'pointerup' | 'pointercancel',
+    x: number,
+    y: number,
+    options: KeyOptions = {}
+): void {
+    const event = new (window as unknown as { Event: typeof Event }).Event(type, { bubbles: true, cancelable: true });
+    Object.assign(event, {
+        clientX: x,
+        clientY: y,
+        button: 0,
+        pointerId: 1,
+        detail: 0,
+        shiftKey: false,
+        altKey: false,
+        ctrlKey: false,
+        metaKey: false,
+        ...options
+    });
+    target.dispatchEvent(event);
+}
+
+export interface MountedEditor {
+    editor: Editor;
+    page: ReturnType<typeof createPage>;
+    input: HTMLTextAreaElement;
+    viewport: HTMLElement;
+    press(key: string, modifiers?: KeyOptions): boolean;
+    release(key: string, modifiers?: KeyOptions): void;
+    type(text: string): void;
+    /* Copies, cuts or pastes through the clipboard; the text a copy or cut left comes back. */
+    clip(type: 'copy' | 'cut' | 'paste', text?: string): string;
+    click(x: number, y: number, options?: KeyOptions): void;
+}
+
+/* An editor on a page of its own, with the helpers that drive it the way a person would. */
+export function mountEditor(options: Partial<EditorOptions> = {}, engineOptions: Partial<SmartEditorEngineOptions> = {}): MountedEditor {
+    const page = createPage();
+    const editor = createSmartEditorEngine({ tokenizer: async () => null, ...engineOptions }).mount(page.host, {
+        text: 'one\ntwo\nthree',
+        theme: 'dark',
+        ...options
+    });
+    const input = page.host.querySelector('textarea')!;
+    const viewport = page.host.querySelector('.se-viewport') as HTMLElement;
+    return {
+        editor,
+        page,
+        input,
+        viewport,
+        press: (key, modifiers = {}) => press(page.window, input, key, modifiers),
+        release: (key, modifiers = {}) => release(page.window, input, key, modifiers),
+        type: (text) => typeInto(page.window, input, text),
+        clip: (type, text = '') => clipboard(page.window, input, type, { text }).text,
+        click: (x, y, modifiers = {}) => {
+            pointer(page.window, viewport, 'pointerdown', x, y, modifiers);
+            pointer(page.window, viewport, 'pointerup', x, y);
+        }
+    };
+}

package/src/theme-scopes.ts

@@ -0,0 +1,75 @@
+import type { ScopeColors, ScopeStyle } from './types.ts';
+
+/* The part of a TextMate theme's token colors this reads. */
+export interface ThemeRule {
+    readonly scope?: string | readonly string[];
+    readonly settings: { readonly foreground?: string; readonly fontStyle?: string };
+}
+
+const FONT_STYLES: Record<string, number> = { italic: 1, bold: 2, underline: 4, strikethrough: 8 };
+
+function fontStyleOf(text: string | undefined): number {
+    return (text ?? '').split(/\s+/).reduce((flags, word) => flags | (FONT_STYLES[word] ?? 0), 0);
+}
+
+function nameMatches(selector: string, scope: string): boolean {
+    return scope === selector || scope.startsWith(`${selector}.`);
+}
+
+/*
+ * How well a selector fits a stack of scopes, outermost first, or null when it does not. As in a
+ * TextMate theme the last word has to match the innermost scope and the words before it some outer
+ * ones in order; the longer the name it matched, the better, and a selector with more words beats one
+ * with fewer.
+ */
+function selectorScore(selector: string, stack: readonly string[]): number | null {
+    const words = selector.trim().split(/\s+/);
+    const last = words[words.length - 1]!;
+    const innermost = stack[stack.length - 1];
+    if (innermost === undefined || !nameMatches(last, innermost)) {
+        return null;
+    }
+    let at = stack.length - 2;
+    for (let word = words.length - 2; word >= 0; word--) {
+        while (at >= 0 && !nameMatches(words[word]!, stack[at]!)) {
+            at--;
+        }
+        if (at < 0) {
+            return null;
+        }
+        at--;
+    }
+    return last.length * 100 + words.length;
+}
+
+/*
+ * What a theme draws a scope stack in, the way its own grammar's tokens are colored: the rule that
+ * fits best wins and a later rule beats an equal one. Undefined where no rule colors it, which is
+ * where the editor keeps what the grammar said.
+ */
+export function scopeColorsOf(rules: readonly ThemeRule[]): ScopeColors {
+    const flat = rules.flatMap((rule, order) =>
+        (typeof rule.scope === 'string' ? rule.scope.split(',') : (rule.scope ?? [])).map((selector) => ({
+            selector: selector.trim(),
+            style: { color: rule.settings.foreground ?? '', fontStyle: fontStyleOf(rule.settings.fontStyle) },
+            order
+        }))
+    );
+    const cache = new Map<string, ScopeStyle | undefined>();
+    return (stack) => {
+        const key = stack.join(' ');
+        if (cache.has(key)) {
+            return cache.get(key);
+        }
+        let best: { score: number; order: number; style: ScopeStyle } | undefined;
+        for (const entry of flat) {
+            const score = entry.selector === '' ? null : selectorScore(entry.selector, stack);
+            if (score !== null && (best === undefined || score > best.score || (score === best.score && entry.order > best.order))) {
+                best = { score, order: entry.order, style: entry.style };
+            }
+        }
+        const style = best !== undefined && best.style.color !== '' ? best.style : undefined;
+        cache.set(key, style);
+        return style;
+    };
+}

package/src/tokens.ts

@@ -0,0 +1,175 @@
+import type { DocumentChange } from '@adecore/editor-core';
+import type { LineToken, LineTokenizer } from './types.ts';
+
+interface Entry {
+    tokens: readonly LineToken[];
+    state: unknown;
+}
+
+/* A line this long is left uncolored: a grammar can spend seconds on it, and nobody reads it in colors. */
+const LONGEST_COLORED_LINE = 20000;
+
+export interface TokenSource {
+    getLineCount(): number;
+    getLineText(line: number): string;
+}
+
+/*
+ * What a grammar made of each line, from the top of the document down. A grammar needs the state the
+ * line above it ended in, so the lines are colored in order from a frontier, and an edit moves the
+ * frontier up to the changed line. Lines below it keep their old colors, shifted to where they now
+ * are, until the recoloring reaches a line that ends in the state it ended in before, which is when
+ * everything under it is right again.
+ */
+export class TokenCache {
+    private tokenizer: LineTokenizer | null = null;
+    private entries: (Entry | undefined)[] = [];
+    /* Every line above it is colored for the text as it is now. */
+    private frontier = 0;
+    /* The last line the edit touched; lines below it may be told right by their old state. */
+    private reusableFrom = Number.POSITIVE_INFINITY;
+    /* The state the last line of the edited stretch ended in before the edit; the same state now means the lines below are right. */
+    private pivotLine = -1;
+    private pivotState: unknown;
+    private lines: number;
+    private readonly source: TokenSource;
+
+    constructor(source: TokenSource) {
+        this.source = source;
+        this.lines = source.getLineCount();
+    }
+
+    get ready(): boolean {
+        return this.tokenizer !== null;
+    }
+
+    /* A new tokenizer, or none, colors everything anew. */
+    setTokenizer(tokenizer: LineTokenizer | null): void {
+        this.tokenizer = tokenizer;
+        this.lines = this.source.getLineCount();
+        this.entries = [];
+        this.frontier = 0;
+        this.reusableFrom = Number.POSITIVE_INFINITY;
+        this.pivotLine = -1;
+    }
+
+    /* How far down the document has colors that are right for the text. */
+    get colored(): number {
+        return this.frontier;
+    }
+
+    /* The colors of text that is not part of the document, a line at a time, or null without a tokenizer. */
+    tokenizeText(lines: readonly string[]): (readonly LineToken[])[] | null {
+        const tokenizer = this.tokenizer;
+        if (tokenizer === null) {
+            return null;
+        }
+        let state: unknown = null;
+        return lines.map((text) => {
+            if (text.length > LONGEST_COLORED_LINE) {
+                return [{ length: text.length, color: '', fontStyle: 0 }];
+            }
+            const result = tokenizer.tokenizeLine(text, state);
+            state = result.state;
+            return result.tokens;
+        });
+    }
+
+    /* The colors of a line, or null while it has none; the old colors of a line the recoloring has not reached yet. */
+    tokensOf(line: number): readonly LineToken[] | null {
+        return this.entries[line]?.tokens ?? null;
+    }
+
+    /* The text changed in these batches; `lineOf` answers for the document as it is now. */
+    edited(batches: readonly (readonly DocumentChange[])[], lineOf: (offset: number) => number): void {
+        const lineCount = this.source.getLineCount();
+        const delta = lineCount - this.lines;
+        this.lines = lineCount;
+        if (this.tokenizer === null) {
+            return;
+        }
+        const first = batches[0]?.[0];
+        if (first === undefined) {
+            this.entries = [];
+            this.frontier = 0;
+            this.reusableFrom = Number.POSITIVE_INFINITY;
+            this.pivotLine = -1;
+            return;
+        }
+        const firstLine = lineOf(first.from);
+        const changes = batches.length === 1 ? batches[0]! : [];
+        let shift = 0;
+        for (let i = 0; i < changes.length - 1; i++) {
+            shift += changes[i]!.insertedLength - (changes[i]!.to - changes[i]!.from);
+        }
+        const last = changes.at(-1);
+        const lastNew = last === undefined ? -1 : lineOf(last.from + shift + last.insertedLength);
+        const lastOld = lastNew - delta;
+        this.frontier = Math.min(this.frontier, firstLine);
+        if (last === undefined || lastOld < firstLine || this.entries.length <= firstLine) {
+            this.entries.length = Math.min(this.entries.length, firstLine);
+            this.reusableFrom = Number.POSITIVE_INFINITY;
+            this.pivotLine = -1;
+            return;
+        }
+        const old = this.entries[lastOld];
+        this.pivotLine = old ? lastNew : -1;
+        this.pivotState = old?.state;
+        this.entries.splice(firstLine, lastOld - firstLine + 1, ...new Array<undefined>(lastNew - firstLine + 1));
+        this.reusableFrom = lastNew + 1;
+    }
+
+    /*
+     * Colors lines from the frontier until `until` is reached, the colors are right for the rest of
+     * the document, or `budget` milliseconds are spent. Returns the first line that changed, so a
+     * view knows what to draw again, or null when nothing did.
+     */
+    advance(until: number, budget: number, now: () => number = () => performance.now()): { from: number; to: number } | null {
+        const tokenizer = this.tokenizer;
+        if (tokenizer === null) {
+            return null;
+        }
+        if (tokenizer.stale?.() === true) {
+            this.frontier = 0;
+            this.reusableFrom = Number.POSITIVE_INFINITY;
+            this.pivotLine = -1;
+        }
+        const count = this.source.getLineCount();
+        const limit = Math.min(until, count - 1);
+        const deadline = now() + budget;
+        const from = this.frontier;
+        while (this.frontier <= limit && this.frontier < count) {
+            const line = this.frontier;
+            const text = this.source.getLineText(line);
+            const state = line === 0 ? null : (this.entries[line - 1]?.state ?? null);
+            const previous = this.entries[line];
+            let next: Entry;
+            if (text.length > LONGEST_COLORED_LINE) {
+                next = { tokens: [{ length: text.length, color: '', fontStyle: 0 }], state };
+            } else {
+                const result = tokenizer.tokenizeLine(text, state);
+                next = { tokens: result.tokens, state: result.state };
+            }
+            this.entries[line] = next;
+            this.frontier = line + 1;
+            const settled =
+                (line === this.pivotLine && tokenizer.sameState(this.pivotState, next.state)) ||
+                (previous !== undefined && line >= this.reusableFrom && tokenizer.sameState(previous.state, next.state));
+            if (settled) {
+                this.frontier = this.entries.length;
+                this.reusableFrom = Number.POSITIVE_INFINITY;
+                this.pivotLine = -1;
+                break;
+            }
+            if (now() > deadline) {
+                break;
+            }
+        }
+        return this.frontier > from ? { from, to: this.frontier - 1 } : null;
+    }
+
+    /* Whether lines through `line` are colored for the text as it is now. */
+    covers(line: number): boolean {
+        return this.frontier > line || this.frontier >= this.source.getLineCount();
+    }
+}

package/src/tracked-range.ts

@@ -0,0 +1,22 @@
+import type { DocumentChange } from '@adecore/editor-core';
+
+/*
+ * Where a tracked range is after a batch of changes, or null once a change touched it. A change
+ * touches a range when it replaces any of the text inside it or inserts strictly between its ends.
+ * Text inserted exactly at an end leaves the range's own text alone: at the start the range moves
+ * behind it, at the end the range stays as it is.
+ */
+export function mapTrackedRange(from: number, to: number, changes: readonly DocumentChange[]): { from: number; to: number } | null {
+    let delta = 0;
+    for (const change of changes) {
+        if (change.to <= from) {
+            delta += change.insertedLength - (change.to - change.from);
+            continue;
+        }
+        if (change.from >= to) {
+            break;
+        }
+        return null;
+    }
+    return { from: from + delta, to: to + delta };
+}