@react-x11/components 0.8.0 → 0.10.0

package/src/code-language/languages/glsl.ts

@@ -105,8 +105,7 @@ function token(stream: StringStream, state: GlslState): string | null {
   return null;
 }
 
-/** GLSL. */
-export function glsl(): Language {
+function build(): Language {
   return streamLanguage<GlslState>({
     name: 'glsl',
     languageData: {
@@ -118,3 +117,14 @@ export function glsl(): Language {
     token,
   });
 }
+
+// One Language per set of options, made the first time it is asked for: a
+// render that writes `language={glsl()}` inline hands the editor the same
+// object each time, where a new one meant a new tokenizer and every line in
+// view laid out again, on every keystroke of a controlled editor.
+let made: Language | undefined;
+
+/** GLSL. */
+export function glsl(): Language {
+  return (made ??= build());
+}

package/src/code-language/languages/javascript.ts

@@ -51,7 +51,7 @@ const SELF = new Set(['this', 'super', 'globalThis']);
  *   one; and, with JSX on: `N` an opening tag before its name, `A` its
  *   attribute zone, `M` a closing tag before its name, `K` a closing tag
  *   awaiting `>`, `E` element children, `J` a `{…}` expression container.
- * - `str` is `'`/`"` while a string continues across a `\` line break.
+ * - `str` is `'`/`"` while a string continues across a `` line break.
  * - `value` is true when the previous significant token produced a value —
  *   which is exactly the bit that disambiguates `/` (divide after a value,
  *   regex otherwise) and `<` (compare after a value, JSX otherwise).
@@ -167,7 +167,7 @@ function eatEscape(stream: StringStream): boolean {
 }
 
 /** Consume string content up to the closing quote / EOL. Returns the type,
- * updating `state.str` when a trailing `\` continues the string. */
+ * updating `state.str` when a trailing `` continues the string. */
 function stringToken(
   stream: StringStream,
   state: JsState,
@@ -391,8 +391,7 @@ export interface JavascriptOptions {
   jsx?: boolean;
 }
 
-/** JavaScript — or TypeScript, JSX and TSX, by options. */
-export function javascript(options: JavascriptOptions = {}): Language {
+function build(options: JavascriptOptions = {}): Language {
   const typescript = options.typescript ?? false;
   const jsx = options.jsx ?? false;
   return streamLanguage<JsState>({
@@ -417,3 +416,21 @@ export function javascript(options: JavascriptOptions = {}): Language {
     token: makeToken(typescript, jsx),
   });
 }
+
+// One Language per set of options, made the first time it is asked for: a
+// render that writes `language={javascript()}` inline hands the editor the same
+// object each time, where a new one meant a new tokenizer and every line in
+// view laid out again, on every keystroke of a controlled editor.
+let made: Map<string, Language> | undefined;
+
+/** JavaScript — or TypeScript, JSX and TSX, by options. */
+export function javascript(options: JavascriptOptions = {}): Language {
+  const key = `${options.typescript ?? false}|${options.jsx ?? false}`;
+  made ??= new Map();
+  let language = made.get(key);
+  if (!language) {
+    language = build(options);
+    made.set(key, language);
+  }
+  return language;
+}

package/src/code-language/languages/json.ts

@@ -61,8 +61,7 @@ function token(stream: StringStream, state: JsonState): string | null {
   return 'invalid';
 }
 
-/** JSON. */
-export function json(): Language {
+function build(): Language {
   return streamLanguage<JsonState>({
     name: 'json',
     languageData: { indentAfter: /[[{]\s*$/ },
@@ -70,3 +69,14 @@ export function json(): Language {
     token,
   });
 }
+
+// One Language per set of options, made the first time it is asked for: a
+// render that writes `language={json()}` inline hands the editor the same
+// object each time, where a new one meant a new tokenizer and every line in
+// view laid out again, on every keystroke of a controlled editor.
+let made: Language | undefined;
+
+/** JSON. */
+export function json(): Language {
+  return (made ??= build());
+}

package/src/code-language/languages/shell.ts

@@ -198,8 +198,7 @@ function dquoteToken(stream: StringStream, state: ShellState): string | null {
   return 'string';
 }
 
-/** POSIX shell / bash. */
-export function shell(): Language {
+function build(): Language {
   return streamLanguage<ShellState>({
     name: 'shell',
     languageData: {
@@ -219,3 +218,14 @@ export function shell(): Language {
     token,
   });
 }
+
+// One Language per set of options, made the first time it is asked for: a
+// render that writes `language={shell()}` inline hands the editor the same
+// object each time, where a new one meant a new tokenizer and every line in
+// view laid out again, on every keystroke of a controlled editor.
+let made: Language | undefined;
+
+/** POSIX shell / bash. */
+export function shell(): Language {
+  return (made ??= build());
+}

package/src/code-language/languages/sql.ts

@@ -77,8 +77,7 @@ export interface SqlOptions {
   hashComments?: boolean;
 }
 
-/** SQL. `sql()` for the generic dialect, options for the house one. */
-export function sql(options: SqlOptions = {}): Language {
+function build(options: SqlOptions = {}): Language {
   const extra = new Set(
     (options.keywords ?? []).map((word) => word.toLowerCase()),
   );
@@ -163,3 +162,24 @@ export function sql(options: SqlOptions = {}): Language {
     token,
   });
 }
+
+// One Language per set of options, made the first time it is asked for: a
+// render that writes `language={sql()}` inline hands the editor the same
+// object each time, where a new one meant a new tokenizer and every line in
+// view laid out again, on every keystroke of a controlled editor.
+let made: Map<string, Language> | undefined;
+
+/** SQL. `sql()` for the generic dialect, options for the house one. */
+export function sql(options: SqlOptions = {}): Language {
+  const key = JSON.stringify([
+    options.keywords ?? [],
+    options.hashComments ?? false,
+  ]);
+  made ??= new Map();
+  let language = made.get(key);
+  if (!language) {
+    language = build(options);
+    made.set(key, language);
+  }
+  return language;
+}

package/src/code-language/stream.ts

@@ -16,7 +16,10 @@ import type {
   LineEdit,
   Token,
   Tokenizer,
+  TokenizerHost,
 } from './types.js';
+import { spliceAll } from '../internal/splice.js';
+import { startTimeout, stopTimeout, type TimerId } from './timers.js';
 
 /**
  * One line of text with a cursor, the surface a mode's `token()` reads
@@ -133,6 +136,56 @@ export interface LineMode<S> {
   stateEquals?(a: S, b: S): boolean;
 }
 
+/**
+ * How much of one line is tokenized; the rest of a longer one is drawn
+ * unstyled, and the next line starts from the state where this one was cut.
+ * CodeMirror's `maxHighlightLength`, and for the same reason: a line is
+ * tokenized again whole on every edit to it, and a minified file is one
+ * line — a keystroke at the end of a million characters ran the language
+ * over all of them. A construct that opens past the cut and closes on a
+ * later line is the price, on lines no one reads by their colours.
+ */
+export const TOKENIZE_LIMIT = 10_000;
+
+/**
+ * How far past the frontier a request walks it on the spot. Further than
+ * that — the end of a file opened a moment ago, a line a search jumped to —
+ * the line is tokenized from a guessed state, and the frontier is walked
+ * there in the background, `SLICE_LINES` a turn, with the host told where a
+ * guess was wrong. CodeMirror 5's answer to the same jump, for the same
+ * reason: tokenizing a 50,000-line file is a fifth of a second, spent on
+ * lines no one is looking at. Decided by the distance rather than by trying:
+ * the lines a guess would take from the cache are the ones the walk would
+ * have found converged, and trying first is the cost this exists to avoid.
+ */
+export const SYNC_LINES = 1_000;
+const SLICE_LINES = 500;
+
+/** How far above a guessed line to look for a state to start from: one a
+ *  guess left, or else the least indented line, taken to be at the top
+ *  level (CodeMirror's `findStartLine`). */
+const GUESS_LOOKBACK = 100;
+
+function cut(text: string): string {
+  return text.length > TOKENIZE_LIMIT ? text.slice(0, TOKENIZE_LIMIT) : text;
+}
+
+function indentOf(text: string): number {
+  let n = 0;
+  while (n < text.length && (text[n] === ' ' || text[n] === '\t')) n++;
+  return n;
+}
+
+function sameTokens(a: readonly Token[], b: readonly Token[]): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    const x = a[i];
+    const y = b[i];
+    if (x.from !== y.from || x.to !== y.to || x.type !== y.type) return false;
+  }
+  return true;
+}
+
 function defaultCopy<S>(state: S): S {
   if (state === null || typeof state !== 'object') return state;
   return { ...(state as object) } as S;
@@ -173,18 +226,43 @@ function defaultEquals(a: unknown, b: unknown): boolean {
  * lines. The `undefined` holes are also what makes convergence sound across
  * multiple pending edits: the fast-forward below can never skip over a line
  * whose text changed, because that line's cache is a hole.
+ *
+ * A guess (`guess`) is one more such pair past the frontier: tokens run
+ * from a state that was not reached from line 0. Convergence is what makes
+ * that safe as well — the frontier's walk takes the rest of a guessed run
+ * as it is where it arrives with the same state, and tokenizes it again
+ * where it does not.
  */
 class StreamTokenizer<S> implements Tokenizer {
   private lines: readonly string[] = [''];
   /** states[i] = state entering line i. Trusted for i <= frontier; a
-   * defined entry beyond that is a pre-edit value, used only as a
-   * convergence candidate. */
+   * defined entry beyond that is a pre-edit value or a guess, used only as
+   * a convergence candidate. Past the frontier every defined `tokens[i]` was
+   * run from the `states[i]` beside it. */
   private states: (S | undefined)[] = [];
   private tokens: (Token[] | undefined)[] = [];
   /** Lines `[0, frontier)` have trusted tokens. */
   private frontier = 0;
+  /** The furthest line answered from a guess that the frontier has not
+   *  reached; -1 when there is none. While there is one, every line past
+   *  the frontier is answered the same way. */
+  private guessedTo = -1;
+  private worker: TimerId = null;
+  /** The first line whose trusted tokens differed from the ones it had —
+   *  what the host is told after a turn of the walk. */
+  private changedFrom = -1;
+  /**
+   * Tokens taken out of `tokens` because the state entering their line was
+   * replaced — kept, by line, for the comparison `run` makes when the line
+   * is run again, and out of the array so that nothing takes them as the
+   * line's meanwhile. See `edit` for how a line gets here.
+   */
+  private stale = new Map<number, Token[]>();
 
-  constructor(private readonly mode: LineMode<S>) {}
+  constructor(
+    private readonly mode: LineMode<S>,
+    private readonly host: TokenizerHost | null = null,
+  ) {}
 
   setLines(lines: readonly string[]): void {
     this.lines = lines;
@@ -192,41 +270,86 @@ class StreamTokenizer<S> implements Tokenizer {
     this.tokens = new Array(lines.length);
     this.states[0] = this.mode.startState();
     this.frontier = 0;
+    this.guessedTo = -1;
+    this.stale.clear();
+    this.stopWorker();
   }
 
   edit({ fromLine, removed, inserted }: LineEdit): void {
     // Align the caches below the edit with the new line numbering (their
     // contents may still be right — convergence will decide), and punch
     // `undefined` holes for the lines whose text actually changed.
-    this.states.splice(
-      fromLine + 1,
-      removed,
-      ...new Array<S | undefined>(inserted),
-    );
-    this.tokens.splice(
-      fromLine,
-      removed,
-      ...new Array<Token[] | undefined>(inserted),
-    );
+    //
+    // …all but one: the state entering the first line after the edit is
+    // put back at that line's new number. It is what convergence compares
+    // the edit's outgoing state with, paired with the tokens that line
+    // still has — punched out with the rest, every edit re-tokenized the
+    // line after it too, and handed back a new token array for text that
+    // had not changed.
+    const after = this.states[fromLine + removed];
+    spliceAll(this.states, fromLine + 1, removed, new Array(inserted));
+    if (inserted > 0) this.states[fromLine + inserted] = after;
+    spliceAll(this.tokens, fromLine, removed, new Array(inserted));
+    const shift = inserted - removed;
+    if (this.stale.size > 0) {
+      const moved = new Map<number, Token[]>();
+      for (const [line, held] of this.stale) {
+        if (line < fromLine) moved.set(line, held);
+        else if (line >= fromLine + removed) moved.set(line + shift, held);
+      }
+      this.stale = moved;
+    }
+    // A walk that stops leaves the frontier's line with the state it gave
+    // it and the tokens it had before, from another: harmless while the
+    // frontier is there, since that line is the first one run. An edit
+    // above it moves the frontier back past it, and convergence would then
+    // take its tokens as the line's — the line under the view, once a
+    // comment opened in it, kept its colours from before and showed them
+    // the moment it was scrolled to. They are set aside instead.
+    const frontier = this.frontier;
+    if (fromLine < frontier && frontier >= fromLine + removed) {
+      this.setAside(frontier + shift);
+    }
     this.frontier = Math.min(this.frontier, fromLine);
+    if (this.guessedTo >= fromLine) {
+      this.guessedTo = Math.max(fromLine, this.guessedTo + inserted - removed);
+    }
   }
 
   lineTokens(line: number): readonly Token[] {
     if (line < 0 || line >= this.lines.length) return [];
-    this.ensure(line);
+    if (line >= this.frontier) {
+      if (this.guessedTo >= 0 || line - this.frontier > SYNC_LINES) {
+        return this.guess(line);
+      }
+      this.ensure(line);
+      // the caller has the tokens: nobody else was holding the old ones
+      this.changedFrom = -1;
+    }
     return this.tokens[line] ?? [];
   }
 
-  private ensure(line: number): void {
+  dispose(): void {
+    this.stopWorker();
+  }
+
+  /**
+   * Walk the frontier to `line`, tokenizing at most `budget` lines on the
+   * way (a converged run skipped is free). False when the budget ran out
+   * first.
+   */
+  private ensure(line: number, budget = Infinity): boolean {
     const copy = this.mode.copyState ?? defaultCopy;
     const equals = this.mode.stateEquals ?? defaultEquals;
+    let spent = 0;
     while (this.frontier <= line) {
+      if (spent++ >= budget) return false;
       const i = this.frontier;
       // `states[i]` is trusted here: it is either below the old frontier or
       // was written by the previous iteration. Copy before running — the
       // mode mutates in place, and the cached entry must stay pristine.
       const state = copy(this.states[i] as S);
-      this.tokens[i] = this.mode.runLine(this.lines[i], state);
+      this.tokens[i] = this.run(i, state);
       const cached = this.states[i + 1];
       if (
         cached !== undefined &&
@@ -251,6 +374,123 @@ class StreamTokenizer<S> implements Tokenizer {
       this.states[i + 1] = state;
       this.frontier = i + 1;
     }
+    return true;
+  }
+
+  /**
+   * Tokens for a line past the frontier, where the frontier is too far to
+   * walk to now: walked from the furthest state up the `GUESS_LOOKBACK`
+   * lines above it — the frontier's, when it is that close, or one a guess
+   * or an edit left — or, when there is none, from the language's start
+   * state at the least indented of those lines. The walk runs what has no
+   * tokens and passes over what has, so an edit's line is run again and the
+   * lines under it follow, the way the frontier walks after an edit; a line
+   * whose entering state it changes loses its tokens, since they ran from
+   * the state it replaced. What it writes are pairs like any other past the
+   * frontier, which the frontier's walk, started here, takes or tokenizes
+   * again when it arrives.
+   */
+  private guess(line: number): readonly Token[] {
+    this.guessedTo = Math.max(this.guessedTo, line);
+    this.schedule();
+    const floor = Math.max(this.frontier, line - GUESS_LOOKBACK);
+    let from = floor;
+    while (from < line && this.states[from] === undefined) from++;
+    if (this.states[from] === undefined) {
+      let least = Infinity;
+      for (let i = line; i >= floor; i--) {
+        const text = this.lines[i];
+        const indent = indentOf(text);
+        if (indent === text.length) continue; // blank says nothing
+        if (indent < least) {
+          least = indent;
+          from = i;
+        }
+      }
+      this.states[from] = this.mode.startState();
+      this.tokens[from] = undefined;
+    }
+    const copy = this.mode.copyState ?? defaultCopy;
+    const equals = this.mode.stateEquals ?? defaultEquals;
+    // whether the state entering line `i` was just replaced, so its tokens
+    // ran from another and it is run again, beside what it had
+    let moved = false;
+    for (let i = from; i <= line; i++) {
+      // a pair past the frontier ran from the state beside it; the
+      // frontier's may be from before the walk replaced its state
+      if (
+        !moved &&
+        i > this.frontier &&
+        this.tokens[i] !== undefined &&
+        this.states[i + 1] !== undefined
+      ) {
+        continue;
+      }
+      const state = copy(this.states[i] as S);
+      this.tokens[i] = this.run(i, state);
+      const next = this.states[i + 1];
+      moved = next === undefined || !equals(state, next);
+      if (moved) this.states[i + 1] = state;
+    }
+    // past the line asked about, what ran from a replaced state is set
+    // aside: run again when asked for, and compared with what it had then
+    if (moved) this.setAside(line + 1);
+    return this.tokens[line] ?? [];
+  }
+
+  /**
+   * Run line `i` from `state`, which it leaves as the line leaves it. The
+   * tokens it had, when they come out the same, are kept rather than
+   * replaced — an editor holds a line's layout by them — and where they do
+   * not, the line is noted for the host.
+   */
+  private run(i: number, state: S): Token[] {
+    let had = this.tokens[i];
+    if (had === undefined && this.stale.size > 0) {
+      had = this.stale.get(i);
+      this.stale.delete(i);
+    }
+    const tokens = this.mode.runLine(cut(this.lines[i]), state);
+    if (had === undefined) return tokens;
+    if (sameTokens(had, tokens)) return had;
+    this.noteChange(i);
+    return tokens;
+  }
+
+  /** Take line `i`'s tokens out: the state entering it was replaced. */
+  private setAside(i: number): void {
+    const held = this.tokens[i];
+    if (held === undefined) return;
+    this.tokens[i] = undefined;
+    this.stale.set(i, held);
+  }
+
+  private noteChange(i: number): void {
+    if (this.changedFrom < 0 || i < this.changedFrom) this.changedFrom = i;
+  }
+
+  private schedule(): void {
+    if (this.worker != null) return;
+    this.worker = startTimeout(() => this.work(), 0);
+  }
+
+  /** One turn of the walk to the furthest guess, and the host told of the
+   *  first line it found a guess wrong on. */
+  private work(): void {
+    this.worker = null;
+    if (this.guessedTo < 0) return;
+    const to = Math.min(this.guessedTo, this.lines.length - 1);
+    const done = this.ensure(to, SLICE_LINES);
+    const changed = this.changedFrom;
+    this.changedFrom = -1;
+    if (done) this.guessedTo = -1;
+    else this.schedule();
+    if (changed >= 0) this.host?.invalidate(changed);
+  }
+
+  private stopWorker(): void {
+    stopTimeout(this.worker);
+    this.worker = null;
   }
 }
 
@@ -289,7 +529,7 @@ export function lineModeLanguage<S>(mode: LineMode<S>): Language {
   return {
     name: mode.name,
     data: mode.languageData,
-    createTokenizer: () => new StreamTokenizer(mode),
+    createTokenizer: (host) => new StreamTokenizer(mode, host),
   };
 }
 

package/src/code-language/types.ts

@@ -117,10 +117,11 @@ export interface LineEdit {
 
 /**
  * What the editor hands an engine at creation time. `invalidate(fromLine)`
- * is for engines that tokenize asynchronously (Lezer reparsing on a debounce,
- * an LSP answering semantic tokens): call it when lines at or below
- * `fromLine` have new tokens, and the editor repaints and pulls again.
- * Synchronous engines never need it.
+ * is for tokens that change after they were asked for (Lezer reparsing on a
+ * debounce, an LSP answering semantic tokens, the stream engine correcting a
+ * line it answered from a guess): call it when lines at or below `fromLine`
+ * have new tokens, and the editor repaints and pulls again. An engine that
+ * only ever answers exactly never needs it.
  */
 export interface TokenizerHost {
   invalidate(fromLine: number): void;

package/src/codeblock/index.ts

@@ -140,7 +140,24 @@ export function codeBlockStyle(look: CodeBlockLook): Style {
   };
 }
 
-/** The style the runs are laid out with, in both components. */
-export function codeTextStyle(look: CodeBlockLook): Style {
-  return { lineHeight: look.lineHeight };
+/**
+ * The style the runs are laid out with, in both components — `wrap` as the
+ * `<richtext>` is given it.
+ *
+ * Unwrapped, the text is as wide as its longest line, and at least as wide
+ * as its viewport. Stretched to the viewport, as a column stretches what it
+ * holds, its box held none of a line past the viewport's edge: the viewport
+ * had nothing to scroll to, and core, asked whether the text reached past
+ * the viewport, heard that it did not and left it unclipped — so a fence,
+ * with nothing round it to clip it either, drew the rest of a long line
+ * across whatever stood beside the document.
+ */
+export function codeTextStyle(look: CodeBlockLook, wrap = true): Style {
+  return wrap
+    ? { lineHeight: look.lineHeight }
+    : {
+        lineHeight: look.lineHeight,
+        alignSelf: 'flex-start',
+        minWidth: '100%',
+      };
 }