@react-x11/components 0.1.0 → 0.2.0

package/src/code-language/stream.ts

@@ -0,0 +1,306 @@
+// The built-in tokenizing engine: line-state stream tokenization.
+//
+// A mode reads one line through a `StringStream` and carries a small state
+// across line boundaries — the shape CodeMirror 5 ran on for a decade and
+// VS Code's Monarch still runs on. It was chosen over a parse-tree engine
+// for the out-of-the-box languages because it matches how the editor paints
+// (per line, lazily, only the visible window), it makes incremental
+// re-tokenization after an edit almost free (recompute from the edited line,
+// stop at the first line whose entry state is unchanged), and it costs zero
+// dependencies. Real parsers plug in through the same `Language` interface —
+// see `./lezer.ts`.
+
+import type {
+  Language,
+  LanguageData,
+  LineEdit,
+  Token,
+  Tokenizer,
+} from './types.js';
+
+/**
+ * One line of text with a cursor, the surface a mode's `token()` reads
+ * through. The mode consumes some characters and names their token type;
+ * the engine turns (start, pos] into a `Token`.
+ */
+export class StringStream {
+  /** Where the current token started. The engine resets it between calls. */
+  start = 0;
+  pos = 0;
+
+  constructor(readonly string: string) {}
+
+  sol(): boolean {
+    return this.pos === 0;
+  }
+  eol(): boolean {
+    return this.pos >= this.string.length;
+  }
+  peek(): string | undefined {
+    return this.string.charAt(this.pos) || undefined;
+  }
+  next(): string | undefined {
+    if (this.pos < this.string.length) return this.string.charAt(this.pos++);
+    return undefined;
+  }
+  /** Consume the next character when it matches (a string or a RegExp). */
+  eat(match: string | RegExp | ((ch: string) => boolean)): string | undefined {
+    const ch = this.string.charAt(this.pos);
+    if (!ch) return undefined;
+    const ok =
+      typeof match === 'string'
+        ? ch === match
+        : match instanceof RegExp
+          ? match.test(ch)
+          : match(ch);
+    if (!ok) return undefined;
+    this.pos++;
+    return ch;
+  }
+  eatWhile(match: RegExp | ((ch: string) => boolean)): boolean {
+    const from = this.pos;
+    while (this.eat(match) !== undefined) {
+      /* consumed in the condition */
+    }
+    return this.pos > from;
+  }
+  eatSpace(): boolean {
+    return this.eatWhile(/[\s ]/);
+  }
+  skipToEnd(): void {
+    this.pos = this.string.length;
+  }
+  /**
+   * Match at the cursor. A string compares directly; a RegExp is applied to
+   * the rest of the line and must match at its start (anchor with `^`).
+   * Consumes on success unless `consume` is `false`.
+   */
+  match(
+    pattern: string | RegExp,
+    consume = true,
+  ): boolean | RegExpMatchArray | null {
+    if (typeof pattern === 'string') {
+      if (!this.string.startsWith(pattern, this.pos)) return null;
+      if (consume) this.pos += pattern.length;
+      return true;
+    }
+    const m = pattern.exec(this.string.slice(this.pos));
+    if (!m || m.index! > 0) return null;
+    if (consume) this.pos += m[0].length;
+    return m;
+  }
+  backUp(n: number): void {
+    this.pos -= n;
+  }
+  current(): string {
+    return this.string.slice(this.start, this.pos);
+  }
+}
+
+/**
+ * A stream mode: `startState` before line 0, `token` consuming one token at
+ * the stream's cursor (return its type, or `null` for plain text), the state
+ * mutated in place as it goes.
+ *
+ * States must be flat bags of primitives (strings, numbers, booleans) —
+ * that is what the default `copyState`/`stateEquals` handle, and what makes
+ * the stop-early check on re-tokenization cheap. A mode with a stack encodes
+ * it as a string (see the JavaScript mode's `ctx`).
+ */
+export interface StreamMode<S> {
+  name: string;
+  languageData?: LanguageData;
+  startState(): S;
+  token(stream: StringStream, state: S): string | null;
+  /** Override when a state is not a flat bag of primitives. */
+  copyState?(state: S): S;
+  /** Override alongside `copyState`, and keep the two consistent. */
+  stateEquals?(a: S, b: S): boolean;
+}
+
+/**
+ * What the engine actually runs on: a whole-line tokenizer with carried
+ * state. `streamLanguage` builds one from a character-level `StreamMode`;
+ * the TextMate adapter builds one directly, because its grammars tokenize
+ * whole lines natively. Both get the same caching and convergence.
+ */
+export interface LineMode<S> {
+  name: string;
+  languageData?: LanguageData;
+  startState(): S;
+  runLine(text: string, state: S): Token[];
+  copyState?(state: S): S;
+  stateEquals?(a: S, b: S): boolean;
+}
+
+function defaultCopy<S>(state: S): S {
+  if (state === null || typeof state !== 'object') return state;
+  return { ...(state as object) } as S;
+}
+
+function defaultEquals(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (
+    a === null ||
+    b === null ||
+    typeof a !== 'object' ||
+    typeof b !== 'object'
+  ) {
+    return false;
+  }
+  const ka = Object.keys(a as object);
+  const kb = Object.keys(b as object);
+  if (ka.length !== kb.length) return false;
+  for (const k of ka) {
+    if (
+      (a as Record<string, unknown>)[k] !== (b as Record<string, unknown>)[k]
+    ) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * The engine. Caches, per line, the state *entering* it (`states[i]`) and
+ * the tokens it produced (`tokens[i]`), the two written together so a cached
+ * pair is always self-consistent. `edit()` splices the caches to the new
+ * line numbering, leaving `undefined` holes exactly where text changed; the
+ * next `lineTokens()` re-tokenizes forward from the edit and stops as soon
+ * as a freshly computed entry state equals a cached one — the classic
+ * convergence trick, and the reason typing at the top of a 10k-line file
+ * re-tokenizes until the edit stops mattering (usually one line), not 10k
+ * lines. The `undefined` holes are also what makes convergence sound across
+ * multiple pending edits: the fast-forward below can never skip over a line
+ * whose text changed, because that line's cache is a hole.
+ */
+class StreamTokenizer<S> implements Tokenizer {
+  private lines: readonly string[] = [''];
+  /** states[i] = state entering line i. Trusted for i <= frontier; a
+   * defined entry beyond that is a pre-edit value, used only as a
+   * convergence candidate. */
+  private states: (S | undefined)[] = [];
+  private tokens: (Token[] | undefined)[] = [];
+  /** Lines `[0, frontier)` have trusted tokens. */
+  private frontier = 0;
+
+  constructor(private readonly mode: LineMode<S>) {}
+
+  setLines(lines: readonly string[]): void {
+    this.lines = lines;
+    this.states = new Array(lines.length + 1);
+    this.tokens = new Array(lines.length);
+    this.states[0] = this.mode.startState();
+    this.frontier = 0;
+  }
+
+  edit({ fromLine, removed, inserted }: LineEdit): void {
+    // Align the caches below the edit with the new line numbering (their
+    // contents may still be right — convergence will decide), and punch
+    // `undefined` holes for the lines whose text actually changed.
+    this.states.splice(
+      fromLine + 1,
+      removed,
+      ...new Array<S | undefined>(inserted),
+    );
+    this.tokens.splice(
+      fromLine,
+      removed,
+      ...new Array<Token[] | undefined>(inserted),
+    );
+    this.frontier = Math.min(this.frontier, fromLine);
+  }
+
+  lineTokens(line: number): readonly Token[] {
+    if (line < 0 || line >= this.lines.length) return [];
+    this.ensure(line);
+    return this.tokens[line] ?? [];
+  }
+
+  private ensure(line: number): void {
+    const copy = this.mode.copyState ?? defaultCopy;
+    const equals = this.mode.stateEquals ?? defaultEquals;
+    while (this.frontier <= line) {
+      const i = this.frontier;
+      // `states[i]` is trusted here: it is either below the old frontier or
+      // was written by the previous iteration. Copy before running — the
+      // mode mutates in place, and the cached entry must stay pristine.
+      const state = copy(this.states[i] as S);
+      this.tokens[i] = this.mode.runLine(this.lines[i], state);
+      const cached = this.states[i + 1];
+      if (
+        cached !== undefined &&
+        this.tokens[i + 1] !== undefined &&
+        equals(state, cached)
+      ) {
+        // Converged: the cached suffix is still right (same entry state,
+        // and every cached pair was computed from the text it sits beside —
+        // an edit would have punched a hole). Fast-forward to the next hole
+        // and resume tokenizing for real there, if the caller needs it.
+        let f = i + 1;
+        while (
+          f < this.lines.length &&
+          this.tokens[f] !== undefined &&
+          this.states[f + 1] !== undefined
+        ) {
+          f++;
+        }
+        this.frontier = f;
+        continue;
+      }
+      this.states[i + 1] = state;
+      this.frontier = i + 1;
+    }
+  }
+}
+
+/** Run one line of a character-level mode, merging same-type runs. */
+function runStreamLine<S>(
+  mode: StreamMode<S>,
+  text: string,
+  state: S,
+): Token[] {
+  const out: Token[] = [];
+  const stream = new StringStream(text);
+  while (!stream.eol()) {
+    stream.start = stream.pos;
+    const type = mode.token(stream, state);
+    if (stream.pos <= stream.start) {
+      // A mode that consumes nothing would loop forever; skip a character
+      // and paint it plain, which also makes mode bugs visible instead of
+      // hanging the app.
+      stream.pos = stream.start + 1;
+    }
+    if (type) {
+      const last = out[out.length - 1];
+      if (last && last.to === stream.start && last.type === type) {
+        last.to = stream.pos; // merge adjacent same-type runs
+      } else {
+        out.push({ from: stream.start, to: stream.pos, type });
+      }
+    }
+  }
+  return out;
+}
+
+/** Wrap a whole-line {@link LineMode} as a {@link Language} — the seam the
+ * TextMate adapter enters through. */
+export function lineModeLanguage<S>(mode: LineMode<S>): Language {
+  return {
+    name: mode.name,
+    data: mode.languageData,
+    createTokenizer: () => new StreamTokenizer(mode),
+  };
+}
+
+/** Wrap a {@link StreamMode} as a {@link Language}. */
+export function streamLanguage<S>(mode: StreamMode<S>): Language {
+  return lineModeLanguage({
+    name: mode.name,
+    languageData: mode.languageData,
+    startState: mode.startState,
+    runLine: (text, state) => runStreamLine(mode, text, state),
+    copyState: mode.copyState,
+    stateEquals: mode.stateEquals,
+  });
+}

package/src/code-language/textmate.ts

@@ -0,0 +1,134 @@
+// The TextMate adapter: plug a grammar from the VS Code / Shiki world into
+// `<CodeEditor language>`.
+//
+// TextMate grammars are the largest grammar collection in existence — every
+// VS Code language extension ships one — and their tokenizer is *exactly*
+// line-state shaped: `tokenizeLine(line, previousState)` returns scoped
+// tokens plus the state the next line starts in, which is this editor's
+// native model. So the adapter is a scope-name mapping over the same
+// line-table engine the built-in modes use — incremental re-tokenization
+// and convergence come for free, with `StateStack.equals` as the
+// convergence test — and this package depends on none of the TextMate
+// machinery. The app brings an initialized grammar:
+//
+// ```ts
+// // via vscode-textmate + vscode-oniguruma, or shiki's core — app policy
+// const grammar = await registry.loadGrammar('source.python');
+// <CodeEditor language={textMateLanguage({ name: 'python', grammar })} />
+// ```
+import type { Language, LanguageData, Token } from './types.js';
+import { lineModeLanguage } from './stream.js';
+
+/** vscode-textmate's `StateStack`, structurally: opaque, with `equals`. */
+export interface TextMateStateLike {
+  equals(other: TextMateStateLike): boolean;
+}
+
+/** The slice of vscode-textmate's `IGrammar` the adapter calls. */
+export interface TextMateGrammarLike {
+  tokenizeLine(
+    line: string,
+    prevState: TextMateStateLike | null,
+  ): {
+    tokens: Array<{ startIndex: number; endIndex?: number; scopes: string[] }>;
+    ruleStack: TextMateStateLike;
+  };
+}
+
+/**
+ * TextMate scope prefix → editor token type, longest dotted prefix wins,
+ * most specific scope first. Extend with `scopeStyles` for a grammar that
+ * invents its own names.
+ */
+const DEFAULT_SCOPE_MAP: Readonly<Record<string, string>> = {
+  comment: 'comment',
+  'comment.block.documentation': 'docComment',
+  string: 'string',
+  'string.regexp': 'string2',
+  'constant.numeric': 'number',
+  'constant.language': 'atom',
+  'constant.character.escape': 'escape',
+  constant: 'atom',
+  keyword: 'keyword',
+  'keyword.operator': 'operator',
+  storage: 'keyword',
+  'storage.modifier': 'modifier',
+  'entity.name.function': 'function',
+  'entity.name.type': 'typeName',
+  'entity.name.class': 'className',
+  'entity.name.namespace': 'namespace',
+  'entity.name.tag': 'typeName',
+  'entity.other.attribute-name': 'propertyName',
+  'support.function': 'function',
+  'support.type': 'typeName',
+  'support.class': 'className',
+  'support.constant': 'atom',
+  variable: 'variableName',
+  'variable.other.property': 'propertyName',
+  'punctuation.definition.comment': 'comment',
+  'punctuation.definition.string': 'string',
+  punctuation: 'punctuation',
+  'meta.preprocessor': 'meta',
+  invalid: 'invalid',
+};
+
+export interface TextMateLanguageOptions {
+  name: string;
+  grammar: TextMateGrammarLike;
+  data?: LanguageData;
+  /** Extra scope-prefix → token-type entries, consulted before the
+   * defaults. */
+  scopeStyles?: Readonly<Record<string, string>>;
+}
+
+function mapScopes(
+  scopes: readonly string[],
+  extra: Readonly<Record<string, string>> | undefined,
+): string | null {
+  // the most specific scope is last; walk backwards and take the first
+  // (longest-prefix) mapping that answers
+  for (let i = scopes.length - 1; i >= 0; i--) {
+    const scope = scopes[i];
+    for (let cut = scope.length; cut > 0;) {
+      const prefix = scope.slice(0, cut);
+      const mapped = extra?.[prefix] ?? DEFAULT_SCOPE_MAP[prefix];
+      if (mapped) return mapped;
+      cut = prefix.lastIndexOf('.');
+    }
+  }
+  return null;
+}
+
+/** Wrap an initialized TextMate grammar as a {@link Language}. */
+export function textMateLanguage(options: TextMateLanguageOptions): Language {
+  interface S {
+    stack: TextMateStateLike | null;
+  }
+  return lineModeLanguage<S>({
+    name: options.name,
+    languageData: options.data,
+    startState: () => ({ stack: null }),
+    copyState: (s) => ({ stack: s.stack }),
+    stateEquals: (a, b) =>
+      a.stack === b.stack ||
+      (a.stack != null && b.stack != null && a.stack.equals(b.stack)),
+    runLine(line: string, state: S): Token[] {
+      const result = options.grammar.tokenizeLine(line, state.stack);
+      state.stack = result.ruleStack;
+      const out: Token[] = [];
+      const tokens = result.tokens;
+      for (let i = 0; i < tokens.length; i++) {
+        const t = tokens[i];
+        const from = t.startIndex;
+        const to = t.endIndex ?? tokens[i + 1]?.startIndex ?? line.length;
+        if (to <= from) continue;
+        const type = mapScopes(t.scopes, options.scopeStyles);
+        if (!type) continue;
+        const last = out[out.length - 1];
+        if (last && last.to === from && last.type === type) last.to = to;
+        else out.push({ from, to, type });
+      }
+      return out;
+    },
+  });
+}

package/src/code-language/theme.ts

@@ -0,0 +1,124 @@
+// Token styles: how the vocabulary in `types.ts` paints.
+//
+// Two built-in palettes, chosen for contrast against the backgrounds
+// react-x11's own light and dark themes give a field. Neither is a clone of
+// a licensed theme; both stay in the familiar register (keywords violet,
+// strings green-in-dark / navy-in-light, comments dim italic) because a code
+// editor that surprises the eye reads as broken. An app overrides per token
+// type, and a value may be a `$token` into the react-x11 theme — resolved at
+// paint against the editor's own `theme`, so `comment: { color: '$textMuted' }`
+// follows the palette the rest of the window uses.
+import type { TokenStyle, TokenStyles } from './types.js';
+import { TOKEN_FALLBACK } from './types.js';
+
+/** For light backgrounds (the default). */
+export const LIGHT_TOKEN_STYLES: TokenStyles = {
+  keyword: { color: '#a626a4' },
+  operator: { color: '#526066' },
+  variableName: { color: '#383a42' },
+  typeName: { color: '#c18401' },
+  propertyName: { color: '#4078f2' },
+  function: { color: '#4078f2' },
+  string: { color: '#50a14f' },
+  string2: { color: '#2f7d3b' },
+  escape: { color: '#0184bc' },
+  number: { color: '#986801' },
+  atom: { color: '#986801' },
+  self: { color: '#e45649' },
+  comment: { color: '#a0a1a7', fontStyle: 'italic' },
+  meta: { color: '#a626a4' },
+  punctuation: { color: '#6a737d' },
+  invalid: { color: '#e45649' },
+};
+
+/** For dark backgrounds. */
+export const DARK_TOKEN_STYLES: TokenStyles = {
+  keyword: { color: '#c678dd' },
+  operator: { color: '#9da5b4' },
+  variableName: { color: '#abb2bf' },
+  typeName: { color: '#e5c07b' },
+  propertyName: { color: '#61afef' },
+  function: { color: '#61afef' },
+  string: { color: '#98c379' },
+  string2: { color: '#7bae62' },
+  escape: { color: '#56b6c2' },
+  number: { color: '#d19a66' },
+  atom: { color: '#d19a66' },
+  self: { color: '#e06c75' },
+  comment: { color: '#5c6370', fontStyle: 'italic' },
+  meta: { color: '#c678dd' },
+  punctuation: { color: '#7f848e' },
+  invalid: { color: '#e06c75' },
+};
+
+/**
+ * Look a token type up, walking {@link TOKEN_FALLBACK} so a theme may style
+ * `string` once and have `string2`/`escape` follow. Null when nothing in the
+ * chain is styled — the editor then paints plain.
+ */
+export function tokenStyleFor(
+  styles: TokenStyles,
+  type: string,
+): TokenStyle | null {
+  let t: string | undefined = type;
+  for (let hops = 0; t !== undefined && hops < 6; hops++) {
+    const style = styles[t];
+    if (style) return style;
+    t = TOKEN_FALLBACK[t];
+  }
+  return null;
+}
+
+/**
+ * Is this a colour a light palette would be unreadable on? `null` when the
+ * string is not a colour this can judge (`'transparent'`, a gradient, an
+ * unresolved `$token`) — the caller then falls back to the next layer down
+ * (the react-x11 theme's `background`).
+ *
+ * Judged by WCAG relative luminance, cut at 0.42 rather than 0.5 because
+ * mid-grey editor themes (Solarized-ish `#657b83` neighbourhoods) read
+ * better with the dark set.
+ */
+export function isDarkBackground(color: string | undefined): boolean | null {
+  if (!color) return null;
+  const rgb = parseColor(color);
+  if (!rgb) return null;
+  const [r, g, b] = rgb.map((v) => {
+    const c = v / 255;
+    return c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4;
+  });
+  return 0.2126 * r + 0.7152 * g + 0.0722 * b < 0.42;
+}
+
+function parseColor(color: string): [number, number, number] | null {
+  const c = color.trim();
+  const hex = /^#([0-9a-fA-F]{3,8})$/.exec(c)?.[1];
+  if (hex) {
+    if (hex.length === 3 || hex.length === 4) {
+      return [
+        parseInt(hex[0] + hex[0], 16),
+        parseInt(hex[1] + hex[1], 16),
+        parseInt(hex[2] + hex[2], 16),
+      ];
+    }
+    if (hex.length === 6 || hex.length === 8) {
+      return [
+        parseInt(hex.slice(0, 2), 16),
+        parseInt(hex.slice(2, 4), 16),
+        parseInt(hex.slice(4, 6), 16),
+      ];
+    }
+    return null;
+  }
+  const fn = /^rgba?\(\s*([\d.]+)\s*,\s*([\d.]+)\s*,\s*([\d.]+)/.exec(c);
+  if (fn) {
+    return [Number(fn[1]), Number(fn[2]), Number(fn[3])];
+  }
+  return null;
+}
+
+/** The built-in palette for a background: dark set on dark, light set
+ * otherwise. What the editor uses when no `tokenStyles` prop is given. */
+export function autoTokenStyles(background: string | undefined): TokenStyles {
+  return isDarkBackground(background) ? DARK_TOKEN_STYLES : LIGHT_TOKEN_STYLES;
+}

package/src/code-language/timers.ts

@@ -0,0 +1,36 @@
+// Timers, reached through `globalThis`. The build compiles `src/` with
+// `types: []` on purpose — a Node global that wanders in must fail the
+// build rather than become an implicit `@types/node` dependency (AGENTS.md)
+// — so the runtime-provided timers are typed here, structurally, once.
+interface TimerGlobals {
+  setInterval?(fn: () => void, ms: number): unknown;
+  clearInterval?(id: unknown): void;
+  setTimeout?(fn: () => void, ms: number): unknown;
+  clearTimeout?(id: unknown): void;
+  queueMicrotask?(fn: () => void): void;
+}
+
+const g = globalThis as TimerGlobals;
+
+export type TimerId = unknown;
+
+export function startInterval(fn: () => void, ms: number): TimerId {
+  return g.setInterval?.(fn, ms) ?? null;
+}
+
+export function stopInterval(id: TimerId): void {
+  if (id != null) g.clearInterval?.(id);
+}
+
+export function startTimeout(fn: () => void, ms: number): TimerId {
+  return g.setTimeout?.(fn, ms) ?? null;
+}
+
+export function stopTimeout(id: TimerId): void {
+  if (id != null) g.clearTimeout?.(id);
+}
+
+export function microtask(fn: () => void): void {
+  if (g.queueMicrotask) g.queueMicrotask(fn);
+  else void Promise.resolve().then(fn);
+}