@react-x11/components 0.1.0 → 0.2.0

package/src/code-language/languages/sql.ts

@@ -0,0 +1,165 @@
+// SQL, as a stream mode. Written here rather than pulled from an ecosystem
+// because the obvious donor (`@codemirror/lang-sql`) depends on the whole
+// CodeMirror editor core (`@codemirror/state`, `language`, `autocomplete`),
+// which is a lot of install closure for a keyword list — see AGENTS.md on
+// why that trade is taken seriously. Tokenizing SQL (not parsing it!) is
+// small: strings, quoted identifiers, comments, numbers, parameters,
+// keywords.
+import { streamLanguage } from '../stream.js';
+import type { StringStream } from '../stream.js';
+import type { Language } from '../types.js';
+
+const KEYWORDS = new Set(
+  (
+    'add all alter and any as asc autoincrement between by cascade case cast ' +
+    'check collate column commit constraint create cross current current_date ' +
+    'current_time current_timestamp database default delete desc distinct drop ' +
+    'else end escape except exists explain fetch filter first following for ' +
+    'foreign from full generated group grouping having if ignore ilike in index ' +
+    'inner insert intersect into is join key last lateral left like limit ' +
+    'materialized natural no not null nulls of offset on only or order outer ' +
+    'over partition preceding primary range recursive references rename ' +
+    'replace restrict returning right rollback row rows select set show some ' +
+    'table temporary then to transaction trigger truncate union unique update ' +
+    'using values view when where window with without'
+  ).split(' '),
+);
+
+const TYPES = new Set(
+  (
+    'bigint binary bit blob boolean char character clob date datetime decimal ' +
+    'double enum float int integer interval json jsonb numeric precision real ' +
+    'serial smallint text time timestamp timestamptz tinyint uuid varbinary ' +
+    'varchar'
+  ).split(' '),
+);
+
+const BUILTINS = new Set(
+  (
+    'abs avg ceil coalesce concat count floor greatest ifnull least length ' +
+    'lower ltrim max min mod now nullif power random round rtrim substr ' +
+    'substring sum trim upper'
+  ).split(' '),
+);
+
+const ATOMS = new Set(['true', 'false', 'unknown']);
+
+const OPERATOR = /[+\-*/<>=~!|&%^]/;
+const PUNCT = /[;,.]/;
+const BRACKET = /[()]/;
+
+interface SqlState {
+  /** Inside a `/* … *​/` comment that has not closed yet. */
+  comment: boolean;
+  /** Inside a `'…'` string that has not closed yet — SQL strings may span
+   * lines. */
+  string: boolean;
+}
+
+/** Consume string content until an un-doubled closing quote or the line
+ * ends; report whether it closed. */
+function eatStringTail(stream: StringStream): boolean {
+  while (!stream.eol()) {
+    if (stream.eat("'")) {
+      if (!stream.eat("'")) return true; // a lone quote closes; '' is an escape
+    } else {
+      stream.next();
+    }
+  }
+  return false;
+}
+
+/** Options for {@link sql}. All additive — the base is generic ANSI-ish. */
+export interface SqlOptions {
+  /** Extra keywords, e.g. a dialect's. Matched case-insensitively. */
+  keywords?: readonly string[];
+  /** Treat `#` as a line comment (MySQL). `--` and `/* *​/` always work. */
+  hashComments?: boolean;
+}
+
+/** SQL. `sql()` for the generic dialect, options for the house one. */
+export function sql(options: SqlOptions = {}): Language {
+  const extra = new Set(
+    (options.keywords ?? []).map((word) => word.toLowerCase()),
+  );
+
+  function token(stream: StringStream, state: SqlState): string | null {
+    if (state.comment) {
+      if (stream.match(/^.*?\*\//)) state.comment = false;
+      else stream.skipToEnd();
+      return 'comment';
+    }
+    if (state.string) {
+      state.string = !eatStringTail(stream);
+      return 'string';
+    }
+    if (stream.eatSpace()) return null;
+
+    if (stream.match('--')) {
+      stream.skipToEnd();
+      return 'comment';
+    }
+    if (options.hashComments && stream.match('#')) {
+      stream.skipToEnd();
+      return 'comment';
+    }
+    if (stream.match('/*')) {
+      state.comment = true;
+      if (stream.match(/^.*?\*\//)) state.comment = false;
+      else stream.skipToEnd();
+      return 'comment';
+    }
+
+    // 'string' — standard escapes the quote by doubling it; may span lines
+    if (stream.eat("'")) {
+      state.string = !eatStringTail(stream);
+      return 'string';
+    }
+    // "quoted identifier" / `mysql identifier`
+    const identQuote = stream.eat('"') ?? stream.eat('`');
+    if (identQuote) {
+      stream.eatWhile((ch) => ch !== identQuote);
+      stream.eat(identQuote);
+      return 'variableName';
+    }
+
+    // parameters: $1, :name, @name, ?
+    if (stream.match(/^\$\d+/) || stream.match(/^[:@][A-Za-z_][\w$]*/)) {
+      return 'atom';
+    }
+    if (stream.eat('?')) return 'atom';
+
+    if (stream.match(/^\d+(\.\d*)?([eE][+-]?\d+)?/) || stream.match(/^\.\d+/)) {
+      return 'number';
+    }
+
+    if (stream.match(/^[A-Za-z_][\w$]*/)) {
+      const word = stream.current().toLowerCase();
+      if (KEYWORDS.has(word) || extra.has(word)) return 'keyword';
+      if (TYPES.has(word)) return 'typeName';
+      if (ATOMS.has(word)) return 'atom';
+      if (BUILTINS.has(word) && stream.match(/^\s*\(/, false)) {
+        return 'function';
+      }
+      return null;
+    }
+
+    if (stream.eat(BRACKET)) return 'bracket';
+    if (stream.eat(PUNCT)) return 'punctuation';
+    if (stream.eatWhile(OPERATOR)) return 'operator';
+    stream.next();
+    return null;
+  }
+
+  return streamLanguage<SqlState>({
+    name: 'sql',
+    languageData: {
+      lineComment: '--',
+      wordChars: '$',
+      completions: [...KEYWORDS, ...TYPES, ...BUILTINS].sort(),
+      indentAfter: /[([]\s*$/,
+    },
+    startState: () => ({ comment: false, string: false }),
+    token,
+  });
+}

package/src/code-language/lezer.ts

@@ -0,0 +1,190 @@
+// The Lezer adapter: plug a CodeMirror-ecosystem grammar (`@lezer/javascript`,
+// `@lezer/python`, `@lezer/rust`, …) into `<CodeEditor language>`.
+//
+// ```ts
+// import { parser } from '@lezer/javascript';
+// <CodeEditor language={lezerLanguage({ name: 'js', parser })} />
+// ```
+//
+// Dependency policy, following the `ical.js` precedent (AGENTS.md): the app
+// installs the grammar package; `@lezer/highlight` — the small (~100 KB,
+// one-dependency) module that turns a Lezer tree into styled ranges — is
+// loaded with a dynamic `import()` that is allowed to fail. It is declared
+// as an *optional peer* rather than an optional dependency because every
+// `@lezer/<lang>` grammar already depends on it: the app that has a parser
+// to pass in has the highlighter on disk, and an optionalDependency would
+// install it even for apps that never touch this file. The parser itself is
+// typed structurally (`LezerParserLike`) so this package's type graph stays
+// free of lezer's.
+//
+// Tokenization is asynchronous by design: a reparse is scheduled on a short
+// debounce after each edit, and the editor repaints when `invalidate` fires
+// — stale colours for a few tens of milliseconds, never a blocked keystroke.
+// (Incremental reparse via `TreeFragments` is a planned refinement; the
+// seam already carries the edit ranges it needs.)
+import { startTimeout, stopTimeout } from './timers.js';
+import type { TimerId } from './timers.js';
+import type {
+  Language,
+  LanguageData,
+  LineEdit,
+  Token,
+  Tokenizer,
+  TokenizerHost,
+} from './types.js';
+
+/** The slice of a Lezer `LRParser` this adapter calls. */
+export interface LezerParserLike {
+  parse(input: string): unknown;
+}
+
+/** The slice of `@lezer/highlight` the adapter uses — also the injection
+ * point tests (and bundlers that dislike dynamic import) use. */
+export interface LezerHighlightLike {
+  highlightTree(
+    tree: unknown,
+    highlighter: unknown,
+    putStyle: (from: number, to: number, classes: string) => void,
+  ): void;
+  classHighlighter: unknown;
+}
+
+export interface LezerLanguageOptions {
+  name: string;
+  parser: LezerParserLike;
+  data?: LanguageData;
+  /** Pass the `@lezer/highlight` module yourself (tests, static bundles);
+   * defaults to `import('@lezer/highlight')`. */
+  highlight?: LezerHighlightLike | Promise<LezerHighlightLike>;
+  /** Reparse debounce, ms (default 30). */
+  delay?: number;
+}
+
+/** `classHighlighter` emits `tok-keyword tok-comment …`; the editor's token
+ * vocabulary *is* those names unprefixed, so the mapping is a strip. */
+function classesToType(classes: string): string {
+  const first = classes.split(' ', 1)[0];
+  return first.startsWith('tok-') ? first.slice(4) : first;
+}
+
+class LezerTokenizer implements Tokenizer {
+  private lines: readonly string[] = [''];
+  private tokens: Token[][] = [];
+  private hl: LezerHighlightLike | null = null;
+  private failed = false;
+  private timer: TimerId = null;
+  private disposed = false;
+  private parseSeq = 0;
+
+  constructor(
+    private readonly options: LezerLanguageOptions,
+    private readonly host: TokenizerHost,
+  ) {
+    const supplied = options.highlight;
+    const loading = supplied
+      ? Promise.resolve(supplied)
+      : // a bare specifier the bundler must not pre-resolve: see header
+        import('@lezer/highlight' as string).then(
+          (m) => m as unknown as LezerHighlightLike,
+        );
+    loading.then(
+      (module_) => {
+        if (this.disposed) return;
+        this.hl = module_;
+        this.schedule(0);
+      },
+      () => {
+        // No highlighter on disk: paint plain, once, quietly — the same
+        // "absence is a normal state" rule the desktop calendar follows.
+        this.failed = true;
+      },
+    );
+  }
+
+  setLines(lines: readonly string[]): void {
+    this.lines = lines;
+    this.tokens = [];
+    this.schedule(0);
+  }
+
+  edit(_edit: LineEdit): void {
+    // Full reparse on the debounce; the old tokens keep painting meanwhile.
+    // The LineEdit is unused today — it is what the TreeFragments-based
+    // incremental version will consume.
+    this.schedule(this.options.delay ?? 30);
+  }
+
+  lineTokens(line: number): readonly Token[] {
+    return this.tokens[line] ?? [];
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    stopTimeout(this.timer);
+    this.timer = null;
+  }
+
+  private schedule(delay: number): void {
+    if (this.failed || !this.hl || this.disposed) return;
+    stopTimeout(this.timer);
+    this.timer = startTimeout(() => {
+      this.timer = null;
+      this.reparse();
+    }, delay);
+  }
+
+  private reparse(): void {
+    const hl = this.hl;
+    if (!hl || this.disposed) return;
+    const seq = ++this.parseSeq;
+    const lines = this.lines;
+    const text = lines.join('\n');
+    let tree: unknown;
+    try {
+      tree = this.options.parser.parse(text);
+    } catch {
+      return; // a grammar that throws paints plain rather than crashing
+    }
+    // line starts, for slicing document offsets into per-line tokens
+    const starts = new Array<number>(lines.length);
+    let offset = 0;
+    for (let i = 0; i < lines.length; i++) {
+      starts[i] = offset;
+      offset += lines[i].length + 1;
+    }
+    const perLine: Token[][] = lines.map(() => []);
+    let line = 0;
+    const put = (from: number, to: number, classes: string): void => {
+      const type = classesToType(classes);
+      while (line + 1 < lines.length && starts[line + 1] <= from) line++;
+      let l = line;
+      let a = from;
+      while (a < to && l < lines.length) {
+        const lineEnd = starts[l] + lines[l].length;
+        const b = Math.min(to, lineEnd);
+        if (b > a) {
+          perLine[l].push({ from: a - starts[l], to: b - starts[l], type });
+        }
+        a = Math.max(b, starts[l] + lines[l].length + 1);
+        l++;
+      }
+    };
+    try {
+      hl.highlightTree(tree, hl.classHighlighter, put);
+    } catch {
+      return;
+    }
+    if (seq !== this.parseSeq || this.disposed || this.lines !== lines) return;
+    this.tokens = perLine;
+    this.host.invalidate(0);
+  }
+}
+
+/** Wrap a Lezer parser as a {@link Language}. */
+export function lezerLanguage(options: LezerLanguageOptions): Language {
+  return {
+    name: options.name,
+    data: options.data,
+    createTokenizer: (host) => new LezerTokenizer(options, host),
+  };
+}

package/src/code-language/registry.ts

@@ -0,0 +1,66 @@
+// Fence-tag → Language. The names are the ones people actually type after
+// three backticks, mapped onto the built-in tokenizers. This is a function
+// of a string, not a table of instances, so a bundle pays only for the
+// languages the call graph can reach — and it can reach all of them, which
+// is the honest cost of highlighting arbitrary fenced code.
+import type { Language } from './types.js';
+import { glsl } from './languages/glsl.js';
+import { javascript } from './languages/javascript.js';
+import { json } from './languages/json.js';
+import { shell } from './languages/shell.js';
+import { sql } from './languages/sql.js';
+
+/**
+ * The `Language` for a markdown fence tag (`js`, `tsx`, `bash`, …), or
+ * null for a tag nothing here tokenizes. Callers decide what null means:
+ * `codeRuns` consults `resolveLanguage` first — `hljsLanguage` is what goes
+ * there for breadth — and paints plain text when that answers null too.
+ */
+export function languageForTag(tag: string): Language | null {
+  switch (tag.toLowerCase()) {
+    case 'js':
+    case 'javascript':
+    case 'mjs':
+    case 'cjs':
+      return javascript();
+    case 'jsx':
+      return javascript({ jsx: true });
+    case 'ts':
+    case 'typescript':
+      return javascript({ typescript: true });
+    case 'tsx':
+      return javascript({ typescript: true, jsx: true });
+    case 'json':
+    case 'jsonc':
+      return json();
+    case 'sh':
+    case 'bash':
+    case 'zsh':
+    case 'shell':
+    case 'console':
+      return shell();
+    case 'sql':
+      return sql();
+    case 'glsl':
+      return glsl();
+    default:
+      return null;
+  }
+}
+
+/**
+ * Run a tokenizer over a whole static text. The editor pulls line by line
+ * while painting; a static code block wants everything once — this is that
+ * adapter. Returns one token array per line.
+ */
+export function tokenizeText(
+  language: Language,
+  text: string,
+): ReadonlyArray<ReadonlyArray<{ from: number; to: number; type: string }>> {
+  const lines = text.split('\n');
+  const tokenizer = language.createTokenizer({ invalidate: () => {} });
+  tokenizer.setLines(lines);
+  const out = lines.map((_, i) => tokenizer.lineTokens(i));
+  tokenizer.dispose?.();
+  return out;
+}

package/src/code-language/runs.ts

@@ -0,0 +1,128 @@
+// Static highlighting: text + fence tag → styled runs, ready for a
+// TextLayout-shaped consumer (`<richtext>` speaks exactly this shape,
+// structurally — this module deliberately imports nothing from it).
+//
+// The chain is: an explicit `language`, else `resolveLanguage(tag)` if the
+// caller supplied one, else a built-in tokenizer for the tag, else one plain
+// run. Every step degrades to readable.
+//
+// The third step used to be ntk's `highlightCode` — a MarkdownView-internal
+// this module reached for by name, so that a fence tagged `python` got
+// highlighted without the app arranging anything. ntk's document widgets are
+// being decommissioned and it goes with them; `resolveLanguage` is what
+// replaces it, and `hljsLanguage` (./hljs.ts) is the same highlight.js
+// breadth as a `Language` the app opts into:
+//
+// ```ts
+// import hljs from 'highlight.js/lib/common';
+// codeRuns(text, tag, { …, resolveLanguage: (t) => hljsLanguage({hljs, name: t}) })
+// ```
+//
+// Explicit rather than automatic, because the alternative is this package
+// depending on highlight.js for every app that renders a fence — see
+// AGENTS.md on tree-shaking, and `textmate.ts` for the same shape.
+import { languageForTag, tokenizeText } from './registry.js';
+import { tokenStyleFor } from './theme.js';
+import type { Language, TokenStyles } from './types.js';
+
+/** One styled run of code. Structurally a `<richtext>` run. */
+export interface CodeRun {
+  text: string;
+  color?: string;
+  weight?: number;
+  style?: 'normal' | 'italic';
+}
+
+export interface CodeRunOptions {
+  /** Token palette; `autoTokenStyles(background)` picks a built-in one. */
+  styles: TokenStyles;
+  /** The plain-text colour for gaps and unstyled tokens. */
+  color: string;
+  /** Resolves a `'$token'` colour a custom palette may use; unresolved
+   *  `$tokens` drop to the plain colour rather than reaching the paint
+   *  path as literal dollar strings. */
+  resolveToken?: (name: string) => string | undefined;
+  /** An explicit `Language` (a Lezer or TextMate adapter, say) — takes
+   *  precedence over whatever the tag would have resolved to. */
+  language?: Language;
+  /**
+   * A `Language` for a fence tag the built-ins do not cover — the seam for
+   * highlight.js (`hljsLanguage`), a Lezer grammar per tag, or a lookup of
+   * the app's own. Consulted before {@link languageForTag}, so it can also
+   * override a built-in; `null` for a tag it does not know.
+   */
+  resolveLanguage?: (tag: string) => Language | null;
+}
+
+function runColor(
+  color: string | undefined,
+  opts: CodeRunOptions,
+): string | undefined {
+  if (!color) return undefined;
+  if (color.startsWith('$'))
+    return opts.resolveToken?.(color.slice(1)) ?? undefined;
+  return color;
+}
+
+/**
+ * Highlight `text` as `tag` (a fence tag: `js`, `bash`, …). The runs
+ * concatenate back to exactly `text`; an empty or unknown tag yields one
+ * plain run.
+ */
+export function codeRuns(
+  text: string,
+  tag: string,
+  opts: CodeRunOptions,
+): CodeRun[] {
+  if (text.length === 0) return [{ text: '', color: opts.color }];
+
+  const language =
+    opts.language ??
+    (tag ? (opts.resolveLanguage?.(tag) ?? languageForTag(tag)) : null);
+  if (language) {
+    const perLine = tokenizeText(language, text);
+    const lines = text.split('\n');
+    const out: CodeRun[] = [];
+    for (let i = 0; i < lines.length; i += 1) {
+      const line = lines[i];
+      if (i > 0) push(out, '\n', undefined, opts);
+      let at = 0;
+      for (const t of perLine[i]) {
+        if (t.from > at) push(out, line.slice(at, t.from), undefined, opts);
+        push(out, line.slice(t.from, t.to), t.type, opts);
+        at = t.to;
+      }
+      if (at < line.length) push(out, line.slice(at), undefined, opts);
+    }
+    return out;
+  }
+
+  return [{ text, color: opts.color }];
+}
+
+function push(
+  out: CodeRun[],
+  text: string,
+  type: string | undefined,
+  opts: CodeRunOptions,
+): void {
+  if (text.length === 0) return;
+  const style = type ? tokenStyleFor(opts.styles, type) : null;
+  const run: CodeRun = {
+    text,
+    color: runColor(style?.color, opts) ?? opts.color,
+  };
+  if (style?.weight) run.weight = style.weight;
+  if (style?.fontStyle) run.style = style.fontStyle;
+  const prev = out[out.length - 1];
+  if (
+    prev &&
+    prev.color === run.color &&
+    prev.weight === run.weight &&
+    prev.style === run.style
+  ) {
+    prev.text += text;
+  } else {
+    out.push(run);
+  }
+}