@blue-patterns/richtext 0.0.0-stage → 0.7.18

package/src/codeLineTokens.ts

@@ -0,0 +1,241 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file codeLineTokens.ts
+ * @input Core CodeBlock's tokenizer (`tokenize`) and token line type.
+ * @output Exports CodeLineTokens, which keeps core CodeBlock's tokens for a
+ *   block of code line by line and re-tokenizes only the lines an edit can
+ *   change.
+ * @position Used by CodeSyntaxPlugin. Its tokens always equal what `tokenize`
+ *   returns for the whole code. An edit re-tokenizes only its own line, and
+ *   at most the line above it, for the built-in TypeScript, JavaScript,
+ *   Python, and shell grammars while every line passes `standsAlone`;
+ *   anything else re-tokenizes all of it. The checks follow those grammars'
+ *   patterns in core CodeBlock's tokenizer.ts, and codeLineTokens.test.ts
+ *   compares edits against a full `tokenize`, so a grammar change that
+ *   breaks them fails there.
+ */
+
+import {
+  tokenize,
+  type CodeLanguage,
+  type TokenLine,
+} from '@blue-patterns/core/CodeBlock';
+
+type Family = 'javascript' | 'python' | 'shell';
+
+const FAMILIES: ReadonlyMap<string, Family> = new Map([
+  ['typescript', 'javascript'],
+  ['javascript', 'javascript'],
+  ['tsx', 'javascript'],
+  ['jsx', 'javascript'],
+  ['ts', 'javascript'],
+  ['js', 'javascript'],
+  ['python', 'python'],
+  ['py', 'python'],
+  ['bash', 'shell'],
+  ['sh', 'shell'],
+  ['zsh', 'shell'],
+  ['shell', 'shell'],
+]);
+
+/** What can open a string or comment that runs on past its line. */
+const OPENERS: Readonly<Record<Family, RegExp>> = {
+  javascript: /['"`]|\/\*/,
+  python: /['"]/,
+  shell: /['"`]|\$\{|\$\(|<</,
+};
+
+/** A line whose last visible character can end a function name. */
+const NAME_END = /[\w$]\s*$/;
+
+/**
+ * Whether the name that ends a line can change color with the code after
+ * it: its last visible character is a name character outside every string,
+ * comment, number, keyword, and constant, whose patterns come before
+ * function names.
+ */
+function endsInOpenName(text: string, tokens: TokenLine): boolean {
+  const last = text.trimEnd().length - 1;
+  return (
+    NAME_END.test(text) &&
+    !tokens.some(
+      token =>
+        token.start <= last &&
+        token.end > last &&
+        token.type !== 'function' &&
+        token.type !== 'type',
+    )
+  );
+}
+
+/** A line's first visible character, or '' when it has none. */
+function firstVisible(line: string): string {
+  return /\S/.exec(line)?.[0] ?? '';
+}
+
+/**
+ * Whether a line's strings and comments all close on it. Their patterns
+ * match line breaks, so an opener left open can run on to a later line.
+ * Only a string or comment the tokenizer emitted shields an opener: one that
+ * another token swallowed, such as `/*` in the operator `+/*`, still counts
+ * as open. Python tries triple quotes first, so `"""` can pair with another
+ * line even where its quotes close as single strings on this one.
+ */
+function standsAlone(text: string, tokens: TokenLine, family: Family): boolean {
+  if (family === 'python' && /'''|"""/.test(text)) {
+    return false;
+  }
+  const opener = OPENERS[family];
+  let position = 0;
+  for (const token of tokens) {
+    if (token.end > text.length) {
+      return false;
+    }
+    if (token.type === 'string' || token.type === 'comment') {
+      if (opener.test(text.slice(position, token.start))) {
+        return false;
+      }
+      position = token.end;
+    }
+  }
+  return !opener.test(text.slice(position));
+}
+
+/**
+ * Core CodeBlock's tokens for a block of code, kept line by line with
+ * line-relative offsets, exactly as `tokenize` returns them for the whole
+ * code.
+ */
+export class CodeLineTokens {
+  readonly language: string | CodeLanguage;
+  /** Where each line starts in the code. */
+  starts: number[] = [];
+  /** Each line's tokens. */
+  tokens: TokenLine[] = [];
+  private lines: string[] = [];
+  private readonly family: Family | null;
+  /** Whether every line stands alone, so an edit re-tokenizes lines alone. */
+  private linesStandAlone = false;
+
+  constructor(text: string, language: string | CodeLanguage) {
+    this.language = language;
+    this.family =
+      typeof language === 'string' ? (FAMILIES.get(language) ?? null) : null;
+    this.tokenizeAll(text);
+  }
+
+  /** The code. */
+  get text(): string {
+    return this.lines.join('\n');
+  }
+
+  /** The line that holds `offset` in the code. */
+  lineAt(offset: number): number {
+    let low = 0;
+    let high = this.starts.length - 1;
+    while (low < high) {
+      const mid = (low + high + 1) >>> 1;
+      if (this.starts[mid] <= offset) {
+        low = mid;
+      } else {
+        high = mid - 1;
+      }
+    }
+    return low;
+  }
+
+  /**
+   * Replaces `removed` characters at `offset` in the code with `inserted`.
+   * Returns the lines it re-tokenized when the edit stays inside one line
+   * and every line still stands alone; otherwise it re-tokenizes all the
+   * code and returns null.
+   */
+  replace(offset: number, removed: number, inserted: string): number[] | null {
+    const line = this.lineAt(offset);
+    const before = this.lines[line];
+    const column = offset - this.starts[line];
+    if (
+      this.family == null ||
+      !this.linesStandAlone ||
+      column + removed > before.length ||
+      inserted.includes('\n')
+    ) {
+      const text = this.text;
+      this.tokenizeAll(
+        text.slice(0, offset) + inserted + text.slice(offset + removed),
+      );
+      return null;
+    }
+    const after =
+      before.slice(0, column) + inserted + before.slice(column + removed);
+    this.lines[line] = after;
+    const tokens = this.tokenizeLine(line);
+    if (!standsAlone(after, tokens, this.family)) {
+      this.tokenizeAll(this.text);
+      return null;
+    }
+    this.tokens[line] = tokens;
+    for (let i = line + 1; i < this.starts.length; i++) {
+      this.starts[i] += inserted.length - removed;
+    }
+    const changed = [line];
+    // A name that ends an earlier line looks past blank lines for `(`, so a
+    // new first visible character here can change that name's color.
+    if (firstVisible(before) !== firstVisible(after)) {
+      let above = line - 1;
+      while (above >= 0 && firstVisible(this.lines[above]) === '') {
+        above--;
+      }
+      if (
+        above >= 0 &&
+        endsInOpenName(this.lines[above], this.tokens[above] ?? [])
+      ) {
+        this.tokens[above] = this.tokenizeLine(above);
+        changed.unshift(above);
+      }
+    }
+    return changed;
+  }
+
+  /**
+   * A line's tokens as `tokenize` finds them in the whole code. A function
+   * name is a name followed by `(` past any whitespace, line breaks
+   * included, so when the line ends in a name, the code up to the next
+   * visible character is tokenized with it.
+   */
+  private tokenizeLine(line: number): TokenLine {
+    let text = this.lines[line];
+    if (NAME_END.test(text)) {
+      for (let next = line + 1; next < this.lines.length; next++) {
+        const visible = /\S/.exec(this.lines[next]);
+        text +=
+          '\n' +
+          (visible == null
+            ? this.lines[next]
+            : this.lines[next].slice(0, visible.index + 1));
+        if (visible != null) {
+          break;
+        }
+      }
+    }
+    return tokenize(text, this.language)[0] ?? [];
+  }
+
+  private tokenizeAll(text: string): void {
+    this.lines = text.split('\n');
+    let start = 0;
+    this.starts = this.lines.map(line => {
+      const lineStart = start;
+      start += line.length + 1;
+      return lineStart;
+    });
+    this.tokens = tokenize(text, this.language);
+    const family = this.family;
+    this.linesStandAlone =
+      family != null &&
+      this.lines.every((line, i) =>
+        standsAlone(line, this.tokens[i] ?? [], family),
+      );
+  }
+}

package/src/doc-mjs.d.ts

@@ -0,0 +1,23 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Ambient module declaration for `*.doc.mjs` imports.
+ *
+ * Package tests import `{Name}.doc.mjs` files directly to assert that their
+ * documented metadata stays in sync with what the component renders (see the
+ * theme-target test in RichTextEditor.test.tsx). Without this declaration,
+ * TypeScript has no way to type a `.mjs` import and the package typecheck
+ * fails with TS7016 ("Could not find a declaration file for module").
+ * Mirrors packages/core/src/doc-mjs.d.ts, which this package cannot see.
+ */
+
+declare module '*.doc.mjs' {
+  import type {
+    ComponentDoc,
+    ComponentTranslationDoc,
+  } from '@blue-patterns/cli/authoring';
+
+  export const docs: ComponentDoc;
+  export const docsZh: ComponentDoc;
+  export const docsDense: ComponentTranslationDoc;
+}

package/src/editorNodes.ts

@@ -0,0 +1,44 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file editorNodes.ts
+ * @input Imports the default Lexical node classes registered by the editor.
+ * @output Exports DEFAULT_NODES — the base OSS node set.
+ * @position Shared by RichTextEditor.tsx (editor config) and
+ *   markdownSerializers.ts (headless conversion), so both register the same
+ *   nodes and Markdown round-trips consistently.
+ *
+ * SYNC: When the default node set changes, both consumers pick it up here.
+ */
+
+import {ListNode, ListItemNode} from '@lexical/list';
+import {HeadingNode, QuoteNode} from '@lexical/rich-text';
+import {LinkNode, AutoLinkNode} from '@lexical/link';
+import {CodeNode, CodeHighlightNode} from '@lexical/code';
+import {TableNode, TableRowNode, TableCellNode} from '@lexical/table';
+import {HorizontalRuleNode} from '@lexical/extension';
+import type {Klass, LexicalNode} from 'lexical';
+import {RichTextExtensionNode} from './markdownExtensionNode';
+
+/**
+ * The default OSS node set registered with the editor: headings, quotes,
+ * lists, links, code, and tables. Extend via the `nodes` prop / option rather
+ * than editing this list.
+ */
+export const DEFAULT_NODES: ReadonlyArray<Klass<LexicalNode>> = [
+  HeadingNode,
+  QuoteNode,
+  ListNode,
+  ListItemNode,
+  LinkNode,
+  AutoLinkNode,
+  CodeNode,
+  CodeHighlightNode,
+  TableNode,
+  TableRowNode,
+  TableCellNode,
+  HorizontalRuleNode,
+  // Markdown plugin nodes (spec:AST-064): registered everywhere so stored
+  // state that holds them always loads.
+  RichTextExtensionNode,
+];