@react-x11/components 0.8.0 → 0.10.0

package/src/{markdown → internal/markdown}/parse.ts

@@ -1,5 +1,8 @@
-// Markdown → AST, written for this component rather than adapted from a
-// DOM-shaped pipeline. GFM-flavoured CommonMark: headings (ATX and setext),
+// Markdown → AST, written for this package rather than adapted from a
+// DOM-shaped pipeline — shared, from `src/internal/`, because two components
+// read it: `<Markdown>` renders what it produces and `<RichTextEditor>` edits
+// it, writing it back out through `./stringify.ts`. GFM-flavoured
+// CommonMark: headings (ATX and setext),
 // paragraphs, fenced and indented code, blockquotes with lazy continuation,
 // nested ordered/bullet/task lists with the tight/loose distinction, tables,
 // thematic breaks; inline emphasis with the real flanking and rule-of-three
@@ -178,31 +181,168 @@ function indentOf(line: string): number {
 
 export function parse(source: string, options: ParseOptions = {}): Document {
   const partial = options.partial !== false;
+  const isComponent = options.isComponent ?? NO_COMPONENTS;
+  const expressions = options.expressions === true;
   const normalized = source.replace(/\r\n?/g, '\n');
   const lines = normalized.split('\n').map(expandLeadingTabs);
+  const previous = options.previous;
+  const was = previous ? resumable.get(previous) : undefined;
+  // Not with components: an open tag looks for its close to the end of the
+  // document (`componentCloseLine`), so a close added anywhere below can
+  // change what it is, and the block before an edit is no longer settled.
+  if (
+    previous &&
+    was &&
+    was.partial === partial &&
+    was.expressions === expressions &&
+    isComponent === NO_COMPONENTS &&
+    was.isComponent === NO_COMPONENTS
+  ) {
+    return resume(previous, was, lines);
+  }
   const blocks: BlockNode[] = [];
   const ranges: Array<[number, number]> = [];
-  parseBlocks(
+  const blanks = new Uint8Array(lines.length);
+  parseBlocks(lines, partial, blocks, ranges, isComponent, expressions, {
+    from: 0,
+    blanks,
+  });
+  return remember(
+    { blocks, raws: ranges.map(([a, b]) => lines.slice(a, b).join('\n')) },
+    { lines, ranges, blanks, partial, isComponent, expressions },
+  );
+}
+
+// --- resuming --------------------------------------------------------------
+
+/**
+ * What a parse leaves for the next parse of the same document to start from
+ * (`ParseOptions.previous`).
+ *
+ * The block loop carries one thing from a block to the next, the paragraph
+ * it is gathering, and it looks only forwards: a block's extent is decided
+ * by the lines up to the first line of the block after it. So `blanks`
+ * marks the lines where the loop stood on a blank line at the top level with
+ * no paragraph open. Two parses standing on such a line, with the same lines
+ * from there to the end, produce the same blocks from there on. That is the
+ * point where an edit's re-parse can stop and take the rest from before.
+ */
+interface Resumable {
+  lines: string[];
+  /** each top-level block's [start, endExcl) line range */
+  ranges: Array<[number, number]>;
+  /** 1 on each line where the top-level loop stood blank with no paragraph */
+  blanks: Uint8Array;
+  partial: boolean;
+  isComponent: (name: string) => boolean;
+  expressions: boolean;
+}
+
+/** Kept beside the documents rather than in them, so a Document stays the
+ *  plain data every caller and test compares. */
+const resumable = new WeakMap<Document, Resumable>();
+
+function remember(doc: Document, state: Resumable): Document {
+  resumable.set(doc, state);
+  return doc;
+}
+
+/**
+ * Parse `lines` as an edit of `previous`: from one block before the first
+ * line that changed, until the loop stands where the previous parse stood
+ * in the lines the edit left alone — or to the end, when it never does (an
+ * unclosed fence swallows the rest of a document either way).
+ */
+function resume(previous: Document, was: Resumable, lines: string[]): Document {
+  const old = was.lines;
+  const oldN = old.length;
+  const n = lines.length;
+  const shortest = Math.min(oldN, n);
+  let first = 0;
+  while (first < shortest && old[first] === lines[first]) first += 1;
+  if (first === oldN && first === n) return previous;
+  // lines at the end the edit did not touch, never counting the prefix twice
+  let same = 0;
+  while (
+    same < shortest - first &&
+    old[oldN - 1 - same] === lines[n - 1 - same]
+  ) {
+    same += 1;
+  }
+  const shift = n - oldN;
+  const suffix = n - same;
+
+  // The first block that reaches the change, and the one before it: a
+  // block's extent is decided by the lines up to the next block's first —
+  // a list looks past blank lines to see whether it goes on — so the block
+  // before the first one touched cannot have depended on the change.
+  const { ranges } = was;
+  let lo = 0;
+  let hi = ranges.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1;
+    if (ranges[mid][1] <= first) lo = mid + 1;
+    else hi = mid;
+  }
+  const kept = Math.max(0, lo - 1);
+  const from = kept === 0 ? 0 : ranges[kept][0];
+
+  const blanks = new Uint8Array(n);
+  blanks.set(was.blanks.subarray(0, from));
+  const blocks: BlockNode[] = previous.blocks.slice(0, kept);
+  const raws = previous.raws.slice(0, kept);
+  const next: Array<[number, number]> = ranges.slice(0, kept);
+  const fresh: Array<[number, number]> = [];
+  const stop = parseBlocks(
     lines,
-    partial,
+    was.partial,
     blocks,
-    ranges,
-    options.isComponent ?? NO_COMPONENTS,
-    options.expressions === true,
+    fresh,
+    was.isComponent,
+    was.expressions,
+    {
+      from,
+      blanks,
+      resync: (i) => i >= suffix && was.blanks[i - shift] === 1,
+    },
   );
-  return {
-    blocks,
-    raws: ranges.map(([a, b]) => lines.slice(a, b).join('\n')),
-  };
+  for (const [a, b] of fresh) {
+    raws.push(lines.slice(a, b).join('\n'));
+    next.push([a, b]);
+  }
+  if (stop < n) {
+    // the previous parse from the same place over the same lines: its
+    // blocks, where they now are
+    const at = stop - shift;
+    blanks.set(was.blanks.subarray(at), stop);
+    let j = ranges.length;
+    while (j > 0 && ranges[j - 1][0] >= at) j -= 1;
+    for (; j < ranges.length; j += 1) {
+      blocks.push(previous.blocks[j]);
+      raws.push(previous.raws[j]);
+      next.push([ranges[j][0] + shift, ranges[j][1] + shift]);
+    }
+  }
+  return remember({ blocks, raws }, { ...was, lines, ranges: next, blanks });
 }
 
 // --- block layer -----------------------------------------------------------
 
+/** The top-level call's extra: where to start, where to record the blank
+ *  lines it stands on, and when to stop (`resume`). */
+interface TopLevel {
+  from: number;
+  blanks: Uint8Array;
+  resync?: (line: number) => boolean;
+}
+
 /**
  * Parse a run of lines into blocks. `tailOpen` says the end of `lines` is
  * the live end of a streaming document — the license for every "complete it
  * anyway" rule. `ranges`, when given, receives each block's [start, endExcl)
  * line range (only the top-level call wants them, for the streaming cache).
+ * Returns the line it stopped at: the end, or where `top.resync` said the
+ * rest would come out as it did before.
  */
 function parseBlocks(
   lines: string[],
@@ -211,8 +351,9 @@ function parseBlocks(
   ranges?: Array<[number, number]>,
   isComponent: (name: string) => boolean = NO_COMPONENTS,
   expressions = false,
-): void {
-  let i = 0;
+  top?: TopLevel,
+): number {
+  let i = top?.from ?? 0;
   const n = lines.length;
 
   let paraStart = -1;
@@ -242,6 +383,10 @@ function parseBlocks(
 
     if (RE_BLANK.test(line)) {
       flushPara(i);
+      if (top) {
+        if (top.resync?.(i)) return i;
+        top.blanks[i] = 1;
+      }
       i += 1;
       continue;
     }
@@ -317,7 +462,8 @@ function parseBlocks(
       const start = i;
       const fenceIndent = fence[1].length;
       const marker = fence[2];
-      const lang = (fence[3].trim().split(/[ \t]/, 1)[0] ?? '').toLowerCase();
+      const info = fence[3].trim();
+      const lang = (info.split(/[ \t]/, 1)[0] ?? '').toLowerCase();
       const body: string[] = [];
       let closed = false;
       i += 1;
@@ -342,7 +488,19 @@ function parseBlocks(
         );
         i += 1;
       }
-      commit({ type: 'code', lang, text: body.join('\n'), closed }, start, i);
+      commit(
+        {
+          type: 'code',
+          lang,
+          // only when it says more than the language does, so a plain
+          // ```js fence parses to the AST it always did
+          ...(info !== lang ? { info } : null),
+          text: body.join('\n'),
+          closed,
+        },
+        start,
+        i,
+      );
       continue;
     }
 
@@ -555,6 +713,7 @@ function parseBlocks(
   }
 
   flushPara(n);
+  return n;
 }
 
 // --- lists -----------------------------------------------------------------
@@ -607,11 +766,27 @@ function parseList(
     const contentIndent = indent + m[2].length + pad;
 
     const inner: string[] = [rest];
+    // The fence open in the item's own content, if one is. A blank line
+    // inside a fence is the code's rather than the list's: it neither ends
+    // the item nor makes the list loose — and a snippet with an empty line
+    // in it, under a numbered step, is the ordinary case.
+    // (a task item's box comes off first: `- [ ] ```js` opens a fence too)
+    let fence = fenceAfter(null, rest.replace(RE_TASK, ''));
     i += 1;
     let pendingBlanks = 0;
     while (i < n) {
       const l = lines[i];
       if (RE_BLANK.test(l)) {
+        if (fence !== null && pendingBlanks === 0) {
+          // …as long as the item goes on after it: an unclosed fence must
+          // not swallow the paragraph the list is followed by
+          let k = i + 1;
+          while (k < n && RE_BLANK.test(lines[k])) k += 1;
+          if (k < n && indentOf(lines[k]) >= contentIndent) {
+            for (; i < k; i += 1) inner.push('');
+            continue;
+          }
+        }
         pendingBlanks += 1;
         if (pendingBlanks > 1) break;
         i += 1;
@@ -623,7 +798,9 @@ function parseList(
           loose = true;
           pendingBlanks = 0;
         }
-        inner.push(l.slice(contentIndent));
+        const content = l.slice(contentIndent);
+        inner.push(content);
+        fence = fenceAfter(fence, content);
         i += 1;
         continue;
       }
@@ -666,6 +843,23 @@ function parseList(
   };
 }
 
+/**
+ * The fence still open after `line`, given the one open before it — the
+ * opening run (```` ``` ````, `~~~~`) or null. Indentation is ignored, so a
+ * fence in a list nested inside this item is seen too.
+ */
+function fenceAfter(open: string | null, line: string): string | null {
+  const m = RE_FENCE_OPEN.exec(line.trimStart());
+  if (open === null) {
+    if (!m || (m[2][0] === '`' && m[3].includes('`'))) return null;
+    return m[2];
+  }
+  if (m && m[2][0] === open[0] && m[2].length >= open.length && m[3] === '') {
+    return null;
+  }
+  return open;
+}
+
 // --- tables ----------------------------------------------------------------
 
 /** Split a row on unescaped pipes; `\|` stays literal in the cell. */
@@ -763,19 +957,45 @@ const NAMED_ENTITIES: Record<string, string> = {
   middot: '·',
 };
 
+/** The same pattern, anchored — for reading one reference at a position. */
+const RE_ENTITY_AT = new RegExp(RE_ENTITY.source, 'y');
+
+function entityText(
+  whole: string,
+  dec?: string,
+  hex?: string,
+  named?: string,
+): string {
+  if (dec) {
+    const cp = parseInt(dec, 10);
+    return cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '�';
+  }
+  if (hex) {
+    const cp = parseInt(hex, 16);
+    return cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '�';
+  }
+  return (named && NAMED_ENTITIES[named]) ?? whole;
+}
+
 function decodeEntities(text: string): string {
   if (!text.includes('&')) return text;
-  return text.replace(RE_ENTITY, (whole, dec, hex, named) => {
-    if (dec) {
-      const cp = parseInt(dec, 10);
-      return cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '�';
-    }
-    if (hex) {
-      const cp = parseInt(hex, 16);
-      return cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '�';
-    }
-    return NAMED_ENTITIES[named] ?? whole;
-  });
+  return text.replace(RE_ENTITY, entityText);
+}
+
+/** The whole character ending at `i`, surrogate pair and all — an emoji is
+ *  a symbol, and half of one is not. */
+function charBefore(text: string, i: number): string | undefined {
+  if (i <= 0) return undefined;
+  const unit = text.charCodeAt(i - 1);
+  return unit >= 0xdc00 && unit <= 0xdfff && i >= 2
+    ? text.slice(i - 2, i)
+    : text[i - 1];
+}
+
+/** The whole character starting at `i`. */
+function charAt(text: string, i: number): string | undefined {
+  const cp = text.codePointAt(i);
+  return cp === undefined ? undefined : String.fromCodePoint(cp);
 }
 
 function classify(ch: string | undefined): 'ws' | 'punct' | 'other' {
@@ -797,11 +1017,14 @@ export function parseInline(
   const items: Item[] = [];
   let buf = '';
 
+  // References are decoded where they are read (below), not on the flushed
+  // buffer: `\&amp;` is an escaped ampersand followed by `amp;`, and
+  // decoding afterwards turned it back into `&`.
   const flushText = (): void => {
     if (buf.length === 0) return;
     items.push({
       kind: 'node',
-      node: { type: 'text', text: decodeEntities(buf) },
+      node: { type: 'text', text: buf },
     });
     buf = '';
   };
@@ -957,7 +1180,7 @@ export function parseInline(
       const suffix = parseLinkSuffix(text, i + 1);
       if (suffix) {
         flushText();
-        closeLink(items, openIdx, suffix.href);
+        closeLink(items, openIdx, suffix.href, suffix.title);
         i = suffix.end;
         continue;
       }
@@ -988,8 +1211,8 @@ export function parseInline(
     if (ch === '*' || ch === '_' || ch === '~') {
       let run = 1;
       while (text[i + run] === ch) run += 1;
-      const before = classify(text[i - 1]);
-      const after = classify(text[i + run]);
+      const before = classify(charBefore(text, i));
+      const after = classify(charAt(text, i + run));
       const leftFlank =
         after !== 'ws' && (after !== 'punct' || before !== 'other');
       const rightFlank =
@@ -1023,6 +1246,16 @@ export function parseInline(
       continue;
     }
 
+    if (ch === '&') {
+      RE_ENTITY_AT.lastIndex = i;
+      const ref = RE_ENTITY_AT.exec(text);
+      if (ref) {
+        buf += entityText(ref[0], ref[1], ref[2], ref[3]);
+        i += ref[0].length;
+        continue;
+      }
+    }
+
     buf += ch;
     i += 1;
   }
@@ -1072,10 +1305,12 @@ function lastBracket(items: Item[]): number {
 interface LinkSuffix {
   href: string;
   end: number;
+  title?: string;
 }
 
 /** `](dest "title")` — destination in `<>` or bare with balanced parens.
- *  The title is tolerated and dropped: nothing here renders one. */
+ *  The title is kept on the node — nothing renders one, but the editor
+ *  writes it back out. */
 function parseLinkSuffix(text: string, from: number): LinkSuffix | null {
   if (text[from] !== '(') return null;
   let i = from + 1;
@@ -1110,22 +1345,33 @@ function parseLinkSuffix(text: string, from: number): LinkSuffix | null {
   while (i < len && (text[i] === ' ' || text[i] === '\n' || text[i] === '\t'))
     i += 1;
   // an optional title in any of the three quote styles
+  let title: string | undefined;
   const q = text[i];
   if (q === '"' || q === "'" || q === '(') {
     const closer = q === '(' ? ')' : q;
     let j = i + 1;
     while (j < len && text[j] !== closer) j += 1;
     if (j >= len) return null;
+    title = decodeEntities(text.slice(i + 1, j));
     i = j + 1;
     while (i < len && (text[i] === ' ' || text[i] === '\n' || text[i] === '\t'))
       i += 1;
   }
   if (text[i] !== ')') return null;
-  return { href: decodeEntities(href), end: i + 1 };
+  return {
+    href: decodeEntities(href),
+    end: i + 1,
+    ...(title !== undefined ? { title } : null),
+  };
 }
 
 /** Fold everything after the bracket opener at `openIdx` into a link node. */
-function closeLink(items: Item[], openIdx: number, href: string | null): void {
+function closeLink(
+  items: Item[],
+  openIdx: number,
+  href: string | null,
+  title?: string,
+): void {
   const bracket = items[openIdx] as { kind: 'bracket'; image: boolean };
   const inner = items.splice(openIdx);
   inner.shift();
@@ -1133,7 +1379,14 @@ function closeLink(items: Item[], openIdx: number, href: string | null): void {
   processEmphasis(inner, href === null);
   items.push({
     kind: 'node',
-    node: { type: 'link', href, image: bracket.image, children: finish(inner) },
+    node: {
+      type: 'link',
+      href,
+      image: bracket.image,
+      children: finish(inner),
+      // only when written, so a link without one keeps its old shape
+      ...(title !== undefined ? { title } : null),
+    },
   });
 }