@astryxdesign/core 0.4.6 → 0.4.7-canary.29be96d

package/src/Markdown/parser.test.ts

@@ -1,8 +1,13 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
 
 import {describe, it, expect} from 'vitest';
-import {parseMarkdown, parseInline} from './parser';
-import type {InlineNode} from './parser';
+import {
+  createIncrementalState,
+  parseInline,
+  parseMarkdown,
+  parseMarkdownIncremental,
+} from './parser';
+import type {BlockNode, InlineNode} from './parser';
 
 describe('parseInline', () => {
   it('parses plain text', () => {
@@ -1209,3 +1214,141 @@ describe('link reference definitions', () => {
     expect(blocks[0].type).toBe('paragraph');
   });
 });
+
+describe('sourceRanges', () => {
+  const slice = (source: string, block: BlockNode) =>
+    block.range == null
+      ? null
+      : source.slice(block.range.start, block.range.end);
+
+  it('is off by default', () => {
+    const [block] = parseMarkdown('# Title');
+    expect(block.range).toBeUndefined();
+  });
+
+  it('addresses a block with named start and end offsets', () => {
+    const source = 'One.\n\nTwo.';
+    const [, second] = parseMarkdown(source, {sourceRanges: true});
+    expect(second.range).toEqual({start: 6, end: 10});
+  });
+
+  it('gives every top-level block the source it came from', () => {
+    const source = [
+      '# Title',
+      '',
+      'A paragraph that',
+      'wraps onto two lines.',
+      '',
+      '- one',
+      '- two',
+      '',
+      '```js',
+      'const x = 1;',
+      '```',
+      '',
+      '| a | b |',
+      '| --- | --- |',
+      '| 1 | 2 |',
+      '',
+      '> quoted',
+      '',
+      '---',
+    ].join('\n');
+    const blocks = parseMarkdown(source, {sourceRanges: true});
+    expect(blocks.map(b => slice(source, b))).toEqual([
+      '# Title',
+      'A paragraph that\nwraps onto two lines.',
+      '- one\n- two',
+      '```js\nconst x = 1;\n```',
+      '| a | b |\n| --- | --- |\n| 1 | 2 |',
+      '> quoted',
+      '---',
+    ]);
+  });
+
+  it('reports offsets into the input, not into the link-definition-stripped text', () => {
+    // The definition lines are removed before the block loop runs, so a naive
+    // offset would drift by their length for everything after them.
+    const source = [
+      '[ref]: https://example.com',
+      '',
+      'See [the docs][ref].',
+      '',
+      'Another paragraph.',
+    ].join('\n');
+    const blocks = parseMarkdown(source, {sourceRanges: true});
+    expect(blocks.map(b => slice(source, b))).toEqual([
+      'See [the docs][ref].',
+      'Another paragraph.',
+    ]);
+  });
+
+  it('reports absolute offsets when the document is parsed incrementally', () => {
+    const source = ['# Title', '', 'One.', '', 'Two.', '', 'Three.'].join('\n');
+    const state = createIncrementalState();
+    let blocks: BlockNode[] = [];
+    for (let end = 1; end <= source.length; end++) {
+      blocks = parseMarkdownIncremental(source.slice(0, end), state, {
+        sourceRanges: true,
+      });
+    }
+    expect(blocks.map(b => slice(source, b))).toEqual([
+      '# Title',
+      'One.',
+      'Two.',
+      'Three.',
+    ]);
+  });
+
+  it('keeps the blank lines an unterminated fence owns', () => {
+    // Mid-stream the closing fence has not arrived, and the blank lines are
+    // part of the code, not spacing between blocks.
+    const source = '```\ncode\n\n';
+    const [block] = parseMarkdown(source, {sourceRanges: true});
+    // The whole thing: the fence consumed those lines as code.
+    expect(slice(source, block)).toBe(source);
+  });
+
+  it('slices to something that re-parses to the same block', () => {
+    // The property a consumer actually needs, and the one that says the
+    // offsets are right: what the range points at is the block.
+    const check = (source: string) => {
+      for (const block of parseMarkdown(source, {sourceRanges: true})) {
+        const {range: _range, ...node} = block;
+        expect(parseMarkdown(slice(source, block)!)).toEqual([node]);
+      }
+    };
+    check('# Title\n\nA paragraph.\n\n- one\n- two\n\n> quoted');
+    // CRLF: the parser keeps the `\r` in its own content, so the range does
+    // too rather than slicing to text that parses differently.
+    check('# Title\r\n\r\nA paragraph.\r\n');
+  });
+
+  it('re-parses when the caller flips the option on an existing state', () => {
+    const source = 'One.\n\nTwo.\n\nThree.';
+    const state = createIncrementalState();
+    const without = parseMarkdownIncremental(source, state);
+    expect(without.every(b => b.range == null)).toBe(true);
+    const with_ = parseMarkdownIncremental(source, state, {
+      sourceRanges: true,
+    });
+    expect(with_.map(b => slice(source, b))).toEqual([
+      'One.',
+      'Two.',
+      'Three.',
+    ]);
+    const back = parseMarkdownIncremental(source, state);
+    expect(back.every(b => b.range == null)).toBe(true);
+  });
+
+  it('spans both halves of a list the incremental parser merged', () => {
+    const source = '1. one\n\n1. two';
+    const state = createIncrementalState();
+    parseMarkdownIncremental('1. one\n\n', state, {sourceRanges: true});
+    const blocks = parseMarkdownIncremental(source, state, {
+      sourceRanges: true,
+    });
+    expect(blocks).toHaveLength(1);
+    expect(slice(source, blocks[0])).toBe(source);
+  });
+});

package/src/Markdown/parser.ts

@@ -24,7 +24,15 @@ export type InlineNode =
   | {type: 'citation'; sourceId: string}
   | {type: 'break'};
 
-export type BlockNode =
+export type BlockNode = BlockNodeKind & {
+  /**
+   * Where this block came from in the source, when parsed with the
+   * `sourceRanges` option. Top-level blocks only.
+   */
+  range?: SourceRange;
+};
+
+type BlockNodeKind =
   | {type: 'heading'; level: 1 | 2 | 3 | 4 | 5 | 6; children: InlineNode[]}
   | {type: 'paragraph'; children: InlineNode[]}
   | {type: 'codeblock'; language: string; content: string}
@@ -47,6 +55,17 @@ export type BlockNode =
   | {type: 'hr'}
   | {type: 'image'; src: string; alt: string};
 
+/**
+ * Where a block sits in the source string handed to `parseMarkdown`:
+ * `source.slice(start, end)` is the block, and `end` excludes the block's
+ * trailing blank lines.
+ *
+ * An object rather than a `[start, end]` tuple so a second way of addressing
+ * the same block — line numbers, once a consumer needs them — can be added as
+ * optional fields without breaking anyone.
+ */
+export type SourceRange = {readonly start: number; readonly end: number};
+
 export type ListItemNode = {checked?: boolean; children: BlockNode[]};
 export type TableCellNode = {children: InlineNode[]};
 export type TableAlignment = 'left' | 'center' | 'right' | null;
@@ -78,11 +97,28 @@ export type ParseOptions = {
    * suffix is not rejected (Astryx accepts any plausible TLD shape).
    */
   autolink?: 'gfm';
+  /**
+   * When true, every top-level block carries a `range` — the offsets it
+   * occupies in the string passed in. Lets a consumer that still holds the
+   * source slice the original markdown for a block instead of reconstructing
+   * it from the parsed node (or from the rendered DOM). Off by default: the
+   * field is absent unless asked for, so nothing that compares nodes changes.
+   *
+   * Blocks nested inside a list item or a blockquote do not carry one.
+   */
+  sourceRanges?: boolean;
 };
 
 type ResolvedOptions = {
   readonly sourceIds: ReadonlySet<string> | undefined;
   readonly autolink: 'gfm' | undefined;
+  readonly sourceRanges?: boolean;
+  /**
+   * Offset of this parse's input within the document the ranges are reported
+   * against. Internal only — the incremental parser parses slices and needs
+   * their blocks' ranges to come out absolute.
+   */
+  readonly baseOffset?: number;
   /**
    * Link reference definitions (`[label]: url`) collected from the whole
    * document, keyed by normalized label. Internal only — populated by the
@@ -109,7 +145,11 @@ function resolveOptions(
     return {sourceIds: arg as ReadonlySet<string>, autolink: undefined};
   }
   const opts = arg as ParseOptions;
-  return {sourceIds: opts.sourceIds, autolink: opts.autolink};
+  return {
+    sourceIds: opts.sourceIds,
+    autolink: opts.autolink,
+    sourceRanges: opts.sourceRanges,
+  };
 }
 
 // ---------------------------------------------------------------------------
@@ -174,6 +214,11 @@ function matchLinkDefinition(
 function extractLinkDefinitions(input: string): {
   defs: ReadonlyMap<string, string>;
   cleaned: string;
+  /**
+   * For each line of `cleaned`, the line of `input` it came from. Undefined
+   * when nothing was stripped and the two are the same text.
+   */
+  lineMap?: number[];
 } {
   const lines = input.split('\n');
   const defs = new Map<string, string>();
@@ -235,8 +280,14 @@ function extractLinkDefinitions(input: string): {
   if (defs.size === 0) {
     return {defs, cleaned: input};
   }
-  const cleaned = lines.filter((_, index) => keep[index]).join('\n');
-  return {defs, cleaned};
+  const lineMap: number[] = [];
+  for (let index = 0; index < lines.length; index++) {
+    if (keep[index]) {
+      lineMap.push(index);
+    }
+  }
+  const cleaned = lineMap.map(index => lines[index]).join('\n');
+  return {defs, cleaned, lineMap};
 }
 
 /** Order-independent signature of a link-definition set, for cache checks. */
@@ -1042,6 +1093,16 @@ function isTableSeparator(line: string): boolean {
   return nonEmpty.length > 0 && nonEmpty.every(cell => /^:?-+:?$/.test(cell));
 }
 
+/**
+ * The same options for content parsed out of an enclosing block. Ranges are a
+ * top-level contract: a list item's or a blockquote's children are parsed from
+ * text the caller reassembled (markers and `>` prefixes stripped), so an
+ * offset into it would not address the document.
+ */
+function nested(opts: ResolvedOptions): ResolvedOptions {
+  return opts.sourceRanges ? {...opts, sourceRanges: false} : opts;
+}
+
 /**
  * Returns true when a line could start a new block — used to stop paragraph
  * continuation.  Every regex here uses bounded or single-class quantifiers
@@ -1213,7 +1274,7 @@ function parseList(
       itemText += '\n' + deindented.join('\n');
     }
 
-    items.push({checked, children: parseMarkdownImpl(itemText, opts)});
+    items.push({checked, children: parseMarkdownImpl(itemText, nested(opts))});
 
     // CommonMark loose list: blank line(s) between items of the same style
     // and indent still form one list. Skip the blanks and continue if the
@@ -1274,7 +1335,7 @@ function parseMarkdownImpl(
   // definitions win on conflict, matching CommonMark's first-definition-wins
   // in document order; locally-nested definitions still resolve within this
   // parse.
-  const {defs, cleaned} = extractLinkDefinitions(input);
+  const {defs, cleaned, lineMap} = extractLinkDefinitions(input);
   const inherited = baseOpts.linkDefs;
   let linkDefs: ReadonlyMap<string, string> | undefined;
   if (defs.size === 0) {
@@ -1288,9 +1349,25 @@ function parseMarkdownImpl(
     linkDefs != null ? {...baseOpts, linkDefs} : baseOpts;
   const lines = cleaned.split('\n');
   const blocks: BlockNode[] = [];
+  // The line each block started on, parallel to `blocks`. Only collected when
+  // ranges were asked for; a block's end is resolved after the loop, since the
+  // branch that produced it has already moved `index` past whatever it read.
+  const blockStartLines: number[] | null = opts.sourceRanges ? [] : null;
+  // Set only by a block that consumes blank lines as content, where the
+  // positional end derivation would trim them away.
+  const blockEndLines: (number | undefined)[] | null = opts.sourceRanges
+    ? []
+    : null;
+  let blockStartLine = 0;
+  const pushBlock = (node: BlockNode, endLine?: number) => {
+    blocks.push(node);
+    blockStartLines?.push(blockStartLine);
+    blockEndLines?.push(endLine);
+  };
   let index = 0;
 
   while (index < lines.length) {
+    blockStartLine = index;
     const line = lines[index];
     if (line.trim() === '') {
       index++;
@@ -1309,14 +1386,20 @@ function parseMarkdownImpl(
         index++;
       }
       index++; // skip closing fence
-      blocks.push({type: 'codeblock', language, content: codeLines.join('\n')});
+      // A fence owns its blank lines, and an unterminated one (mid-stream)
+      // can end on them, so it states its own end rather than letting the
+      // positional derivation trim them off.
+      pushBlock(
+        {type: 'codeblock', language, content: codeLines.join('\n')},
+        Math.min(index, lines.length) - 1,
+      );
       continue;
     }
 
     // --- Heading ---
     const headingMatch = line.match(/^(#{1,6}) +(.*)/);
     if (headingMatch) {
-      blocks.push({
+      pushBlock({
         type: 'heading',
         level: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,
         children: parseInlineEntry(headingMatch[2], opts),
@@ -1327,7 +1410,7 @@ function parseMarkdownImpl(
 
     // --- HR (must precede list check to handle `- - -`, `* * *`, `_ _ _`) ---
     if (isHorizontalRule(line)) {
-      blocks.push({type: 'hr'});
+      pushBlock({type: 'hr'});
       index++;
       continue;
     }
@@ -1335,7 +1418,7 @@ function parseMarkdownImpl(
     // --- Standalone image ---
     const imageMatch = line.match(/^!\[([^\]]*)\]\(([^)]+)\)/);
     if (imageMatch && line.trim() === imageMatch[0]) {
-      blocks.push({type: 'image', alt: imageMatch[1], src: imageMatch[2]});
+      pushBlock({type: 'image', alt: imageMatch[1], src: imageMatch[2]});
       index++;
       continue;
     }
@@ -1347,7 +1430,7 @@ function parseMarkdownImpl(
       isTableSeparator(lines[index + 1])
     ) {
       const tableResult = parseTable(lines, index, opts);
-      blocks.push(tableResult.node);
+      pushBlock(tableResult.node);
       index = tableResult.nextIndex;
       continue;
     }
@@ -1362,9 +1445,9 @@ function parseMarkdownImpl(
         quoteLines.push(lines[index].replace(/^> ?/, ''));
         index++;
       }
-      blocks.push({
+      pushBlock({
         type: 'blockquote',
-        children: parseMarkdownImpl(quoteLines.join('\n'), opts),
+        children: parseMarkdownImpl(quoteLines.join('\n'), nested(opts)),
       });
       continue;
     }
@@ -1372,7 +1455,7 @@ function parseMarkdownImpl(
     // --- Unordered list ---
     if (/^ {0,9}[-*+] /.test(line)) {
       const listResult = parseList(lines, index, false, opts);
-      blocks.push(listResult.node);
+      pushBlock(listResult.node);
       index = listResult.nextIndex;
       continue;
     }
@@ -1380,7 +1463,7 @@ function parseMarkdownImpl(
     // --- Ordered list ---
     if (/^ {0,9}\d+[.)] /.test(line)) {
       const listResult = parseList(lines, index, true, opts);
-      blocks.push(listResult.node);
+      pushBlock(listResult.node);
       index = listResult.nextIndex;
       continue;
     }
@@ -1396,14 +1479,76 @@ function parseMarkdownImpl(
       paraLines.push(lines[index]);
       index++;
     }
-    blocks.push({
+    pushBlock({
       type: 'paragraph',
       children: parseInlineEntry(paraLines.join('\n'), opts),
     });
   }
+  if (blockStartLines != null) {
+    stampSourceRanges(
+      blocks,
+      blockStartLines,
+      blockEndLines ?? [],
+      lines,
+      lineMap,
+      input,
+      opts,
+    );
+  }
   return blocks;
 }
 
+/**
+ * Give each block the offsets it occupies in the original input.
+ *
+ * Blocks are contiguous and in source order, so a block runs from its own
+ * first line to the line before the next block starts, minus the blank lines
+ * between them. Offsets are computed against the *input*, not the text the
+ * block loop saw: link reference definitions are stripped before parsing, and
+ * `lineMap` says which input line each surviving line came from.
+ */
+function stampSourceRanges(
+  blocks: BlockNode[],
+  blockStartLines: number[],
+  blockEndLines: (number | undefined)[],
+  lines: string[],
+  lineMap: number[] | undefined,
+  input: string,
+  opts: ResolvedOptions,
+): void {
+  const base = opts.baseOffset ?? 0;
+  // Offset of the first character of every line of the input.
+  const inputLineStarts = [0];
+  for (let i = 0; i < input.length; i++) {
+    if (input[i] === '\n') {
+      inputLineStarts.push(i + 1);
+    }
+  }
+  // Stripping removes whole lines and never edits one, so a parsed line's
+  // length is its input line's length.
+  const lineStart = (line: number): number =>
+    base + inputLineStarts[lineMap != null ? lineMap[line] : line];
+
+  for (let i = 0; i < blocks.length; i++) {
+    const startLine = blockStartLines[i];
+    let endLine = blockEndLines[i];
+    if (endLine == null) {
+      const nextStart =
+        i + 1 < blocks.length ? blockStartLines[i + 1] : lines.length;
+      endLine = nextStart - 1;
+      while (endLine > startLine && lines[endLine].trim() === '') {
+        endLine--;
+      }
+    }
+    // Exactly the block's own lines, verbatim — a CRLF document's trailing
+    // `\r` included, since the parser reads it as part of the line too and a
+    // range that dropped it would slice to something that re-parses
+    // differently.
+    const end = lineStart(endLine) + lines[endLine].length;
+    blocks[i] = {...blocks[i], range: {start: lineStart(startLine), end}};
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Incremental parsing
 // ---------------------------------------------------------------------------
@@ -1420,6 +1565,12 @@ export interface IncrementalState {
    * with newly-arriving content.
    */
   autolink?: 'gfm';
+  /**
+   * The `sourceRanges` option the cached `settledBlocks` were parsed with.
+   * Flipping it invalidates them the same way `autolink` does: they either
+   * lack the ranges the caller now asks for, or carry ones it did not.
+   */
+  sourceRanges?: boolean;
   /**
    * Signature of the link reference definitions the cached `settledBlocks`
    * were parsed with. Definitions are document-global and typically arrive
@@ -1689,6 +1840,15 @@ function trimUnsettledStructural(text: string): string {
   return lines.join('\n');
 }
 
+/**
+ * The same options, for parsing a slice that starts at `offset` of the
+ * document — so the slice's blocks report ranges into the whole document
+ * rather than into the slice.
+ */
+function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {
+  return opts.sourceRanges ? {...opts, baseOffset: offset} : opts;
+}
+
 /**
  * Concatenate freshly-parsed delta blocks with previously-settled blocks,
  * merging adjacent same-style lists into a single loose list. The boundary
@@ -1718,6 +1878,11 @@ function mergeSettledBlocks(
       delimiter: prevLast.delimiter,
       loose: true,
       items: [...prevLast.items, ...deltaFirst.items],
+      // One list now, so one range: from where the first half started to
+      // where the second half ended.
+      ...(prevLast.range != null && deltaFirst.range != null
+        ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}
+        : null),
     };
     return [...prev.slice(0, -1), merged, ...delta.slice(1)];
   }
@@ -1740,15 +1905,19 @@ export function parseMarkdownIncremental(
   arg?: ReadonlySet<string> | ParseOptions,
 ): BlockNode[] {
   const opts = resolveOptions(arg);
-  // Invalidate cache when the autolink option flips — cached settled blocks
-  // were parsed with the previous setting and would otherwise be reused
-  // unchanged.
-  if (state.autolink !== opts.autolink) {
+  // Invalidate cache when the autolink or sourceRanges option flips — cached
+  // settled blocks were parsed with the previous setting and would otherwise
+  // be reused unchanged.
+  if (
+    state.autolink !== opts.autolink ||
+    Boolean(state.sourceRanges) !== Boolean(opts.sourceRanges)
+  ) {
     state.prevInput = '';
     state.settledText = '';
     state.settledBlocks = [];
     state.settledUpTo = 0;
     state.autolink = opts.autolink;
+    state.sourceRanges = opts.sourceRanges;
   }
   if (input === '') {
     state.prevInput = '';
@@ -1799,15 +1968,32 @@ export function parseMarkdownIncremental(
   ) {
     // Settled portion grew — parse only the new delta
     const delta = settledText.slice(state.settledText.length);
-    const deltaBlocks = parseMarkdownImpl(delta, parseOpts);
+    const deltaBlocks = parseMarkdownImpl(
+      delta,
+      atOffset(parseOpts, state.settledText.length),
+    );
     settledBlocks = mergeSettledBlocks(state.settledBlocks, deltaBlocks);
   } else {
     // Content before the boundary changed — full re-parse of settled portion
     settledBlocks = parseMarkdownImpl(settledText, parseOpts);
   }
 
+  // The unsettled tail is trimmed before parsing, so its offset in the
+  // document is where that trimmed text actually starts — not the boundary,
+  // which is a line index. If it somehow can't be located, parse it without
+  // ranges rather than report wrong ones. Only worth searching for when
+  // ranges were asked for: this runs on every streamed chunk.
+  const unsettledStart =
+    unsettledText && opts.sourceRanges
+      ? input.indexOf(unsettledText, settledText.length)
+      : -1;
   const unsettledBlocks = unsettledText
-    ? parseMarkdownImpl(unsettledText, parseOpts)
+    ? parseMarkdownImpl(
+        unsettledText,
+        unsettledStart >= 0
+          ? atOffset(parseOpts, unsettledStart)
+          : nested(parseOpts),
+      )
     : [];
 
   state.settledText = settledText;

package/src/MoreMenu/MoreMenu.doc.mjs

@@ -46,7 +46,8 @@ export const docs = {
       type: 'boolean',
       description: 'Whether the menu trigger is disabled.',
       default: 'false',
-    },    {
+    },
+    {
       name: 'placement',
       type: "'above' | 'below' | 'start' | 'end'",
       description:
@@ -60,6 +61,11 @@ export const docs = {
         "Alignment along the placement axis. Use 'end' to align the menu with the trigger's trailing edge, which is usually what an overflow menu wants.",
       default: "'start'",
     },
+    {
+      name: 'onOpenChange',
+      type: '(isOpen: boolean) => void',
+      description: 'Callback fired when the menu opens or closes.',
+    },
     {
       name: 'xstyle',
       type: 'StyleXStyles',

package/src/MultiSelector/MultiSelector.doc.mjs

@@ -103,6 +103,12 @@ export const docs = {
           description: 'How to display selected items in the trigger.',
           default: "'count'",
         },
+        {
+          name: 'formatValue',
+          type: '(items: {value: string; label: string}[]) => string',
+          description:
+            'Formats the trigger text when triggerDisplay="count" or "labels". Receives the selected items (value plus resolved label); the count is items.length. Not used by triggerDisplay="badges".',
+        },
         {
           name: 'maxBadges',
           type: 'number',
@@ -284,6 +290,8 @@ export const docsZh = {
         placeholder: '未选择值时显示的占位文本。',
         size: '选择器的尺寸变体。',
         triggerDisplay: '在触发器中显示选中项的方式。',
+        formatValue:
+          '格式化 triggerDisplay="count" 或 "labels" 时的触发器文本。接收选中项（value 及解析后的 label），数量为 items.length。triggerDisplay="badges" 不使用此属性。',
         maxBadges:
           '显示"+N"之前的最大徽章数。仅适用于 triggerDisplay="badges"。',
         hasSelectAll: '是否显示全选复选框。',
@@ -420,6 +428,8 @@ export const docsDense = {
         variant:
           'visual trigger style: input bordered control or ghost toolbar control',
         triggerDisplay: 'how to show selected in trigger',
+        formatValue:
+          'formats count/labels trigger text; receives selected items',
         maxBadges: 'max badges before "+N"; badges mode only',
         hasSelectAll: 'show select-all checkbox',
         selectAllLabel: 'select-all label',