@astryxdesign/core 0.6.2 → 0.6.3-canary.01fa439

package/src/Markdown/ast.test.ts

@@ -0,0 +1,307 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {
+  createIncrementalState,
+  parseMarkdown,
+  parseMarkdownAst,
+} from './parser';
+import {createMarkdownPlugin} from './plugins';
+import type {
+  MarkdownExtensionNode,
+  MarkdownSyntaxPluginDefinition,
+} from './plugins';
+
+type DemoNoteNode = MarkdownExtensionNode<
+  'demo-notes',
+  'note',
+  {readonly body: string},
+  'block'
+>;
+
+const demoNotes = createMarkdownPlugin<'demo-notes', DemoNoteNode>({
+  name: 'demo-notes',
+  apiVersion: 1,
+  parseKey: 'v1',
+  syntax: {
+    block: [
+      {
+        startsWith: [':::note'],
+        maxSpan: 200,
+        tokenize({source, offset, end, isFinal}) {
+          const close = source.indexOf('\n:::', offset + 7);
+          if (close < 0 || close + 4 > end) {
+            return isFinal ? {status: 'no-match'} : {status: 'defer'};
+          }
+          return {
+            status: 'match',
+            end: close + 4,
+            node: {
+              type: 'extension',
+              plugin: 'demo-notes',
+              name: 'note',
+              display: 'block',
+              data: {body: source.slice(offset + 7, close).trim()},
+            },
+          };
+        },
+      },
+    ],
+  },
+  renderers: {
+    note: {
+      render: ({node}) => node.data.body,
+      toText: node => node.data.body,
+    },
+  },
+} satisfies MarkdownSyntaxPluginDefinition<'demo-notes', DemoNoteNode>);
+
+describe('canonical Markdown AST', () => {
+  it('uses MDAST-aligned names and fields for every built-in structure', () => {
+    const ast = parseMarkdownAst(
+      '# Title\n\n**bold** and *emphasis* with `code` and [link](/docs).\n\n- item\n\n| A | B |\n| - | - |\n| 1 | 2 |\n\n```ts\nconst value = 1;\n```\n\n---',
+    );
+
+    expect(ast).toMatchObject({
+      type: 'root',
+      children: [
+        {
+          type: 'heading',
+          depth: 1,
+          children: [{type: 'text', value: 'Title'}],
+        },
+        {
+          type: 'paragraph',
+          children: [
+            {type: 'strong'},
+            {type: 'text'},
+            {type: 'emphasis'},
+            {type: 'text'},
+            {type: 'inlineCode', value: 'code'},
+            {type: 'text'},
+            {type: 'link', url: '/docs'},
+            {type: 'text', value: '.'},
+          ],
+        },
+        {
+          type: 'list',
+          ordered: false,
+          children: [{type: 'listItem'}],
+        },
+        {
+          type: 'table',
+          align: [null, null],
+          children: [
+            {
+              type: 'tableRow',
+              children: [{type: 'tableCell'}, {type: 'tableCell'}],
+            },
+            {
+              type: 'tableRow',
+              children: [{type: 'tableCell'}, {type: 'tableCell'}],
+            },
+          ],
+        },
+        {type: 'code', lang: 'ts', value: 'const value = 1;'},
+        {type: 'thematicBreak'},
+      ],
+    });
+  });
+
+  it('represents remaining built-ins without legacy aliases', () => {
+    const ast = parseMarkdownAst(
+      '> quote\n\n![Block](/block.png)\n\nParagraph ![Inline](/inline.png) [src] $x$  \nnext\n\n$$\ny\n$$',
+      {math: true, sourceIds: new Set(['src'])},
+    );
+
+    expect(ast.children).toMatchObject([
+      {
+        type: 'blockquote',
+        children: [
+          {type: 'paragraph', children: [{type: 'text', value: 'quote'}]},
+        ],
+      },
+      {type: 'image', url: '/block.png', alt: 'Block'},
+      {
+        type: 'paragraph',
+        children: [
+          {type: 'text', value: 'Paragraph '},
+          {type: 'image', url: '/inline.png', alt: 'Inline'},
+          {type: 'text', value: ' '},
+          {type: 'citation', sourceId: 'src'},
+          {type: 'text', value: ' '},
+          {type: 'inlineMath', value: 'x'},
+          {type: 'break'},
+          {type: 'text', value: 'next'},
+        ],
+      },
+      {type: 'math', value: 'y'},
+    ]);
+    expect(parseMarkdownAst('```\nplain\n```').children[0]).toMatchObject({
+      type: 'code',
+      lang: null,
+      value: 'plain',
+    });
+    expect(parseMarkdown('```\nplain\n```')[0]).toEqual({
+      type: 'codeblock',
+      language: 'plaintext',
+      content: 'plain',
+    });
+  });
+
+  it('keeps source provenance canonical while projecting the released range', () => {
+    const source = '# Heading\n\nParagraph';
+    const ast = parseMarkdownAst(source, {sourceRanges: true});
+    const legacy = parseMarkdown(source, {sourceRanges: true});
+
+    expect(ast.children[0].position).toEqual({
+      start: {offset: 0},
+      end: {offset: 9},
+    });
+    expect(legacy[0]).toEqual({
+      type: 'heading',
+      level: 1,
+      children: [{type: 'text', content: 'Heading'}],
+      range: {start: 0, end: 9},
+    });
+    expect('position' in legacy[0]).toBe(false);
+  });
+
+  it('authors canonical positions without the sourceRanges option', () => {
+    const source = '# Heading\n\nParagraph\n\n> Quoted';
+    const ast = parseMarkdownAst(source);
+
+    // Transforms observe provenance on every top-level block whether or not
+    // the caller opted into the released `range` projection.
+    expect(ast.children.map(block => block.position)).toEqual([
+      {start: {offset: 0}, end: {offset: 9}},
+      {start: {offset: 11}, end: {offset: 20}},
+      {start: {offset: 22}, end: {offset: 30}},
+    ]);
+    expect(
+      parseMarkdownAst(source, {sourceRanges: true}).children.map(
+        block => block.position,
+      ),
+    ).toEqual(ast.children.map(block => block.position));
+
+    // Nested blocks remain a top-level-only contract.
+    const quote = ast.children[2];
+    if (quote.type !== 'blockquote') {
+      throw new Error('Expected blockquote');
+    }
+    expect(quote.children[0].position).toBeUndefined();
+
+    // The released projection is unchanged: no `range` unless asked for.
+    expect(parseMarkdown(source)).toEqual([
+      {
+        type: 'heading',
+        level: 1,
+        children: [{type: 'text', content: 'Heading'}],
+      },
+      {type: 'paragraph', children: [{type: 'text', content: 'Paragraph'}]},
+      {
+        type: 'blockquote',
+        children: [
+          {type: 'paragraph', children: [{type: 'text', content: 'Quoted'}]},
+        ],
+      },
+    ]);
+  });
+
+  it('keeps canonical positions off released extension nodes', () => {
+    const source = ':::note\nBody\n:::\n\nTail prose.';
+    const plugins = [demoNotes];
+
+    // The projection rebuilds every built-in block field by field, but passes
+    // an extension node through — so it is the one kind that could carry a
+    // canonical position out. Without the option the released node is exactly
+    // what it has always been.
+    const released = parseMarkdown(source, {plugins});
+    expect(released[0]).toEqual({
+      type: 'extension',
+      plugin: 'demo-notes',
+      name: 'note',
+      display: 'block',
+      data: {body: 'Body'},
+      source: ':::note\nBody\n:::',
+    });
+    expect('position' in released[0]).toBe(false);
+    expect(Object.keys(released[0])).toEqual([
+      'type',
+      'plugin',
+      'name',
+      'display',
+      'data',
+      'source',
+    ]);
+
+    // With the option the node keeps the exact provenance it has always
+    // exposed: `position` with offsets, and no `range`.
+    const withRanges = parseMarkdown(source, {sourceRanges: true, plugins});
+    expect(withRanges[0]).toEqual({
+      type: 'extension',
+      plugin: 'demo-notes',
+      name: 'note',
+      display: 'block',
+      data: {body: 'Body'},
+      source: ':::note\nBody\n:::',
+      position: {start: {offset: 0}, end: {offset: 16}},
+    });
+    expect('range' in withRanges[0]).toBe(false);
+
+    // The canonical tree carries provenance either way.
+    expect(parseMarkdownAst(source, {plugins}).children[0].position).toEqual({
+      start: {offset: 0},
+      end: {offset: 16},
+    });
+  });
+
+  it('preserves optional legacy keys and insertion order exactly', () => {
+    const [image, list] = parseMarkdown('![Alt](/image.png)\n\n- item');
+
+    expect(Object.keys(image)).toEqual(['type', 'alt', 'src']);
+    expect(Object.keys(list)).toEqual([
+      'type',
+      'ordered',
+      'start',
+      'delimiter',
+      'loose',
+      'items',
+    ]);
+    if (list.type !== 'list') {
+      throw new Error('Expected list');
+    }
+    expect(Object.keys(list.items[0])).toEqual(['checked', 'children']);
+    expect(list).toStrictEqual({
+      type: 'list',
+      ordered: false,
+      start: undefined,
+      delimiter: undefined,
+      loose: undefined,
+      items: [
+        {
+          checked: undefined,
+          children: [
+            {
+              type: 'paragraph',
+              children: [{type: 'text', content: 'item'}],
+            },
+          ],
+        },
+      ],
+    });
+  });
+
+  it('does not expose canonical fields through released parser results', () => {
+    const state = createIncrementalState();
+    const blocks = parseMarkdown('# Heading\n\nParagraph');
+
+    expect(blocks[0]).toEqual({
+      type: 'heading',
+      level: 1,
+      children: [{type: 'text', content: 'Heading'}],
+    });
+    expect('depth' in blocks[0]).toBe(false);
+    expect(state.settledBlocks).toEqual([]);
+  });
+});

package/src/Markdown/ast.ts

@@ -0,0 +1,337 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file ast.ts
+ * @input Parsed Markdown structure and optional Astryx extension nodes
+ * @output Canonical immutable MDAST-aligned Markdown node types and text projection
+ * @position Public canonical Markdown tree shared by parsing, transforms, rendering, and Outline
+ */
+
+export type MarkdownAstDataValue =
+  | null
+  | boolean
+  | number
+  | string
+  | ReadonlyArray<MarkdownAstDataValue>
+  | {readonly [key: string]: MarkdownAstDataValue};
+
+export interface MarkdownAstPoint {
+  readonly line?: number;
+  readonly column?: number;
+  readonly offset?: number;
+}
+
+export interface MarkdownAstPosition {
+  readonly start: MarkdownAstPoint;
+  readonly end: MarkdownAstPoint;
+}
+
+export interface MarkdownAstNodeBase {
+  readonly position?: MarkdownAstPosition;
+  readonly data?: MarkdownAstDataValue;
+}
+
+export interface MarkdownAstExtensionNode<
+  Plugin extends string = string,
+  Name extends string = string,
+  Data extends MarkdownAstDataValue = MarkdownAstDataValue,
+  Display extends 'inline' | 'block' = 'inline' | 'block',
+> extends MarkdownAstNodeBase {
+  readonly type: 'extension';
+  readonly plugin: Plugin;
+  readonly name: Name;
+  readonly data: Data;
+  readonly display: Display;
+  readonly source?: string;
+}
+
+export interface MarkdownAstText extends MarkdownAstNodeBase {
+  readonly type: 'text';
+  readonly value: string;
+}
+
+export interface MarkdownAstParent<
+  Type extends 'strong' | 'emphasis' | 'delete',
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: Type;
+  readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstInlineCode extends MarkdownAstNodeBase {
+  readonly type: 'inlineCode';
+  readonly value: string;
+}
+
+export interface MarkdownAstInlineMath extends MarkdownAstNodeBase {
+  readonly type: 'inlineMath';
+  readonly value: string;
+}
+
+export interface MarkdownAstLink<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'link';
+  readonly url: string;
+  readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstImage extends MarkdownAstNodeBase {
+  readonly type: 'image';
+  readonly url: string;
+  readonly alt: string;
+}
+
+export interface MarkdownAstCitation extends MarkdownAstNodeBase {
+  readonly type: 'citation';
+  readonly sourceId: string;
+}
+
+export interface MarkdownAstBreak extends MarkdownAstNodeBase {
+  readonly type: 'break';
+}
+
+export type MarkdownAstPhrasingContent<
+  Extension extends MarkdownAstExtensionNode = never,
+> =
+  | MarkdownAstText
+  | MarkdownAstParent<'strong', Extension>
+  | MarkdownAstParent<'emphasis', Extension>
+  | MarkdownAstParent<'delete', Extension>
+  | MarkdownAstInlineCode
+  | MarkdownAstInlineMath
+  | MarkdownAstLink<Extension>
+  | MarkdownAstImage
+  | MarkdownAstCitation
+  | MarkdownAstBreak
+  | (Extension & {readonly display: 'inline'});
+
+export interface MarkdownAstHeading<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'heading';
+  readonly depth: 1 | 2 | 3 | 4 | 5 | 6;
+  readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstParagraph<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'paragraph';
+  readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+const markdownAstLegacyCodeLanguage = Symbol('MarkdownAstLegacyCodeLanguage');
+
+export interface MarkdownAstCode extends MarkdownAstNodeBase {
+  readonly type: 'code';
+  readonly lang: string | null;
+  readonly meta?: string;
+  readonly value: string;
+}
+
+type MarkdownAstCodeWithLegacyLanguage = MarkdownAstCode & {
+  readonly [markdownAstLegacyCodeLanguage]?: string | null;
+};
+
+/** @internal Preserves the released parser/renderer language projection. */
+export function markMarkdownAstLegacyCodeLanguage<Node extends MarkdownAstCode>(
+  node: Node,
+  language: string | null,
+): Node {
+  if (language === node.lang) {
+    return node;
+  }
+  Object.defineProperty(node, markdownAstLegacyCodeLanguage, {
+    configurable: false,
+    enumerable: true,
+    value: language,
+    writable: false,
+  });
+  return node;
+}
+
+/** @internal Reads the released language projection for a canonical code node. */
+export function getMarkdownAstLegacyCodeLanguage(
+  node: MarkdownAstCode,
+): string | null {
+  const legacyLanguage = (node as MarkdownAstCodeWithLegacyLanguage)[
+    markdownAstLegacyCodeLanguage
+  ];
+  return legacyLanguage === undefined ? node.lang : legacyLanguage;
+}
+
+export interface MarkdownAstMath extends MarkdownAstNodeBase {
+  readonly type: 'math';
+  readonly value: string;
+}
+
+export interface MarkdownAstBlockquote<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'blockquote';
+  readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export interface MarkdownAstListItem<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'listItem';
+  readonly checked?: boolean;
+  readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export interface MarkdownAstList<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'list';
+  readonly ordered: boolean;
+  readonly start?: number;
+  readonly spread?: boolean;
+  /** Astryx-preserved ordered-list marker delimiter. */
+  readonly delimiter?: '.' | ')';
+  readonly children: ReadonlyArray<MarkdownAstListItem<Extension>>;
+}
+
+export type MarkdownAstTableAlignment = 'left' | 'center' | 'right' | null;
+
+export interface MarkdownAstTableCell<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'tableCell';
+  readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstTableRow<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'tableRow';
+  readonly children: ReadonlyArray<MarkdownAstTableCell<Extension>>;
+}
+
+export interface MarkdownAstTable<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'table';
+  readonly align: ReadonlyArray<MarkdownAstTableAlignment>;
+  readonly children: ReadonlyArray<MarkdownAstTableRow<Extension>>;
+}
+
+export interface MarkdownAstThematicBreak extends MarkdownAstNodeBase {
+  readonly type: 'thematicBreak';
+}
+
+export type MarkdownAstBlockContent<
+  Extension extends MarkdownAstExtensionNode = never,
+> =
+  | MarkdownAstHeading<Extension>
+  | MarkdownAstParagraph<Extension>
+  | MarkdownAstCode
+  | MarkdownAstMath
+  | MarkdownAstBlockquote<Extension>
+  | MarkdownAstList<Extension>
+  | MarkdownAstTable<Extension>
+  | MarkdownAstThematicBreak
+  | MarkdownAstImage
+  | (Extension & {readonly display: 'block'});
+
+export interface MarkdownAstRoot<
+  Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+  readonly type: 'root';
+  readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export interface MarkdownAstNodeMap<
+  Extension extends MarkdownAstExtensionNode = MarkdownAstExtensionNode,
+> {
+  readonly root: MarkdownAstRoot<Extension>;
+  readonly text: MarkdownAstText;
+  readonly strong: MarkdownAstParent<'strong', Extension>;
+  readonly emphasis: MarkdownAstParent<'emphasis', Extension>;
+  readonly delete: MarkdownAstParent<'delete', Extension>;
+  readonly inlineCode: MarkdownAstInlineCode;
+  readonly inlineMath: MarkdownAstInlineMath;
+  readonly break: MarkdownAstBreak;
+  readonly link: MarkdownAstLink<Extension>;
+  readonly image: MarkdownAstImage;
+  readonly citation: MarkdownAstCitation;
+  readonly heading: MarkdownAstHeading<Extension>;
+  readonly paragraph: MarkdownAstParagraph<Extension>;
+  readonly code: MarkdownAstCode;
+  readonly math: MarkdownAstMath;
+  readonly blockquote: MarkdownAstBlockquote<Extension>;
+  readonly list: MarkdownAstList<Extension>;
+  readonly listItem: MarkdownAstListItem<Extension>;
+  readonly table: MarkdownAstTable<Extension>;
+  readonly tableRow: MarkdownAstTableRow<Extension>;
+  readonly tableCell: MarkdownAstTableCell<Extension>;
+  readonly thematicBreak: MarkdownAstThematicBreak;
+  readonly extension: Extension;
+}
+
+export type MarkdownAstNode<
+  Extension extends MarkdownAstExtensionNode = MarkdownAstExtensionNode,
+  Type extends keyof MarkdownAstNodeMap<Extension> =
+    keyof MarkdownAstNodeMap<Extension>,
+> = MarkdownAstNodeMap<Extension>[Type];
+
+export function visitMarkdownNodes<
+  Extension extends MarkdownAstExtensionNode,
+  Type extends keyof MarkdownAstNodeMap<Extension>,
+>(
+  root: MarkdownAstRoot<Extension>,
+  type: Type,
+  visitor: (node: MarkdownAstNodeMap<Extension>[Type]) => void,
+): void {
+  const visit = (node: MarkdownAstNode<Extension>): void => {
+    if (node.type === type) {
+      visitor(node as MarkdownAstNodeMap<Extension>[Type]);
+    }
+    if ('children' in node) {
+      for (const child of node.children) {
+        visit(child);
+      }
+    }
+  };
+  visit(root);
+}
+
+export function markdownAstText<
+  Extension extends MarkdownAstExtensionNode = never,
+>(
+  nodes: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>,
+  extensionText: (
+    node: Extension & {readonly display: 'inline'},
+  ) => string = node => node.source ?? '',
+): string {
+  let text = '';
+  for (const node of nodes) {
+    switch (node.type) {
+      case 'text':
+      case 'inlineCode':
+      case 'inlineMath':
+        text += node.value;
+        break;
+      case 'strong':
+      case 'emphasis':
+      case 'delete':
+      case 'link':
+        text += markdownAstText(node.children, extensionText);
+        break;
+      case 'image':
+        text += node.alt;
+        break;
+      case 'citation':
+      case 'break':
+        break;
+      case 'extension':
+        text += extensionText(node);
+        break;
+      default: {
+        node satisfies never;
+      }
+    }
+  }
+  return text;
+}

package/src/Markdown/incremental.test.ts

@@ -47,6 +47,18 @@ describe('parseMarkdownIncremental', () => {
     }
   });
 
+  it('keeps projected settled-list identity when a following block settles too', () => {
+    const state = createIncrementalState();
+    parseMarkdownIncremental('1. a\n\nTail', state);
+    const result = parseMarkdownIncremental(
+      '1. a\n\n1. b\n\nAfter\n\nTail',
+      state,
+    );
+
+    expect(state.settledBlocks).toEqual(parseMarkdown(state.settledText));
+    expect(result[0]).toBe(state.settledBlocks[0]);
+  });
+
   it('joins loose ) ordered list across streamed chunks', () => {
     const text = '1) apple\n\n1) banana\n\n1) cherry\n';
     const {final} = simulateStreaming(text, 5);