@jvs-milkdown/transformer 1.0.0

package/src/parser/state.ts

@@ -0,0 +1,217 @@
+import type {
+  Attrs,
+  MarkType,
+  Node,
+  NodeType,
+  Schema,
+} from '@jvs-milkdown/prose/model'
+
+import {
+  createNodeInParserFail,
+  parserMatchError,
+  stackOverFlow,
+} from '@jvs-milkdown/exception'
+import { Mark } from '@jvs-milkdown/prose/model'
+
+import type {
+  MarkSchema,
+  MarkdownNode,
+  NodeSchema,
+  RemarkParser,
+} from '../utility'
+import type { Parser } from './types'
+
+import { Stack } from '../utility'
+import { ParserStackElement } from './stack-element'
+
+/// A state machine for parser. Transform remark AST into prosemirror state.
+export class ParserState extends Stack<Node, ParserStackElement> {
+  /// The schema in current editor.
+  readonly schema: Schema
+
+  /// @internal
+  #marks: readonly Mark[] = Mark.none
+
+  /// Create a parser from schema and remark instance.
+  ///
+  /// ```typescript
+  /// const parser = ParserState.create(schema, remark)
+  /// const prosemirrorNode = parser(SomeMarkdownText)
+  /// ```
+  static create = (schema: Schema, remark: RemarkParser): Parser => {
+    const state = new this(schema)
+    return (text) => {
+      state.run(remark, text)
+      return state.toDoc()
+    }
+  }
+
+  /// @internal
+  constructor(schema: Schema) {
+    super()
+    this.schema = schema
+  }
+
+  /// @internal
+  #hasText = (node: Node): node is Node & { text: string } => node.isText
+
+  /// @internal
+  #maybeMerge = (a: Node, b: Node): Node | undefined => {
+    if (this.#hasText(a) && this.#hasText(b) && Mark.sameSet(a.marks, b.marks))
+      return this.schema.text(a.text + b.text, a.marks)
+
+    return undefined
+  }
+
+  /// @internal
+  #matchTarget = (node: MarkdownNode): NodeType | MarkType => {
+    const result = Object.values({
+      ...this.schema.nodes,
+      ...this.schema.marks,
+    }).find((x): x is NodeType | MarkType => {
+      const spec = x.spec as NodeSchema | MarkSchema
+      return spec.parseMarkdown.match(node)
+    })
+
+    if (!result) throw parserMatchError(node)
+
+    return result
+  }
+
+  /// @internal
+  #runNode = (node: MarkdownNode) => {
+    const type = this.#matchTarget(node)
+    const spec = type.spec as NodeSchema | MarkSchema
+
+    spec.parseMarkdown.runner(this, node, type as NodeType & MarkType)
+  }
+
+  /// Inject root node for prosemirror state.
+  injectRoot = (node: MarkdownNode, nodeType: NodeType, attrs?: Attrs) => {
+    this.openNode(nodeType, attrs)
+    this.next(node.children)
+
+    return this
+  }
+
+  /// Open a new node, the next operations will
+  /// add nodes into that new node until `closeNode` is called.
+  openNode = (nodeType: NodeType, attrs?: Attrs) => {
+    this.open(ParserStackElement.create(nodeType, [], attrs))
+    return this
+  }
+
+  /// @internal
+  #closeNodeAndPush = (): Node => {
+    this.#marks = Mark.none
+    const element = this.close()
+
+    return this.#addNodeAndPush(element.type, element.attrs, element.content)
+  }
+
+  /// Close the current node and push it into the parent node.
+  closeNode = () => {
+    try {
+      this.#closeNodeAndPush()
+    } catch (e) {
+      console.error(e)
+    }
+    return this
+  }
+
+  /// @internal
+  #addNodeAndPush = (
+    nodeType: NodeType,
+    attrs?: Attrs,
+    content?: Node[]
+  ): Node => {
+    const node = nodeType.createAndFill(attrs, content, this.#marks)
+    if (!node) throw createNodeInParserFail(nodeType, attrs, content)
+
+    this.push(node)
+
+    return node
+  }
+
+  /// Add a node into current node.
+  addNode = (nodeType: NodeType, attrs?: Attrs, content?: Node[]) => {
+    try {
+      this.#addNodeAndPush(nodeType, attrs, content)
+    } catch (e) {
+      console.error(e)
+    }
+    return this
+  }
+
+  /// Open a new mark, the next nodes added will have that mark.
+  openMark = (markType: MarkType, attrs?: Attrs) => {
+    const mark = markType.create(attrs)
+
+    this.#marks = mark.addToSet(this.#marks)
+    return this
+  }
+
+  /// Close a opened mark.
+  closeMark = (markType: MarkType) => {
+    this.#marks = markType.removeFromSet(this.#marks)
+    return this
+  }
+
+  /// Add a text node into current node.
+  addText = (text: string) => {
+    try {
+      const topElement = this.top()
+      if (!topElement) throw stackOverFlow()
+
+      const prevNode = topElement.pop()
+      const currNode = this.schema.text(text, this.#marks)
+
+      if (!prevNode) {
+        topElement.push(currNode)
+        return this
+      }
+
+      const merged = this.#maybeMerge(prevNode, currNode)
+      if (merged) {
+        topElement.push(merged)
+        return this
+      }
+      topElement.push(prevNode, currNode)
+      return this
+    } catch (e) {
+      console.error(e)
+      return this
+    }
+  }
+
+  /// @internal
+  build = (): Node => {
+    let doc: Node | undefined
+
+    do doc = this.#closeNodeAndPush()
+    while (this.size())
+
+    return doc
+  }
+
+  /// Give the node or node list back to the state and
+  /// the state will find a proper runner (by `match` method in parser spec) to handle it.
+  next = (nodes: MarkdownNode | MarkdownNode[] = []) => {
+    ;[nodes].flat().forEach((node) => this.#runNode(node))
+    return this
+  }
+
+  /// Build the current state into a [prosemirror document](https://prosemirror.net/docs/ref/#model.Document_Structure).
+  toDoc = () => this.build()
+
+  /// Transform a markdown string into prosemirror state.
+  run = (remark: RemarkParser, markdown: string) => {
+    const tree = remark.runSync(
+      remark.parse(markdown),
+      markdown
+    ) as MarkdownNode
+    this.next(tree)
+
+    return this
+  }
+}

package/src/parser/types.ts

@@ -0,0 +1,55 @@
+import type { MarkType, Node, NodeType } from '@jvs-milkdown/prose/model'
+
+import type { MarkdownNode } from '../utility/types'
+import type { ParserState } from './state'
+
+/// The parser type which is used to transform markdown text into prosemirror node.
+export type Parser = (text: string) => Node
+
+/// The spec for node parser in schema.
+export interface NodeParserSpec {
+  /// The match function to check if the node is the target node.
+  /// For example:
+  ///
+  /// ```typescript
+  /// match: (node) => node.type === 'paragraph'
+  /// ```
+  match: (node: MarkdownNode) => boolean
+  /// The runner function to transform the node into prosemirror node.
+  /// Generally, you should call methods in `state` to add node to state.
+  /// For example:
+  ///
+  /// ```typescript
+  /// runner: (state, node, type) => {
+  ///   state
+  ///     .openNode(type)
+  ///     .next(node.children)
+  ///     .closeNode();
+  /// }
+  /// ```
+  runner: (state: ParserState, node: MarkdownNode, proseType: NodeType) => void
+}
+
+/// The spec for mark parser in schema.
+export interface MarkParserSpec {
+  /// The match function to check if the node is the target mark.
+  /// For example:
+  ///
+  /// ```typescript
+  /// match: (mark) => mark.type === 'emphasis'
+  /// ```
+  match: (node: MarkdownNode) => boolean
+  /// The runner function to transform the node into prosemirror mark.
+  /// Generally, you should call methods in `state` to add mark to state.
+  /// For example:
+  ///
+  /// ```typescript
+  /// runner: (state, node, type) => {
+  ///   state
+  ///     .openMark(type)
+  ///     .next(node.children)
+  ///     .closeMark(type)
+  /// }
+  /// ```
+  runner: (state: ParserState, node: MarkdownNode, proseType: MarkType) => void
+}

package/src/serializer/index.ts

@@ -0,0 +1,2 @@
+export * from './types'
+export * from './state'

package/src/serializer/stack-element.spec.ts

@@ -0,0 +1,17 @@
+import { expect, it } from 'vitest'
+
+import { SerializerStackElement } from './stack-element'
+
+it('serializer-stack-element', () => {
+  const element = SerializerStackElement.create('type')
+
+  expect(element.props).toEqual({})
+
+  element.push({ type: 'text' })
+
+  expect(element.children).toEqual([{ type: 'text' }])
+
+  const node = element.pop()
+
+  expect(node).toEqual({ type: 'text' })
+})

package/src/serializer/stack-element.ts

@@ -0,0 +1,30 @@
+import type { MarkdownNode } from '..'
+import type { JSONRecord } from '../utility'
+
+import { StackElement } from '../utility'
+
+export class SerializerStackElement extends StackElement<MarkdownNode> {
+  constructor(
+    public type: string,
+    public children?: MarkdownNode[],
+    public value?: string,
+    public props: JSONRecord = {}
+  ) {
+    super()
+  }
+
+  static create = (
+    type: string,
+    children?: MarkdownNode[],
+    value?: string,
+    props: JSONRecord = {}
+  ) => new SerializerStackElement(type, children, value, props)
+
+  push = (node: MarkdownNode, ...rest: MarkdownNode[]) => {
+    if (!this.children) this.children = []
+
+    this.children.push(node, ...rest)
+  }
+
+  pop = (): MarkdownNode | undefined => this.children?.pop()
+}

package/src/serializer/state.spec.ts

@@ -0,0 +1,287 @@
+import type { Mark, Schema } from '@jvs-milkdown/prose/model'
+
+import { describe, expect, it } from 'vitest'
+
+import { SerializerState } from './state'
+
+const boldMark = {
+  isInSet: (arr: string[]) => arr.includes('bold'),
+  addToSet: (arr: string[]) => arr.concat('bold'),
+  type: {
+    removeFromSet: (arr: string[]) => arr.filter((x) => x !== 'bold'),
+  },
+} as unknown as Mark
+
+const italicMark = {
+  isInSet: (arr: string[]) => arr.includes('italic'),
+  addToSet: (arr: string[]) => arr.concat('italic'),
+  type: {
+    removeFromSet: (arr: string[]) => arr.filter((x) => x !== 'italic'),
+  },
+} as unknown as Mark
+
+const inlineCodeMark = {
+  isInSet: (arr: string[]) => arr.includes('inlineCode'),
+  addToSet: (arr: string[]) => arr.concat('inlineCode'),
+  type: {
+    removeFromSet: (arr: string[]) => arr.filter((x) => x !== 'inlineCode'),
+  },
+} as unknown as Mark
+
+const schema = {
+  nodes: {
+    paragraph: {
+      spec: {
+        toMarkdown: {
+          match: (node: { type: string }) => node.type === 'paragraph',
+          runner: (state: SerializerState, node: { value: string }) => {
+            state.addNode('text', [], node.value)
+          },
+        },
+      },
+    },
+    blockquote: {
+      spec: {
+        toMarkdown: {
+          match: (node: { type: string }) => node.type === 'blockquote',
+          runner: (state: SerializerState, node: { content: never }) => {
+            state.openNode('blockquote')
+            state.next(node.content)
+            state.closeNode()
+          },
+        },
+      },
+    },
+  },
+  marks: {},
+  text: (text: string, marks: string[]) => ({ text, marks, isText: true }),
+} as unknown as Schema
+
+describe('serializer-state', () => {
+  it('node', () => {
+    const state = new SerializerState(schema)
+    state.openNode('doc')
+    state.openNode('paragraph', 'paragraph node value', { foo: 'bar' })
+    state.addNode('text', [], 'text node value')
+    state.closeNode()
+
+    expect(state.top()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          value: 'paragraph node value',
+          children: [
+            {
+              type: 'text',
+              value: 'text node value',
+            },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('maybe merge children for same mark', () => {
+    const state = new SerializerState(schema)
+    state.openNode('doc')
+    state.openNode('paragraph')
+    state.withMark(boldMark, 'bold')
+    state.addNode('text', [], 'The lunatic is on the grass.')
+    state.closeMark(boldMark)
+    state.withMark(boldMark, 'bold')
+    state.addNode('text', [], 'The lunatic is in the hell.')
+    state.closeMark(boldMark)
+    state.closeNode()
+
+    expect(state.top()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          children: [
+            {
+              type: 'bold',
+              isMark: true,
+              children: [
+                {
+                  type: 'text',
+                  value: 'The lunatic is on the grass.',
+                },
+                {
+                  type: 'text',
+                  value: 'The lunatic is in the hell.',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('build', () => {
+    const state = new SerializerState(schema)
+    state.openNode('doc')
+    state.openNode('paragraph', 'paragraph node value', { foo: 'bar' })
+    state.addNode('text', [], 'text node value')
+    state.closeNode()
+
+    expect(state.build()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          value: 'paragraph node value',
+          children: [
+            {
+              type: 'text',
+              value: 'text node value',
+            },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('next', () => {
+    const state = new SerializerState(schema)
+    state.openNode('doc')
+    state.next({
+      type: 'blockquote',
+      marks: [],
+      content: {
+        type: 'paragraph',
+        marks: [],
+        value: 'The lunatic is on the grass.',
+      },
+    } as any)
+
+    expect(state.build()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'blockquote',
+          children: [
+            {
+              type: 'text',
+              value: 'The lunatic is on the grass.',
+            },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('trim spaces around marks', () => {
+    const state = new SerializerState(schema)
+
+    state.openNode('doc')
+    state.openNode('paragraph')
+    // Open a bold mark, add a text node with surrounding spaces, then close the mark
+    state.withMark(boldMark, 'bold')
+    state.addNode('text', [], ' hello ')
+    state.closeMark(boldMark)
+    state.closeNode() // close paragraph
+
+    // Expect SerializerState to move the spaces outside of the mark node.
+    expect(state.top()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          children: [
+            { type: 'text', value: ' ' },
+            {
+              type: 'bold',
+              isMark: true,
+              children: [{ type: 'text', value: 'hello' }],
+            },
+            { type: 'text', value: ' ' },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('should not corrupt value-based marks during merge', () => {
+    const state = new SerializerState(schema)
+
+    state.openNode('doc')
+    state.openNode('paragraph')
+    // Simulate: inlineCode("Hello") followed by bold > inlineCode("World")
+    state.withMark(inlineCodeMark, 'inlineCode', 'Hello')
+    state.closeMark(inlineCodeMark)
+    state.withMark(boldMark, 'bold')
+    state.withMark(inlineCodeMark, 'inlineCode', 'World')
+    state.closeMark(inlineCodeMark)
+    state.closeMark(boldMark)
+    state.closeNode()
+
+    // The bold wrapper around inlineCode should be preserved,
+    // not restructured by searchType
+    expect(state.top()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          children: [
+            {
+              type: 'inlineCode',
+              isMark: true,
+              value: 'Hello',
+            },
+            {
+              type: 'bold',
+              isMark: true,
+              children: [
+                {
+                  type: 'inlineCode',
+                  isMark: true,
+                  value: 'World',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    })
+  })
+
+  it('try to merge marks', () => {
+    const state = new SerializerState(schema)
+
+    state.openNode('doc')
+    state.openNode('paragraph')
+    state.withMark(boldMark, 'bold')
+    state.withMark(italicMark, 'italic')
+    state.addNode('text', [], 'hello')
+    state.closeMark(italicMark)
+    state.addNode('text', [], 'world')
+    state.closeMark(boldMark)
+    state.closeNode()
+
+    expect(state.top()).toMatchObject({
+      type: 'doc',
+      children: [
+        {
+          type: 'paragraph',
+          children: [
+            {
+              type: 'bold',
+              isMark: true,
+              children: [
+                {
+                  type: 'italic',
+                  isMark: true,
+                  children: [{ type: 'text', value: 'hello' }],
+                },
+                { type: 'text', value: 'world' },
+              ],
+            },
+          ],
+        },
+      ],
+    })
+  })
+})