mdom 0.2.0 → 0.4.0

data/lib/mdom/parser.rb

@@ -1,285 +1,335 @@
 # frozen_string_literal: true
 
+require_relative "error"
+require_relative "inline_parser"
+require_relative "node/nodes"
+require_relative "tokenizer"
+
 module MDOM
-  module Parser
-    extend self
-
-    HEADING_RE = /\A(\#{1,6})[ \t]+(.*?)[ \t]*\z/.freeze
-    SETEXT_RE = /\A\s*([=-]+)\s*\z/.freeze
-    FENCE_RE = /\A([ ]{0,3})(`{3,}|~{3,})(.*)\z/.freeze
-    HRULE_RE = /\A\s*((\*\s*){3,}|(-\s*){3,}|(_\s*){3,})\z/.freeze
-    QUOTE_RE = /\A[ ]{0,3}>[ \t]?(.*)\z/.freeze
-    LIST_RE = /\A([ ]*)([-*+])([ \t]+)(.*)\z/.freeze
-    ORDERED_RE = /\A([ ]*)(\d{1,9})([.)])([ \t]+)(.*)\z/.freeze
-    TASK_RE = /\A\[([ xX])\][ \t]+(.*)\z/.freeze
-
-    def read(source)
-      Reader.new(normalize(source)).parse
+  # Token Stream → AST1（docs/IMPLEMENTATION.md §2.4）。
+  #
+  # 只负责**块结构**：按 CSM 缩进（CSM §1.1）把 token 挂成树。段落、标题等的
+  # 正文以原文保存，行内解析留给后续的 InlineParser。嵌入 token 生成
+  # +Node::RubyEmbed+ 节点，存源码原文与位置，**不求值**。
+  #
+  #   MDOM::Parser.parse("# 标题\n\n正文\n")
+  class Parser
+    # 容器栈的一层。+indent+ 是该容器所在行的 CSM 缩进（根与节块用 -1，
+    # 表示不会被同级或更深的块弹出）。
+    Frame = Struct.new(:indent, :node, :kind, keyword_init: true)
+
+    # 行内容器：内容策略为 **PHRASING** 的节点（`docs/CSM.md` 元素表），它们的
+    # +attributes[:text]+ 会被 InlineParser 解析成行内节点。
+    #
+    # 内容策略为 FLOW 的容器（`blockquote` / `li` / `dd`）**不在此列**——它们的正文
+    # 先成为一个段落，直接子节点一律是块级节点。表格单元格（`th` / `td`）也是
+    # PHRASING。
+    PHRASING_CONTAINERS = %i[paragraph heading term cite table_header_cell table_cell].freeze
+
+    # 解析 CSM 源码，返回 +Node::Document+。+inline: false+ 可跳过行内解析。
+    def self.parse(source, **options)
+      new(**options).parse(source)
     end
 
-    def normalize(source)
-      s = source.to_s
-      s = s.gsub(/\r\n?/, "\n").sub(/\A\n+/, "").chomp
-      return [] if s.empty?
-
-      s.split("\n", -1).map { |line| expand_leading_tabs(line) }
+    # 参数：
+    # * +detector:+ —— 块判定器，默认 +BlockDetector.new+。**自定义判定器时，A 类的
+    #   开关与白名单要配在判定器自己身上**，不要再传下面两个选项（同时给出会报错）。
+    # * +inline:+ —— 是否做行内解析；+false+ 只做块结构。
+    # * +free_blocks:+ —— **A 类（自由块）判定开关**（CSM §3.5），默认 +true+。
+    #   +false+ 时不做任何 A 类判定，命中行按**排版**处理（CSM §3.5.3，不报错）；
+    #   B 类（引导关键字）与 `~` 行不受影响。
+    # * +free_blocks_allowed:+ —— A 类判定的**白名单**，默认 +nil+（不限制）。给出后
+    #   只有行首标识符命中的行才尝试 A 类判定：+String+ 精确相等，+Regexp+ 对已剥离
+    #   缩进的整行匹配。空数组等价于 +free_blocks: false+。
+    #
+    # 后两个选项服务于英文散文文档：那里「以字母开头的行」比比皆是，逐一交给 Prism
+    # 增量 parse 会产生几百到上万次调用（docs/IMPLEMENTATION.md §2.2.4）。
+    def initialize(detector: nil, inline: true, free_blocks: true, free_blocks_allowed: nil)
+      @tokenizer = Tokenizer.new(detector: resolve_detector(detector, free_blocks, free_blocks_allowed))
+      @inline = inline
     end
 
-    # Expand leading tab characters in +line+ into spaces at 4-column tab
-    # stops. This normalizes list/quote indentation so the block parser can
-    # reason about column widths, while leaving the non-indentation content
-    # (including tabs inside code or text) untouched. A tab stop positions the
-    # next character at the next multiple of 4.
-    def expand_leading_tabs(line)
-      i = 0
-      col = 0
-      out = +""
-      while i < line.length
-        ch = line[i]
-        case ch
-        when " "
-          out << " "
-          col += 1
-          i += 1
-        when "\t"
-          target = (col / 4 + 1) * 4
-          out << " " * (target - col)
-          col = target
-          i += 1
-        else
-          out << line[i..]
+    def parse(source)
+      root = Node::Document.new
+      stack = [Frame.new(indent: -1, node: root, kind: :document)]
+      @previous = nil
+
+      @tokenizer.tokenize(source).each do |token|
+        if token.type == :end_marker
+          stack.pop while stack.size > 1
+          root << Node::EndMarker.new(token.value[:line], token.value[:data])
           break
         end
-      end
-      out
-    end
-
-    # Line-oriented block scanner. parse_blocks returns a flat array of sibmling
-    # block nodes (and nothing else), so callers don't juggle an index.
-    class Reader
-      def initialize(lines)
-        @lines = lines
-        @pos = 0
-      end
 
-      def parse
-        Document.new(children: parse_blocks)
+        dispatch(token, root, stack)
+        @previous = token
       end
 
-      def self.parse_lines(lines)
-        new(lines).send(:parse_blocks)
-      end
-
-      private
+      drop_empty_divisions(root)
+      resolve_inline(root) if @inline
+      root
+    end
 
-      def eof?
-        @pos >= @lines.length
+    private
+
+    # 组装块判定器：没给 +detector+ 就把两个 A 类选项用在默认判定器上；给了就要求
+    # 那两个选项保持默认——配置的**单一来源**，避免「传了却不生效」的静默失效。
+    def resolve_detector(detector, free_blocks, free_blocks_allowed)
+      if detector.nil?
+        BlockDetector.new(free_blocks: free_blocks, free_blocks_allowed: free_blocks_allowed)
+      else
+        unless free_blocks == true && free_blocks_allowed.nil?
+          raise ArgumentError,
+                "`detector:` cannot be combined with `free_blocks:` / `free_blocks_allowed:` " \
+                "— configure the detector itself instead"
+        end
+        detector
       end
+    end
 
-      def current
-        @lines[@pos]
+    def dispatch(token, root, stack)
+      case token.type
+      when :blank
+        handle_blank(token, root, stack)
+      when :text
+        handle_text(token, stack)
+      when :heading
+        place_leaf(token, stack, build_heading(token))
+      when :hrule
+        place_leaf(token, stack, Node::Hrule.new)
+      when :fence
+        place_leaf(token, stack, build_code_block(token))
+      when :math_block
+        place_leaf(token, stack, Node::MathBlock.new(token.value[:tex]))
+      when :table
+        place_leaf(token, stack, build_table(token))
+      when :ruby_block
+        place_leaf(token, stack, build_embed(token))
+      when :blockquote
+        handle_blockquote(token, stack)
+      when :cite
+        handle_cite(token, stack)
+      when :list_item
+        handle_list_item(token, stack)
+      when :definition
+        handle_definition(token, stack)
+      else
+        raise ParseError, "unknown token type #{token.type.inspect} (line #{token.line})"
       end
+    end
 
-      def peek(offset = 1)
-        @lines[@pos + offset]
-      end
+    # 段落：连续的排版文本行合并为一个段落。软换行的折叠（CSM §1.1）与行尾
+    # `\` 的硬换行由 InlineParser 处理，这里只把原始行用 `\n` 连接。
+    def handle_text(token, stack)
+      parent = pop_to_parent(stack, token.indent)
+      last = parent.children.last
 
-      def blank?(line)
-        line.nil? || line.strip.empty?
+      if @previous&.type == :text && @previous.indent == token.indent && last.is_a?(Node::Paragraph)
+        last.text = "#{last.text}\n#{token.value[:text]}"
+      else
+        parent << Node::Paragraph.new(token.value[:text])
       end
+    end
 
-      def parse_blocks
-        blocks = []
-        while !eof?
-          if blank?(current)
-            @pos += 1
-            next
-          end
-
-          block = parse_block
-          blocks << block if block
-        end
-        blocks
+    # 块引用（内容策略 FLOW）：相邻的引用行合并进**同一个段落**（软换行语义，
+    # CSM §1.1）；更深的缩进开启块级子节点。
+    def handle_blockquote(token, stack)
+      indent = token.indent
+      pop_deeper(stack, indent)
+      stack.pop while stack.size > 1 && stack.last.indent == indent && stack.last.kind != :quote
+
+      if stack.last.kind == :quote && stack.last.indent == indent
+        quote = stack.last.node
+      else
+        quote = Node::Blockquote.new
+        stack.last.node << quote
+        stack.push(Frame.new(indent: indent, node: quote, kind: :quote))
       end
 
-      def parse_block
-        line = current
+      append_body_line(quote, token.value[:text])
+      quote
+    end
 
-        if (m = FENCE_RE.match(line))
-          return parse_fenced_code(m) if fence_safe?(m)
-        end
+    # 把一行正文并进 FLOW 容器的最后一个段落（软换行折叠交给 InlineParser）；
+    # 上一个子节点不是段落（如嵌套块）时另起一个段落。空行不产生内容。
+    def append_body_line(node, text)
+      return if text.empty?
 
-        if (om = ORDERED_RE.match(line))
-          return parse_list(om, ordered: true)
-        end
-        if (um = LIST_RE.match(line))
-          return parse_list(um, ordered: false)
-        end
+      paragraph = node.children.last
+      if paragraph.is_a?(Node::Paragraph)
+        paragraph.text = "#{paragraph.text}\n#{text}"
+      else
+        node << Node::Paragraph.new(text)
+      end
+    end
 
-        if HRULE_RE.match(line)
-          @pos += 1
-          return Hrule.new
-        end
+    # `>-` 收尾：作为 +Node::Cite+ 子节点挂到同级引用的末尾（内容策略
+    # PHRASING，正文会被行内解析，CSM §1.4）。
+    def handle_cite(token, stack)
+      indent = token.indent
+      pop_deeper(stack, indent)
+
+      if stack.last.kind == :quote && stack.last.indent == indent
+        quote = stack.last.node
+      else
+        stack.pop while stack.size > 1 && stack.last.indent == indent
+        quote = Node::Blockquote.new
+        stack.last.node << quote
+      end
 
-        if (q = QUOTE_RE.match(line))
-          return parse_blockquote(q)
-        end
+      quote << Node::Cite.new(token.value[:text])
+    end
 
-        if (h = HEADING_RE.match(line))
-          text = h[2].sub(/[ \t]+#+[ \t]*\z/, "").strip
-          @pos += 1
-          return Heading.new(h[1].length, children: inline_children(text))
-        end
+    # 列表项（内容策略 FLOW）：同缩进的连续项复用同一个 +Node::List+，更深缩进则
+    # 嵌入到上一个列表项之下（CSM §1.3 的「嵌套通过缩进控制」）。正文成为段落。
+    def handle_list_item(token, stack)
+      indent = token.indent
+      pop_deeper(stack, indent)
+      stack.pop while stack.size > 1 && stack.last.indent == indent && stack.last.kind != :list
+
+      if stack.last.kind == :list && stack.last.indent == indent
+        list = stack.last.node
+      else
+        list = Node::List.new(ordered: token.value[:ordered], task: token.value[:task],
+                              marker: token.value[:marker])
+        stack.last.node << list
+        stack.push(Frame.new(indent: indent, node: list, kind: :list))
+      end
 
-        if setext_heading?
-          return parse_setext_heading
-        end
+      list.attributes[:task] = true if token.value[:task]
+      item = Node::ListItem.new(marker: token.value[:marker], task: token.value[:task],
+                                checked: token.value[:checked])
+      item << Node::Paragraph.new(token.value[:text])
+      list << item
+      stack.push(Frame.new(indent: indent, node: item, kind: :item))
+      item
+    end
 
-        parse_paragraph
+    # 定义列表（CSM §1.3）：`term` 行 + 其后的 `: 定义` 行。第一个 `:` 行把前一个
+    # 同级段落提升为定义术语，再建 +Node::DefinitionList+。
+    #
+    # +dd+ 的内容策略是 FLOW，所以正文成为段落；并且它自己成为容器栈的一层——
+    # 缩进的子块因此挂在**这个** +Definition+ 下（而不是 `dl` 下）。
+    def handle_definition(token, stack)
+      indent = token.indent
+      pop_deeper(stack, indent)
+      stack.pop while stack.size > 1 && stack.last.indent == indent && stack.last.kind != :deflist
+
+      if stack.last.kind == :deflist && stack.last.indent == indent
+        list = stack.last.node
+      else
+        list = promote_to_definition_list(stack.last.node)
+        stack.push(Frame.new(indent: indent, node: list, kind: :deflist))
       end
 
-      def fence_safe?(m)
-        return true unless m[2].start_with?("`")
+      definition = Node::Definition.new
+      definition << Node::Paragraph.new(token.value[:text])
+      list << definition
+      stack.push(Frame.new(indent: indent, node: definition, kind: :definition))
+      definition
+    end
 
-        !m[3].include?("`")
+    # 空行终止当前块（CSM §1.1）。连续四个 `\n` 另起一个节块（`<div>`）。
+    def handle_blank(token, root, stack)
+      if token.value[:count] >= 3
+        stack.pop while stack.size > 1
+        division = Node::Div.new
+        root << division
+        stack.push(Frame.new(indent: -1, node: division, kind: :div))
+      else
+        stack.pop while stack.size > 1 && stack.last.kind != :div
       end
+    end
 
-      def parse_fenced_code(m)
-        indent = m[1].length
-        marker = m[2][0]
-        fence_len = m[2].length
-        info = m[3].strip
-        lang = info.empty? ? nil : info.split(/\s+/).first
-
-        @pos += 1
-        content = []
-        while !eof?
-          line = current
-          if (cm = FENCE_RE.match(line)) && cm[2][0] == marker && cm[2].length >= fence_len && cm[1].length <= 3
-            @pos += 1
-            break
-          end
-          content << line[indent..].to_s
-          @pos += 1
-        end
-        CodeBlock.new(content.join("\n"), lang: lang, fenced: true)
-      end
+    # 叶子块：挂到「缩进严格小于自己」的最近容器下。
+    def place_leaf(token, stack, node)
+      parent = pop_to_parent(stack, token.indent)
+      parent << node
+      node
+    end
 
-      def setext_heading?
-        !blank?(current) && !HRULE_RE.match(current) && SETEXT_RE.match(peek(1).to_s)
-      end
+    # 弹出所有「缩进不小于 +indent+」的容器，返回合适的父容器。
+    def pop_to_parent(stack, indent)
+      stack.pop while stack.size > 1 && stack.last.indent >= indent
+      stack.last.node
+    end
 
-      def parse_setext_heading
-        text = current.strip
-        level = SETEXT_RE.match(peek(1))[1].start_with?("=") ? 1 : 2
-        @pos += 2
-        Heading.new(level, children: inline_children(text))
-      end
+    # 只弹出「缩进严格大于 +indent+」的容器，保留同级的可复用容器。
+    def pop_deeper(stack, indent)
+      stack.pop while stack.size > 1 && stack.last.indent > indent
+    end
 
-      def parse_blockquote(quote)
-        quote_lines = []
-        while !eof? && (m = QUOTE_RE.match(current))
-          quote_lines << m[1]
-          @pos += 1
-        end
-        Blockquote.new(children: Reader.parse_lines(quote_lines))
+    # 把容器里最后一个同级段落提升为定义术语。
+    def promote_to_definition_list(parent)
+      list = Node::DefinitionList.new
+      last = parent.children.last
+
+      if last.is_a?(Node::Paragraph)
+        list << Node::Term.new(last.text)
+        parent.replace(last, list)
+      else
+        parent << list
       end
+      list
+    end
 
-      def parse_list(marker, ordered:)
-        start = ordered ? marker[2].to_i : nil
-        items = []
-        while !eof?
-          line = current
-          lm = ORDERED_RE.match(line) || LIST_RE.match(line)
-          break if lm.nil?
-          break if (ORDERED_RE.match(line).nil?) == ordered
-
-          marker_len = marker_width(lm)
-          content = lm[5] || lm[4]
-          task = false
-          checked = false
-          if (tm = TASK_RE.match(content))
-            task = true
-            checked = tm[1].downcase == "x"
-            content = tm[2]
-          end
-          @pos += 1
-
-          item_lines = [content]
-          while !eof? && !blank?(current)
-            l = current
-            child = ORDERED_RE.match(l) || LIST_RE.match(l)
-            break if child && marker_width(child) <= marker_len
-
-            item_lines << l
-            @pos += 1
-          end
-
-          li = ListItem.new
-          Reader.parse_lines(item_lines).each { |n| li.append(n) }
-          li.attributes[:task] = true if task
-          li.attributes[:checked] = checked if task
-          items << li
-        end
+    def build_heading(token)
+      Node::Heading.new(token.value[:level], token.value[:text], closed: token.value[:closed])
+    end
 
-        list = List.new(ordered: ordered, tight: true, start: start)
-        items.each { |li| list.append(li) }
-        list
-      end
+    def build_code_block(token)
+      Node::CodeBlock.new(
+        token.value[:code],
+        lang: token.value[:lang],
+        params: token.value[:params],
+        fence: token.value[:fence]
+      )
+    end
 
-      # Width in columns of a list marker prefix (indent + bullet/number + space).
-      def marker_width(m)
-        content = m[5] || m[4]
-        m[0].length - content.length
-      end
+    # 表格（CSM §1.12）：行 → 单元格，表头行用 +TableHeaderCell+，其余用
+    # +TableCell+；对齐逐列取自该行生效的分隔行。
+    def build_table(token)
+      Node::Table.new(children: token.value[:rows].map { |row| build_table_row(row) })
+    end
 
-      def parse_paragraph
-        collected = []
-        while !eof? && !blank?(current) && !block_start?(current)
-          collected << current.strip
-          @pos += 1
-        end
-        Paragraph.new(children: inline_children(collected.join("\n")))
-      end
+    def build_table_row(row)
+      klass = row[:header] ? Node::TableHeaderCell : Node::TableCell
+      align = row[:align] || []
+      cells = row[:cells].each_with_index.map { |text, column| klass.new(text, align: align[column]) }
+      Node::TableRow.new(children: cells)
+    end
 
-      def block_start?(line)
-        return true if FENCE_RE.match(line)
-        return true if LIST_RE.match(line) || ORDERED_RE.match(line)
-        return true if HRULE_RE.match(line)
-        return true if QUOTE_RE.match(line)
-        return true if HEADING_RE.match(line)
+    def build_embed(token)
+      Node::RubyEmbed.new(
+        source: token.value[:source],
+        form: token.value[:form],
+        lines: token.value[:lines],
+        indent: token.value[:indent],
+        render: token.value[:render]
+      )
+    end
 
-        false
-      end
+    # 把行内容器（内容策略 PHRASING）的 +text+ 原文解析为行内节点，插到
+    # +children+ 之前。
+    #
+    # +attributes[:text]+ 保留原文（供保真回吐与调试），行内节点才是向下游交付的
+    # 结构。FLOW 容器不直接承载行内内容，所以这里不碰它们——它们的正文是段落，
+    # 段落自己会被递归解析到。
+    def resolve_inline(node)
+      node.children.each { |child| resolve_inline(child) }
+      return unless PHRASING_CONTAINERS.include?(node.type)
 
-      # Parse a run of inline text into inline nodes.
-      def inline_children(text)
-        return [] if text.to_s.empty?
+      text = node.attributes[:text]
+      return if text.nil? || text.empty?
 
-        nodes = InlineParser.parse(text)
-        trim_hardbreak_text(nodes)
-      end
+      InlineParser.parse(text).reverse_each { |inline| node.prepend(inline) }
+    end
 
-      # A hard break is represented by two trailing spaces before the newline.
-      # The text node emitted before it still carries those spaces; trim them so
-      # the model has the break (not literal spaces) as its semantic marker.
-      def trim_hardbreak_text(nodes)
-        out = []
-        nodes.each_with_index do |node, i|
-          if node.text? && nodes[i + 1]&.type == :hardbreak
-            value = node.value.sub(/ {2}\z/, "")
-            out << (value.empty? ? nil : Text.new(value))
-          else
-            out << node
-          end
-        end
-        out.compact
+    # 文档末尾由空行开出的空节块没有意义，解析完清掉。
+    def drop_empty_divisions(root)
+      root.children.dup.each do |child|
+        root.remove(child) if child.is_a?(Node::Div) && child.children.empty?
       end
     end
   end
-
-  # Public entry point: parse +source+ into an MDOM::Document tree.
-  def self.parse(source)
-    Parser.read(source)
-  end
 end