prosemirror 0.1.0 → 0.3.0

data/lib/prosemirror/json_parser.rb

@@ -68,18 +68,7 @@ module Prosemirror
       nodes = content.each_with_index.filter_map do |child, index|
         build_node(child, "#{path}.content[#{index}]", depth + 1)
       end
-      join_text(nodes)
-    end
-
-    def join_text(nodes)
-      nodes.each_with_object([]) do |node, joined|
-        previous = joined.last
-        if previous&.text? && node.text? && previous.marks == node.marks
-          joined[-1] = previous.with(text: previous.text + node.text)
-        else
-          joined << node
-        end
-      end
+      Node.join_text(nodes)
     end
 
     def build_marks(marks, path)

data/lib/prosemirror/mark_type.rb

@@ -27,6 +27,17 @@ module Prosemirror
         inherited_setting(:@excludes, nil)
       end
 
+      # Whether this mark can't coexist with marks of the other type.
+      def excludes?(other)
+        case excludes
+        when nil then other.type_name == type_name
+        when "_" then true
+        else
+          names = excludes.split.map(&:to_sym)
+          names.include?(other.type_name) || names.intersect?(other.group)
+        end
+      end
+
       def group(*names)
         return @group = names.flat_map { |n| n.to_s.split }.map(&:to_sym) unless names.empty?
 

data/lib/prosemirror/markdown/builder.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module Markdown
+    # Builds the Elements and Texts of HTML import from Markdown, with commonmarker's syntax tree
+    # (https://github.com/gjtorikian/commonmarker), so that the schema's parse rules apply to Markdown too.
+    # Each Markdown node becomes the element commonmarker renders for it, without going through HTML.
+    #
+    # - GitHub Flavored Markdown: tables, strikethrough and autolinks. Task list items stay as text (`[ ] `).
+    # - Raw HTML can span several Markdown blocks, like `<del>` around a paragraph: with the herb gem, a
+    #   document holding raw HTML is rendered to HTML by commonmarker and imported. Without it, raw HTML
+    #   stays as text.
+    # - Code blocks lose their final newline, as in prosemirror-markdown.
+    # - YAML front matter, between `---` lines at the start, is dropped.
+    class Builder
+      MAX_DEPTH = HTML::Builder::MAX_DEPTH
+      EXTENSIONS = {
+        strikethrough: true, table: true, autolink: true, tasklist: false, shortcodes: false, header_ids: nil,
+        front_matter_delimiter: "---"
+      }.freeze
+      RENDER_OPTIONS = {extension: EXTENSIONS, render: {unsafe: true, hardbreaks: false, github_pre_lang: false}}.freeze
+      TAGS = {
+        paragraph: "p", block_quote: "blockquote", item: "li", thematic_break: "hr", table_row: "tr",
+        emph: "em", strong: "strong", strikethrough: "s", linebreak: "br"
+      }.freeze
+      # A table's delimiter row, like `| :-- | --: |`, at the end of its line.
+      DELIMITER_ROW = /\|?(?:\s*:?-+:?\s*\|)*\s*:?-+:?\s*\|?\s*\z/
+      # The bytes commonmarker escapes in URLs, as comrak's `escape_href`: all but these, and `%` when it
+      # doesn't start an escape. `&` and `'` are kept: comrak writes them as HTML references.
+      URL_UNSAFE = %r{%(?!\h\h)|[^-_.+!*(),#@?=;:/$~A-Za-z0-9&'%]}n
+      # An IPv6 host at the start of a URL, like `http://[::1]`, kept as it is.
+      IPV6_HOST = %r{\A[A-Za-z][A-Za-z0-9+.-]*://\[[\h:.]+\]}
+
+      def self.load_commonmarker
+        require "commonmarker"
+      rescue LoadError
+        raise Error, 'Markdown import needs the commonmarker gem: add `gem "commonmarker"` to your Gemfile'
+      end
+
+      # A root Element holding the content.
+      def call(markdown)
+        self.class.load_commonmarker
+        markdown = UTF8.string(markdown)
+        document = parse(markdown)
+        return from_html(document) if raw_html?(document) && HTML::Builder.herb?
+
+        @lines = markdown.lines
+        root = HTML::Element.new("#document")
+        build(document, root)
+        root
+      end
+
+      private
+
+      def parse(markdown)
+        Commonmarker.parse(markdown, options: {extension: EXTENSIONS})
+      rescue Commonmarker::NestingDepthExceededError => error
+        raise Error, "Markdown nested too deeply: #{error.message}"
+      end
+
+      # Walks the tree without recursion. Each item of the stack is a node, the element to add it to, and
+      # the depth of that element. Elements nested deeper than MAX_DEPTH are flattened.
+      def build(document, root)
+        stack = document.to_a.reverse.map { |node| [node, root, 0] }
+        until stack.empty?
+          node, target, depth = stack.pop
+          element = add(node, target)
+          next unless element
+
+          if depth >= MAX_DEPTH
+            element = target
+          else
+            target.append(element)
+          end
+          node.to_a.reverse_each { |child| stack << [child, element, element.equal?(target) ? depth : depth + 1] }
+        end
+      end
+
+      # Adds a node to the target. Returns the element to add its children to, when they need adding.
+      def add(node, target)
+        case node.type
+        when *TAGS.keys then HTML::Element.new(TAGS[node.type])
+        when :heading then HTML::Element.new("h#{node.header_level}")
+        when :list then list(node)
+        when :code_block then code_block(node, target)
+        when :table then table(node)
+        when :table_cell then table_cell(node)
+        when :link then HTML::Element.new("a", {"href" => escape_url(node.url), "title" => presence(node.title)}.compact)
+        when :image then image(node, target)
+        when :code then text(target, node.string_content, "code")
+        when :text then text(target, node.string_content)
+        when :softbreak then text(target, "\n")
+        when :html_block then text(target.append(HTML::Element.new("p")), node.literal.chomp)
+        when :html_inline then text(target, node.literal)
+        when :frontmatter then nil
+        else HTML::Element.new("span") # Unwrapped by HTML import, like escaped characters (`\*`).
+        end
+      end
+
+      def list(node)
+        return HTML::Element.new("ul") unless node.list_type == :ordered
+
+        HTML::Element.new("ol", (node.list_start == 1) ? {} : {"start" => node.list_start.to_s})
+      end
+
+      def code_block(node, target)
+        language = node.fence_info.to_s.split.first
+        code = HTML::Element.new("code", language ? {"class" => "language-#{language}"} : {})
+        code.append(HTML::Text.new(node.string_content.chomp))
+        target.append(HTML::Element.new("pre")).append(code)
+        nil
+      end
+
+      def table(node)
+        @alignments = alignments(node)
+        @row = 0
+        HTML::Element.new("table")
+      end
+
+      # The first row holds header cells.
+      def table_cell(node)
+        index = cell_index(node)
+        @row += 1 if index.zero?
+        align = @alignments[index]
+        HTML::Element.new((@row == 1) ? "th" : "td", align ? {"style" => "text-align: #{align}"} : {})
+      end
+
+      def cell_index(node)
+        index = 0
+        index += 1 while (node = node.previous_sibling)
+        index
+      end
+
+      # commonmarker doesn't give the alignments of a table's columns: they are read from its delimiter row,
+      # the line after its first line.
+      def alignments(node)
+        row = @lines[node.source_position[:start_line]].to_s.chomp[DELIMITER_ROW].to_s
+        row.strip.delete_prefix("|").delete_suffix("|").split("|").map do |cell|
+          cell = cell.strip
+          if cell.start_with?(":") && cell.end_with?(":") then "center"
+          elsif cell.end_with?(":") then "right"
+          elsif cell.start_with?(":") then "left"
+          end
+        end
+      end
+
+      def image(node, target)
+        attributes = {"src" => escape_url(node.url), "alt" => plain_text(node), "title" => presence(node.title)}.compact
+        target.append(HTML::Element.new("img", attributes))
+        nil
+      end
+
+      def text(target, value, tag = nil)
+        target = target.append(HTML::Element.new(tag)) if tag
+        target.append(HTML::Text.new(value))
+        nil
+      end
+
+      def raw_html?(document)
+        document.walk { |node| return true if node.type == :html_block || node.type == :html_inline }
+        false
+      end
+
+      def from_html(document)
+        root = HTML::Builder.new.call(document.to_html(options: RENDER_OPTIONS, plugins: {syntax_highlighter: nil}))
+        chomp_code_blocks(root)
+        root
+      end
+
+      # The code blocks commonmarker renders, `<pre><code>`, not the `<pre>` elements of raw HTML.
+      def chomp_code_blocks(element)
+        element.children.each do |child|
+          next unless child.element?
+          next chomp_code_blocks(child) unless child.name == "pre"
+
+          code = child.children.first
+          next unless code&.element? && code.name == "code" && child.children.size == 1
+
+          last = code.children.last
+          code.children[-1] = HTML::Text.new(last.value.chomp).tap { |text| text.parent = code } if last.is_a?(HTML::Text)
+        end
+      end
+
+      # The URL as commonmarker renders it, with the characters URLs can't hold percent-encoded.
+      def escape_url(url)
+        host = url[IPV6_HOST].to_s
+        rest = url.delete_prefix(host).b.gsub(URL_UNSAFE) { |byte| byte.ord.zero? ? "%EF%BF%BD" : format("%%%02X", byte.ord) }
+        host + rest.force_encoding(Encoding::UTF_8)
+      end
+
+      def plain_text(node)
+        node.to_a.map { |child| %i[text code].include?(child.type) ? child.string_content : plain_text(child) }.join
+      end
+
+      def presence(value)
+        value unless value.nil? || value.empty?
+      end
+    end
+  end
+end

data/lib/prosemirror/marks/basic/code.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module Marks
+    module Basic
+      # ProseMirror's inline code, which can be combined with other marks.
+      class Code < MarkType
+        code
+
+        parse_html tag: "code"
+
+        def view_template(&) = code(&)
+      end
+    end
+  end
+end

data/lib/prosemirror/marks/basic/em.rb

@@ -5,6 +5,10 @@ module Prosemirror
     module Basic
       class Em < MarkType
         aliases :italic
+        parse_html tag: "i"
+        parse_html tag: "em"
+        parse_html style: "font-style=italic"
+        parse_html style: "font-style=normal", clear_mark: true
 
         def view_template(&) = em(&)
       end

data/lib/prosemirror/marks/basic/link.rb

@@ -7,6 +7,7 @@ module Prosemirror
       class Link < MarkType
         prop :href, _String(length: 1..)
         prop :title, _String?
+        parse_html(tag: "a[href]") { |element, schema| schema.allowed_url?(element["href"]) && {} }
 
         def view_template(&)
           href = safe_url(@href)

data/lib/prosemirror/marks/basic/strong.rb

@@ -5,6 +5,10 @@ module Prosemirror
     module Basic
       class Strong < MarkType
         aliases :bold
+        parse_html tag: "strong"
+        parse_html(tag: "b") { |element| element.style["font-weight"] != "normal" && {} }
+        parse_html style: "font-weight=400", clear_mark: true
+        parse_html(style: "font-weight") { |value| value.match?(Bold::BOLD) && {} }
 
         def view_template(&) = strong(&)
       end

data/lib/prosemirror/marks/bold.rb

@@ -3,8 +3,16 @@
 module Prosemirror
   module Marks
     class Bold < MarkType
+      # The `font-weight` values that mean bold.
+      BOLD = /\A(bold(er)?|[5-9]\d{2,})\z/
+
       aliases :strong
 
+      parse_html tag: "strong"
+      parse_html(tag: "b") { |element| element.style["font-weight"] != "normal" && {} }
+      parse_html style: "font-weight=400", clear_mark: true
+      parse_html(style: "font-weight") { |value| value.match?(BOLD) && {} }
+
       def view_template(&) = strong(&)
     end
   end

data/lib/prosemirror/marks/code.rb

@@ -2,8 +2,13 @@
 
 module Prosemirror
   module Marks
+    # Tiptap's inline code, which can't be combined with other marks.
     class Code < MarkType
       code
+      excludes "_"
+
+      parse_html tag: "code"
+      parse_html tag: "tt"
 
       def view_template(&) = code(&)
     end

data/lib/prosemirror/marks/highlight.rb

@@ -6,7 +6,9 @@ module Prosemirror
     class Highlight < MarkType
       COLOR = %r{\A(\#\h{3,8}|[a-z]+|(rgb|hsl)a?\([\d\s.,%/]+\))\z}i
 
-      prop :color, _String?(COLOR)
+      prop :color, _String?(COLOR), parse_html: ->(element) { element["data-color"] || element.style["background-color"] }
+
+      parse_html tag: "mark"
 
       def view_template(&)
         mark(data_color: @color, style: @color && "background-color: #{@color}; color: inherit", &)

data/lib/prosemirror/marks/italic.rb

@@ -4,6 +4,10 @@ module Prosemirror
   module Marks
     class Italic < MarkType
       aliases :em
+      parse_html tag: "em"
+      parse_html(tag: "i") { |element| element.style["font-style"] != "normal" && {} }
+      parse_html style: "font-style=normal", clear_mark: true
+      parse_html style: "font-style=italic"
 
       def view_template(&) = em(&)
     end

data/lib/prosemirror/marks/link.rb

@@ -9,6 +9,7 @@ module Prosemirror
       prop :rel, _String?
       prop :css_class, _String?, json: "class"
       prop :title, _String?
+      parse_html(tag: "a[href]") { |element, schema| schema.allowed_url?(element["href"]) && {} }
 
       def view_template(&)
         href = safe_url(@href)

data/lib/prosemirror/marks/strike.rb

@@ -3,6 +3,11 @@
 module Prosemirror
   module Marks
     class Strike < MarkType
+      parse_html tag: "s"
+      parse_html tag: "del"
+      parse_html tag: "strike"
+      parse_html(style: "text-decoration", consuming: false) { |value| value.include?("line-through") ? {} : false }
+
       def view_template(&) = s(&)
     end
   end

data/lib/prosemirror/marks/subscript.rb

@@ -4,6 +4,8 @@ module Prosemirror
   module Marks
     class Subscript < MarkType
       excludes "superscript"
+      parse_html tag: "sub"
+      parse_html(style: "vertical-align") { |value| (value == "sub") ? {} : false }
 
       def view_template(&) = sub(&)
     end

data/lib/prosemirror/marks/superscript.rb

@@ -4,6 +4,8 @@ module Prosemirror
   module Marks
     class Superscript < MarkType
       excludes "subscript"
+      parse_html tag: "sup"
+      parse_html(style: "vertical-align") { |value| (value == "super") ? {} : false }
 
       def view_template(&) = sup(&)
     end

data/lib/prosemirror/marks/underline.rb

@@ -3,6 +3,9 @@
 module Prosemirror
   module Marks
     class Underline < MarkType
+      parse_html tag: "u"
+      parse_html(style: "text-decoration", consuming: false) { |value| value.include?("underline") ? {} : false }
+
       def view_template(&) = u(&)
     end
   end

data/lib/prosemirror/node.rb

@@ -18,6 +18,18 @@ module Prosemirror
     prop :marks, _Array(Mark), default: [].freeze, &Coerce::FROZEN
     prop :text, _String?, &Coerce::FROZEN
 
+    # The nodes with adjacent text nodes that have the same marks joined, as in ProseMirror.
+    def self.join_text(nodes)
+      nodes.each_with_object([]) do |node, joined|
+        previous = joined.last
+        if previous&.text? && node.text? && previous.marks == node.marks
+          joined[-1] = previous.with(text: previous.text + node.text)
+        else
+          joined << node
+        end
+      end
+    end
+
     # A copy with some properties changed.
     def with(**changes)
       self.class.new(**to_h, **changes)

data/lib/prosemirror/node_type.rb

@@ -86,6 +86,28 @@ module Prosemirror
         value = leaf_text
         value.respond_to?(:call) ? value.call(node).to_s : value.to_s
       end
+
+      # This inline node stands for a newline, as ProseMirror's `linebreakReplacement`: when importing
+      # HTML whose whitespace is preserved, newlines become nodes of this type.
+      def linebreak_replacement(value = true)
+        @linebreak_replacement = value
+      end
+
+      def linebreak_replacement?
+        inherited_setting(:@linebreak_replacement, false)
+      end
+
+      # Whether marks of this type can be applied inside this node, according to its `marks` option.
+      def allows_mark?(mark_type, schema)
+        case marks
+        when nil then schema.content_match(self).inline_content?
+        when "_" then true
+        when "" then false
+        else
+          names = marks.split.map(&:to_sym)
+          names.include?(mark_type.type_name) || names.intersect?(mark_type.group)
+        end
+      end
     end
 
     # The node being rendered.

data/lib/prosemirror/nodes/basic/code_block.rb

@@ -9,6 +9,7 @@ module Prosemirror
         content "text*"
         marks ""
         code
+        parse_html tag: "pre", preserve_whitespace: :full
 
         def view_template(&)
           pre { code(&) }

data/lib/prosemirror/nodes/basic/image.rb

@@ -11,6 +11,7 @@ module Prosemirror
         prop :src, _String(length: 1..)
         prop :alt, _String?
         prop :title, _String?
+        parse_html tag: "img[src]"
 
         def view_template
           src = safe_url(@src)

data/lib/prosemirror/nodes/basic/ordered_list.rb

@@ -8,7 +8,10 @@ module Prosemirror
         group :block, :list
         content "list_item+"
 
-        prop :order, Integer, default: 1, aliases: ["start"], &Coerce::INTEGER
+        prop :order, Integer, default: 1, aliases: ["start"],
+          parse_html: ->(element) { element.key?("start") ? element["start"].to_i : 1 }, &Coerce::INTEGER
+
+        parse_html tag: "ol"
 
         def view_template(&)
           ol(start: (@order unless @order == 1), &)

data/lib/prosemirror/nodes/blockquote.rb

@@ -5,6 +5,7 @@ module Prosemirror
     class Blockquote < NodeType
       group :block
       content "block+"
+      parse_html tag: "blockquote"
 
       def view_template(&) = blockquote(&)
     end

data/lib/prosemirror/nodes/bullet_list.rb

@@ -5,6 +5,7 @@ module Prosemirror
     class BulletList < NodeType
       group :block, :list
       content "list_item+"
+      parse_html tag: "ul"
 
       def view_template(&) = ul(&)
     end

data/lib/prosemirror/nodes/code_block.rb

@@ -9,7 +9,12 @@ module Prosemirror
       marks ""
       code
 
-      prop :language, _String?(/\A[\w#+.-]+\z/), aliases: ["params"]
+      # From the `language-*` class of the `<code>` inside `<pre>`.
+      prop :language, _String?(/\A[\w#+.-]+\z/), aliases: ["params"], parse_html: lambda { |element|
+        element.first_element_child&.classes&.find { |name| name.start_with?("language-") }&.delete_prefix("language-")
+      }
+
+      parse_html tag: "pre", preserve_whitespace: :full
 
       def view_template(&)
         pre { code(class: @language && "language-#{@language}", &) }

data/lib/prosemirror/nodes/hard_break.rb

@@ -6,6 +6,8 @@ module Prosemirror
       group :inline
       inline
       leaf_text "\n"
+      linebreak_replacement
+      parse_html tag: "br"
 
       def view_template = br
     end

data/lib/prosemirror/nodes/heading.rb

@@ -7,6 +7,7 @@ module Prosemirror
       content "inline*"
 
       prop :level, _Integer(1..6), default: 1, &Coerce::INTEGER
+      (1..6).each { |level| parse_html tag: "h#{level}", attrs: {level:} }
 
       def view_template(&) = tag(:"h#{@level}", &)
     end

data/lib/prosemirror/nodes/horizontal_rule.rb

@@ -4,6 +4,7 @@ module Prosemirror
   module Nodes
     class HorizontalRule < NodeType
       group :block
+      parse_html tag: "hr"
 
       def view_template = hr
     end

data/lib/prosemirror/nodes/image.rb

@@ -11,6 +11,7 @@ module Prosemirror
       prop :title, _String?
       prop :width, _Integer?(1..), &Coerce::INTEGER
       prop :height, _Integer?(1..), &Coerce::INTEGER
+      parse_html tag: %(img[src]:not([src^="data:"]))
 
       def view_template
         src = safe_url(@src)

data/lib/prosemirror/nodes/list_item.rb

@@ -4,6 +4,7 @@ module Prosemirror
   module Nodes
     class ListItem < NodeType
       content "paragraph block*"
+      parse_html tag: "li"
 
       def view_template(&) = li(&)
     end

data/lib/prosemirror/nodes/ordered_list.rb

@@ -4,11 +4,29 @@ module Prosemirror
   module Nodes
     # Tiptap's ordered list. ProseMirror's `order` attribute is read as `start`.
     class OrderedList < NodeType
+      # CSS list styles, used by Google Docs and Word, and their `type`.
+      LIST_STYLE_TYPES = {
+        "upper-roman" => "I", "lower-roman" => "i", "upper-alpha" => "A", "upper-latin" => "A",
+        "lower-alpha" => "a", "lower-latin" => "a"
+      }.freeze
+
       group :block, :list
       content "list_item+"
 
-      prop :start, Integer, default: 1, aliases: ["order"], &Coerce::INTEGER
-      prop :type, _Union?("1", "a", "A", "i", "I")
+      prop :start, Integer, default: 1, aliases: ["order"],
+        parse_html: ->(element) { element.key?("start") ? element["start"].to_i : 1 }, &Coerce::INTEGER
+      prop :type, _Union?("1", "a", "A", "i", "I"), parse_html: ->(element) { OrderedList.type_from_html(element) }
+
+      parse_html tag: "ol"
+
+      # From the `type` attribute, or the list style of the list or of its first item.
+      def self.type_from_html(element)
+        element["type"] || list_style_type(element) || element.query("li")&.then { |item| list_style_type(item) }
+      end
+
+      def self.list_style_type(element)
+        LIST_STYLE_TYPES[element.style["list-style-type"].to_s.downcase]
+      end
 
       def view_template(&)
         ol(start: (@start unless @start == 1), type: @type, &)

data/lib/prosemirror/nodes/paragraph.rb

@@ -5,6 +5,7 @@ module Prosemirror
     class Paragraph < NodeType
       group :block
       content "inline*"
+      parse_html tag: "p"
 
       def view_template(&) = p(&)
     end

data/lib/prosemirror/nodes/table.rb

@@ -5,6 +5,7 @@ module Prosemirror
     class Table < NodeType
       group :block
       content "table_row+"
+      parse_html tag: "table"
 
       def view_template(&)
         table { tbody(&) }

data/lib/prosemirror/nodes/table_cell.rb

@@ -3,11 +3,20 @@
 module Prosemirror
   module Nodes
     class TableCell < NodeType
+      ALIGNMENTS = %w[left right center].freeze
+
       content "block+"
 
       prop :colspan, _Integer(1..), default: 1, &Coerce::INTEGER
       prop :rowspan, _Integer(1..), default: 1, &Coerce::INTEGER
-      prop :colwidth, _Array?(_Integer(1..))
+      prop :colwidth, _Array?(_Integer(1..)),
+        parse_html: ->(element) { element["colwidth"]&.split(",")&.map(&:to_i) }
+      # From the `text-align` style, or the old `align` attribute.
+      prop :align, _Union?(*ALIGNMENTS), parse_html: lambda { |element|
+        (element.style["text-align"] || element["align"]).to_s.strip.downcase.then { |value| value if ALIGNMENTS.include?(value) }
+      }
+
+      parse_html tag: "td"
 
       def view_template(&)
         td(**cell_attributes, &)
@@ -16,7 +25,8 @@ module Prosemirror
       private
 
       def cell_attributes
-        {colspan: (@colspan unless @colspan == 1), rowspan: (@rowspan unless @rowspan == 1)}
+        {colspan: (@colspan unless @colspan == 1), rowspan: (@rowspan unless @rowspan == 1),
+         style: @align && "text-align: #{@align}"}
       end
     end
   end

data/lib/prosemirror/nodes/table_header.rb

@@ -5,6 +5,8 @@ module Prosemirror
     class TableHeader < TableCell
       type_name :table_header
 
+      parse_html tag: "th"
+
       def view_template(&)
         th(**cell_attributes, &)
       end

data/lib/prosemirror/nodes/table_row.rb

@@ -4,6 +4,7 @@ module Prosemirror
   module Nodes
     class TableRow < NodeType
       content "(table_cell | table_header)*"
+      parse_html tag: "tr"
 
       def view_template(&) = tr(&)
     end