prosemirror 0.1.0 → 0.3.0

data/lib/prosemirror/content_match.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  # A state of a node type's content expression, compiled into a deterministic automaton, as in
+  # ProseMirror's ContentMatch: https://prosemirror.net/docs/ref/#model.ContentMatch
+  #
+  # Edges go from one state to the next for a node type. Their order matters: the first generatable
+  # type is used to fill required content.
+  class ContentMatch
+    attr_reader :edges
+
+    def initialize(valid_end)
+      @valid_end = valid_end
+      @edges = []
+      @wrappings = {}
+    end
+
+    # Compiles a content expression for a schema. Returns EMPTY for leaf types.
+    def self.compile(expression, schema)
+      return EMPTY if expression.nil?
+
+      Compiler.new(expression, schema).call
+    end
+
+    # Whether the content can end here.
+    def valid_end?
+      @valid_end
+    end
+
+    # The state after a node of this type, or nil when it can't come here.
+    def match_type(type)
+      @edges.find { |edge_type, _| edge_type == type }&.last
+    end
+
+    # The state after these node types, or nil when they don't match.
+    def match_types(types)
+      types.reduce(self) { |match, type| match&.match_type(type) }
+    end
+
+    def inline_content?
+      !@edges.empty? && @edges.first.first.inline?
+    end
+
+    # The first type that can come here and can be created empty, or nil.
+    def default_type(schema)
+      @edges.map(&:first).find { |type| generatable?(type, schema) }
+    end
+
+    # The node types to insert before `types` so that they match, possibly none, or nil when no
+    # insertion works. With `to_end`, the content must also be able to end after them.
+    def fill_before(types, schema, to_end: false)
+      seen = [self]
+      search = lambda do |match, filled|
+        finished = match.match_types(types)
+        return filled if finished && (!to_end || finished.valid_end?)
+
+        match.edges.each do |type, following|
+          next if !generatable?(type, schema) || seen.include?(following)
+
+          seen << following
+          found = search.call(following, filled + [type])
+          return found if found
+        end
+        nil
+      end
+      search.call(self, [])
+    end
+
+    # The wrapper types that let a node of this type come here, possibly none, or nil when none do.
+    def find_wrapping(target, schema)
+      return @wrappings[target] if @wrappings.key?(target)
+
+      @wrappings[target] = compute_wrapping(target, schema)
+    end
+
+    def generatable?(type, schema)
+      type.type_name != :text && !type.required_attrs?
+    end
+
+    private
+
+    def compute_wrapping(target, schema)
+      seen = {}
+      active = [[self, nil, nil]]
+      until active.empty?
+        current = active.shift
+        match, type, = current
+        return wrapping_path(current) if match.match_type(target)
+
+        match.edges.each do |next_type, following|
+          next if next_type.leaf? || next_type.required_attrs? || seen[next_type]
+          next if type && !following.valid_end?
+
+          active << [schema.content_match(next_type), next_type, current]
+          seen[next_type] = true
+        end
+      end
+      nil
+    end
+
+    def wrapping_path(step)
+      path = []
+      while step && step[1]
+        path.unshift(step[1])
+        step = step[2]
+      end
+      path
+    end
+
+    EMPTY = new(true)
+
+    # Compiles an expression tree into a non-deterministic automaton, then a deterministic one.
+    class Compiler
+      def initialize(expression, schema)
+        @expression = expression
+        @schema = schema
+        @nfa = [[]]
+      end
+
+      def call
+        connect(compile(@expression.tree, 0), new_state)
+        explore(null_from(0), {})
+      end
+
+      private
+
+      Edge = Struct.new(:term, :to)
+
+      def new_state
+        @nfa << []
+        @nfa.size - 1
+      end
+
+      def edge(from, to = nil, term = nil)
+        Edge.new(term, to).tap { |created| @nfa[from] << created }
+      end
+
+      def connect(edges, to)
+        edges.each { |created| created.to = to }
+      end
+
+      def compile(tree, from)
+        case tree.first
+        when :choice then tree[1].flat_map { |branch| compile(branch, from) }
+        when :sequence then compile_sequence(tree[1], from)
+        when :repeat then compile_repeat(*tree.drop(1), from)
+        when :name then resolve(tree[1]).map { |type| edge(from, nil, type) }
+        end
+      end
+
+      def compile_sequence(parts, from)
+        parts.each_with_index do |part, index|
+          edges = compile(part, from)
+          return edges if index == parts.size - 1
+
+          connect(edges, from = new_state)
+        end
+      end
+
+      def compile_repeat(tree, min, max, from)
+        if min.zero? && max.nil? # *
+          loop_state = new_state
+          edge(from, loop_state)
+          connect(compile(tree, loop_state), loop_state)
+          [edge(loop_state)]
+        elsif min == 1 && max.nil? # +
+          loop_state = new_state
+          connect(compile(tree, from), loop_state)
+          connect(compile(tree, loop_state), loop_state)
+          [edge(loop_state)]
+        elsif min.zero? && max == 1 # ?
+          [edge(from)] + compile(tree, from)
+        else
+          compile_range(tree, min, max, from)
+        end
+      end
+
+      def compile_range(tree, min, max, from)
+        current = from
+        min.times do
+          following = new_state
+          connect(compile(tree, current), following)
+          current = following
+        end
+        if max.nil?
+          connect(compile(tree, current), current)
+        else
+          (max - min).times do
+            following = new_state
+            edge(current, following)
+            connect(compile(tree, current), following)
+            current = following
+          end
+        end
+        [edge(current)]
+      end
+
+      def resolve(name)
+        types = @schema.node_types_named(name)
+        raise ArgumentError, "no node type or group #{name} in content expression #{@expression.source.inspect}" if types.empty?
+
+        types
+      end
+
+      # The states reachable from `state` without consuming a node, sorted.
+      def null_from(state)
+        result = []
+        scan = lambda do |current|
+          edges = @nfa[current]
+          return scan.call(edges.first.to) if edges.size == 1 && edges.first.term.nil?
+
+          result << current
+          edges.each { |created| scan.call(created.to) if created.term.nil? && !result.include?(created.to) }
+        end
+        scan.call(state)
+        result.sort.reverse
+      end
+
+      def explore(states, labeled)
+        out = []
+        states.each do |state|
+          @nfa[state].each do |created|
+            next unless created.term
+
+            set = out.find { |type, _| type == created.term }&.last
+            null_from(created.to).each do |reached|
+              out << [created.term, set = []] unless set
+              set << reached unless set.include?(reached)
+            end
+          end
+        end
+        match = labeled[states.join(",")] = ContentMatch.new(states.include?(@nfa.size - 1))
+        out.each do |type, set|
+          set = set.sort.reverse
+          match.edges << [type, labeled[set.join(",")] || explore(set, labeled)]
+        end
+        match
+      end
+    end
+  end
+end

data/lib/prosemirror/html/builder.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # Builds Elements and Texts from HTML, with Herb's parser (https://herb-tools.dev), which never raises
+    # on malformed HTML. It also does what Herb leaves to its users:
+    # - decodes character references, in text and attribute values;
+    # - keeps ERB tags (`<% %>`) as text;
+    # - applies an opening tag without a closing tag to the rest of its parent, as browsers do;
+    # - drops comments (including Word's `<![if …]>`), doctypes and CDATA;
+    # - walks the tree without recursion, and flattens elements nested deeper than MAX_DEPTH.
+    class Builder
+      MAX_DEPTH = 256
+      # The size of the pieces Herb parses, in bytes. See Splitter.
+      PIECE_SIZE = 65_536
+      # What browsers read as a comment, and Herb leaves in text, like Word's `<![if !supportLists]>`.
+      BOGUS_COMMENT = /<![^>]*>/
+
+      def self.load_herb
+        require "herb"
+      rescue LoadError
+        raise Error, 'HTML import needs the herb gem: add `gem "herb"` to your Gemfile'
+      end
+
+      # Whether the herb gem is installed.
+      def self.herb?
+        load_herb
+        true
+      rescue Error
+        false
+      end
+
+      # A root Element holding the parsed content.
+      def call(html)
+        self.class.load_herb
+        root = Element.new("#document")
+        Splitter.new(PIECE_SIZE).call(UTF8.string(html)).each do |piece|
+          # Without `timeout: 0`, Herb stops after a second and returns what it parsed so far.
+          build(Herb.parse(piece, track_locations: false, strict: false, timeout: 0).value.children, root)
+        end
+        root
+      end
+
+      private
+
+      # Each item of the stack is a list of Herb nodes, the position in it, the element to add them to,
+      # and its depth.
+      def build(nodes, root)
+        stack = [[nodes, 0, root, 0]]
+        until stack.empty?
+          frame = stack.last
+          list, index, target, depth = frame
+          if index >= list.size
+            stack.pop
+            next
+          end
+          frame[1] += 1
+          child = add(list[index], target, depth)
+          next unless child
+
+          element, children = child
+          if element.equal?(target)
+            stack << [children, 0, target, depth]
+          elsif children.nil? # an opening tag without a closing tag: the rest of the list goes in it
+            frame[2] = element
+            frame[3] = depth + 1
+          else
+            stack << [children, 0, element, depth + 1]
+          end
+        end
+      end
+
+      # Adds a Herb node to the target. Returns the element to fill and the Herb nodes to fill it with,
+      # when the node has children.
+      def add(node, target, depth)
+        case node
+        when Herb::AST::HTMLElementNode then add_element(node, target, depth)
+        when Herb::AST::HTMLOpenTagNode then add_open_tag(node, target, depth)
+        when Herb::AST::HTMLTextNode then add_text(Entities.decode(node.content.to_s.gsub(BOGUS_COMMENT, "")), target)
+        when Herb::AST::LiteralNode then add_text(node.content.to_s, target)
+        when Herb::AST::WhitespaceNode then add_text(node.value&.value.to_s, target)
+        when Herb::AST::HTMLCommentNode, Herb::AST::HTMLDoctypeNode, Herb::AST::XMLDeclarationNode,
+             Herb::AST::XMLProcessingInstructionNode, Herb::AST::CDATANode, Herb::AST::HTMLCloseTagNode
+          nil
+        else add_other(node, target)
+        end
+      end
+
+      def add_element(node, target, depth)
+        open_tag = node.open_tag
+        return [target, node.body] unless open_tag.respond_to?(:tag_name) && open_tag.tag_name
+
+        element = new_element(open_tag)
+        return [target, node.body] if depth >= MAX_DEPTH
+
+        target.append(element)
+        [element, node.body]
+      end
+
+      def add_open_tag(node, target, depth)
+        element = new_element(node)
+        return nil if depth >= MAX_DEPTH
+
+        target.append(element)
+        node.is_void ? nil : [element, nil]
+      end
+
+      # ERB tags become text. Other nodes give their children, if any.
+      def add_other(node, target)
+        if node.respond_to?(:tag_opening) && node.respond_to?(:content) && node.respond_to?(:tag_closing)
+          add_text([node.tag_opening, node.content, node.tag_closing].map { |token| token&.value.to_s }.join, target)
+        end
+        children = node.respond_to?(:child_nodes) ? node.child_nodes.compact : []
+        children.empty? ? nil : [target, children]
+      end
+
+      def add_text(value, target)
+        return nil if value.empty?
+
+        last = target.children.last
+        if last.is_a?(Text)
+          target.children[-1] = Text.new(last.value + value).tap { |text| text.parent = target }
+        else
+          target.append(Text.new(value))
+        end
+        nil
+      end
+
+      def new_element(open_tag)
+        Element.new(open_tag.tag_name.value.downcase, attributes(open_tag.children))
+      end
+
+      # The first value wins when an attribute is repeated, as in browsers.
+      def attributes(nodes)
+        nodes.grep(Herb::AST::HTMLAttributeNode).each_with_object({}) do |attribute, result|
+          name = text_of(attribute.name&.children).downcase
+          next if name.empty? || result.key?(name)
+
+          result[name] = Entities.decode(text_of(attribute.value&.children), attribute: true)
+        end
+      end
+
+      def text_of(nodes)
+        Array(nodes).map do |node|
+          if node.respond_to?(:tag_opening) && node.respond_to?(:tag_closing)
+            [node.tag_opening, node.content, node.tag_closing].map { |token| token&.value.to_s }.join
+          else
+            node.content.to_s
+          end
+        end.join
+      end
+    end
+  end
+end

data/lib/prosemirror/html/element.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # An element, with a lowercase name, attributes with lowercase names and decoded values, and
+    # children. The import works on these rather than on the parser's own tree.
+    class Element
+      attr_reader :name, :attributes, :children
+      attr_accessor :parent
+
+      def initialize(name, attributes = {})
+        @name = name
+        @attributes = attributes
+        @children = []
+      end
+
+      def append(child)
+        child.parent = self
+        @children << child
+        child
+      end
+
+      def element?
+        true
+      end
+
+      def [](attribute)
+        @attributes[attribute]
+      end
+
+      def key?(attribute)
+        @attributes.key?(attribute)
+      end
+
+      def classes
+        self["class"].to_s.split
+      end
+
+      # The declarations of the `style` attribute, with lowercase properties: `{"font-weight" => "bold"}`.
+      def style
+        @style ||= self["style"].to_s.split(";").each_with_object({}) do |declaration, styles|
+          property, value = declaration.split(":", 2)
+          next if value.nil? || property.strip.empty?
+
+          styles[property.strip.downcase] = value.strip.sub(/\s*!important\z/i, "")
+        end
+      end
+
+      def text_content
+        @children.map(&:text_content).join
+      end
+
+      def element_children
+        @children.select(&:element?)
+      end
+
+      def first_element_child
+        @children.find(&:element?)
+      end
+
+      def previous_sibling(child)
+        index = @children.index { |candidate| candidate.equal?(child) }
+        @children[index - 1] if index&.positive?
+      end
+
+      def matches?(selector)
+        Selector.parse(selector).match?(self)
+      end
+
+      # The first descendant matching the selector, depth first.
+      def query(selector)
+        compiled = Selector.parse(selector)
+        stack = element_children.reverse
+        until stack.empty?
+          element = stack.pop
+          return element if compiled.match?(element)
+
+          stack.concat(element.element_children.reverse)
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/prosemirror/html/entities.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # Decodes character references (`&`, `é`, `é`, `é`) as HTML5 does, with the
+    # WHATWG list of named references: https://html.spec.whatwg.org/multipage/named-characters.html
+    module Entities
+      # Code points 0x80 to 0x9F, read as Windows-1252, as browsers do.
+      WINDOWS_1252 = {
+        0x80 => 0x20AC, 0x82 => 0x201A, 0x83 => 0x0192, 0x84 => 0x201E, 0x85 => 0x2026, 0x86 => 0x2020,
+        0x87 => 0x2021, 0x88 => 0x02C6, 0x89 => 0x2030, 0x8A => 0x0160, 0x8B => 0x2039, 0x8C => 0x0152,
+        0x8E => 0x017D, 0x91 => 0x2018, 0x92 => 0x2019, 0x93 => 0x201C, 0x94 => 0x201D, 0x95 => 0x2022,
+        0x96 => 0x2013, 0x97 => 0x2014, 0x98 => 0x02DC, 0x99 => 0x2122, 0x9A => 0x0161, 0x9B => 0x203A,
+        0x9C => 0x0153, 0x9E => 0x017E, 0x9F => 0x0178
+      }.freeze
+
+      # The length of the longest reference without `;`, like `middot`.
+      LONGEST_LEGACY_NAME = 6
+      REFERENCE = /&(?:#(\d+);?|#[xX](\h+);?|([A-Za-z][A-Za-z0-9]*;?))/
+
+      module_function
+
+      # In an attribute, a named reference without a semicolon followed by a letter, a digit or `=`
+      # stays as it is, as in browsers (`?a=1&copy=2` keeps `&copy`).
+      def decode(string, attribute: false)
+        return string unless string.include?("&")
+
+        string.gsub(REFERENCE) do
+          if (decimal = Regexp.last_match(1))
+            character(decimal.to_i)
+          elsif (hexadecimal = Regexp.last_match(2))
+            character(hexadecimal.to_i(16))
+          else
+            named(Regexp.last_match(3), Regexp.last_match.post_match, attribute) || Regexp.last_match(0)
+          end
+        end
+      end
+
+      # The characters of a named reference ending with `;`, or nil. `entities.txt` has one reference per
+      # line, with its code points in hexadecimal (`nvlt; 3C,20D2`). It is kept as one string, searched with
+      # `index`, which takes a tenth of the memory of a Hash. The references found are cached, but not the
+      # unknown names, which documents could make up without limit.
+      def lookup(name)
+        @found ||= {}
+        @found[name] ||= begin
+          start = list.index("\n#{name} ") or return nil
+          characters(list[start + name.length + 2...list.index("\n", start + 1)])
+        end
+      end
+
+      # The references without `;`, which browsers also read at the start of a longer name (`&copy2`).
+      def legacy
+        @legacy ||= list.scan(/^([A-Za-z0-9]+) ([\h,]+)$/).to_h { |name, codes| [name, characters(codes)] }.freeze
+      end
+
+      def list
+        @list ||= "\n#{File.read(File.join(__dir__, "entities.txt"))}".freeze
+      end
+
+      def characters(codes)
+        codes.split(",").map { |code| code.to_i(16) }.pack("U*").freeze
+      end
+
+      def character(code)
+        return "�" if code.zero? || code > 0x10FFFF || (0xD800..0xDFFF).cover?(code)
+
+        [WINDOWS_1252.fetch(code, code)].pack("U")
+      end
+
+      # The longest reference starting the name, followed by the rest of the name. A `;` can only end the
+      # name, so a shorter reference is one without `;`.
+      def named(name, following, attribute)
+        return lookup(name) if name.end_with?(";") && lookup(name)
+
+        [name.length, LONGEST_LEGACY_NAME].min.downto(1) do |length|
+          next unless (characters = legacy[name[0, length]])
+
+          rest = name[length..]
+          return nil if attribute && "#{rest}#{following}".match?(/\A[A-Za-z0-9=]/)
+
+          return characters + rest
+        end
+        nil
+      end
+    end
+  end
+end