prosemirror 0.0.1 → 0.2.0

data/lib/prosemirror/html/builder.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # Builds Elements and Texts from HTML, with Herb's parser (https://herb-tools.dev), which never raises
+    # on malformed HTML. It also does what Herb leaves to its users:
+    # - decodes character references, in text and attribute values;
+    # - keeps ERB tags (`<% %>`) as text;
+    # - applies an opening tag without a closing tag to the rest of its parent, as browsers do;
+    # - drops comments (including Word's `<![if …]>`), doctypes and CDATA;
+    # - walks the tree without recursion, and flattens elements nested deeper than MAX_DEPTH.
+    class Builder
+      MAX_DEPTH = 256
+      # The size of the pieces Herb parses, in bytes. See Splitter.
+      PIECE_SIZE = 65_536
+      # What browsers read as a comment, and Herb leaves in text, like Word's `<![if !supportLists]>`.
+      BOGUS_COMMENT = /<![^>]*>/
+
+      def self.load_herb
+        require "herb"
+      rescue LoadError
+        raise Error, 'HTML import needs the herb gem: add `gem "herb"` to your Gemfile'
+      end
+
+      # A root Element holding the parsed content.
+      def call(html)
+        self.class.load_herb
+        root = Element.new("#document")
+        Splitter.new(PIECE_SIZE).call(html.to_s).each do |piece|
+          # Without `timeout: 0`, Herb stops after a second and returns what it parsed so far.
+          build(Herb.parse(piece, track_locations: false, strict: false, timeout: 0).value.children, root)
+        end
+        root
+      end
+
+      private
+
+      # Each item of the stack is a list of Herb nodes, the position in it, the element to add them to,
+      # and its depth.
+      def build(nodes, root)
+        stack = [[nodes, 0, root, 0]]
+        until stack.empty?
+          frame = stack.last
+          list, index, target, depth = frame
+          if index >= list.size
+            stack.pop
+            next
+          end
+          frame[1] += 1
+          child = add(list[index], target, depth)
+          next unless child
+
+          element, children = child
+          if element.equal?(target)
+            stack << [children, 0, target, depth]
+          elsif children.nil? # an opening tag without a closing tag: the rest of the list goes in it
+            frame[2] = element
+            frame[3] = depth + 1
+          else
+            stack << [children, 0, element, depth + 1]
+          end
+        end
+      end
+
+      # Adds a Herb node to the target. Returns the element to fill and the Herb nodes to fill it with,
+      # when the node has children.
+      def add(node, target, depth)
+        case node
+        when Herb::AST::HTMLElementNode then add_element(node, target, depth)
+        when Herb::AST::HTMLOpenTagNode then add_open_tag(node, target, depth)
+        when Herb::AST::HTMLTextNode then add_text(Entities.decode(node.content.to_s.gsub(BOGUS_COMMENT, "")), target)
+        when Herb::AST::LiteralNode then add_text(node.content.to_s, target)
+        when Herb::AST::WhitespaceNode then add_text(node.value&.value.to_s, target)
+        when Herb::AST::HTMLCommentNode, Herb::AST::HTMLDoctypeNode, Herb::AST::XMLDeclarationNode,
+             Herb::AST::XMLProcessingInstructionNode, Herb::AST::CDATANode, Herb::AST::HTMLCloseTagNode
+          nil
+        else add_other(node, target)
+        end
+      end
+
+      def add_element(node, target, depth)
+        open_tag = node.open_tag
+        return [target, node.body] unless open_tag.respond_to?(:tag_name) && open_tag.tag_name
+
+        element = new_element(open_tag)
+        return [target, node.body] if depth >= MAX_DEPTH
+
+        target.append(element)
+        [element, node.body]
+      end
+
+      def add_open_tag(node, target, depth)
+        element = new_element(node)
+        return nil if depth >= MAX_DEPTH
+
+        target.append(element)
+        node.is_void ? nil : [element, nil]
+      end
+
+      # ERB tags become text. Other nodes give their children, if any.
+      def add_other(node, target)
+        if node.respond_to?(:tag_opening) && node.respond_to?(:content) && node.respond_to?(:tag_closing)
+          add_text([node.tag_opening, node.content, node.tag_closing].map { |token| token&.value.to_s }.join, target)
+        end
+        children = node.respond_to?(:child_nodes) ? node.child_nodes.compact : []
+        children.empty? ? nil : [target, children]
+      end
+
+      def add_text(value, target)
+        return nil if value.empty?
+
+        last = target.children.last
+        if last.is_a?(Text)
+          target.children[-1] = Text.new(last.value + value).tap { |text| text.parent = target }
+        else
+          target.append(Text.new(value))
+        end
+        nil
+      end
+
+      def new_element(open_tag)
+        Element.new(open_tag.tag_name.value.downcase, attributes(open_tag.children))
+      end
+
+      # The first value wins when an attribute is repeated, as in browsers.
+      def attributes(nodes)
+        nodes.grep(Herb::AST::HTMLAttributeNode).each_with_object({}) do |attribute, result|
+          name = text_of(attribute.name&.children).downcase
+          next if name.empty? || result.key?(name)
+
+          result[name] = Entities.decode(text_of(attribute.value&.children), attribute: true)
+        end
+      end
+
+      def text_of(nodes)
+        Array(nodes).map do |node|
+          if node.respond_to?(:tag_opening) && node.respond_to?(:tag_closing)
+            [node.tag_opening, node.content, node.tag_closing].map { |token| token&.value.to_s }.join
+          else
+            node.content.to_s
+          end
+        end.join
+      end
+    end
+  end
+end

data/lib/prosemirror/html/element.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # An element, with a lowercase name, attributes with lowercase names and decoded values, and
+    # children. The import works on these rather than on the parser's own tree.
+    class Element
+      attr_reader :name, :attributes, :children
+      attr_accessor :parent
+
+      def initialize(name, attributes = {})
+        @name = name
+        @attributes = attributes
+        @children = []
+      end
+
+      def append(child)
+        child.parent = self
+        @children << child
+        child
+      end
+
+      def element?
+        true
+      end
+
+      def [](attribute)
+        @attributes[attribute]
+      end
+
+      def key?(attribute)
+        @attributes.key?(attribute)
+      end
+
+      def classes
+        self["class"].to_s.split
+      end
+
+      # The declarations of the `style` attribute, with lowercase properties: `{"font-weight" => "bold"}`.
+      def style
+        @style ||= self["style"].to_s.split(";").each_with_object({}) do |declaration, styles|
+          property, value = declaration.split(":", 2)
+          next if value.nil? || property.strip.empty?
+
+          styles[property.strip.downcase] = value.strip.sub(/\s*!important\z/i, "")
+        end
+      end
+
+      def text_content
+        @children.map(&:text_content).join
+      end
+
+      def element_children
+        @children.select(&:element?)
+      end
+
+      def first_element_child
+        @children.find(&:element?)
+      end
+
+      def previous_sibling(child)
+        index = @children.index { |candidate| candidate.equal?(child) }
+        @children[index - 1] if index&.positive?
+      end
+
+      def matches?(selector)
+        Selector.parse(selector).match?(self)
+      end
+
+      # The first descendant matching the selector, depth first.
+      def query(selector)
+        compiled = Selector.parse(selector)
+        stack = element_children.reverse
+        until stack.empty?
+          element = stack.pop
+          return element if compiled.match?(element)
+
+          stack.concat(element.element_children.reverse)
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/prosemirror/html/entities.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Prosemirror
+  module HTML
+    # Decodes character references (`&`, `é`, `é`, `é`) as HTML5 does, with the
+    # WHATWG list of named references: https://html.spec.whatwg.org/multipage/named-characters.html
+    module Entities
+      # Code points 0x80 to 0x9F, read as Windows-1252, as browsers do.
+      WINDOWS_1252 = {
+        0x80 => 0x20AC, 0x82 => 0x201A, 0x83 => 0x0192, 0x84 => 0x201E, 0x85 => 0x2026, 0x86 => 0x2020,
+        0x87 => 0x2021, 0x88 => 0x02C6, 0x89 => 0x2030, 0x8A => 0x0160, 0x8B => 0x2039, 0x8C => 0x0152,
+        0x8E => 0x017D, 0x91 => 0x2018, 0x92 => 0x2019, 0x93 => 0x201C, 0x94 => 0x201D, 0x95 => 0x2022,
+        0x96 => 0x2013, 0x97 => 0x2014, 0x98 => 0x02DC, 0x99 => 0x2122, 0x9A => 0x0161, 0x9B => 0x203A,
+        0x9C => 0x0153, 0x9E => 0x017E, 0x9F => 0x0178
+      }.freeze
+
+      # The length of the longest reference without `;`, like `middot`.
+      LONGEST_LEGACY_NAME = 6
+      REFERENCE = /&(?:#(\d+);?|#[xX](\h+);?|([A-Za-z][A-Za-z0-9]*;?))/
+
+      module_function
+
+      # In an attribute, a named reference without a semicolon followed by a letter, a digit or `=`
+      # stays as it is, as in browsers (`?a=1&copy=2` keeps `&copy`).
+      def decode(string, attribute: false)
+        return string unless string.include?("&")
+
+        string.gsub(REFERENCE) do
+          if (decimal = Regexp.last_match(1))
+            character(decimal.to_i)
+          elsif (hexadecimal = Regexp.last_match(2))
+            character(hexadecimal.to_i(16))
+          else
+            named(Regexp.last_match(3), Regexp.last_match.post_match, attribute) || Regexp.last_match(0)
+          end
+        end
+      end
+
+      # The characters of a named reference ending with `;`, or nil. `entities.txt` has one reference per
+      # line, with its code points in hexadecimal (`nvlt; 3C,20D2`). It is kept as one string, searched with
+      # `index`, which takes a tenth of the memory of a Hash. The references found are cached, but not the
+      # unknown names, which documents could make up without limit.
+      def lookup(name)
+        @found ||= {}
+        @found[name] ||= begin
+          start = list.index("\n#{name} ") or return nil
+          characters(list[start + name.length + 2...list.index("\n", start + 1)])
+        end
+      end
+
+      # The references without `;`, which browsers also read at the start of a longer name (`&copy2`).
+      def legacy
+        @legacy ||= list.scan(/^([A-Za-z0-9]+) ([\h,]+)$/).to_h { |name, codes| [name, characters(codes)] }.freeze
+      end
+
+      def list
+        @list ||= "\n#{File.read(File.join(__dir__, "entities.txt"))}".freeze
+      end
+
+      def characters(codes)
+        codes.split(",").map { |code| code.to_i(16) }.pack("U*").freeze
+      end
+
+      def character(code)
+        return "�" if code.zero? || code > 0x10FFFF || (0xD800..0xDFFF).cover?(code)
+
+        [WINDOWS_1252.fetch(code, code)].pack("U")
+      end
+
+      # The longest reference starting the name, followed by the rest of the name. A `;` can only end the
+      # name, so a shorter reference is one without `;`.
+      def named(name, following, attribute)
+        return lookup(name) if name.end_with?(";") && lookup(name)
+
+        [name.length, LONGEST_LEGACY_NAME].min.downto(1) do |length|
+          next unless (characters = legacy[name[0, length]])
+
+          rest = name[length..]
+          return nil if attribute && "#{rest}#{following}".match?(/\A[A-Za-z0-9=]/)
+
+          return characters + rest
+        end
+        nil
+      end
+    end
+  end
+end