markbridge 0.4.2 → 0.5.0

data/lib/markbridge/renderers/discourse/postprocessor.rb

@@ -10,10 +10,13 @@ module Markbridge
       #    ZWNBSP/BOM). Deliberately excludes ASCII space and tab so
       #    Markdown's "two trailing spaces = hard line break" rule
       #    still works. Off by default.
-      # 2. collapses runs of 3+ newlines down to two,
-      # 3. clears whitespace-only lines,
+      # 2. clears whitespace-only lines,
+      # 3. collapses runs of 3+ newlines down to two,
       # 4. trims leading/trailing whitespace from the whole document.
       #
+      # Steps 1 to 3 leave fenced code blocks alone: inside a fence every
+      # line is code, and a blank line or a line of spaces is part of it.
+      #
       # Subclass to customize. The +call+ method is the entry point.
       class Postprocessor
         # NBSP plus zero-width format chars. Spelled with explicit
@@ -29,6 +32,22 @@ module Markbridge
         #   U+FEFF  ZWNBSP/BOM  zero-width no-break space / byte-order mark
         TRAILING_INVISIBLE_RE = /[\u{00A0 200B 200C 200D 2060 FEFF}]+$/
 
+        # A run that may open or close a fence. Found with a plain search
+        # over the whole text; the line around it is then checked with the
+        # anchored patterns below. An anchored pattern alone is tried at
+        # every byte of the text and costs many times more.
+        BACKTICK_FENCE = "```"
+        TILDE_FENCE = "~~~"
+        # A line that opens a fence: optional indentation and list markers
+        # (a fence inside a list item is indented, and one that starts an
+        # item follows the marker on the same line) and a run of at least
+        # three backticks or tildes.
+        OPENING_LINE = /\A[ \t]*(?:(?:[-+*]|\d+[.)])[ \t]+)*(`{3,}|~{3,})/
+        # A line that closes a fence: a run of backticks or tildes alone,
+        # whitespace around it allowed.
+        CLOSING_LINE = /\A[ \t]*(`{3,}|~{3,})[ \t]*\z/
+        private_constant :BACKTICK_FENCE, :TILDE_FENCE, :OPENING_LINE, :CLOSING_LINE
+
         # @param strip_trailing_invisibles [Boolean] when true, strips
         #   trailing invisible characters (NBSP and zero-width format
         #   chars) from each line before the standard cleanup pass.
@@ -39,11 +58,168 @@ module Markbridge
         # @param text [String]
         # @return [String]
         def call(text)
-          text = text.gsub(TRAILING_INVISIBLE_RE, "") if @strip_trailing_invisibles
-          text
-            .gsub(/\n{3,}/, "\n\n") # Max 2 consecutive newlines
-            .gsub(/^[ \t]+$/, "") # Remove whitespace-only lines
-            .strip # Trim leading/trailing whitespace
+          clean_document(text).strip # Trim leading/trailing whitespace
+        end
+
+        private
+
+        # Text without a fence run takes the plain path; the fence scan gives
+        # the same result for it, only slower. The escaper writes a backslash
+        # in front of every backtick and tilde in text, so three in a row
+        # only occur in a fence or in a code span with a long delimiter.
+        def clean_document(text)
+          if text.include?(BACKTICK_FENCE) || text.include?(TILDE_FENCE)
+            clean_around_fences(text)
+          else
+            clean(text)
+          end
+        end
+
+        # Three or more newlines in a row; the gsub brings them down to two.
+        NEWLINE_RUN = /\n{3,}/
+        # The shortest run the gsub replaces, as a string for include?.
+        NEWLINE_RUN_START = "\n\n\n"
+        # A line with nothing but spaces and tabs on it.
+        WHITESPACE_LINE = /^[ \t]+$/
+        SPACE = 32
+        TAB = 9
+        NEWLINE = 10
+        private_constant :NEWLINE_RUN, :NEWLINE_RUN_START, :WHITESPACE_LINE, :SPACE, :TAB, :NEWLINE
+
+        # The two checks in front of the gsubs save the copy that gsub makes
+        # also when nothing matches. Most text has neither three newlines
+        # in a row nor a whitespace-only line.
+        def clean(prose)
+          prose = prose.gsub(TRAILING_INVISIBLE_RE, "") if @strip_trailing_invisibles
+          # Whitespace-only lines are cleared first, so the blank lines
+          # they leave are part of the newline run that is collapsed next.
+          prose = prose.gsub(WHITESPACE_LINE, "") if whitespace_line?(prose)
+          prose = prose.gsub(NEWLINE_RUN, "\n\n") if prose.include?(NEWLINE_RUN_START)
+          prose
+        end
+
+        # Whether +clean+ would change anything in +document+. The segments
+        # the fence scan hands to +clean+ each start at the beginning of a
+        # line and end at the end of one, so a line of a segment is a line
+        # of the document at the same place. A document that holds nothing
+        # for +clean+ therefore holds nothing for any of its segments.
+        #
+        # An instance that strips invisibles says yes without looking at the
+        # text: that pattern costs a scan of its own and the option is off
+        # by default.
+        # @param document [String]
+        # @return [Boolean]
+        def needs_cleaning?(document)
+          @strip_trailing_invisibles || document.include?(NEWLINE_RUN_START) ||
+            document.match?(WHITESPACE_LINE)
+        end
+
+        # Cleans the text between fenced code blocks and copies the blocks
+        # as they are. Works with byte offsets and searches over the whole
+        # text, so the lines inside a block cost nothing each. A run of
+        # newlines in front of a fence is part of the text before it and
+        # is collapsed there; the newline that ends a closing fence belongs
+        # to the text after the block. Text with nothing to clean is handed
+        # back as it is — the scan would put the same bytes into a new
+        # string, and a document whose fences are the only reason it comes
+        # here is the common one.
+        # Whether a line of +prose+ holds nothing but spaces and tabs. A
+        # walk over the line starts; the regex would scan the whole text,
+        # and scan it once more for its encoding first.
+        def whitespace_line?(prose)
+          start = 0
+          while start
+            index = start
+            index += 1 while (byte = prose.getbyte(index)) == SPACE || byte == TAB
+            return true if index > start && (byte.nil? || byte == NEWLINE)
+
+            newline = prose.byteindex("\n", index)
+            start = newline && newline + 1
+          end
+          false
+        end
+
+        def clean_around_fences(text)
+          return text unless needs_cleaning?(text)
+
+          result = +""
+          prose_start = 0
+          search_from = 0
+
+          while (candidate = fence_index(text, from: search_from))
+            line_start = (text.byterindex("\n", candidate) || -1) + 1
+            line_end = text.byteindex("\n", candidate) || text.bytesize
+            run = opening_run(text.byteslice(line_start, line_end - line_start))
+
+            # A fence starts a line, so a run elsewhere on it opens none.
+            # Keep looking on the next line.
+            unless run
+              search_from = line_end
+              next
+            end
+
+            block_end = closing_fence_end(text, from: line_end, run:)
+            result << clean(text.byteslice(prose_start, line_start - prose_start))
+            result << text.byteslice(line_start, block_end - line_start)
+            prose_start = search_from = block_end
+          end
+
+          result << clean(text.byteslice(prose_start, text.bytesize - prose_start))
+        end
+
+        # The offset of the next fence run of either kind at or after
+        # +from+, else nil. Two string searches beat one regex with an
+        # alternation: a string search runs as a byte search, the regex
+        # walks the text with the engine.
+        # @param text [String]
+        # @param from [Integer] byte offset to start at
+        # @return [Integer, nil]
+        def fence_index(text, from:)
+          backtick = text.byteindex(BACKTICK_FENCE, from)
+          tilde = text.byteindex(TILDE_FENCE, from)
+          return tilde unless backtick
+          return backtick unless tilde
+
+          backtick < tilde ? backtick : tilde
+        end
+
+        # The run when +line+ opens a fenced code block, else nil. A
+        # backtick fence cannot have a backtick after its run (CommonMark
+        # 4.5), which keeps an inline code span from opening a fence.
+        # @param line [String]
+        # @return [String, nil]
+        def opening_run(line)
+          match = OPENING_LINE.match(line)
+          return nil unless match
+
+          run = match[1]
+          run unless run.include?("`") && match.post_match.include?("`")
+        end
+
+        # The byte offset just after the closing fence of the block opened
+        # by +run+, searching from +from+; the end of the text when the
+        # block never closes. A closing fence is a run of the same
+        # character, at least as long as the opening run, alone on its
+        # line. Every such line contains +run+ itself, so a string search
+        # finds the candidates and the pattern checks the whole line.
+        # @param text [String]
+        # @param from [Integer] byte offset to search from
+        # @param run [String] the opening run
+        # @return [Integer]
+        def closing_fence_end(text, from:, run:)
+          while (candidate = text.byteindex(run, from))
+            # +from+ starts at the newline that ends the opening line, so
+            # there is always a newline in front of a candidate.
+            line_start = text.byterindex("\n", candidate) + 1
+            line_end = text.byteindex("\n", candidate) || text.bytesize
+            if CLOSING_LINE.match?(text.byteslice(line_start, line_end - line_start))
+              return line_end
+            end
+
+            from = line_end
+          end
+
+          text.bytesize
         end
 
         DEFAULT = new

data/lib/markbridge/renderers/discourse/render_context.rb

@@ -20,24 +20,36 @@ module Markbridge
         # @return [RenderContext, nil] the enclosing context (nil at root)
         attr_reader :parent_context
 
+        # @return [AST::Element, nil] the element whose children are
+        #   rendered at the root, when it has no tag of its own (the
+        #   Document, as a rule). It is not on the parent chain, so the
+        #   chain walks stay short; {RenderingInterface#previous_sibling}
+        #   and {RenderingInterface#next_sibling} fall back to it.
+        attr_reader :root
+
         # @param parents [Array<AST::Element>] parent elements in document
         #   order (outermost first); convenience form for building a
         #   context from scratch. Ignored when +element:+ is given.
         # @param html_mode [Boolean] see {#html_mode?}
-        # @param parent [RenderContext, nil] enclosing context (chain form)
+        # @param parent [RenderContext, nil] enclosing context (chain form);
+        #   nil gives the element an empty root context as its parent
         # @param element [AST::Element, nil] nearest parent element (chain form)
-        def initialize(parents = [], html_mode: false, parent: nil, element: nil)
+        # @param root [AST::Element, nil] see {#root}
+        def initialize(parents = [], html_mode: false, parent: nil, element: nil, root: nil)
           @html_mode = html_mode
+          @root = root
           if element
-            @parent_context = parent
+            # Without a parent the chain still ends in a root context, the
+            # walks in Renderer#render_text rely on that.
+            @parent_context = parent || self.class.new(html_mode:, root:)
             @element = element
-            @depth = (parent ? parent.depth : 0) + 1
+            @depth = @parent_context.depth + 1
           elsif parents.empty?
             # Root context: @element and @parent_context stay unset and
             # read as nil.
             @depth = 0
           else
-            @parent_context = self.class.new(parents[0, parents.size - 1], html_mode:)
+            @parent_context = self.class.new(parents[0, parents.size - 1], html_mode:, root:)
             @element = parents.last
             @depth = parents.size
           end
@@ -51,7 +63,7 @@ module Markbridge
         def parents
           result = []
           context = self
-          while context && (parent = context.element)
+          while (parent = context.element)
             result << parent
             context = context.parent_context
           end
@@ -63,14 +75,26 @@ module Markbridge
         # @param element [AST::Element]
         # @return [RenderContext]
         def with_parent(element)
-          self.class.new(html_mode: @html_mode, parent: self, element:)
+          self.class.new(html_mode: @html_mode, parent: self, element:, root: @root)
         end
 
         # Create new context with html_mode toggled.
         # @param value [Boolean]
         # @return [RenderContext]
         def with_html_mode(value)
-          self.class.new(html_mode: value, parent: @parent_context, element: @element)
+          self.class.new(html_mode: value, parent: @parent_context, element: @element, root: @root)
+        end
+
+        # Create new context with the root element set, see {#root}.
+        # @param element [AST::Element]
+        # @return [RenderContext]
+        def with_root(element)
+          self.class.new(
+            html_mode: @html_mode,
+            parent: @parent_context,
+            element: @element,
+            root: element,
+          )
         end
 
         # @return [Boolean]
@@ -81,13 +105,15 @@ module Markbridge
         # Find closest parent that is_a? klass (handles subclasses).
         # The chain walks are inlined in each query (instead of a shared
         # yielding helper) — these run several times per rendered text
-        # node, and a plain while loop avoids the block invocation.
+        # node, and a plain while loop avoids the block invocation. Every
+        # chain ends in a root context whose element is nil (see
+        # #initialize), so the walks stop there without a nil check.
         # @param klass [Class]
         # @return [AST::Element, nil] nil when no parent matches (implicit
         #   from the exhausted while loop)
         def find_parent(klass)
           context = self
-          while context && (parent = context.element)
+          while (parent = context.element)
             return parent if parent.is_a?(klass)
             context = context.parent_context
           end
@@ -99,7 +125,7 @@ module Markbridge
         def count_parents(klass)
           count = 0
           context = self
-          while context && (parent = context.element)
+          while (parent = context.element)
             count += 1 if parent.is_a?(klass)
             context = context.parent_context
           end
@@ -111,7 +137,7 @@ module Markbridge
         # @return [Boolean]
         def has_parent?(klass)
           context = self
-          while context && (parent = context.element)
+          while (parent = context.element)
             return true if parent.is_a?(klass)
             context = context.parent_context
           end