markbridge 0.4.2 → 0.5.0

data/lib/markbridge/renderers/discourse/inline/tokens.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Markbridge
+  module Renderers
+    module Discourse
+      module Inline
+        # The tokens a tag gives the {Serializer} instead of finished
+        # Markdown. A token is a plain Array with its kind first, so
+        # building one costs a single allocation, or none for the tokens
+        # that carry no value; the constructors here are the only place
+        # that knows the layout. Finished Markdown is a token too: a plain
+        # String among the tokens is written as it is, and the serializer
+        # joins it with its neighbors by its first and last character.
+        module Tokens
+          # Text the serializer escapes.
+          TEXT = :text
+          # Finished Markdown from a tag, written as it is. Such a token is
+          # the String itself, see {.raw}.
+          RAW = :raw
+          # The start of a delimiter pair, see {DELIMITERS}.
+          OPEN = :open
+          # The end of a delimiter pair.
+          CLOSE = :close
+          # A code span. The serializer picks the backtick run.
+          CODE = :code
+          # The start of a link with the destination.
+          LINK_OPEN = :link_open
+          # The end of a link.
+          LINK_CLOSE = :link_close
+          # A link whose text is its href. The serializer decides from the
+          # neighbors whether it can stay bare.
+          BARE_URL = :bare_url
+          # An image.
+          IMAGE = :image
+          # A hard line break.
+          HARD_BREAK = :hard_break
+          # Raw HTML that stays inline HTML in Markdown.
+          HTML = :html
+          # Text that a later step replaces, with what will most likely
+          # take its place, see {.placeholder}.
+          PLACEHOLDER = :placeholder
+
+          # How a text token is escaped: as Markdown text, as the label of
+          # a link (also `]`), or not at all (inside a code span).
+          ESCAPE = :escape
+          LABEL = :label
+          VERBATIM = :verbatim
+
+          # The delimiter pairs {.open} and {.close} name.
+          DELIMITERS = %i[strong em s u].freeze
+
+          # @param string [String]
+          # @param mode [Symbol] {ESCAPE}, {LABEL} or {VERBATIM}
+          # @return [Array]
+          def self.text(string, mode = ESCAPE)
+            [TEXT, string, mode]
+          end
+
+          # @param markdown [String]
+          # @return [String] the Markdown itself
+          def self.raw(markdown)
+            markdown
+          end
+
+          OPEN_TOKENS = DELIMITERS.to_h { |delimiter| [delimiter, [OPEN, delimiter].freeze] }.freeze
+          CLOSE_TOKENS =
+            DELIMITERS.to_h { |delimiter| [delimiter, [CLOSE, delimiter].freeze] }.freeze
+          private_constant :OPEN_TOKENS, :CLOSE_TOKENS
+
+          # @param delimiter [Symbol] one of {DELIMITERS}
+          # @return [Array] the same frozen token on every call
+          def self.open(delimiter)
+            OPEN_TOKENS.fetch(delimiter)
+          end
+
+          # @param delimiter [Symbol] one of {DELIMITERS}
+          # @return [Array] the same frozen token on every call
+          def self.close(delimiter)
+            CLOSE_TOKENS.fetch(delimiter)
+          end
+
+          # @param content [String] the code, not escaped
+          # @return [Array]
+          def self.code(content)
+            [CODE, content]
+          end
+
+          # @param destination [String] the href, not escaped
+          # @return [Array]
+          def self.link_open(destination)
+            [LINK_OPEN, destination]
+          end
+
+          LINK_CLOSE_TOKEN = [LINK_CLOSE].freeze
+          private_constant :LINK_CLOSE_TOKEN
+
+          # @return [Array]
+          def self.link_close
+            LINK_CLOSE_TOKEN
+          end
+
+          # @param href [String]
+          # @return [Array]
+          def self.bare_url(href)
+            [BARE_URL, href]
+          end
+
+          # @param src [String]
+          # @param alt [String, nil]
+          # @param width [Integer, nil]
+          # @param height [Integer, nil]
+          # @return [Array]
+          def self.image(src:, alt: nil, width: nil, height: nil)
+            [IMAGE, src, alt, width, height]
+          end
+
+          HARD_BREAK_TOKEN = [HARD_BREAK].freeze
+          private_constant :HARD_BREAK_TOKEN
+
+          # @return [Array]
+          def self.hard_break
+            HARD_BREAK_TOKEN
+          end
+
+          # @param string [String] HTML, already escaped
+          # @return [Array]
+          def self.html(string)
+            [HTML, string]
+          end
+
+          # Text for a later step to replace, like `[link|17]` that an
+          # importer swaps for the real URL. The serializer writes +text+
+          # as it is, but joins it with its neighbors as if +like+ stood
+          # there: whether a marker next to it needs `<!---->`, whether a
+          # bare URL in front of it can stay bare, whether a `!` in front
+          # of it needs a backslash. In an HTML block +text+ is written as
+          # it is. See "Placeholders" in the Extending guide.
+          #
+          # @param text [String] the placeholder, written as it is
+          # @param like [String] Markdown that will most likely replace
+          #   it, such as `https://example.com/t/17` or `@alice`; its first
+          #   and its last character decide the joins
+          # @return [Array]
+          # @raise [ArgumentError] when +like+ is no String or is empty
+          # @example
+          #   Tokens.placeholder("[link|17]", like: "https://example.com/t/17")
+          def self.placeholder(text, like:)
+            unless like.instance_of?(String) && !like.empty?
+              raise ArgumentError, "like: must be a non-empty String, got #{like.inspect}"
+            end
+
+            [PLACEHOLDER, text, like]
+          end
+
+          # Whether the tokens put something on the page: text or Markdown
+          # with something other than whitespace in it, code, a link, a
+          # bare URL, an image or a placeholder whose +like+ is visible.
+          # Delimiters and hard breaks alone do not.
+          #
+          # @param tokens [Array<Array>]
+          # @return [Boolean]
+          def self.visible?(tokens)
+            tokens.any? do |token|
+              next Joins.visible?(token) if token.instance_of?(String)
+
+              case token.first
+              when TEXT, HTML
+                Joins.visible?(token.fetch(1))
+              when PLACEHOLDER
+                Joins.visible?(token.fetch(2))
+              when CODE
+                !token.fetch(1).empty?
+              when LINK_OPEN, BARE_URL, IMAGE
+                true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/markbridge/renderers/discourse/link_destination.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Markbridge
+  module Renderers
+    module Discourse
+      # Writes a URL as the destination of a Markdown link or image.
+      module LinkDestination
+        SPECIAL = /[()\\]/
+        WHITESPACE = /\s/
+        ANGLE_BRACKETS = /[<>]/
+        private_constant :SPECIAL, :WHITESPACE, :ANGLE_BRACKETS
+
+        # An unbalanced parenthesis ends a CommonMark destination early, so
+        # every parenthesis gets a backslash; balanced ones would be fine,
+        # but the check is not worth it. A backslash is doubled, or it
+        # would escape the parenthesis after it, also the one that closes
+        # the destination. A destination with whitespace is wrapped in
+        # <>, relevant for relative targets like MediaWiki page names
+        # ("Main Page") or image files with a space in the name; a `<` or
+        # `>` inside the brackets gets a backslash, or it ends them.
+        #
+        # The match? in front of the gsub saves the copy that gsub makes
+        # also without a match: links are frequent, these characters are
+        # rare.
+        # @param destination [String, nil]
+        # @return [String, nil] nil for nil
+        def self.escape(destination)
+          return nil if destination.nil?
+
+          escaped =
+            if destination.match?(SPECIAL)
+              destination.gsub(SPECIAL) { |char| "\\#{char}" }
+            else
+              destination
+            end
+          return escaped unless escaped.match?(WHITESPACE)
+
+          "<#{escaped.gsub(ANGLE_BRACKETS) { |char| "\\#{char}" }}>"
+        end
+      end
+    end
+  end
+end

data/lib/markbridge/renderers/discourse/markdown_escaper.rb

@@ -17,6 +17,8 @@ module Markbridge
       # - Fast path returns original string for plain text (no allocations)
       # - Pre-allocated result buffers with estimated capacity
       # - Byte-level processing for inline escaping (YJIT-friendly tight loops)
+      # - A byte table decides which bytes need the inline dispatch, so an
+      #   ordinary byte is copied without walking the dispatch arms
       # - Simplified escaping rules: [ breaks links, so ] doesn't need escaping
       #
       # @example Basic escaping
@@ -67,9 +69,10 @@ module Markbridge
         # Block-level patterns
         ATX_HEADING = /\A\#{1,6}(?=[ \t]|$)/
         BLOCK_QUOTE = /\A>/
-        # List markers followed by space, tab, or end of line
+        # List markers followed by space, tab, or end of line. A marker
+        # alone on a line is an empty list item.
         BULLET_LIST = /\A[-+*](?=[ \t]|$)/
-        ORDERED_LIST = /\A(\d+)([.)])(?=[ \t])/
+        ORDERED_LIST = /\A(\d+)([.)])(?=[ \t]|$)/
         THEMATIC_BREAK_DASH = /\A(?:-[ \t]*){3,}$/
         THEMATIC_BREAK_STAR = /\A(?:\*[ \t]*){3,}$/
         THEMATIC_BREAK_UNDERSCORE = /\A(?:_[ \t]*){3,}$/
@@ -88,8 +91,12 @@ module Markbridge
         # HTML tag pattern (we escape these, but NOT autolinks)
         # Handles quoted attributes which can contain > characters
         # Attribute patterns: name="value" | name='value' | name=value | name
-        HTML_ATTR = /(?:\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?)/
-        HTML_TAG = %r{\A</?[a-zA-Z][a-zA-Z0-9-]*#{HTML_ATTR}*\s*/?>}
+        # Whitespace as markdown-it's JavaScript regexes read it (\s): the
+        # no-break space and the other Unicode spaces count too, so a tag
+        # name followed by a no-break space starts an HTML block or tag.
+        HTML_ATTR =
+          /(?:[[:space:]]+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:[[:space:]]*=[[:space:]]*(?:"[^"]*"|'[^']*'|[^[:space:]"'=<>`]+))?)/
+        HTML_TAG = %r{\A</?[a-zA-Z][a-zA-Z0-9-]*#{HTML_ATTR}*[[:space:]]*/?>}
 
         # Autolink pattern - we pass these through entirely unchanged
         # Matches <http://...>, <https://...>, <mailto:...>, and email addresses
@@ -102,7 +109,7 @@ module Markbridge
         # - Incomplete/multi-line HTML tags: <div followed by attributes on next line
         # - Custom elements: <my-component>, <responsive-image>
         # The (?:[\s/]|$) ensures we don't match comparisons like "a < b"
-        HTML_TAG_START = %r{\A<(?:[?!]|/?\s*[a-zA-Z][a-zA-Z0-9-]*(?:[\s/]|$))}
+        HTML_TAG_START = %r{\A<(?:[?!]|/?[[:space:]]*[a-zA-Z][a-zA-Z0-9-]*(?:[[:space:]/]|$))}
 
         # Byte constants for inline processing
         BACKSLASH = 92 # \
@@ -125,6 +132,11 @@ module Markbridge
         DIGIT_0 = 48
         DIGIT_9 = 57
 
+        # `DIGIT_0..DIGIT_9` written out in the `when` arm is not a
+        # literal range, so Ruby builds a new Range for every line that
+        # reaches it. The frozen constant is looked up instead.
+        DIGITS = (DIGIT_0..DIGIT_9).freeze
+
         # Escapes markdown special characters in the given text.
         #
         # Handles both block-level constructs (headings, lists, code blocks, HTML blocks)
@@ -225,8 +237,13 @@ module Markbridge
           has_indent = indent_len > 0
           content = has_indent ? line[indent_len..] : line
 
-          escaped, skip_inline = escape_block_level(content)
-          escaped = escape_inline(escaped) unless skip_inline
+          # escape_block_level hands the content back unchanged when the
+          # line starts no block construct, and the content then only
+          # needs inline escaping. Answering with the same object instead
+          # of a [line, skip_inline] pair keeps an Array allocation off
+          # every line.
+          escaped = escape_block_level(content)
+          escaped = escape_inline(content) if escaped.equal?(content)
 
           if has_indent
             result = String.new(encoding: line.encoding)
@@ -272,6 +289,9 @@ module Markbridge
           "#{nbsp_indent}#{escape_inline(content)}"
         end
 
+        # @return [String] the escaped line, or +content+ itself when the
+        #   line starts no block construct and the caller has to
+        #   inline-escape it.
         def escape_block_level(content)
           first_byte = content.getbyte(0)
 
@@ -295,7 +315,7 @@ module Markbridge
             return escape_block_star(content)
           when UNDERSCORE
             if THEMATIC_BREAK_UNDERSCORE.match?(content)
-              return escape_all_chars(content, UNDERSCORE, "\\_"), true
+              return escape_all_chars(content, UNDERSCORE, "\\_")
             end
           when EQUALS
             # A line of only `=` is a setext heading underline when a
@@ -309,46 +329,54 @@ module Markbridge
             # Discourse renders `\=` as a literal `=`, so the result
             # looks the same.
             if SETEXT_UNDERLINE_EQUALS.match?(content)
-              return escape_all_chars(content, EQUALS, "\\="), true
+              return escape_all_chars(content, EQUALS, "\\=")
             end
           when BACKTICK
-            if FENCED_CODE_BACKTICK.match?(content)
-              return escape_all_chars(content, BACKTICK, "\\`"), true
-            end
+            return escape_fence(content, BACKTICK, "\\`") if FENCED_CODE_BACKTICK.match?(content)
           when TILDE
-            return "\\#{content}", true if FENCED_CODE_TILDE.match?(content)
+            return escape_fence(content, TILDE, "\\~") if FENCED_CODE_TILDE.match?(content)
           when BRACKET_OPEN
             return escape_first_char_inline(content, "\\[")
           when PIPE
             return escape_first_char_inline(content, "\\|")
-          when DIGIT_0..DIGIT_9
+          when DIGITS
             return escape_block_ordered_list(content)
           end
 
-          [content, false]
+          content
+        end
+
+        # Every character of the fence run gets a backslash, so no part of
+        # the run can open a code span or strikethrough either, and the
+        # info string after the run is inline-escaped like any text.
+        def escape_fence(content, byte_val, escaped)
+          # The caller matched the fence, so the first byte is one.
+          length = 1
+          length += 1 while content.getbyte(length) == byte_val
+          "#{escaped * length}#{escape_inline(content[length..])}"
         end
 
         # Escape the first character and inline-escape the rest.
         def escape_first_char_inline(content, escaped_char)
-          ["#{escaped_char}#{escape_inline(content[1..])}", true]
+          "#{escaped_char}#{escape_inline(content[1..])}"
         end
 
         def escape_block_dash(content)
-          return escape_all_chars(content, DASH, "\\-"), true if THEMATIC_BREAK_DASH.match?(content)
+          return escape_all_chars(content, DASH, "\\-") if THEMATIC_BREAK_DASH.match?(content)
           if BULLET_LIST.match?(content)
             return pass_first_char_inline(content) if @allow.include?(:bullet_list)
             return escape_first_char_inline(content, "\\-")
           end
-          [content, false]
+          content
         end
 
         def escape_block_star(content)
-          return escape_all_chars(content, STAR, "\\*"), true if THEMATIC_BREAK_STAR.match?(content)
+          return escape_all_chars(content, STAR, "\\*") if THEMATIC_BREAK_STAR.match?(content)
           if BULLET_LIST.match?(content)
             return pass_first_char_inline(content) if @allow.include?(:bullet_list)
             return escape_first_char_inline(content, "\\*")
           end
-          [content, false]
+          content
         end
 
         def escape_block_ordered_list(content)
@@ -356,23 +384,23 @@ module Markbridge
             rest = content[match[0].length..]
             return pass_marker_inline(content, match[0].length) if @allow.include?(:ordered_list)
 
-            return "#{match[1]}\\#{match[2]}#{escape_inline(rest)}", true
+            return "#{match[1]}\\#{match[2]}#{escape_inline(rest)}"
           end
-          [content, false]
+          content
         end
 
         # Like {#escape_first_char_inline} but the leading character is
         # preserved verbatim (used when allow: lets a single-byte
         # marker like `-`, `+`, `*`, or `>` through).
         def pass_first_char_inline(content)
-          ["#{content[0]}#{escape_inline(content[1..])}", true]
+          "#{content[0]}#{escape_inline(content[1..])}"
         end
 
         # Preserve a multi-byte marker (e.g. `1.`, `99)`, `##`) and
         # inline-escape the rest. Used when allow: lets ordered lists
         # or ATX headings through.
         def pass_marker_inline(content, marker_length)
-          ["#{content[0, marker_length]}#{escape_inline(content[marker_length..])}", true]
+          "#{content[0, marker_length]}#{escape_inline(content[marker_length..])}"
         end
 
         def escape_all_chars(str, byte_val, escaped)
@@ -396,21 +424,56 @@ module Markbridge
           @inline_len = bytesize
           pos = 0
 
-          # No loop-progress guard: every `dispatch_inline_byte` branch
-          # returns `pos + N` for N >= 1 by construction, so the loop
-          # is provably terminating. Mutations that break this
+          # No loop-progress guard: the copy branch below advances by
+          # one and every `dispatch_inline_byte` branch returns
+          # `pos + N` for N >= 1 by construction, so the loop always
+          # ends. Mutations that break this
           # (`while true`, body drops, selector swaps that short-circuit
           # the dispatch) surface as timeouts rather than alive
           # mutations, and the inline guard would otherwise cost ~15%
           # on this hot path per benchmark.
           while pos < @inline_len
             byte = @inline_content.getbyte(pos)
-            pos = dispatch_inline_byte(byte, pos)
+            if DISPATCH_BYTE[byte]
+              pos = dispatch_inline_byte(byte, pos)
+            else
+              # An ASCII byte that starts no construct. Writing it here
+              # instead of walking the whole case in dispatch_inline_byte
+              # saves eleven comparisons and two calls per byte, and most
+              # bytes of ordinary text land here.
+              @inline_result << byte
+              pos += 1
+            end
           end
 
           @inline_result
         end
 
+        # Every byte dispatch_inline_byte has an arm for, plus the lead
+        # byte of a multi-byte character (128 and up), which its `else`
+        # copies through. A new arm needs its byte in here too, or the
+        # arm never runs.
+        DISPATCH_BYTE =
+          begin
+            table = Array.new(256, false)
+            [
+              BACKSLASH,
+              DASH,
+              TILDE,
+              STAR,
+              UNDERSCORE,
+              BACKTICK,
+              BANG,
+              BRACKET_OPEN,
+              PIPE,
+              LT,
+              AMP,
+              *(128..255),
+            ].each { |byte| table[byte] = true }
+            table.freeze
+          end
+        private_constant :DISPATCH_BYTE
+
         def dispatch_inline_byte(byte, pos)
           case byte
           when BACKSLASH
@@ -438,7 +501,7 @@ module Markbridge
           when AMP
             escape_amp(pos)
           else
-            escape_regular_char(byte, pos)
+            copy_multibyte_char(byte, pos)
           end
         end
 
@@ -505,11 +568,10 @@ module Markbridge
             matched = match[0]
             @inline_result << matched
             pos + matched.bytesize
-          elsif (match = HTML_TAG.match(remaining))
-            matched = match[0]
-            @inline_result << "\\" << matched.gsub("`") { "\\`" }
-            pos + matched.bytesize
-          elsif HTML_TAG_START.match?(remaining)
+          elsif HTML_TAG.match?(remaining) || HTML_TAG_START.match?(remaining)
+            # The backslash keeps the tag text. What is inside the tag
+            # goes on through the loop like any text: a backslash, a
+            # backtick or an entity in an attribute needs its escape too.
             @inline_result << "\\<"
             pos + 1
           else
@@ -536,17 +598,15 @@ module Markbridge
           @inline_content.byteslice(pos, @inline_len - pos)
         end
 
-        # Handle regular characters including multi-byte UTF-8.
-        def escape_regular_char(byte, pos)
-          if byte < 128
-            @inline_result << byte
-            pos + 1
-          else
-            char_len = utf8_char_length(byte)
-            end_pos = [pos + char_len, @inline_len].min
-            @inline_result << @inline_content.byteslice(pos, end_pos - pos)
-            end_pos
-          end
+        # Copy one multi-byte UTF-8 character through unchanged. The loop
+        # in escape_inline sends every byte of 128 and up here. A lead
+        # byte that promises more bytes than the content holds needs no
+        # clamp: byteslice copies what is left, and the returned
+        # position is past the end, which ends the loop.
+        def copy_multibyte_char(byte, pos)
+          char_len = utf8_char_length(byte)
+          @inline_result << @inline_content.byteslice(pos, char_len)
+          pos + char_len
         end
 
         def ascii_punctuation?(byte)

data/lib/markbridge/renderers/discourse/paragraphs.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Markbridge
+  module Renderers
+    module Discourse
+      # Inline markup like `**` or `[u]` can't span a blank line: the blank
+      # line ends the paragraph and the markers stay visible as text. Forums
+      # allow `[b]` around several paragraphs, so the markup has to be
+      # closed before every blank line and opened again after it.
+      #
+      # The serializer does that for emphasis, strikethrough and underline.
+      # A custom tag that writes its own markers into a String can use
+      # {.wrap} or {.map} for the same.
+      module Paragraphs
+        # One or more blank lines.
+        BREAK = /\n[[:blank:]]*\n/
+        # The same with all the blank lines of a run, captured so that
+        # split keeps them.
+        SPLIT = /(\n[[:blank:]]*\n(?:[[:blank:]]*\n)*)/
+        # Whitespace at the start and at the end, and what is between.
+        EDGES = /\A([[:space:]]*)(.*?)([[:space:]]*)\z/m
+        private_constant :BREAK, :SPLIT, :EDGES
+
+        # Whether +content+ has a blank line that markup would have to be
+        # closed and opened again around. A blank line inside a code fence
+        # belongs to the code, so content with a fence is one paragraph.
+        #
+        # @param content [String]
+        # @return [Boolean]
+        def self.many?(content)
+          content.match?(BREAK) && !content.include?("```") && !content.include?("~~~")
+        end
+
+        # Yields each paragraph of +content+ on its own and joins the
+        # results with the original blank lines. Content without a blank
+        # line is yielded as a whole. A blank line at the start or at the
+        # end is kept, and the empty String in front of a blank line at
+        # the start is yielded like a paragraph.
+        #
+        # @param content [String]
+        # @yieldparam paragraph [String] content without a blank line
+        # @yieldparam first [Boolean] whether it is the first paragraph
+        # @yieldparam last [Boolean] whether it is the last paragraph
+        # @yieldreturn [String] the wrapped paragraph
+        # @return [String]
+        # @example
+        #   Paragraphs.map("a\n\nb") { |paragraph| "**#{paragraph}**" }
+        #   # => "**a**\n\n**b**"
+        def self.map(content)
+          return yield(content, true, true) unless many?(content)
+
+          parts = content.split(SPLIT)
+          # Even indexes are paragraphs, odd ones the blank lines between.
+          last = parts.size - 1
+          last -= 1 if last.odd?
+          parts
+            .each_with_index
+            .map { |part, index| index.odd? ? part : yield(part, index.zero?, index == last) }
+            .join
+        end
+
+        # Puts +open+ and +close+ around every paragraph of +content+.
+        # Whitespace at the edges of a paragraph stays outside, and a
+        # paragraph with only whitespace stays as it is.
+        #
+        # @param content [String]
+        # @param open [String]
+        # @param close [String]
+        # @return [String]
+        # @example
+        #   Paragraphs.wrap("a \n\nb", "[color=red]", "[/color]")
+        #   # => "[color=red]a[/color] \n\n[color=red]b[/color]"
+        def self.wrap(content, open, close)
+          map(content) do |paragraph|
+            lead, core, trail = paragraph.match(EDGES).captures
+            next paragraph if core.empty?
+
+            # A BBCode tag at the start of a line with a `:` right after
+            # it reads as a link reference definition, `[color=red]:x`,
+            # and cooks to nothing; the backslash keeps the colon text.
+            core = "\\#{core}" if open.start_with?("[") && core.start_with?(":")
+            "#{lead}#{open}#{core}#{close}#{trail}"
+          end
+        end
+      end
+    end
+  end
+end