markbridge 0.2.1 → 0.3.1

data/lib/markbridge/normalizer.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require_relative "normalizer/rule_set"
+require_relative "normalizer/report"
+require_relative "normalizer/text_projection"
+require_relative "normalizer/walker"
+
+module Markbridge
+  # Rewrites an AST so the renderer only gets markup the target format can
+  # express. It runs once, between parse and render. The default rules are
+  # CommonMark legality: no link inside a link, no block element inside an
+  # inline container, and an inline-only code span. Each match resolves to a
+  # strategy (+:keep+, +:hoist_after+, +:unwrap+, +:textify+, +:drop+, or a
+  # callable) that the {Walker} applies.
+  #
+  # Discourse-specific policy (for example, moving an image out of a link) is
+  # not built in. A consumer adds those with {#rule}.
+  #
+  # @example The default, reused across conversions
+  #   Markbridge::Normalizer.shared_default
+  #
+  # @example A customized normalizer
+  #   n = Markbridge::Normalizer.default
+  #   n.rule(parent: AST::Url, child: AST::Image, strategy: :hoist_after)
+  #   Markbridge.convert(input, format: :bbcode, normalize: n)
+  #
+  # @example List what would change, without changing it
+  #   Markbridge::Normalizer.default.violations(ast) # => [...]
+  class Normalizer
+    # The strategy symbols a rule may resolve to.
+    STRATEGIES = %i[keep hoist_after unwrap textify drop].freeze
+
+    EMPTY_STACK = [].freeze
+    private_constant :EMPTY_STACK
+
+    # Containers that hold inline content only: a link's text (CommonMark
+    # §6.3), and emphasis and heading content.
+    INLINE_CONTAINERS = [
+      AST::Url,
+      AST::Bold,
+      AST::Italic,
+      AST::Strikethrough,
+      AST::Underline,
+      AST::Superscript,
+      AST::Subscript,
+      AST::Heading,
+    ].freeze
+
+    # AST nodes the Discourse renderer prints as block-level Markdown (their
+    # output has blank lines around it). One inside an inline container breaks
+    # that container, so it is moved out. Spoiler and single-line Code stay
+    # inline and are not listed (Code is handled by {KEEP_INLINE_CODE}).
+    BLOCK_NODES = [
+      AST::Quote,
+      AST::Heading,
+      AST::List,
+      AST::ListItem,
+      AST::Table,
+      AST::TableRow,
+      AST::TableCell,
+      AST::Details,
+      AST::Paragraph,
+      AST::HorizontalRule,
+      AST::Align,
+      AST::Poll,
+      AST::Event,
+    ].freeze
+
+    # A code span may stay inside an inline container while it is on one line.
+    # A fenced or multi-line block is moved out. This matches
+    # +RenderingInterface#block_context?+: Code prints as a fenced block when a
+    # Text child has a newline (the language alone does not make it a block).
+    KEEP_INLINE_CODE =
+      lambda do |_boundary, node|
+        block = node.children.any? { |c| c.instance_of?(AST::Text) && c.text.include?("\n") }
+        block ? :hoist_after : :keep
+      end
+
+    class << self
+      # A fresh, customizable normalizer with the default rules. Add more with
+      # {#rule}.
+      # @return [Normalizer]
+      def default
+        new(build_rules)
+      end
+
+      # The default normalizer, built once and frozen, reused across
+      # conversions. +#normalize+ and +#violations+ keep no state on the
+      # instance, so one frozen instance is safe to reuse, also across threads.
+      # @return [Normalizer] the same frozen instance on every call
+      def shared_default
+        @shared_default ||= default.freeze
+      end
+
+      private
+
+      def build_rules
+        rules = RuleSet.new
+
+        # §6.3 A link may not contain another link, at any depth. Unwrap the
+        # inner link and keep its text.
+        rules.add(parent: AST::Url, child: AST::Url, strategy: :unwrap)
+
+        INLINE_CONTAINERS.each do |container|
+          rules.add(parent: container, child: AST::Code, strategy: KEEP_INLINE_CODE)
+
+          BLOCK_NODES.each do |block|
+            next if container == block
+
+            rules.add(parent: container, child: block, strategy: :hoist_after)
+          end
+        end
+
+        rules
+      end
+    end
+
+    # @param rule_set [RuleSet]
+    def initialize(rule_set)
+      @rules = rule_set
+    end
+
+    # Add or override a rule. Chainable. A rule for a +(parent, child)+ pair
+    # that already exists is replaced. Raises on a frozen ({shared_default})
+    # instance; build a fresh one with {.default}.
+    #
+    # @param parent [Class] ancestor AST class
+    # @param child [Class] contained AST class
+    # @param strategy [Symbol, #call] one of {STRATEGIES} or a callable
+    # @return [self]
+    def rule(parent:, child:, strategy:)
+      @rules.add(parent:, child:, strategy:)
+      self
+    end
+
+    # Rewrite +ast+ in place so it satisfies the rules.
+    #
+    # @param ast [AST::Document, AST::Element]
+    # @return [Array<Hash>] a report of what changed (empty when nothing did),
+    #   one +{parent:, child:, strategy:, count:}+ row per distinct change.
+    def normalize(ast)
+      report = Report.new
+      Walker.new(@rules, report).call(ast)
+      report.to_a
+    end
+
+    # List the violations in +ast+ without changing it.
+    #
+    # @param ast [AST::Document, AST::Element]
+    # @return [Array<Hash>] +{parent:, child:, strategy:}+ per occurrence
+    def violations(ast)
+      found = []
+      collect_violations(ast, EMPTY_STACK, found)
+      found
+    end
+
+    def freeze
+      @rules.freeze
+      super
+    end
+
+    private
+
+    def collect_violations(element, ancestors, found)
+      stack = ancestors + [element]
+      element.children.each do |child|
+        strategy, boundary = @rules.resolve(child, stack)
+        strategy = strategy.call(boundary, child) if strategy.respond_to?(:call)
+        unless strategy.nil? || strategy == :keep
+          found << { parent: demodulize(boundary.class), child: demodulize(child.class), strategy: }
+        end
+        collect_violations(child, stack, found) if child.is_a?(AST::Element)
+      end
+    end
+
+    def demodulize(klass)
+      klass.name.split("::").last
+    end
+  end
+end

data/lib/markbridge/parsers/bbcode/handler_registry.rb

@@ -62,7 +62,11 @@ module Markbridge
         # @param tag_name [String]
         # @return [BaseHandler, nil]
         def [](tag_name)
-          @handlers[tag_name.to_s.downcase]
+          # Keys are always downcased strings and the scanner emits
+          # downcased tags, so the fetch hits directly on the hot path;
+          # the block normalizes only unusual lookups (mixed case,
+          # symbols) instead of allocating a downcased copy per token.
+          @handlers.fetch(tag_name) { @handlers[tag_name.to_s.downcase] }
         end
 
         # Get handler for an element instance
@@ -176,6 +180,28 @@ module Markbridge
           registry
         end
 
+        # Shared, deep-frozen default registry for the no-customization
+        # fast path. Built once per process; {Parser} falls back to it
+        # when no +handlers:+ are given, skipping the full registry
+        # construction on every parse. Handlers and closing strategies
+        # are stateless after construction, so sharing is safe across
+        # parsers and threads.
+        #
+        # @return [HandlerRegistry] the same frozen instance on every call
+        def self.shared_default
+          @shared_default ||= default.freeze
+        end
+
+        # Freeze the registry together with its internal collections so
+        # that registration on a shared instance fails loudly instead of
+        # silently mutating state visible to every parser.
+        def freeze
+          @handlers.freeze
+          @element_handlers.freeze
+          @auto_closeable_elements.freeze
+          super
+        end
+
         # Build a registry from the default configuration with optional customization
         # @yield [HandlerRegistry] the registry to customize
         # @return [HandlerRegistry]

data/lib/markbridge/parsers/bbcode/handlers/quote_handler.rb

@@ -10,7 +10,14 @@ module Markbridge
         # - [quote=author]text[/quote]
         # - [quote="author"]text[/quote]
         # - [quote author=username]text[/quote]
-        # - [quote="username, post:123, topic:456"]text[/quote] (Discourse format)
+        # - [quote="username, post:2, topic:456"]text[/quote] (Discourse format)
+        #
+        # Attribution semantics are Discourse's: `post:` is the post's
+        # number within its topic and `topic:` is the topic id. Beware
+        # when feeding other dialects through this handler — XenForo
+        # attributions ("name, post: 12345, member: 678") also match the
+        # `post:` pattern, but there the value is a database post id, not
+        # a post number.
         class QuoteHandler < BaseHandler
           def initialize
             @element_class = AST::Quote
@@ -27,13 +34,13 @@ module Markbridge
           private
 
           def extract_quote_attrs(token)
-            author, post, topic, username = extract_from_option(token)
+            author, post_number, topic_id, username = extract_from_option(token)
             author ||= token.attrs[:author]
 
             {
               author:,
-              post: token.attrs[:post] || post,
-              topic: token.attrs[:topic] || topic,
+              post_number: integer_or_nil(token.attrs[:post]) || post_number,
+              topic_id: integer_or_nil(token.attrs[:topic]) || topic_id,
               username: token.attrs[:username] || username,
             }
           end
@@ -42,15 +49,31 @@ module Markbridge
             option = token.attrs[:option]
             return nil, nil, nil, nil unless option
 
-            post = option[/,\s*post:(\d+)/, 1]
-            return option, nil, nil, nil unless post
+            post_number = option[/,\s*post:(\d+)/, 1]
+            return option, nil, nil, nil unless post_number
 
-            # Discourse format: "username, post:123, topic:456" (topic optional,
+            # Discourse format: "username, post:2, topic:456" (topic optional,
             # order irrelevant between post: and topic:).
             username = option.split(",").first.strip
-            topic = option[/,\s*topic:(\d+)/, 1]
+            topic_id = option[/,\s*topic:(\d+)/, 1]
 
-            [username, post, topic, username]
+            # The regex captures are guaranteed digit runs, so strict
+            # Integer() applies. Base 10 is explicit: without it a
+            # leading zero switches Integer() to octal and "099" raises.
+            [username, Integer(post_number, 10), topic_id && Integer(topic_id, 10), username]
+          end
+
+          # Coerce an attribute value to Integer; nil and non-numeric
+          # values (which would make a useless attribution anyway)
+          # become nil. Base 10 is explicit so leading zeros ("099")
+          # don't trip Integer()'s octal mode and prefix forms ("0x1A")
+          # don't smuggle in surprise values. The result is an unbounded
+          # Ruby Integer — a runaway digit run like
+          # "post:77777777777777777789999" parses to a bignum, so
+          # consumers binding these into fixed-width storage (int64
+          # columns) need their own bounds check.
+          def integer_or_nil(value)
+            Integer(value, 10, exception: false)
           end
         end
       end

data/lib/markbridge/parsers/bbcode/handlers/raw_handler.rb

@@ -11,6 +11,13 @@ module Markbridge
           def initialize(element_class, collector: RawContentCollector.new)
             @element_class = element_class
             @collector = collector
+            # Computed once here: handlers are long-lived, elements are
+            # per-tag, and the reflection is too costly to repeat per element.
+            @accepts_language =
+              element_class
+                .instance_method(:initialize)
+                .parameters
+                .any? { |_kind, name| name == :language }
           end
 
           def on_open(token:, context:, registry:, tokens:)
@@ -43,10 +50,7 @@ module Markbridge
           end
 
           def accepts_language?
-            @element_class
-              .instance_method(:initialize)
-              .parameters
-              .any? { |_kind, name| name == :language }
+            @accepts_language
           end
         end
       end

data/lib/markbridge/parsers/bbcode/parser.rb

@@ -26,7 +26,7 @@ module Markbridge
             if block_given?
               HandlerRegistry.build_from_default(&block)
             else
-              handlers || HandlerRegistry.default
+              handlers || HandlerRegistry.shared_default
             end
           @unknown_tags = Hash.new(0)
         end
@@ -53,11 +53,14 @@ module Markbridge
 
         private
 
+        LINE_ENDING_RE = /\r\n?|[\u2028\u2029]+/
+        private_constant :LINE_ENDING_RE
+
         # Normalize line endings (CR, CRLF, and Unicode separators)
         # @param input [String]
-        # @return [String]
+        # @return [String] the input itself when already normalized (LF-only)
         def normalize_line_endings(input)
-          input.gsub(/\r\n?|[\u2028\u2029]+/, "\n")
+          input.match?(LINE_ENDING_RE) ? input.gsub(LINE_ENDING_RE, "\n") : input
         end
 
         # Parse tokens using scanner

data/lib/markbridge/parsers/bbcode/scanner.rb

@@ -3,26 +3,39 @@
 module Markbridge
   module Parsers
     module BBCode
-      # High-performance character-by-character BBCode scanner
+      # High-performance BBCode scanner
       # Tokenizes BBCode in O(n) time with minimal allocations and bounded backtracking
+      #
+      # The scanner works entirely in *byte* offsets (byteindex/byteslice/
+      # getbyte). CRuby has no character-index cache, so character-index
+      # operations like `str[pos]` walk the string from the start on any
+      # input containing a multibyte character — O(pos) per call and
+      # superlinear per document. Byte offsets keep multibyte input at
+      # near-ASCII cost. Invariant: `@current_pos` always sits on a
+      # character boundary — jumps land on matches of ASCII-only patterns
+      # and advances step over ASCII bytes or whole matches. Token
+      # positions are therefore byte offsets into the input.
       class Scanner
         def initialize(input)
           @input = input
-          @length = input.length
+          @length = input.bytesize
           @current_pos = 0
         end
 
         def next_token
           return nil if end_of_input?
           start_pos = @current_pos
-          bracket_index = @input.index("[", @current_pos)
+          # `byteindex` returns 0 for a match at the start — nil-check, never
+          # truthiness-check. Can't be 0 here though: callers guarantee
+          # @current_pos <= bracket_index.
+          bracket_index = @input.byteindex("[", @current_pos)
 
           if bracket_index.nil?
-            text = @input[@current_pos..]
+            text = @input.byteslice(@current_pos, @length - @current_pos)
             @current_pos = @length
             TextToken.new(text:, pos: start_pos)
           elsif bracket_index > @current_pos
-            text = @input[@current_pos...bracket_index]
+            text = @input.byteslice(@current_pos, bracket_index - @current_pos)
             @current_pos = bracket_index
             TextToken.new(text:, pos: start_pos)
           elsif (tag_token = parse_tag_at_cursor)
@@ -35,31 +48,65 @@ module Markbridge
 
         private
 
-        TAG_INITIAL_CHAR = /[a-z*]/i
-        TAG_NAME_CHAR = /[a-z0-9]/i
-        UID_HEX_CHAR = /[0-9a-f]/i
-        ATTR_NAME_CHAR = /\w/
-        WHITESPACE_CHAR = /\s/
+        # Byte constants for tag scanning. A byte >= 0x80 (multibyte
+        # lead/continuation) never satisfies any of the ASCII predicates
+        # below, so no encoding-awareness is needed at probe sites.
+        TAB = 9 # \t
+        CR = 13 # \r  (\s == [ \t\n\v\f\r] == 0x20, 0x09..0x0D)
+        SPACE = 32
+        DOUBLE_QUOTE = 34 # "
+        SINGLE_QUOTE = 39 # '
+        STAR = 42 # *
+        SLASH = 47 # /
+        DIGIT_0 = 48
+        DIGIT_9 = 57
+        COLON = 58 # :
+        EQUALS = 61 # =
+        UPPER_A = 65
+        UPPER_F = 70
+        UPPER_Z = 90
+        BRACKET_CLOSE = 93 # ]
+        UNDERSCORE = 95 # _
+        LOWER_A = 97
+        LOWER_F = 102
+        LOWER_Z = 122
+
+        # Characters an unquoted attribute value stops at. ASCII-only, so a
+        # byteindex match always lands on a character boundary.
         UNQUOTED_VALUE_STOP = /[\[\]\s]/
 
-        private_constant :TAG_INITIAL_CHAR,
-                         :TAG_NAME_CHAR,
-                         :UID_HEX_CHAR,
-                         :ATTR_NAME_CHAR,
-                         :WHITESPACE_CHAR,
+        private_constant :TAB,
+                         :CR,
+                         :SPACE,
+                         :DOUBLE_QUOTE,
+                         :SINGLE_QUOTE,
+                         :STAR,
+                         :SLASH,
+                         :DIGIT_0,
+                         :DIGIT_9,
+                         :COLON,
+                         :EQUALS,
+                         :UPPER_A,
+                         :UPPER_F,
+                         :UPPER_Z,
+                         :BRACKET_CLOSE,
+                         :UNDERSCORE,
+                         :LOWER_A,
+                         :LOWER_F,
+                         :LOWER_Z,
                          :UNQUOTED_VALUE_STOP
 
         # @return [Token, nil] tag token or nil if not a valid tag (caller rolls back)
-        # Precondition: caller has verified current_char == "[".
+        # Precondition: caller has verified the byte at the cursor is "[".
         def parse_tag_at_cursor
           tag_start_pos = @current_pos
           @current_pos += 1 # skip '['
-          closing = consume("/")
+          closing = consume(SLASH)
           tag_name = scan_tag_name
           attrs = (closing || tag_name.nil?) ? {} : scan_attributes
-          return rollback(tag_start_pos) unless tag_name && consume("]")
+          return rollback(tag_start_pos) unless tag_name && consume(BRACKET_CLOSE)
 
-          source = @input[tag_start_pos...@current_pos]
+          source = @input.byteslice(tag_start_pos, @current_pos - tag_start_pos)
           build_token(closing:, tag: tag_name.downcase, attrs:, pos: tag_start_pos, source:)
         end
 
@@ -76,27 +123,26 @@ module Markbridge
           nil
         end
 
-        # Scan a tag name: [a-z*][a-z0-9]*(:hex*)?
+        # Scan a tag name: [a-z*][a-z0-9]*(:hex*)?  (case-insensitive)
         #
-        # Char-by-char rather than a single regex over `@input[pos..]`
+        # Byte-predicate loop rather than a regex over `@input[pos..]`
         # because the regex form allocates a substring for every tag,
-        # which is a dominant cost on tag-heavy input. The char-based
-        # loop is ~3x faster under YJIT.
+        # which is a dominant cost on tag-heavy input.
         # @return [String, nil]
         def scan_tag_name
           start = @current_pos
 
-          return nil unless current_char&.match?(TAG_INITIAL_CHAR)
+          return nil unless tag_initial_byte?(current_byte)
           @current_pos += 1
 
-          @current_pos += 1 while current_char&.match?(TAG_NAME_CHAR)
+          @current_pos += 1 while tag_name_byte?(current_byte)
 
-          if current_char == ":"
+          if current_byte == COLON
             @current_pos += 1
-            @current_pos += 1 while current_char&.match?(UID_HEX_CHAR)
+            @current_pos += 1 while uid_hex_byte?(current_byte)
           end
 
-          @input[start...@current_pos]
+          @input.byteslice(start, @current_pos - start)
         end
 
         # Scan tag attributes
@@ -107,7 +153,7 @@ module Markbridge
           attrs = {}
           skip_whitespace
 
-          if current_char == "="
+          if current_byte == EQUALS
             @current_pos += 1
             skip_whitespace
             if (val = scan_attribute_value)
@@ -116,9 +162,9 @@ module Markbridge
             skip_whitespace
           end
 
-          while (name = scan_while(ATTR_NAME_CHAR))
+          while (name = scan_attr_name)
             skip_whitespace
-            break unless consume("=")
+            break unless consume(EQUALS)
 
             skip_whitespace
             value = scan_attribute_value
@@ -129,16 +175,16 @@ module Markbridge
           attrs
         end
 
-        def consume(char)
-          return false if current_char != char
+        def consume(byte)
+          return false if current_byte != byte
 
           @current_pos += 1
           true
         end
 
         def scan_attribute_value
-          char = current_char
-          if char == '"' || char == "'"
+          byte = current_byte
+          if byte == DOUBLE_QUOTE || byte == SINGLE_QUOTE
             scan_quoted_string
           else
             scan_unquoted_value
@@ -161,52 +207,86 @@ module Markbridge
         #
         # @return [String, nil] the unescaped attribute value, or nil if unterminated
         def scan_quoted_string
-          quote_char = current_char
+          quote = current_byte == DOUBLE_QUOTE ? "\"" : "'"
           start = (@current_pos += 1) # skip opening quote
-          closing_index = @input.index(quote_char, start)
+          closing_index = @input.byteindex(quote, start)
           return nil unless closing_index
 
           @current_pos = closing_index + 1
-          @input[start...closing_index]
+          @input.byteslice(start, closing_index - start)
         end
 
         def scan_unquoted_value
-          scan_until(UNQUOTED_VALUE_STOP)
+          consume_range(@input.byteindex(UNQUOTED_VALUE_STOP, @current_pos) || @length)
         end
 
-        # Consumes characters matching +pattern+; returns substring or nil if empty
-        def scan_while(pattern)
+        # Consumes attribute-name characters (\w == [A-Za-z0-9_]); returns
+        # substring or nil if empty
+        def scan_attr_name
           stop_index = @current_pos
-          stop_index += 1 while stop_index < @length && @input[stop_index].match?(pattern)
+          stop_index += 1 while stop_index < @length && attr_name_byte?(@input.getbyte(stop_index))
           consume_range(stop_index)
         end
 
-        # Consumes characters until +pattern+ matches (or end of input); returns substring or nil if empty
-        def scan_until(pattern)
-          consume_range(@input.index(pattern, @current_pos) || @length)
-        end
-
         # Slice [@current_pos, stop_index), advance the cursor, or return nil for empty.
         def consume_range(stop_index)
           return nil if stop_index == @current_pos
 
-          value = @input[@current_pos...stop_index]
+          value = @input.byteslice(@current_pos, stop_index - @current_pos)
           @current_pos = stop_index
           value
         end
 
-        def current_char
-          @input[@current_pos]
+        # @return [Integer, nil] byte at the cursor, nil at end of input
+        def current_byte
+          @input.getbyte(@current_pos)
         end
 
         def skip_whitespace
           @current_pos += 1 while @current_pos < @length &&
-            @input[@current_pos].match?(WHITESPACE_CHAR)
+            whitespace_byte?(@input.getbyte(@current_pos))
+        end
+
+        # [a-z*]/i — first character of a tag name; nil (end of input) is
+        # not a tag byte
+        def tag_initial_byte?(byte)
+          byte &&
+            (
+              (byte >= LOWER_A && byte <= LOWER_Z) || (byte >= UPPER_A && byte <= UPPER_Z) ||
+                byte == STAR
+            )
+        end
+
+        # [a-z0-9]/i — rest of a tag name
+        def tag_name_byte?(byte)
+          return false if byte.nil?
+
+          (byte >= LOWER_A && byte <= LOWER_Z) || (byte >= UPPER_A && byte <= UPPER_Z) ||
+            (byte >= DIGIT_0 && byte <= DIGIT_9)
+        end
+
+        # [0-9a-f]/i — uid suffix after ':'
+        def uid_hex_byte?(byte)
+          return false if byte.nil?
+
+          (byte >= DIGIT_0 && byte <= DIGIT_9) || (byte >= LOWER_A && byte <= LOWER_F) ||
+            (byte >= UPPER_A && byte <= UPPER_F)
+        end
+
+        # \w — attribute name character
+        def attr_name_byte?(byte)
+          (byte >= LOWER_A && byte <= LOWER_Z) || (byte >= UPPER_A && byte <= UPPER_Z) ||
+            (byte >= DIGIT_0 && byte <= DIGIT_9) || byte == UNDERSCORE
+        end
+
+        # \s — exactly [ \t\n\v\f\r]
+        def whitespace_byte?(byte)
+          byte == SPACE || (byte >= TAB && byte <= CR)
         end
 
         def end_of_input?
-          # All callers maintain @current_pos <= @length (scan_while
-          # bounds on @length; scan_until uses `index || @length`;
+          # All callers maintain @current_pos <= @length (scan_attr_name
+          # bounds on @length; scan_unquoted_value uses `byteindex || @length`;
           # consume is a no-op at EOF); `==` and `>=` are observably
           # identical here.
           @current_pos == @length