regexp_parser 2.12.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa5734a20a0705226c021d9a0efef48a6ca24b5b18f15f93b34c12ffe5021d1e
-  data.tar.gz: fa77ac98b3bd17d6bdba43d387ebbbd84a6a0a5fa86123c7eee68646e90ce37c
+  metadata.gz: 3064d283e8ab829bdaffd9312b9e975ff3c47ace80c2add1dd9336c4cefb955b
+  data.tar.gz: 78c30fac8b319e64ba6cadf096b03e88f65fd87171f706bb4ad6f1d2abbcd9c5
 SHA512:
-  metadata.gz: 4bb62063aaa64e3828c5abf551fbf5ce83f726b5ebeb56af3e9210fd7e8c110c7f8f9ba1fcafd0dd0f66b8e8f36f04c1d72e5a739c4ace8fa19978dfc1a794b1
-  data.tar.gz: a82384912534ec3ca98d7665434c655468b15e4d75d6c1f13405f0b777d818746c6686d36049a158f0df9b829c129026487e5c3e6cf8a8f7c7ca65155a937db3
+  metadata.gz: d3d1a370ef4671e0dfb50e30612af9364e26f82b0facdc3e526dcf403c06e9fefa52e31f68f7d1949ca28d213603a46af2385ff8831822494b29cc9d945b2f90
+  data.tar.gz: b81338ab447b5f03156770c6ba7f165dcc18f4d0ff0c9c3e00a64baedae62da927d508e1fb68c7893eb19738bf6db9edec204bc248b89a92af0f81a60d31df16

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2012-2025,  Ammar Ali
+Copyright (c) 2010-2026,  Ammar Ali
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

data/lib/regexp_parser/expression/base.rb

@@ -26,24 +26,17 @@ module Regexp::Expression
 
     # Deprecated. Prefer `#repetitions` which has a more uniform interface.
     def quantity
-      return [nil,nil] unless quantified?
+      return [nil, nil] unless quantified?
+
       [quantifier.min, quantifier.max]
     end
 
     def repetitions
-      @repetitions ||=
-        if quantified?
-          min = quantifier.min
-          max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
-          range = min..max
-          # fix Range#minmax on old Rubies - https://bugs.ruby-lang.org/issues/15807
-          if RUBY_VERSION.to_f < 2.7
-            range.define_singleton_method(:minmax) { [min, max] }
-          end
-          range
-        else
-          1..1
-        end
+      return 1..1 unless quantified?
+
+      min = quantifier.min
+      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
+      min..max
     end
 
     def greedy?
@@ -74,5 +67,48 @@ module Regexp::Expression
       }
     end
     alias :attributes :to_h
+
+    # Recalculates the nesting_level for this node and children, which reflects
+    # how deep a node is in the tree. Do this at the end of parsing to account
+    # for tree rewrites.
+    # This must be safe against overflows and performant - check the benchmarks.
+    def recursively_update_nesting_levels(base = nesting_level)
+      queue = [base, self]
+
+      until queue.empty?
+        exp = queue.pop
+        exp_lvl = queue.pop
+        exp.nesting_level = exp_lvl
+        exp.quantifier.nesting_level = exp_lvl if exp.quantifier
+        exp.terminal? || exp.expressions.each do |subexp|
+          queue.push(exp_lvl + 1, subexp)
+        end
+      end
+    end
+
+    # Assigns referenced expressions to referring expressions, e.g. if there is
+    # an instance of Backreference::Number, its #referenced_expression is set to
+    # the instance of Group::Capture that it refers to via its number.
+    def assign_referenced_expressions
+      # find all referenceable and referring expressions
+      targets = {}
+      targets[0] = [self] if instance_of?(Root)
+      referrers = []
+      each_expression do |exp|
+        if exp.referential?
+          referrers << exp
+        elsif exp.is_a?(Group::Capture)
+          (targets[exp.identifier] ||= []) << exp
+        end
+      end
+      # assign referenced expressions to referring expressions
+      # (in a second iteration because there might be forward references)
+      referrers.each do |exp|
+        exp.referenced_expressions = targets[exp.reference] || raise(
+          Regexp::Parser::ParserError,
+          "Invalid reference #{exp.reference} at pos #{exp.ts}"
+        )
+      end
+    end
   end
 end

data/lib/regexp_parser/expression/classes/character_set/range.rb

@@ -14,7 +14,7 @@ module Regexp::Expression
       end
 
       def complete?
-        count == 2
+        length == 2
       end
     end
   end

data/lib/regexp_parser/expression/classes/character_set.rb

@@ -22,4 +22,4 @@ module Regexp::Expression
 
   # alias for symmetry between token symbol and Expression class name
   Set = CharacterSet
-end # module Regexp::Expression
+end

data/lib/regexp_parser/expression/classes/unicode_property.rb

@@ -118,4 +118,4 @@ module Regexp::Expression
   # alias for symmetry between token symbol and Expression class name
   Property    = UnicodeProperty
   Nonproperty = UnicodeProperty
-end # module Regexp::Expression
+end

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -112,9 +112,9 @@ module Regexp::Expression
   class Subexpression
     def match_length
       MatchLength.new(self,
-                       base_min: map { |exp| exp.match_length.min }.inject(0, :+),
-                       base_max: map { |exp| exp.match_length.max }.inject(0, :+),
-                       reify: ->{ map { |exp| exp.match_length.to_re }.join })
+                      base_min: map { |exp| exp.match_length.min }.inject(0, :+),
+                      base_max: map { |exp| exp.match_length.max }.inject(0, :+),
+                      reify: ->{ map { |exp| exp.match_length.to_re }.join })
     end
 
     def inner_match_length
@@ -132,9 +132,9 @@ module Regexp::Expression
     klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
       def match_length
         MatchLength.new(self,
-                         base_min: map { |exp| exp.match_length.min }.min,
-                         base_max: map { |exp| exp.match_length.max }.max,
-                         reify: ->{ map { |exp| exp.match_length.to_re }.join('|') })
+                        base_min: map { |exp| exp.match_length.min }.min,
+                        base_max: map { |exp| exp.match_length.max }.max,
+                        reify: ->{ map { |exp| exp.match_length.to_re }.join('|') })
       end
     RUBY
   end
@@ -158,7 +158,15 @@ module Regexp::Expression
       if referenced_expression.nil?
         raise ArgumentError, 'Missing referenced_expression - not parsed?'
       end
-      referenced_expression.unquantified_clone.match_length
+      # The targets' own quantifiers do not apply: a backref/subexp match what
+      # the target captures in one repetition. The backref/call's own quantifier
+      # does apply, and they can be multiplexed, i.e. refer to several targets.
+      targets = referenced_expressions
+                .map { |exp| exp.unquantified_clone.match_length }
+      MatchLength.new(self,
+                      base_min: targets.map(&:min).min,
+                      base_max: targets.map(&:max).max,
+                      reify: ->{ targets.map(&:to_re).join('|') })
     end
   end
 

data/lib/regexp_parser/expression/methods/tests.rb

@@ -101,14 +101,30 @@ module Regexp::Expression
     # When changing the conditions, please make sure to update
     # #pretty_print_instance_variables so that it includes all relevant values.
     def ==(other)
+      return shallow_equal?(other) if terminal?
+
+      stack = [[self, other]]
+      until stack.empty?
+        exp1, exp2 = stack.pop
+        return false unless exp1.shallow_equal?(exp2)
+
+        next if exp1.terminal?
+        return false unless exp1.length == exp2.length
+
+        stack.concat(exp1.zip(exp2))
+      end
+      true
+    end
+    alias :=== :==
+    alias :eql? :==
+
+    # Compares two expressions without considering their subexpressions.
+    def shallow_equal?(other)
       self.class   == other.class &&
         text       == other.text &&
         quantifier == other.quantifier &&
-        options    == other.options &&
-        (terminal? || expressions == other.expressions)
+        options    == other.options
     end
-    alias :=== :==
-    alias :eql? :==
 
     def optional?
       quantified? && quantifier.min == 0

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
-
+#
+# This file implements several tree traversal methods.
+#
+# Note: The methods are optimized for performance and avoid stack overflows on
+#       large trees. Check tasks/benchmark/traversal.rb when working in here.
 module Regexp::Expression
   class Subexpression < Regexp::Expression::Base
 
@@ -36,14 +40,30 @@ module Regexp::Expression
 
       block.call(:enter, self, 0) if include_self
 
-      each_with_index do |exp, index|
-        if exp.terminal?
-          block.call(:visit, exp, index)
-        else
-          block.call(:enter, exp, index)
-          exp.traverse(&block)
-          block.call(:exit, exp, index)
+      stack = []
+      parent, children, index = self, expressions, 0
+
+      loop do
+        while (exp = children[index])
+          if exp.terminal?
+            block.call(:visit, exp, index)
+            index += 1
+          else
+            block.call(:enter, exp, index)
+            # Resume at this child to emit its exit event after descending.
+            stack.push(parent, index)
+            parent, children, index = exp, exp.expressions, 0
+          end
         end
+
+        break if stack.empty?
+
+        exp = parent
+        index = stack.pop
+        parent = stack.pop
+        children = parent.expressions
+        block.call(:exit, exp, index)
+        index += 1
       end
 
       block.call(:exit, self, 0) if include_self
@@ -66,16 +86,34 @@ module Regexp::Expression
     protected
 
     def each_expression_with_index(&block)
-      each_with_index do |exp, index|
-        block.call(exp, index)
-        exp.each_expression_with_index(&block) unless exp.terminal?
+      stack = []
+      children, index = expressions, 0
+
+      loop do
+        while (exp = children[index])
+          block.call(exp, index)
+          index += 1
+          unless exp.terminal?
+            # Resume at the next sibling after visiting this child's subtree.
+            stack.push(children, index)
+            children, index = exp.expressions, 0
+          end
+        end
+
+        break if stack.empty?
+
+        index = stack.pop
+        children = stack.pop
       end
     end
 
     def each_expression_without_index(&block)
-      each do |exp|
+      queue = expressions.reverse
+
+      until queue.empty?
+        exp = queue.pop
         block.call(exp)
-        exp.each_expression_without_index(&block) unless exp.terminal?
+        queue.concat(exp.expressions.reverse) unless exp.terminal?
       end
     end
   end

data/lib/regexp_parser/expression/quantifier.rb

@@ -11,7 +11,7 @@ module Regexp::Expression
     MODES = %i[greedy possessive reluctant].freeze
 
     def initialize(*args)
-      deprecated_old_init(*args) and return if args.count == 4 || args.count == 5
+      deprecated_old_init(*args) and return if args.length == 4 || args.length == 5
 
       init_from_token_and_options(*args)
       # TODO: remove in v3.0.0, stop removing parts of #token (?)

data/lib/regexp_parser/expression/shared.rb

@@ -9,11 +9,9 @@ module Regexp::Expression
         extend Shared::ClassMethods
 
         attr_accessor :type, :token, :text, :ts, :te,
-                      :level, :set_level, :conditional_level,
-                      :options, :parent,
+                      :level, :set_level, :conditional_level, :nesting_level,
+                      :options, :parent, :quantifier,
                       :custom_to_s_handling, :pre_quantifier_decorations
-
-        attr_reader   :nesting_level, :quantifier
       end
     end
 
@@ -72,15 +70,30 @@ module Regexp::Expression
     # lit.to_s(:original) # => 'a +' # with quantifier AND intermittent decorations
     #
     def to_s(format = :full)
-      base = ''.dup
-      parts.each do |part|
-        if part.instance_of?(String)
-          base << part
-        elsif !part.custom_to_s_handling
-          base << part.to_s(:original)
+      result = ''.dup
+      stack = []
+      exp, current_parts, index = self, parts, 0
+
+      loop do
+        while index < current_parts.length
+          part = current_parts[index]
+          index += 1
+          if part.instance_of?(String)
+            result << part
+          elsif !part.custom_to_s_handling
+            stack.push([exp, format, current_parts, index])
+            exp, format, current_parts, index = part, :original, part.parts, 0
+          end
         end
+
+        result << exp.pre_quantifier_decoration(format).to_s
+        result << exp.quantifier_affix(format).to_s
+        break if stack.empty?
+
+        exp, format, current_parts, index = stack.pop
       end
-      "#{base}#{pre_quantifier_decoration(format)}#{quantifier_affix(format)}"
+
+      result
     end
     alias :to_str :to_s
 
@@ -99,16 +112,5 @@ module Regexp::Expression
     def coded_offset
       '@%d+%d' % offset
     end
-
-    def nesting_level=(lvl)
-      @nesting_level = lvl
-      quantifier && quantifier.nesting_level = lvl
-      terminal? || each { |subexp| subexp.nesting_level = lvl + 1 }
-    end
-
-    def quantifier=(qtf)
-      @quantifier = qtf
-      @repetitions = nil # clear memoized value
-    end
   end
 end

data/lib/regexp_parser/expression/subexpression.rb

@@ -24,7 +24,7 @@ module Regexp::Expression
       expressions << exp
     end
 
-    %w[[] at each empty? fetch index join last length values_at].each do |method|
+    %w[[] at each empty? fetch index join last length values_at zip].each do |method|
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{method}(*args, &block)
           expressions.#{method}(*args, &block)
@@ -51,7 +51,7 @@ module Regexp::Expression
 
     def extract_quantifier_target(quantifier_description)
       pre_quantifier_decorations = []
-      target = expressions.reverse.find do |exp|
+      target = expressions.reverse_each.find do |exp|
         if exp.decorative?
           exp.custom_to_s_handling = true
           pre_quantifier_decorations << exp.text

data/lib/regexp_parser/lexer.rb

@@ -5,19 +5,17 @@
 # normalizes tokens for the parser, and checks if they are implemented by the
 # given syntax flavor.
 class Regexp::Lexer
-
   OPENING_TOKENS = %i[
     capture passive lookahead nlookahead lookbehind nlookbehind
     atomic options options_switch named absence open
   ].freeze
 
-  CLOSING_TOKENS = %i[close].freeze
-
   CONDITION_TOKENS = %i[condition condition_close].freeze
 
   def self.lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
     new.lex(input, syntax, options: options, collect_tokens: collect_tokens, &block)
   end
+  class << self; alias :scan :lex end
 
   def lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
     syntax = syntax ? Regexp::Syntax.for(syntax) : Regexp::Syntax::CURRENT
@@ -33,19 +31,21 @@ class Regexp::Lexer
     self.shift = 0
 
     Regexp::Scanner.scan(input, options: options, collect_tokens: false) do |type, token, text, ts, te|
-      type, token = *syntax.normalize(type, token)
+      type, token = syntax.normalize(type, token)
       syntax.check! type, token
 
       ascend(type, token)
 
-      if (last = prev_token) &&
-         type == :quantifier &&
-         (
-           (last.type == :literal         && (parts = break_literal(last))) ||
-           (last.token == :codepoint_list && (parts = break_codepoint_list(last)))
-         )
-        emit(parts[0])
-        last = parts[1]
+      if (last = prev_token) && type == :quantifier
+        shortened_run, quantified_node =
+          if    last.type  == :literal        then break_literal_run(last)
+          elsif last.token == :codepoint_list then break_codepoint_list(last)
+          end
+
+        if shortened_run
+          emit(shortened_run)
+          last = quantified_node
+        end
       end
 
       current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
@@ -80,10 +80,6 @@ class Regexp::Lexer
     end
   end
 
-  class << self
-    alias :scan :lex
-  end
-
   private
 
   attr_accessor :block,
@@ -91,7 +87,7 @@ class Regexp::Lexer
                 :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
-    return unless CLOSING_TOKENS.include?(token)
+    return unless token == :close
 
     case type
     when :group, :assertion
@@ -122,7 +118,7 @@ class Regexp::Lexer
 
   # called by scan to break a literal run that is longer than one character
   # into two separate tokens when it is followed by a quantifier
-  def break_literal(token)
+  def break_literal_run(token)
     lead, last, _ = token.text.partition(/.\z/mu)
     return if lead.empty?
 
@@ -141,7 +137,7 @@ class Regexp::Lexer
 
   # if a codepoint list is followed by a quantifier, that quantifier applies
   # to the last codepoint, e.g. /\u{61 62 63}{3}/ =~ 'abccc'
-  # c.f. #break_literal.
+  # c.f. #break_literal_run.
   def break_codepoint_list(token)
     lead, _, tail = token.text.rpartition(' ')
     return if lead.empty?
@@ -153,7 +149,11 @@ class Regexp::Lexer
               (token.ts + lead.length + 1), (token.te + 3),
               nesting, set_nesting, conditional_nesting)
 
-    self.shift = shift + 3 # one space less, but extra \, u, {, and }
+    # Splitting the codepoint list into two parts changes the correct offsets
+    # of any following tokens because it requires extra chars to represent the
+    # 'virtual' token, e.g. `\u{61 62 63}{3}` is lexed as `\u{61 62}\u{63}{3}`,
+    # which is 3 chars longer.
+    self.shift = shift + 3
 
     token_1.previous = preprev_token
     token_1.next = token_2
@@ -167,5 +167,4 @@ class Regexp::Lexer
     token.previous = preprev_token # .next will be set by #lex
     token
   end
-
-end # module Regexp::Lexer
+end

data/lib/regexp_parser/parser.rb

@@ -36,15 +36,14 @@ class Regexp::Parser
     self.conditional_nesting = []
 
     self.captured_group_counts = Hash.new(0)
+    self.total_captured_group_count = 0
 
-    Regexp::Lexer.scan(input, syntax, options: options, collect_tokens: false) do |token|
+    Regexp::Lexer.new.lex(input, syntax, options: options, collect_tokens: false) do |token|
       parse_token(token)
     end
 
-    # Trigger recursive setting of #nesting_level, which reflects how deep
-    # a node is in the tree. Do this at the end to account for tree rewrites.
-    root.nesting_level = 0
-    assign_referenced_expressions
+    root.assign_referenced_expressions
+    root.recursively_update_nesting_levels
 
     if block_given?
       block.call(root)
@@ -57,7 +56,7 @@ class Regexp::Parser
 
   attr_accessor :root, :node, :nesting,
                 :options_stack, :switching_options, :conditional_nesting,
-                :captured_group_counts
+                :captured_group_counts, :total_captured_group_count
 
   def extract_options(input, options)
     if options && !input.is_a?(String)
@@ -197,16 +196,13 @@ class Regexp::Parser
     nest(group)
   end
 
-  def total_captured_group_count
-    captured_group_counts.values.reduce(0, :+)
-  end
-
   def captured_group_count_at_level
     captured_group_counts[node]
   end
 
   def count_captured_group
     captured_group_counts[node] += 1
+    self.total_captured_group_count = total_captured_group_count + 1
   end
 
   def close_group
@@ -496,7 +492,10 @@ class Regexp::Parser
       )
       new_group.implicit = true
       new_group << target_node
-      increase_group_level(target_node)
+      increment_group_level(target_node)
+      unless target_node.terminal?
+        target_node.each_expression { |exp| increment_group_level(exp) }
+      end
       node.expressions[node.expressions.index(target_node)] = new_group
       target_node = new_group
     end
@@ -509,10 +508,9 @@ class Regexp::Parser
     target_node.quantify(token, active_opts)
   end
 
-  def increase_group_level(exp)
+  def increment_group_level(exp)
     exp.level += 1
     exp.quantifier.level += 1 if exp.quantifier
-    exp.terminal? || exp.each { |subexp| increase_group_level(subexp) }
   end
 
   def set(token)
@@ -577,26 +575,4 @@ class Regexp::Parser
   def active_opts
     options_stack.last
   end
-
-  # Assigns referenced expressions to referring expressions, e.g. if there is
-  # an instance of Backreference::Number, its #referenced_expression is set to
-  # the instance of Group::Capture that it refers to via its number.
-  def assign_referenced_expressions
-    # find all referenceable and referring expressions
-    targets = { 0 => [root] }
-    referrers = []
-    root.each_expression do |exp|
-      if exp.referential?
-        referrers << exp
-      elsif exp.is_a?(Group::Capture)
-        (targets[exp.identifier] ||= []) << exp
-      end
-    end
-    # assign referenced expressions to referring expressions
-    # (in a second iteration because there might be forward references)
-    referrers.each do |exp|
-      exp.referenced_expressions = targets[exp.reference] ||
-        raise(ParserError, "Invalid reference #{exp.reference} at pos #{exp.ts}")
-    end
-  end
-end # module Regexp::Parser
+end