regexp_parser 2.9.3 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e429c2cc03b2c9c31e3bf5c3dc71ffc15c5032a35f52c3abae9134d02c45496f
-  data.tar.gz: 6a89f8618748c8ab479c4d81ff44c9fabfb461337993fcc641da23d6c349a1ec
+  metadata.gz: 4cb66cfbf1c78a46f36cb24a7cbc9e04b0bc96aa1285fe81de79cec4bfd1c2c1
+  data.tar.gz: f650a1b30acac1298186dce0818eede9944e3b5117e794801abd0576d7b37b9e
 SHA512:
-  metadata.gz: 8f9cb8133b24db6f8bb2199356101c234960839ae1251a77da6fe4faeaafc2ab1d6f679f5a6e081860d4a9137a91aeb7793dbc617f04c9747b9110d64134d45f
-  data.tar.gz: bdfe1c9a13fef4f891c28787588c92be37c1c8a61e0d473d05482a67207675be090ec5a74d12b46cf93c8d2565388c6680acd89ad65bc63454514ee720181c82
+  metadata.gz: 1b87b74cafd00c2a8a3fe5a44942a005a4756974363c916c650c18e74df719920537bfaecdf21080aed339d24f1988444940d96fd66dc6af847498c04efbc033
+  data.tar.gz: 40e7f8357bd2ff7485c7d7105d852b6c615eaf8902b787b611953df637e246a53990acdc6e1b7f7ff2dd350edf749bd012352981b908de2bbcbee0bc59714513

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2012-2024,  Ammar Ali
+Copyright (c) 2010, 2012-2025,  Ammar Ali
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

data/lib/regexp_parser/expression/classes/backreference.rb

@@ -1,25 +1,6 @@
 module Regexp::Expression
   module Backreference
-    class Base < Regexp::Expression::Base
-      attr_accessor :referenced_expression
-
-      def initialize_copy(orig)
-        exp_id = [self.class, self.starts_at]
-
-        # prevent infinite recursion for recursive subexp calls
-        copied = @@copied ||= {}
-        self.referenced_expression =
-          if copied[exp_id]
-            orig.referenced_expression
-          else
-            copied[exp_id] = true
-            orig.referenced_expression.dup
-          end
-        copied.clear
-
-        super
-      end
-    end
+    class Base < Regexp::Expression::Base; end
 
     class Number < Backreference::Base
       attr_reader :number

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -7,26 +7,17 @@ module Regexp::Expression
     end
 
     class Condition < Regexp::Expression::Base
-      attr_accessor :referenced_expression
-
       # Name or number of the referenced capturing group that determines state.
       # Returns a String if reference is by name, Integer if by number.
       def reference
         ref = text.tr("'<>()", "")
         ref =~ /\D/ ? ref : Integer(ref)
       end
-
-      def initialize_copy(orig)
-        self.referenced_expression = orig.referenced_expression.dup
-        super
-      end
     end
 
     class Branch < Regexp::Expression::Sequence; end
 
     class Expression < Regexp::Expression::Subexpression
-      attr_accessor :referenced_expression
-
       def <<(exp)
         expressions.last << exp
       end
@@ -54,11 +45,6 @@ module Regexp::Expression
       def reference
         condition.reference
       end
-
-      def initialize_copy(orig)
-        self.referenced_expression = orig.referenced_expression.dup
-        super
-      end
     end
   end
 end

data/lib/regexp_parser/expression/classes/escape_sequence.rb

@@ -1,100 +1,29 @@
 module Regexp::Expression
   module EscapeSequence
-    class Base < Regexp::Expression::Base
-      def codepoint
-        char.ord
-      end
-
-      if ''.respond_to?(:undump)
-        def char
-          %("#{text}").undump
-        end
-      else
-        # poor man's unescape without using eval
-        require 'yaml'
-        def char
-          YAML.load(%Q(---\n"#{text}"\n))
-        end
-      end
-    end
-
-    class Literal < EscapeSequence::Base
-      def char
-        text[1..-1]
-      end
-    end
-
-    class AsciiEscape   < EscapeSequence::Base; end
-    class Backspace     < EscapeSequence::Base; end
-    class Bell          < EscapeSequence::Base; end
-    class FormFeed      < EscapeSequence::Base; end
-    class Newline       < EscapeSequence::Base; end
-    class Return        < EscapeSequence::Base; end
-    class Tab           < EscapeSequence::Base; end
-    class VerticalTab   < EscapeSequence::Base; end
-
-    class Hex           < EscapeSequence::Base; end
-    class Codepoint     < EscapeSequence::Base; end
-
-    class CodepointList < EscapeSequence::Base
-      def char
-        raise NoMethodError, 'CodepointList responds only to #chars'
-      end
-
-      def codepoint
-        raise NoMethodError, 'CodepointList responds only to #codepoints'
-      end
-
-      def chars
-        codepoints.map { |cp| cp.chr('utf-8') }
-      end
-
-      def codepoints
-        text.scan(/\h+/).map(&:hex)
-      end
-    end
-
-    class Octal < EscapeSequence::Base
-      def char
-        text[1..-1].to_i(8).chr('utf-8')
-      end
-    end
-
-    class AbstractMetaControlSequence < EscapeSequence::Base
-      def char
-        codepoint.chr('utf-8')
-      end
-
-      private
-
-      def control_sequence_to_s(control_sequence)
-        five_lsb = control_sequence.unpack('B*').first[-5..-1]
-        ["000#{five_lsb}"].pack('B*')
-      end
-
-      def meta_char_to_codepoint(meta_char)
-        byte_value = meta_char.ord
-        byte_value < 128 ? byte_value + 128 : byte_value
-      end
-    end
-
-    class Control < AbstractMetaControlSequence
-      def codepoint
-        control_sequence_to_s(text).ord
-      end
-    end
-
-    class Meta < AbstractMetaControlSequence
-      def codepoint
-        meta_char_to_codepoint(text[-1])
-      end
-    end
-
-    class MetaControl < AbstractMetaControlSequence
-      def codepoint
-        meta_char_to_codepoint(control_sequence_to_s(text))
-      end
-    end
+    Base        = Class.new(Regexp::Expression::Base)
+
+    AsciiEscape = Class.new(Base) # \e
+    Backspace   = Class.new(Base) # \b
+    Bell        = Class.new(Base) # \a
+    FormFeed    = Class.new(Base) # \f
+    Newline     = Class.new(Base) # \n
+    Return      = Class.new(Base) # \r
+    Tab         = Class.new(Base) # \t
+    VerticalTab = Class.new(Base) # \v
+
+    Literal     = Class.new(Base) # e.g. \j, \@, \😀 (ineffectual escapes)
+
+    Octal       = Class.new(Base) # e.g. \012
+    Hex         = Class.new(Base) # e.g. \x0A
+    Codepoint   = Class.new(Base) # e.g. \u000A
+
+    CodepointList = Class.new(Base) # e.g. \u{A B}
+    UTF8Hex       = Class.new(Base) # e.g. \xE2\x82\xAC
+
+    AbstractMetaControlSequence = Class.new(Base)
+    Control                     = Class.new(AbstractMetaControlSequence) # e.g. \cB
+    Meta                        = Class.new(AbstractMetaControlSequence) # e.g. \M-Z
+    MetaControl                 = Class.new(AbstractMetaControlSequence) # e.g. \M-\cX
   end
 
   # alias for symmetry between Token::* and Expression::*

data/lib/regexp_parser/expression/methods/escape_sequence_char.rb

@@ -0,0 +1,5 @@
+Regexp::Expression::EscapeSequence::Base.class_eval do
+  def char
+    codepoint.chr('utf-8')
+  end
+end

data/lib/regexp_parser/expression/methods/escape_sequence_codepoint.rb

@@ -0,0 +1,74 @@
+module Regexp::Expression::EscapeSequence
+  AsciiEscape.class_eval { def codepoint; 0x1B end }
+  Backspace.class_eval   { def codepoint; 0x8  end }
+  Bell.class_eval        { def codepoint; 0x7  end }
+  FormFeed.class_eval    { def codepoint; 0xC  end }
+  Newline.class_eval     { def codepoint; 0xA  end }
+  Return.class_eval      { def codepoint; 0xD  end }
+  Tab.class_eval         { def codepoint; 0x9  end }
+  VerticalTab.class_eval { def codepoint; 0xB  end }
+
+  Literal.class_eval     { def codepoint; text[1].ord end }
+
+  Octal.class_eval       { def codepoint; text[/\d+/].to_i(8) end }
+
+  Hex.class_eval         { def codepoint; text[/\h+/].hex end }
+  Codepoint.class_eval   { def codepoint; text[/\h+/].hex end }
+
+  UTF8Hex.class_eval do
+    def codepoint
+      text.scan(/\h+/).map(&:hex).pack('C*').force_encoding('utf-8').ord
+    end
+  end
+
+  CodepointList.class_eval do
+    # Maybe this should be a unique top-level expression class?
+    def char
+      raise NoMethodError, 'CodepointList responds only to #chars'
+    end
+
+    def codepoint
+      raise NoMethodError, 'CodepointList responds only to #codepoints'
+    end
+
+    def chars
+      codepoints.map { |cp| cp.chr('utf-8') }
+    end
+
+    def codepoints
+      text.scan(/\h+/).map(&:hex)
+    end
+  end
+
+  AbstractMetaControlSequence.class_eval do
+    private
+
+    def control_sequence_to_s(control_sequence)
+      five_lsb = control_sequence.unpack('B*').first[-5..-1]
+      ["000#{five_lsb}"].pack('B*')
+    end
+
+    def meta_char_to_codepoint(meta_char)
+      byte_value = meta_char.ord
+      byte_value < 128 ? byte_value + 128 : byte_value
+    end
+  end
+
+  Control.class_eval do
+    def codepoint
+      control_sequence_to_s(text).ord
+    end
+  end
+
+  Meta.class_eval do
+    def codepoint
+      meta_char_to_codepoint(text[-1])
+    end
+  end
+
+  MetaControl.class_eval do
+    def codepoint
+      meta_char_to_codepoint(control_sequence_to_s(text))
+    end
+  end
+end

data/lib/regexp_parser/expression/methods/referenced_expressions.rb

@@ -0,0 +1,28 @@
+module Regexp::Expression
+  module ReferencedExpressions
+    attr_accessor :referenced_expressions
+
+    def referenced_expression
+      referenced_expressions && referenced_expressions.first
+    end
+
+    def initialize_copy(orig)
+      exp_id = [self.class, self.starts_at]
+
+      # prevent infinite recursion for recursive subexp calls
+      copied = self.class.instance_eval { @copied_ref_exps ||= {} }
+      self.referenced_expressions =
+        if copied[exp_id]
+          orig.referenced_expressions
+        else
+          copied[exp_id] = true
+          orig.referenced_expressions && orig.referenced_expressions.map(&:dup)
+        end
+      copied.clear
+
+      super
+    end
+  end
+
+  Base.include ReferencedExpressions
+end

data/lib/regexp_parser/expression.rb

@@ -25,6 +25,8 @@ require_relative 'expression/classes/root'
 require_relative 'expression/classes/unicode_property'
 
 require_relative 'expression/methods/construct'
+require_relative 'expression/methods/escape_sequence_char'
+require_relative 'expression/methods/escape_sequence_codepoint'
 require_relative 'expression/methods/human_name'
 require_relative 'expression/methods/match'
 require_relative 'expression/methods/match_length'
@@ -32,6 +34,7 @@ require_relative 'expression/methods/negative'
 require_relative 'expression/methods/options'
 require_relative 'expression/methods/parts'
 require_relative 'expression/methods/printing'
+require_relative 'expression/methods/referenced_expressions'
 require_relative 'expression/methods/strfregexp'
 require_relative 'expression/methods/tests'
 require_relative 'expression/methods/traverse'

data/lib/regexp_parser/parser.rb

@@ -319,6 +319,7 @@ class Regexp::Parser
     when :codepoint_list; node << EscapeSequence::CodepointList.new(token, active_opts)
     when :hex;            node << EscapeSequence::Hex.new(token, active_opts)
     when :octal;          node << EscapeSequence::Octal.new(token, active_opts)
+    when :utf8_hex;       node << EscapeSequence::UTF8Hex.new(token, active_opts)
 
     when :control
       if token.text =~ /\A(?:\\C-\\M|\\c\\M)/
@@ -580,16 +581,19 @@ class Regexp::Parser
   # the instance of Group::Capture that it refers to via its number.
   def assign_referenced_expressions
     # find all referenceable and referring expressions
-    targets = { 0 => root }
+    targets = { 0 => [root] }
     referrers = []
     root.each_expression do |exp|
-      exp.is_a?(Group::Capture) && targets[exp.identifier] = exp
-      referrers << exp if exp.referential?
+      if exp.referential?
+        referrers << exp
+      elsif exp.is_a?(Group::Capture)
+        (targets[exp.identifier] ||= []) << exp
+      end
     end
-    # assign reference expression to referring expressions
+    # assign referenced expressions to referring expressions
     # (in a second iteration because there might be forward references)
     referrers.each do |exp|
-      exp.referenced_expression = targets[exp.reference] ||
+      exp.referenced_expressions = targets[exp.reference] ||
         raise(ParserError, "Invalid reference #{exp.reference} at pos #{exp.ts}")
     end
   end

data/lib/regexp_parser/scanner/scanner.rl

@@ -37,7 +37,8 @@
   octal_sequence        = [0-7]{1,3};
 
   hex_sequence          = 'x' . xdigit{1,2};
-  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
+  hex_sequence_err      = 'x' . [^0-9A-Fa-f];
+  high_hex_sequence     = 'x' . [89A-Fa-f] . xdigit . ( '\\x' . [89A-Fa-f] . xdigit )*;
 
   codepoint_single      = 'u' . xdigit{4};
   codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
@@ -210,7 +211,7 @@
         type = :nonposixclass
       end
 
-      unless self.class.posix_classes.include?(class_name)
+      unless POSIX_CLASSES[class_name]
         raise ValidationError.for(:posix_class, text)
       end
 
@@ -256,9 +257,21 @@
   # escape sequence scanner
   # --------------------------------------------------------------------------
   escape_sequence := |*
-    [1-9] {
+    [1-9] . [0-9]* {
       text = copy(data, ts-1, te)
-      emit(:backref, :number, text)
+
+      # If not enough groups have been opened, there is a fallback to either an
+      # octal or literal interpretation for 2+ digit numerical escapes.
+      digits = text[1..-1]
+      if digits.size == 1 || digits.to_i <= self.capturing_group_count
+        emit(:backref, :number, text)
+      elsif digits =~ /\A[0-7]{2,}\z/
+        emit(:escape, :octal, text)
+      else
+        emit(:escape, :literal, text[0..1])
+        emit(:literal, :literal, text[2..-1])
+      end
+
       fret;
     };
 
@@ -321,6 +334,16 @@
       fret;
     };
 
+    high_hex_sequence > (escaped_alpha, 5) {
+      text = copy(data, ts-1, te)
+      if regexp_encoding == Encoding::BINARY
+        text.split(/(?=\\)/).each { |part| emit(:escape, :hex, part) }
+      else
+        emit(:escape, :utf8_hex, text)
+      end
+      fret;
+    };
+
     hex_sequence > (escaped_alpha, 5) @eof(premature_end_error) {
       emit(:escape, :hex, copy(data, ts-1, te))
       fret;
@@ -514,6 +537,7 @@
     };
 
     group_open @group_opened {
+      self.capturing_group_count += 1
       text = copy(data, ts, te)
       emit(:group, :capture, text)
     };
@@ -662,6 +686,7 @@ class Regexp::Scanner
 
     input = input_object.is_a?(Regexp) ? input_object.source : input_object
     self.free_spacing = free_spacing?(input_object, options)
+    self.regexp_encoding = input_object.encoding if input_object.is_a?(::Regexp)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*")
@@ -672,6 +697,7 @@ class Regexp::Scanner
 
     self.set_depth = 0
     self.group_depth = 0
+    self.capturing_group_count = 0
     self.conditional_stack = []
     self.char_pos = 0
 
@@ -711,10 +737,9 @@ class Regexp::Scanner
     File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
   end
 
-  def self.posix_classes
+  POSIX_CLASSES =
     %w[alnum alpha ascii blank cntrl digit graph
-       lower print punct space upper word xdigit]
-  end
+       lower print punct space upper word xdigit].to_h { |c| [c, true] }.freeze
 
   # Emits an array with the details of the scanned pattern
   def emit(type, token, text)
@@ -749,7 +774,9 @@ class Regexp::Scanner
   attr_accessor :block,
                 :collect_tokens, :tokens, :prev_token,
                 :free_spacing, :spacing_stack,
+                :regexp_encoding,
                 :group_depth, :set_depth, :conditional_stack,
+                :capturing_group_count,
                 :char_pos
 
   def free_spacing?(input_object, options)