regexp_parser 2.10.0 → 2.12.0

data/lib/regexp_parser/scanner/properties/short.csv

@@ -11,6 +11,7 @@ bamu,bamum
 bass,bassa_vah
 batk,batak
 beng,bengali
+berf,beria_erfe
 bhks,bhaiksuki
 bidic,bidi_control
 bopo,bopomofo
@@ -58,6 +59,7 @@ epres,emoji_presentation
 ethi,ethiopic
 ext,extender
 extpict,extended_pictographic
+gara,garay
 geor,georgian
 glag,glagolitic
 gong,gunjala_gondi
@@ -69,6 +71,7 @@ grek,greek
 grext,grapheme_extend
 grlink,grapheme_link
 gujr,gujarati
+gukh,gurung_khema
 guru,gurmukhi
 hang,hangul
 hani,han
@@ -97,6 +100,7 @@ khmr,khmer
 khoj,khojki
 kits,khitan_small_script
 knda,kannada
+krai,kirat_rai
 kthi,kaithi
 l,letter
 lana,tai_tham
@@ -122,6 +126,7 @@ mand,mandaic
 mani,manichaean
 marc,marchen
 mc,spacing_mark
+mcm,modifier_combining_mark
 me,enclosing_mark
 medf,medefaidrin
 mend,mende_kikakui
@@ -154,6 +159,7 @@ oids,other_id_start
 olck,ol_chiki
 olower,other_lowercase
 omath,other_math
+onao,ol_onal
 orkh,old_turkic
 orya,oriya
 osge,osage
@@ -197,6 +203,7 @@ sgnw,signwriting
 shaw,shavian
 shrd,sharada
 sidd,siddham
+sidt,sidetic
 sind,khudawadi
 sinh,sinhala
 sk,modifier_symbol
@@ -208,6 +215,7 @@ sora,sora_sompeng
 soyo,soyombo
 sterm,sentence_terminal
 sund,sundanese
+sunu,sunuwar
 sylo,syloti_nagri
 syrc,syriac
 tagb,tagbanwa
@@ -217,6 +225,7 @@ talu,new_tai_lue
 taml,tamil
 tang,tangut
 tavt,tai_viet
+tayo,tai_yo
 telu,telugu
 term,terminal_punctuation
 tfng,tifinagh
@@ -225,6 +234,9 @@ thaa,thaana
 tibt,tibetan
 tirh,tirhuta
 tnsa,tangsa
+todr,todhri
+tols,tolong_siki
+tutg,tulu_tigalari
 ugar,ugaritic
 uideo,unified_ideograph
 vaii,vai

data/lib/regexp_parser/scanner/scanner.rl

@@ -37,7 +37,8 @@
   octal_sequence        = [0-7]{1,3};
 
   hex_sequence          = 'x' . xdigit{1,2};
-  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
+  hex_sequence_err      = 'x' . [^0-9A-Fa-f];
+  high_hex_sequence     = 'x' . [89A-Fa-f] . xdigit . ( '\\x' . [89A-Fa-f] . xdigit )*;
 
   codepoint_single      = 'u' . xdigit{4};
   codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
@@ -210,7 +211,7 @@
         type = :nonposixclass
       end
 
-      unless self.class.posix_classes.include?(class_name)
+      unless POSIX_CLASSES[class_name]
         raise ValidationError.for(:posix_class, text)
       end
 
@@ -246,7 +247,7 @@
     # Treat all remaining escapes - those not supported in sets - as literal.
     # (This currently includes \^, \-, \&, \:, although these could potentially
     # be meta chars when not escaped, depending on their position in the set.)
-    any > (escaped_set_alpha, 1) {
+    (any | utf8_multibyte) > (escaped_set_alpha, 1) {
       emit(:escape, :literal, copy(data, ts-1, te))
       fret;
     };
@@ -256,9 +257,21 @@
   # escape sequence scanner
   # --------------------------------------------------------------------------
   escape_sequence := |*
-    [1-9] {
+    [1-9] . [0-9]* {
       text = copy(data, ts-1, te)
-      emit(:backref, :number, text)
+
+      # If not enough groups have been opened, there is a fallback to either an
+      # octal or literal interpretation for 2+ digit numerical escapes.
+      digits = text[1..-1]
+      if digits.size == 1 || digits.to_i <= capturing_group_count
+        emit(:backref, :number, text)
+      elsif digits =~ /\A[0-7]{2,}\z/
+        emit(:escape, :octal, text)
+      else
+        emit(:escape, :literal, text[0..1])
+        emit(:literal, :literal, text[2..-1])
+      end
+
       fret;
     };
 
@@ -321,6 +334,16 @@
       fret;
     };
 
+    high_hex_sequence > (escaped_alpha, 5) {
+      text = copy(data, ts-1, te)
+      if regexp_encoding == Encoding::BINARY
+        text.split(/(?=\\)/).each { |part| emit(:escape, :hex, part) }
+      else
+        emit(:escape, :utf8_hex, text)
+      end
+      fret;
+    };
+
     hex_sequence > (escaped_alpha, 5) @eof(premature_end_error) {
       emit(:escape, :hex, copy(data, ts-1, te))
       fret;
@@ -514,6 +537,7 @@
     };
 
     group_open @group_opened {
+      self.capturing_group_count = capturing_group_count + 1
       text = copy(data, ts, te)
       emit(:group, :capture, text)
     };
@@ -662,6 +686,7 @@ class Regexp::Scanner
 
     input = input_object.is_a?(Regexp) ? input_object.source : input_object
     self.free_spacing = free_spacing?(input_object, options)
+    self.regexp_encoding = extract_encoding(input_object, options)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*")
@@ -672,6 +697,7 @@ class Regexp::Scanner
 
     self.set_depth = 0
     self.group_depth = 0
+    self.capturing_group_count = 0
     self.conditional_stack = []
     self.char_pos = 0
 
@@ -711,10 +737,11 @@ class Regexp::Scanner
     File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
   end
 
-  def self.posix_classes
+  # Use each_with_object for required_ruby_version >= 2.2, or #to_h for >= 2.6
+  POSIX_CLASSES =
     %w[alnum alpha ascii blank cntrl digit graph
        lower print punct space upper word xdigit]
-  end
+      .inject({}) { |o, e| o.merge(e => true) }.freeze
 
   # Emits an array with the details of the scanned pattern
   def emit(type, token, text)
@@ -742,16 +769,25 @@ class Regexp::Scanner
     end
   end
 
-  attr_accessor :literal_run # only public for #||= to work on ruby <= 2.5
+  attr_accessor :capturing_group_count, :literal_run # only public for #||= to work on ruby <= 2.5
 
   private
 
   attr_accessor :block,
                 :collect_tokens, :tokens, :prev_token,
                 :free_spacing, :spacing_stack,
+                :regexp_encoding,
                 :group_depth, :set_depth, :conditional_stack,
                 :char_pos
 
+  def extract_encoding(input_object, options)
+    if input_object.is_a?(::Regexp)
+      input_object.encoding
+    elsif options && (options & Regexp::NOENCODING)
+      Encoding::BINARY
+    end
+  end
+
   def free_spacing?(input_object, options)
     if options && !input_object.is_a?(String)
       raise ArgumentError, 'options cannot be supplied unless scanning a String'