regexp_parser 1.3.0 → 1.7.0

data/lib/regexp_parser/scanner/char_type.rl

@@ -21,9 +21,6 @@
       when '\W'; emit(:type, :nonword,    text, ts - 1, te)
       when '\R'; emit(:type, :linebreak,  text, ts - 1, te)
       when '\X'; emit(:type, :xgrapheme,  text, ts - 1, te)
-      else
-        raise ScannerError.new(
-          "Unexpected character in type at #{text} (char #{ts})")
       end
       fret;
     };

data/lib/regexp_parser/scanner/properties/long.yml

@@ -5,6 +5,9 @@
 adlam: adlam
 age=1.1: age=1.1
 age=10.0: age=10.0
+age=11.0: age=11.0
+age=12.0: age=12.0
+age=12.1: age=12.1
 age=2.0: age=2.0
 age=2.1: age=2.1
 age=3.0: age=3.0
@@ -63,7 +66,6 @@ changeswhenuppercased: changes_when_uppercased
 cherokee: cherokee
 closepunctuation: close_punctuation
 cntrl: cntrl
-combiningmark: combining_mark
 common: common
 connectorpunctuation: connector_punctuation
 control: control
@@ -81,9 +83,11 @@ deseret: deseret
 devanagari: devanagari
 diacritic: diacritic
 digit: digit
+dogra: dogra
 duployan: duployan
 egyptianhieroglyphs: egyptian_hieroglyphs
 elbasan: elbasan
+elymaic: elymaic
 emoji: emoji
 emojicomponent: emoji_component
 emojimodifier: emoji_modifier
@@ -104,9 +108,11 @@ graphemeextend: grapheme_extend
 graphemelink: grapheme_link
 greek: greek
 gujarati: gujarati
+gunjalagondi: gunjala_gondi
 gurmukhi: gurmukhi
 han: han
 hangul: hangul
+hanifirohingya: hanifi_rohingya
 hanunoo: hanunoo
 hatran: hatran
 hebrew: hebrew
@@ -160,6 +166,7 @@ inchakma: in_chakma
 incham: in_cham
 incherokee: in_cherokee
 incherokeesupplement: in_cherokee_supplement
+inchesssymbols: in_chess_symbols
 incjkcompatibility: in_cjk_compatibility
 incjkcompatibilityforms: in_cjk_compatibility_forms
 incjkcompatibilityideographs: in_cjk_compatibility_ideographs
@@ -197,11 +204,14 @@ indeseret: in_deseret
 indevanagari: in_devanagari
 indevanagariextended: in_devanagari_extended
 indingbats: in_dingbats
+indogra: in_dogra
 indominotiles: in_domino_tiles
 induployan: in_duployan
 inearlydynasticcuneiform: in_early_dynastic_cuneiform
+inegyptianhieroglyphformatcontrols: in_egyptian_hieroglyph_format_controls
 inegyptianhieroglyphs: in_egyptian_hieroglyphs
 inelbasan: in_elbasan
+inelymaic: in_elymaic
 inemoticons: in_emoticons
 inenclosedalphanumerics: in_enclosed_alphanumerics
 inenclosedalphanumericsupplement: in_enclosed_alphanumeric_supplement
@@ -215,6 +225,7 @@ ingeneralpunctuation: in_general_punctuation
 ingeometricshapes: in_geometric_shapes
 ingeometricshapesextended: in_geometric_shapes_extended
 ingeorgian: in_georgian
+ingeorgianextended: in_georgian_extended
 ingeorgiansupplement: in_georgian_supplement
 inglagolitic: in_glagolitic
 inglagoliticsupplement: in_glagolitic_supplement
@@ -223,6 +234,7 @@ ingrantha: in_grantha
 ingreekandcoptic: in_greek_and_coptic
 ingreekextended: in_greek_extended
 ingujarati: in_gujarati
+ingunjalagondi: in_gunjala_gondi
 ingurmukhi: in_gurmukhi
 inhalfwidthandfullwidthforms: in_halfwidth_and_fullwidth_forms
 inhangulcompatibilityjamo: in_hangul_compatibility_jamo
@@ -230,6 +242,7 @@ inhanguljamo: in_hangul_jamo
 inhanguljamoextendeda: in_hangul_jamo_extended_a
 inhanguljamoextendedb: in_hangul_jamo_extended_b
 inhangulsyllables: in_hangul_syllables
+inhanifirohingya: in_hanifi_rohingya
 inhanunoo: in_hanunoo
 inhatran: in_hatran
 inhebrew: in_hebrew
@@ -240,6 +253,7 @@ inhiragana: in_hiragana
 inideographicdescriptioncharacters: in_ideographic_description_characters
 inideographicsymbolsandpunctuation: in_ideographic_symbols_and_punctuation
 inimperialaramaic: in_imperial_aramaic
+inindicsiyaqnumbers: in_indic_siyaq_numbers
 ininscriptionalpahlavi: in_inscriptional_pahlavi
 ininscriptionalparthian: in_inscriptional_parthian
 inipaextensions: in_ipa_extensions
@@ -279,6 +293,7 @@ inlycian: in_lycian
 inlydian: in_lydian
 inmahajani: in_mahajani
 inmahjongtiles: in_mahjong_tiles
+inmakasar: in_makasar
 inmalayalam: in_malayalam
 inmandaic: in_mandaic
 inmanichaean: in_manichaean
@@ -286,6 +301,8 @@ inmarchen: in_marchen
 inmasaramgondi: in_masaram_gondi
 inmathematicalalphanumericsymbols: in_mathematical_alphanumeric_symbols
 inmathematicaloperators: in_mathematical_operators
+inmayannumerals: in_mayan_numerals
+inmedefaidrin: in_medefaidrin
 inmeeteimayek: in_meetei_mayek
 inmeeteimayekextensions: in_meetei_mayek_extensions
 inmendekikakui: in_mende_kikakui
@@ -309,12 +326,14 @@ inmyanmar: in_myanmar
 inmyanmarextendeda: in_myanmar_extended_a
 inmyanmarextendedb: in_myanmar_extended_b
 innabataean: in_nabataean
+innandinagari: in_nandinagari
 innewa: in_newa
 innewtailue: in_new_tai_lue
 innko: in_nko
 innoblock: in_no_block
 innumberforms: in_number_forms
 innushu: in_nushu
+innyiakengpuachuehmong: in_nyiakeng_puachue_hmong
 inogham: in_ogham
 inolchiki: in_ol_chiki
 inoldhungarian: in_old_hungarian
@@ -322,6 +341,7 @@ inolditalic: in_old_italic
 inoldnortharabian: in_old_north_arabian
 inoldpermic: in_old_permic
 inoldpersian: in_old_persian
+inoldsogdian: in_old_sogdian
 inoldsoutharabian: in_old_south_arabian
 inoldturkic: in_old_turkic
 inopticalcharacterrecognition: in_optical_character_recognition
@@ -329,6 +349,7 @@ inoriya: in_oriya
 inornamentaldingbats: in_ornamental_dingbats
 inosage: in_osage
 inosmanya: in_osmanya
+inottomansiyaqnumbers: in_ottoman_siyaq_numbers
 inpahawhhmong: in_pahawh_hmong
 inpalmyrene: in_palmyrene
 inpaucinhau: in_pau_cin_hau
@@ -354,6 +375,8 @@ insiddham: in_siddham
 insinhala: in_sinhala
 insinhalaarchaicnumbers: in_sinhala_archaic_numbers
 insmallformvariants: in_small_form_variants
+insmallkanaextension: in_small_kana_extension
+insogdian: in_sogdian
 insorasompeng: in_sora_sompeng
 insoyombo: in_soyombo
 inspacingmodifierletters: in_spacing_modifier_letters
@@ -371,6 +394,7 @@ insupplementaryprivateuseareaa: in_supplementary_private_use_area_a
 insupplementaryprivateuseareab: in_supplementary_private_use_area_b
 insuttonsignwriting: in_sutton_signwriting
 insylotinagri: in_syloti_nagri
+insymbolsandpictographsextendeda: in_symbols_and_pictographs_extended_a
 insyriac: in_syriac
 insyriacsupplement: in_syriac_supplement
 intagalog: in_tagalog
@@ -382,6 +406,7 @@ intaiviet: in_tai_viet
 intaixuanjingsymbols: in_tai_xuan_jing_symbols
 intakri: in_takri
 intamil: in_tamil
+intamilsupplement: in_tamil_supplement
 intangut: in_tangut
 intangutcomponents: in_tangut_components
 intelugu: in_telugu
@@ -399,6 +424,7 @@ invariationselectors: in_variation_selectors
 invariationselectorssupplement: in_variation_selectors_supplement
 invedicextensions: in_vedic_extensions
 inverticalforms: in_vertical_forms
+inwancho: in_wancho
 inwarangciti: in_warang_citi
 inyijinghexagramsymbols: in_yijing_hexagram_symbols
 inyiradicals: in_yi_radicals
@@ -431,6 +457,7 @@ lowercaseletter: lowercase_letter
 lycian: lycian
 lydian: lydian
 mahajani: mahajani
+makasar: makasar
 malayalam: malayalam
 mandaic: mandaic
 manichaean: manichaean
@@ -439,6 +466,7 @@ mark: mark
 masaramgondi: masaram_gondi
 math: math
 mathsymbol: math_symbol
+medefaidrin: medefaidrin
 meeteimayek: meetei_mayek
 mendekikakui: mende_kikakui
 meroiticcursive: meroitic_cursive
@@ -452,6 +480,7 @@ mro: mro
 multani: multani
 myanmar: myanmar
 nabataean: nabataean
+nandinagari: nandinagari
 newa: newa
 newline: newline
 newtailue: new_tai_lue
@@ -460,6 +489,7 @@ noncharactercodepoint: noncharacter_code_point
 nonspacingmark: nonspacing_mark
 number: number
 nushu: nushu
+nyiakengpuachuehmong: nyiakeng_puachue_hmong
 ogham: ogham
 olchiki: ol_chiki
 oldhungarian: old_hungarian
@@ -467,6 +497,7 @@ olditalic: old_italic
 oldnortharabian: old_north_arabian
 oldpermic: old_permic
 oldpersian: old_persian
+oldsogdian: old_sogdian
 oldsoutharabian: old_south_arabian
 oldturkic: old_turkic
 openpunctuation: open_punctuation
@@ -515,6 +546,7 @@ siddham: siddham
 signwriting: signwriting
 sinhala: sinhala
 softdotted: soft_dotted
+sogdian: sogdian
 sorasompeng: sora_sompeng
 soyombo: soyombo
 space: space
@@ -550,6 +582,7 @@ uppercase: uppercase
 uppercaseletter: uppercase_letter
 vai: vai
 variationselector: variation_selector
+wancho: wancho
 warangciti: warang_citi
 whitespace: white_space
 word: word

data/lib/regexp_parser/scanner/properties/short.yml

@@ -31,6 +31,7 @@ cher: cherokee
 ci: case_ignorable
 cn: unassigned
 co: private_use
+combiningmark: mark
 copt: coptic
 cprt: cypriot
 cs: surrogate
@@ -44,14 +45,17 @@ dep: deprecated
 deva: devanagari
 di: default_ignorable_code_point
 dia: diacritic
+dogr: dogra
 dsrt: deseret
 dupl: duployan
 egyp: egyptian_hieroglyphs
 elba: elbasan
+elym: elymaic
 ethi: ethiopic
 ext: extender
 geor: georgian
 glag: glagolitic
+gong: gunjala_gondi
 gonm: masaram_gondi
 goth: gothic
 gran: grantha
@@ -70,6 +74,7 @@ hex: hex_digit
 hira: hiragana
 hluw: anatolian_hieroglyphs
 hmng: pahawh_hmong
+hmnp: nyiakeng_puachue_hmong
 hung: old_hungarian
 idc: id_continue
 ideo: ideographic
@@ -105,11 +110,13 @@ lyci: lycian
 lydi: lydian
 m: mark
 mahj: mahajani
+maka: makasar
 mand: mandaic
 mani: manichaean
 marc: marchen
 mc: spacing_mark
 me: enclosing_mark
+medf: medefaidrin
 mend: mende_kikakui
 merc: meroitic_cursive
 mero: meroitic_hieroglyphs
@@ -121,6 +128,7 @@ mtei: meetei_mayek
 mult: multani
 mymr: myanmar
 n: number
+nand: nandinagari
 narb: old_north_arabian
 nbat: nabataean
 nchar: noncharacter_code_point
@@ -168,6 +176,7 @@ qaai: inherited
 qmark: quotation_mark
 ri: regional_indicator
 rjng: rejang
+rohg: hanifi_rohingya
 runr: runic
 s: symbol
 samr: samaritan
@@ -184,6 +193,8 @@ sinh: sinhala
 sk: modifier_symbol
 sm: math_symbol
 so: other_symbol
+sogd: sogdian
+sogo: old_sogdian
 sora: sora_sompeng
 soyo: soyombo
 sterm: sentence_terminal
@@ -209,6 +220,7 @@ uideo: unified_ideograph
 vaii: vai
 vs: variation_selector
 wara: warang_citi
+wcho: wancho
 wspace: white_space
 xidc: xid_continue
 xids: xid_start

data/lib/regexp_parser/scanner/scanner.rl

@@ -49,9 +49,9 @@
   codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
   codepoint_sequence    = codepoint_single | codepoint_list;
 
-  control_sequence      = ('c' | 'C-') . (backslash . 'M-')?;
+  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;
 
-  meta_sequence         = 'M-' . (backslash . control_sequence)?;
+  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;
 
   zero_or_one           = '?' | '??' | '?+';
   zero_or_more          = '*' | '*?' | '*+';
@@ -82,7 +82,8 @@
   assertion_lookbehind  = '?<=';
   assertion_nlookbehind = '?<!';
 
-  group_options         = '?' . [\-mixdau];
+  # try to treat every other group head as options group, like Ruby
+  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;
 
   group_ref             = [gk];
   group_name_char       = (alnum | '_');
@@ -135,41 +136,35 @@
   # Invalid sequence error, used from sequences, like escapes and sets
   action invalid_sequence_error {
     text = ts ? copy(data, ts-1..-1) : data.pack('c*')
-    raise InvalidSequenceError.new('sequence', text)
+    validation_error(:sequence, 'sequence', text)
   }
 
   # group (nesting) and set open/close actions
-  action group_opened { self.group_depth = group_depth + 1; in_group = true }
-  action group_closed { self.group_depth = group_depth - 1; in_group = group_depth > 0 ? true : false }
+  action group_opened { self.group_depth = group_depth + 1 }
+  action group_closed { self.group_depth = group_depth - 1 }
+  action set_opened   { self.set_depth   = set_depth   + 1 }
+  action set_closed   { self.set_depth   = set_depth   - 1 }
 
   # Character set scanner, continues consuming characters until it meets the
   # closing bracket of the set.
   # --------------------------------------------------------------------------
   character_set := |*
-    set_close > (set_meta, 2) {
-      set_depth -= 1
-      in_set = set_depth > 0 ? true : false
-
+    set_close > (set_meta, 2) @set_closed {
       emit(:set, :close, *text(data, ts, te))
-
-      if set_depth == 0
-        fgoto main;
-      else
+      if in_set?
         fret;
+      else
+        fgoto main;
       end
     };
 
-    '-]' { # special case, emits two tokens
-      set_depth -= 1
-      in_set = set_depth > 0 ? true : false
-
-      emit(:literal, :literal, copy(data, ts..te-2), ts, te)
-      emit(:set, :close, copy(data, ts+1..te-1), ts, te)
-
-      if set_depth == 0
-        fgoto main;
-      else
+    '-]' @set_closed { # special case, emits two tokens
+      emit(:literal, :literal, copy(data, ts..te-2), ts, te - 1)
+      emit(:set, :close, copy(data, ts+1..te-1), ts + 1, te)
+      if in_set?
         fret;
+      else
+        fgoto main;
       end
     };
 
@@ -207,14 +202,12 @@
       fcall set_escape_sequence;
     };
 
-    set_open >(open_bracket, 1) {
-      set_depth += 1
-
+    set_open >(open_bracket, 1) >set_opened {
       emit(:set, :open, *text(data, ts, te))
       fcall character_set;
     };
 
-    class_posix >(open_bracket, 1) @eof(premature_end_error) {
+    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       text = text(data, ts, te).first
 
       type = :posixclass
@@ -227,11 +220,11 @@
       emit(type, class_name.to_sym, text, ts, te)
     };
 
-    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
+    collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       emit(:set, :collation, *text(data, ts, te))
     };
 
-    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
+    character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       emit(:set, :equivalent, *text(data, ts, te))
     };
 
@@ -337,44 +330,24 @@
     };
 
     control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
-      if data[te]
-        c = data[te].chr
-        if c =~ /[\x00-\x7F]/
-          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
-          p += 1
-        else
-          raise InvalidSequenceError.new("control sequence")
-        end
-      else
-        raise PrematureEndError.new("control sequence")
-      end
+      emit_meta_control_sequence(data, ts, te, :control)
       fret;
     };
 
     meta_sequence >(backslashed, 3) $eof(premature_end_error) {
-      if data[te]
-        c = data[te].chr
-        if c =~ /[\x00-\x7F]/
-          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
-          p += 1
-        else
-          raise InvalidSequenceError.new("meta sequence")
-        end
-      else
-        raise PrematureEndError.new("meta sequence")
-      end
+      emit_meta_control_sequence(data, ts, te, :meta_sequence)
       fret;
     };
 
     char_type_char > (escaped_alpha, 2) {
       fhold;
-      fnext *(in_set ? fentry(character_set) : fentry(main));
+      fnext *(in_set? ? fentry(character_set) : fentry(main));
       fcall char_type;
     };
 
     property_char > (escaped_alpha, 2) {
       fhold;
-      fnext *(in_set ? fentry(character_set) : fentry(main));
+      fnext *(in_set? ? fentry(character_set) : fentry(main));
       fcall unicode_property;
     };
 
@@ -412,8 +385,7 @@
     };
 
     alternation {
-      if in_conditional and conditional_stack.length > 0 and
-         conditional_stack.last[1] == group_depth
+      if conditional_stack.last == group_depth
         emit(:conditional, :separator, *text(data, ts, te))
       else
         emit(:meta, :alternation, *text(data, ts, te))
@@ -442,18 +414,12 @@
       when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
       when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
       when '\\G'; emit(:anchor, :match_start,        text, ts, te)
-      else
-        raise ScannerError.new(
-          "Unexpected character in anchor at #{text} (char #{ts})")
       end
     };
 
     # Character sets
     # ------------------------------------------------------------------------
-    set_open {
-      set_depth += 1
-      in_set = true
-
+    set_open >set_opened {
       emit(:set, :open, *text(data, ts, te))
       fcall character_set;
     };
@@ -465,9 +431,7 @@
     conditional {
       text = text(data, ts, te).first
 
-      in_conditional = true unless in_conditional
-      conditional_depth += 1
-      conditional_stack << [conditional_depth, group_depth]
+      conditional_stack << group_depth
 
       emit(:conditional, :open, text[0..-2], ts, te-1)
       emit(:conditional, :condition_open, '(', te-1, te)
@@ -496,7 +460,11 @@
     #   (?imxdau-imx:subexp)  option on/off for subexp
     # ------------------------------------------------------------------------
     group_open . group_options >group_opened {
-      p = scan_options(p, data, ts, te)
+      text = text(data, ts, te).first
+      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
+        raise InvalidGroupOption.new($1 || "-#{$2}", text)
+      end
+      emit_options(text, ts, te)
     };
 
     # Assertions
@@ -528,19 +496,15 @@
       when '(?>';  emit(:group, :atomic,       text, ts, te)
       when '(?~';  emit(:group, :absence,      text, ts, te)
 
-      when /^\(\?<(\w*)>/
-        empty_name_error(:group, 'named group (ab)') if $1.empty?
+      when /^\(\?(?:<>|'')/
+        validation_error(:group, 'named group', 'name is empty')
 
+      when /^\(\?<\w*>/
         emit(:group, :named_ab,  text, ts, te)
 
-      when /^\(\?'(\w*)'/
-        empty_name_error(:group, 'named group (sq)') if $1.empty?
-
+      when /^\(\?'\w*'/
         emit(:group, :named_sq,  text, ts, te)
 
-      else
-        raise ScannerError.new(
-          "Unknown subexpression group format '#{text}'")
       end
     };
 
@@ -550,20 +514,13 @@
     };
 
     group_close @group_closed {
-      if in_conditional and conditional_stack.last and
-         conditional_stack.last[1] == (group_depth + 1)
-
-        emit(:conditional, :close, *text(data, ts, te))
+      if conditional_stack.last == group_depth + 1
         conditional_stack.pop
-
-        if conditional_stack.length == 0
-          in_conditional = false
-        end
+        emit(:conditional, :close, *text(data, ts, te))
       else
-        if spacing_stack.length > 1 and
-          spacing_stack.last[:depth] == (group_depth + 1)
+        if spacing_stack.length > 1 &&
+           spacing_stack.last[:depth] == group_depth + 1
           spacing_stack.pop
-
           self.free_spacing = spacing_stack.last[:free_spacing]
         end
 
@@ -576,11 +533,8 @@
     # ------------------------------------------------------------------------
     backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
       case text = text(data, ts, te).first
-      when /^\\([gk])<>/ # angle brackets
-        empty_backref_error("ref/call (ab)")
-
-      when /^\\([gk])''/ # single quotes
-        empty_backref_error("ref/call (sq)")
+      when /^\\([gk])(<>|'')/ # angle brackets
+        validation_error(:backref, 'ref/call', 'ref ID is empty')
 
       when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
         if $1 == 'k'
@@ -636,9 +590,6 @@
       when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
         emit(:backref, :number_recursion_ref_sq, text, ts, te)
 
-      else
-        raise ScannerError.new(
-          "Unknown backreference format '#{text}'")
       end
     };
 
@@ -786,7 +737,7 @@ class Regexp::Scanner
       input = input_object
       self.free_spacing = false
     end
-
+    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*") if input.is_a?(String)
     eof   = data.length
@@ -794,15 +745,9 @@ class Regexp::Scanner
     self.tokens = []
     self.block  = block_given? ? block : nil
 
-    self.in_group = false
+    self.set_depth = 0
     self.group_depth = 0
-    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
-
-    in_set = false
-    set_depth = 0
-    in_conditional = false
-    conditional_depth = 0
-    conditional_stack = []
+    self.conditional_stack = []
 
     %% write data;
     %% write init;
@@ -817,9 +762,9 @@ class Regexp::Scanner
     end
 
     raise PrematureEndError.new("(missing group closing paranthesis) "+
-          "[#{in_group}:#{group_depth}]") if in_group
+          "[#{group_depth}]") if in_group?
     raise PrematureEndError.new("(missing set closing bracket) "+
-          "[#{in_set}:#{set_depth}]") if in_set
+          "[#{set_depth}]") if in_set?
 
     # when the entire expression is a literal run
     emit_literal if literal
@@ -854,62 +799,15 @@ class Regexp::Scanner
 
   private
 
-  attr_accessor :tokens, :literal, :block,
-                :in_group, :group_depth,
-                :free_spacing, :spacing_stack
-
-  # Ragel's regex-based scan of the group options introduced a lot of
-  # ambiguity, so we just ask it to find the beginning of what looks
-  # like an options run and handle the rest in here.
-  def scan_options(p, data, ts, te)
-    text = text(data, ts, te).first
-
-    options_char, options_length = true, 0
-
-    # Copy while we have option characters. There is no maximum length,
-    # as ruby allows things like '(?xxxxxxxxx-xxxxxxxxxxxxx:abc)'.
-    negative_options = false
-    while options_char
-      if data[te + options_length]
-        c = data[te + options_length].chr
-
-        if c =~ /[-mixdau]/
-          negative_options = true if c == '-'
+  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
+                :group_depth, :set_depth, :conditional_stack
 
-          raise InvalidGroupOption.new(c, text) if negative_options and
-            c =~ /[dau]/
-
-          text << c ; p += 1 ; options_length += 1
-        else
-          options_char = false
-        end
-      else
-        raise PrematureEndError.new("expression options `#{text}'")
-      end
-    end
-
-    if data[te + options_length]
-      c = data[te + options_length].chr
-
-      if c == ':'
-        # Include the ':' in the options text
-        text << c ; p += 1 ; options_length += 1
-        emit_options(text, ts, te + options_length)
-
-      elsif c == ')'
-        # Don't include the closing ')', let group_close handle it.
-        emit_options(text, ts, te + options_length)
-
-      else
-        # Plain Regexp reports this as 'undefined group option'
-        raise ScannerError.new(
-          "Unexpected `#{c}' in options sequence, ':' or ')' expected")
-      end
-    else
-      raise PrematureEndError.new("expression options `#{text}'")
-    end
+  def in_group?
+    group_depth > 0
+  end
 
-    p # return the new value of the data pointer
+  def in_set?
+    set_depth > 0
   end
 
   # Copy from ts to te from data as text
@@ -945,32 +843,39 @@ class Regexp::Scanner
   def emit_options(text, ts, te)
     token = nil
 
-    if text =~ /\(\?([mixdau]*)-?([mix]*)(:)?/
-      positive, negative, group_local = $1, $2, $3
+    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
+    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
+    positive, negative, group_local = $1, $2, $3
 
-      if positive.include?('x')
-        self.free_spacing = true
-      end
+    if positive.include?('x')
+      self.free_spacing = true
+    end
 
-      # If the x appears in both, treat it like ruby does, the second cancels
-      # the first.
-      if negative.include?('x')
-        self.free_spacing = false
-      end
+    # If the x appears in both, treat it like ruby does, the second cancels
+    # the first.
+    if negative && negative.include?('x')
+      self.free_spacing = false
+    end
 
-      if group_local
-        spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
-        token = :options
-      else
-        # switch for parent group level
-        spacing_stack.last[:free_spacing] = free_spacing
-        token = :options_switch
-      end
+    if group_local
+      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
+      token = :options
+    else
+      # switch for parent group level
+      spacing_stack.last[:free_spacing] = free_spacing
+      token = :options_switch
     end
 
     emit(:group, token, text, ts, te)
   end
 
+  def emit_meta_control_sequence(data, ts, te, token)
+    if data.last < 0x00 || data.last > 0x7F
+      validation_error(:sequence, 'escape', token.to_s)
+    end
+    emit(:escape, token, *text(data, ts, te, 1))
+  end
+
   # Centralizes and unifies the handling of validation related
   # errors.
   def validation_error(type, what, reason)
@@ -981,21 +886,8 @@ class Regexp::Scanner
       error = InvalidBackrefError.new(what, reason)
     when :sequence
       error = InvalidSequenceError.new(what, reason)
-    else
-      error = ValidationError.new('expression')
     end
 
     raise error # unless @@config.validation_ignore
   end
-
-  # Used for references with an empty name or number
-  def empty_backref_error(type, what)
-    validation_error(:backref, what, 'ref ID is empty')
-  end
-
-  # Used for named expressions with an empty name
-  def empty_name_error(type, what)
-    validation_error(type, what, 'name is empty')
-  end
-
 end # module Regexp::Scanner