regexp_parser 1.5.0 → 1.5.1

data/lib/regexp_parser/scanner/char_type.rl

@@ -21,9 +21,6 @@
       when '\W'; emit(:type, :nonword,    text, ts - 1, te)
       when '\R'; emit(:type, :linebreak,  text, ts - 1, te)
       when '\X'; emit(:type, :xgrapheme,  text, ts - 1, te)
-      else
-        raise ScannerError.new(
-          "Unexpected character in type at #{text} (char #{ts})")
       end
       fret;
     };

data/lib/regexp_parser/scanner/scanner.rl

@@ -49,9 +49,9 @@
   codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
   codepoint_sequence    = codepoint_single | codepoint_list;
 
-  control_sequence      = ('c' | 'C-') . (backslash . 'M-')?;
+  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;
 
-  meta_sequence         = 'M-' . (backslash . control_sequence)?;
+  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;
 
   zero_or_one           = '?' | '??' | '?+';
   zero_or_more          = '*' | '*?' | '*+';
@@ -82,7 +82,8 @@
   assertion_lookbehind  = '?<=';
   assertion_nlookbehind = '?<!';
 
-  group_options         = '?' . [\-mixdau];
+  # try to treat every other group head as options group, like Ruby
+  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;
 
   group_ref             = [gk];
   group_name_char       = (alnum | '_');
@@ -135,41 +136,35 @@
   # Invalid sequence error, used from sequences, like escapes and sets
   action invalid_sequence_error {
     text = ts ? copy(data, ts-1..-1) : data.pack('c*')
-    raise InvalidSequenceError.new('sequence', text)
+    validation_error(:sequence, 'sequence', text)
   }
 
   # group (nesting) and set open/close actions
-  action group_opened { self.group_depth = group_depth + 1; in_group = true }
-  action group_closed { self.group_depth = group_depth - 1; in_group = group_depth > 0 ? true : false }
+  action group_opened { self.group_depth = group_depth + 1 }
+  action group_closed { self.group_depth = group_depth - 1 }
+  action set_opened   { self.set_depth   = set_depth   + 1 }
+  action set_closed   { self.set_depth   = set_depth   - 1 }
 
   # Character set scanner, continues consuming characters until it meets the
   # closing bracket of the set.
   # --------------------------------------------------------------------------
   character_set := |*
-    set_close > (set_meta, 2) {
-      set_depth -= 1
-      in_set = set_depth > 0 ? true : false
-
+    set_close > (set_meta, 2) @set_closed {
       emit(:set, :close, *text(data, ts, te))
-
-      if set_depth == 0
-        fgoto main;
-      else
+      if in_set?
         fret;
+      else
+        fgoto main;
       end
     };
 
-    '-]' { # special case, emits two tokens
-      set_depth -= 1
-      in_set = set_depth > 0 ? true : false
-
-      emit(:literal, :literal, copy(data, ts..te-2), ts, te)
-      emit(:set, :close, copy(data, ts+1..te-1), ts, te)
-
-      if set_depth == 0
-        fgoto main;
-      else
+    '-]' @set_closed { # special case, emits two tokens
+      emit(:literal, :literal, copy(data, ts..te-2), ts, te - 1)
+      emit(:set, :close, copy(data, ts+1..te-1), ts + 1, te)
+      if in_set?
         fret;
+      else
+        fgoto main;
       end
     };
 
@@ -207,14 +202,12 @@
       fcall set_escape_sequence;
     };
 
-    set_open >(open_bracket, 1) {
-      set_depth += 1
-
+    set_open >(open_bracket, 1) >set_opened {
       emit(:set, :open, *text(data, ts, te))
       fcall character_set;
     };
 
-    class_posix >(open_bracket, 1) @eof(premature_end_error) {
+    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       text = text(data, ts, te).first
 
       type = :posixclass
@@ -227,11 +220,11 @@
       emit(type, class_name.to_sym, text, ts, te)
     };
 
-    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
+    collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       emit(:set, :collation, *text(data, ts, te))
     };
 
-    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
+    character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
       emit(:set, :equivalent, *text(data, ts, te))
     };
 
@@ -337,44 +330,24 @@
     };
 
     control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
-      if data[te]
-        c = data[te].chr
-        if c =~ /[\x00-\x7F]/
-          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
-          p += 1
-        else
-          raise InvalidSequenceError.new("control sequence")
-        end
-      else
-        raise PrematureEndError.new("control sequence")
-      end
+      emit_meta_control_sequence(data, ts, te, :control)
       fret;
     };
 
     meta_sequence >(backslashed, 3) $eof(premature_end_error) {
-      if data[te]
-        c = data[te].chr
-        if c =~ /[\x00-\x7F]/
-          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
-          p += 1
-        else
-          raise InvalidSequenceError.new("meta sequence")
-        end
-      else
-        raise PrematureEndError.new("meta sequence")
-      end
+      emit_meta_control_sequence(data, ts, te, :meta_sequence)
       fret;
     };
 
     char_type_char > (escaped_alpha, 2) {
       fhold;
-      fnext *(in_set ? fentry(character_set) : fentry(main));
+      fnext *(in_set? ? fentry(character_set) : fentry(main));
       fcall char_type;
     };
 
     property_char > (escaped_alpha, 2) {
       fhold;
-      fnext *(in_set ? fentry(character_set) : fentry(main));
+      fnext *(in_set? ? fentry(character_set) : fentry(main));
       fcall unicode_property;
     };
 
@@ -412,8 +385,7 @@
     };
 
     alternation {
-      if in_conditional and conditional_stack.length > 0 and
-         conditional_stack.last[1] == group_depth
+      if conditional_stack.last == group_depth
         emit(:conditional, :separator, *text(data, ts, te))
       else
         emit(:meta, :alternation, *text(data, ts, te))
@@ -442,18 +414,12 @@
       when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
       when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
       when '\\G'; emit(:anchor, :match_start,        text, ts, te)
-      else
-        raise ScannerError.new(
-          "Unexpected character in anchor at #{text} (char #{ts})")
       end
     };
 
     # Character sets
     # ------------------------------------------------------------------------
-    set_open {
-      set_depth += 1
-      in_set = true
-
+    set_open >set_opened {
       emit(:set, :open, *text(data, ts, te))
       fcall character_set;
     };
@@ -465,9 +431,7 @@
     conditional {
       text = text(data, ts, te).first
 
-      in_conditional = true unless in_conditional
-      conditional_depth += 1
-      conditional_stack << [conditional_depth, group_depth]
+      conditional_stack << group_depth
 
       emit(:conditional, :open, text[0..-2], ts, te-1)
       emit(:conditional, :condition_open, '(', te-1, te)
@@ -496,7 +460,11 @@
     #   (?imxdau-imx:subexp)  option on/off for subexp
     # ------------------------------------------------------------------------
     group_open . group_options >group_opened {
-      p = scan_options(p, data, ts, te)
+      text = text(data, ts, te).first
+      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
+        raise InvalidGroupOption.new($1 || "-#{$2}", text)
+      end
+      emit_options(text, ts, te)
     };
 
     # Assertions
@@ -528,19 +496,15 @@
       when '(?>';  emit(:group, :atomic,       text, ts, te)
       when '(?~';  emit(:group, :absence,      text, ts, te)
 
-      when /^\(\?<(\w*)>/
-        empty_name_error(:group, 'named group (ab)') if $1.empty?
+      when /^\(\?(?:<>|'')/
+        validation_error(:group, 'named group', 'name is empty')
 
+      when /^\(\?<\w*>/
         emit(:group, :named_ab,  text, ts, te)
 
-      when /^\(\?'(\w*)'/
-        empty_name_error(:group, 'named group (sq)') if $1.empty?
-
+      when /^\(\?'\w*'/
         emit(:group, :named_sq,  text, ts, te)
 
-      else
-        raise ScannerError.new(
-          "Unknown subexpression group format '#{text}'")
       end
     };
 
@@ -550,20 +514,13 @@
     };
 
     group_close @group_closed {
-      if in_conditional and conditional_stack.last and
-         conditional_stack.last[1] == (group_depth + 1)
-
-        emit(:conditional, :close, *text(data, ts, te))
+      if conditional_stack.last == group_depth + 1
         conditional_stack.pop
-
-        if conditional_stack.length == 0
-          in_conditional = false
-        end
+        emit(:conditional, :close, *text(data, ts, te))
       else
-        if spacing_stack.length > 1 and
-          spacing_stack.last[:depth] == (group_depth + 1)
+        if spacing_stack.length > 1 &&
+           spacing_stack.last[:depth] == group_depth + 1
           spacing_stack.pop
-
           self.free_spacing = spacing_stack.last[:free_spacing]
         end
 
@@ -576,11 +533,8 @@
     # ------------------------------------------------------------------------
     backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
       case text = text(data, ts, te).first
-      when /^\\([gk])<>/ # angle brackets
-        empty_backref_error("ref/call (ab)")
-
-      when /^\\([gk])''/ # single quotes
-        empty_backref_error("ref/call (sq)")
+      when /^\\([gk])(<>|'')/ # angle brackets
+        validation_error(:backref, 'ref/call', 'ref ID is empty')
 
       when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
         if $1 == 'k'
@@ -636,9 +590,6 @@
       when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
         emit(:backref, :number_recursion_ref_sq, text, ts, te)
 
-      else
-        raise ScannerError.new(
-          "Unknown backreference format '#{text}'")
       end
     };
 
@@ -786,7 +737,7 @@ class Regexp::Scanner
       input = input_object
       self.free_spacing = false
     end
-
+    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*") if input.is_a?(String)
     eof   = data.length
@@ -794,15 +745,9 @@ class Regexp::Scanner
     self.tokens = []
     self.block  = block_given? ? block : nil
 
-    self.in_group = false
+    self.set_depth = 0
     self.group_depth = 0
-    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
-
-    in_set = false
-    set_depth = 0
-    in_conditional = false
-    conditional_depth = 0
-    conditional_stack = []
+    self.conditional_stack = []
 
     %% write data;
     %% write init;
@@ -817,9 +762,9 @@ class Regexp::Scanner
     end
 
     raise PrematureEndError.new("(missing group closing paranthesis) "+
-          "[#{in_group}:#{group_depth}]") if in_group
+          "[#{group_depth}]") if in_group?
     raise PrematureEndError.new("(missing set closing bracket) "+
-          "[#{in_set}:#{set_depth}]") if in_set
+          "[#{set_depth}]") if in_set?
 
     # when the entire expression is a literal run
     emit_literal if literal
@@ -854,62 +799,15 @@ class Regexp::Scanner
 
   private
 
-  attr_accessor :tokens, :literal, :block,
-                :in_group, :group_depth,
-                :free_spacing, :spacing_stack
-
-  # Ragel's regex-based scan of the group options introduced a lot of
-  # ambiguity, so we just ask it to find the beginning of what looks
-  # like an options run and handle the rest in here.
-  def scan_options(p, data, ts, te)
-    text = text(data, ts, te).first
-
-    options_char, options_length = true, 0
-
-    # Copy while we have option characters. There is no maximum length,
-    # as ruby allows things like '(?xxxxxxxxx-xxxxxxxxxxxxx:abc)'.
-    negative_options = false
-    while options_char
-      if data[te + options_length]
-        c = data[te + options_length].chr
-
-        if c =~ /[-mixdau]/
-          negative_options = true if c == '-'
+  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
+                :group_depth, :set_depth, :conditional_stack
 
-          raise InvalidGroupOption.new(c, text) if negative_options and
-            c =~ /[dau]/
-
-          text << c ; p += 1 ; options_length += 1
-        else
-          options_char = false
-        end
-      else
-        raise PrematureEndError.new("expression options `#{text}'")
-      end
-    end
-
-    if data[te + options_length]
-      c = data[te + options_length].chr
-
-      if c == ':'
-        # Include the ':' in the options text
-        text << c ; p += 1 ; options_length += 1
-        emit_options(text, ts, te + options_length)
-
-      elsif c == ')'
-        # Don't include the closing ')', let group_close handle it.
-        emit_options(text, ts, te + options_length)
-
-      else
-        # Plain Regexp reports this as 'undefined group option'
-        raise ScannerError.new(
-          "Unexpected `#{c}' in options sequence, ':' or ')' expected")
-      end
-    else
-      raise PrematureEndError.new("expression options `#{text}'")
-    end
+  def in_group?
+    group_depth > 0
+  end
 
-    p # return the new value of the data pointer
+  def in_set?
+    set_depth > 0
   end
 
   # Copy from ts to te from data as text
@@ -945,32 +843,39 @@ class Regexp::Scanner
   def emit_options(text, ts, te)
     token = nil
 
-    if text =~ /\(\?([mixdau]*)-?([mix]*)(:)?/
-      positive, negative, group_local = $1, $2, $3
+    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
+    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
+    positive, negative, group_local = $1, $2, $3
 
-      if positive.include?('x')
-        self.free_spacing = true
-      end
+    if positive.include?('x')
+      self.free_spacing = true
+    end
 
-      # If the x appears in both, treat it like ruby does, the second cancels
-      # the first.
-      if negative.include?('x')
-        self.free_spacing = false
-      end
+    # If the x appears in both, treat it like ruby does, the second cancels
+    # the first.
+    if negative && negative.include?('x')
+      self.free_spacing = false
+    end
 
-      if group_local
-        spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
-        token = :options
-      else
-        # switch for parent group level
-        spacing_stack.last[:free_spacing] = free_spacing
-        token = :options_switch
-      end
+    if group_local
+      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
+      token = :options
+    else
+      # switch for parent group level
+      spacing_stack.last[:free_spacing] = free_spacing
+      token = :options_switch
     end
 
     emit(:group, token, text, ts, te)
   end
 
+  def emit_meta_control_sequence(data, ts, te, token)
+    if data.last < 0x00 || data.last > 0x7F
+      validation_error(:sequence, 'escape', token.to_s)
+    end
+    emit(:escape, token, *text(data, ts, te, 1))
+  end
+
   # Centralizes and unifies the handling of validation related
   # errors.
   def validation_error(type, what, reason)
@@ -981,21 +886,8 @@ class Regexp::Scanner
       error = InvalidBackrefError.new(what, reason)
     when :sequence
       error = InvalidSequenceError.new(what, reason)
-    else
-      error = ValidationError.new('expression')
     end
 
     raise error # unless @@config.validation_ignore
   end
-
-  # Used for references with an empty name or number
-  def empty_backref_error(type, what)
-    validation_error(:backref, what, 'ref ID is empty')
-  end
-
-  # Used for named expressions with an empty name
-  def empty_name_error(type, what)
-    validation_error(type, what, 'name is empty')
-  end
-
 end # module Regexp::Scanner