regexp_parser 2.3.0 → 2.5.0

data/lib/regexp_parser/expression/shared.rb

@@ -0,0 +1,85 @@
+module Regexp::Expression
+  module Shared
+    module ClassMethods; end # filled in ./methods/*.rb
+
+    def self.included(mod)
+      mod.class_eval do
+        extend Shared::ClassMethods
+
+        attr_accessor :type, :token, :text, :ts, :te,
+                      :level, :set_level, :conditional_level,
+                      :options, :quantifier
+
+        attr_reader   :nesting_level
+      end
+    end
+
+    def init_from_token_and_options(token, options = {})
+      self.type              = token.type
+      self.token             = token.token
+      self.text              = token.text
+      self.ts                = token.ts
+      self.te                = token.te
+      self.level             = token.level
+      self.set_level         = token.set_level
+      self.conditional_level = token.conditional_level
+      self.nesting_level     = 0
+      self.options           = options || {}
+    end
+    private :init_from_token_and_options
+
+    def initialize_copy(orig)
+      self.text       = orig.text.dup         if orig.text
+      self.options    = orig.options.dup      if orig.options
+      self.quantifier = orig.quantifier.clone if orig.quantifier
+      super
+    end
+
+    def starts_at
+      ts
+    end
+
+    def base_length
+      to_s(:base).length
+    end
+
+    def full_length
+      to_s.length
+    end
+
+    def to_s(format = :full)
+      "#{parts.join}#{quantifier_affix(format)}"
+    end
+    alias :to_str :to_s
+
+    def parts
+      [text.dup]
+    end
+
+    def quantifier_affix(expression_format)
+      quantifier.to_s if quantified? && expression_format != :base
+    end
+
+    def quantified?
+      !quantifier.nil?
+    end
+
+    def offset
+      [starts_at, full_length]
+    end
+
+    def coded_offset
+      '@%d+%d' % offset
+    end
+
+    def terminal?
+      !respond_to?(:expressions)
+    end
+
+    def nesting_level=(lvl)
+      @nesting_level = lvl
+      quantifier && quantifier.nesting_level = lvl
+      terminal? || each { |subexp| subexp.nesting_level = lvl + 1 }
+    end
+  end
+end

data/lib/regexp_parser/expression/subexpression.rb

@@ -5,9 +5,8 @@ module Regexp::Expression
     attr_accessor :expressions
 
     def initialize(token, options = {})
-      super
-
       self.expressions = []
+      super
     end
 
     # Override base method to clone the expressions as well.
@@ -43,16 +42,21 @@ module Regexp::Expression
       ts + to_s.length
     end
 
-    def to_s(format = :full)
-      # Note: the format does not get passed down to subexpressions.
-      "#{expressions.join}#{quantifier_affix(format)}"
+    def parts
+      expressions
     end
 
     def to_h
-      attributes.merge({
+      attributes.merge(
         text:        to_s(:base),
         expressions: expressions.map(&:to_h)
-      })
+      )
+    end
+
+    private
+
+    def intersperse(expressions, separator)
+      expressions.flat_map { |exp| [exp, separator] }.slice(0...-1)
     end
   end
 end

data/lib/regexp_parser/expression.rb

@@ -1,5 +1,6 @@
 require 'regexp_parser/error'
 
+require 'regexp_parser/expression/shared'
 require 'regexp_parser/expression/base'
 require 'regexp_parser/expression/quantifier'
 require 'regexp_parser/expression/subexpression'
@@ -12,6 +13,7 @@ require 'regexp_parser/expression/classes/backreference'
 require 'regexp_parser/expression/classes/character_set'
 require 'regexp_parser/expression/classes/character_set/intersection'
 require 'regexp_parser/expression/classes/character_set/range'
+require 'regexp_parser/expression/classes/character_type'
 require 'regexp_parser/expression/classes/conditional'
 require 'regexp_parser/expression/classes/escape_sequence'
 require 'regexp_parser/expression/classes/free_space'
@@ -19,10 +21,10 @@ require 'regexp_parser/expression/classes/group'
 require 'regexp_parser/expression/classes/keep'
 require 'regexp_parser/expression/classes/literal'
 require 'regexp_parser/expression/classes/posix_class'
-require 'regexp_parser/expression/classes/property'
 require 'regexp_parser/expression/classes/root'
-require 'regexp_parser/expression/classes/type'
+require 'regexp_parser/expression/classes/unicode_property'
 
+require 'regexp_parser/expression/methods/construct'
 require 'regexp_parser/expression/methods/match'
 require 'regexp_parser/expression/methods/match_length'
 require 'regexp_parser/expression/methods/options'

data/lib/regexp_parser/parser.rb

@@ -23,7 +23,7 @@ class Regexp::Parser
   end
 
   def parse(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
-    root = Root.build(extract_options(input, options))
+    root = Root.construct(options: extract_options(input, options))
 
     self.root = root
     self.node = root
@@ -39,6 +39,9 @@ class Regexp::Parser
       parse_token(token)
     end
 
+    # Trigger recursive setting of #nesting_level, which reflects how deep
+    # a node is in the tree. Do this at the end to account for tree rewrites.
+    root.nesting_level = 0
     assign_referenced_expressions
 
     if block_given?
@@ -197,11 +200,11 @@ class Regexp::Parser
   end
 
   def captured_group_count_at_level
-    captured_group_counts[node.level]
+    captured_group_counts[node]
   end
 
   def count_captured_group
-    captured_group_counts[node.level] += 1
+    captured_group_counts[node] += 1
   end
 
   def close_group
@@ -286,17 +289,9 @@ class Regexp::Parser
   def nest(exp)
     nesting.push(exp)
     node << exp
-    update_transplanted_subtree(exp, node)
     self.node = exp
   end
 
-  # subtrees are transplanted to build Alternations, Intersections, Ranges
-  def update_transplanted_subtree(exp, new_parent)
-    exp.nesting_level = new_parent.nesting_level + 1
-    exp.respond_to?(:each) &&
-      exp.each { |subexp| update_transplanted_subtree(subexp, exp) }
-  end
-
   def escape(token)
     case token.token
 
@@ -480,79 +475,33 @@ class Regexp::Parser
     # description of the problem: https://github.com/ammar/regexp_parser/issues/3
     # rationale for this solution: https://github.com/ammar/regexp_parser/pull/69
     if target_node.quantified?
-      new_token = Regexp::Token.new(
-        :group,
-        :passive,
-        '', # text
-        target_node.ts,
-        nil, # te (unused)
-        target_node.level,
-        target_node.set_level,
-        target_node.conditional_level
+      new_group = Group::Passive.construct(
+        token:             :passive,
+        ts:                target_node.ts,
+        level:             target_node.level,
+        set_level:         target_node.set_level,
+        conditional_level: target_node.conditional_level,
+        options:           active_opts,
       )
-      new_group = Group::Passive.new(new_token, active_opts)
       new_group.implicit = true
       new_group << target_node
-      increase_level(target_node)
+      increase_group_level(target_node)
       node.expressions[node.expressions.index(target_node)] = new_group
       target_node = new_group
     end
 
-    case token.token
-    when :zero_or_one
-      target_node.quantify(:zero_or_one, token.text, 0, 1, :greedy)
-    when :zero_or_one_reluctant
-      target_node.quantify(:zero_or_one, token.text, 0, 1, :reluctant)
-    when :zero_or_one_possessive
-      target_node.quantify(:zero_or_one, token.text, 0, 1, :possessive)
-
-    when :zero_or_more
-      target_node.quantify(:zero_or_more, token.text, 0, -1, :greedy)
-    when :zero_or_more_reluctant
-      target_node.quantify(:zero_or_more, token.text, 0, -1, :reluctant)
-    when :zero_or_more_possessive
-      target_node.quantify(:zero_or_more, token.text, 0, -1, :possessive)
-
-    when :one_or_more
-      target_node.quantify(:one_or_more, token.text, 1, -1, :greedy)
-    when :one_or_more_reluctant
-      target_node.quantify(:one_or_more, token.text, 1, -1, :reluctant)
-    when :one_or_more_possessive
-      target_node.quantify(:one_or_more, token.text, 1, -1, :possessive)
-
-    when :interval
-      interval(target_node, token)
-
-    else
+    unless token.token =~ /\A(?:zero_or_one|zero_or_more|one_or_more|interval)
+                             (?:_greedy|_reluctant|_possessive)?\z/x
       raise UnknownTokenError.new('Quantifier', token)
     end
+
+    target_node.quantify(token, active_opts)
   end
 
-  def increase_level(exp)
+  def increase_group_level(exp)
     exp.level += 1
-    exp.respond_to?(:each) && exp.each { |subexp| increase_level(subexp) }
-  end
-
-  def interval(target_node, token)
-    text = token.text
-    mchr = text[text.length-1].chr =~ /[?+]/ ? text[text.length-1].chr : nil
-    case mchr
-    when '?'
-      range_text = text[0...-1]
-      mode = :reluctant
-    when '+'
-      range_text = text[0...-1]
-      mode = :possessive
-    else
-      range_text = text
-      mode = :greedy
-    end
-
-    range = range_text.gsub(/\{|\}/, '').split(',', 2)
-    min = range[0].empty? ? 0 : range[0]
-    max = range[1] ? (range[1].empty? ? -1 : range[1]) : min
-
-    target_node.quantify(:interval, text, min.to_i, max.to_i, mode)
+    exp.quantifier.level += 1 if exp.quantifier
+    exp.terminal? || exp.each { |subexp| increase_group_level(subexp) }
   end
 
   def set(token)

data/lib/regexp_parser/scanner/property.rl

@@ -20,7 +20,7 @@
       name = data[ts+2..te-2].pack('c*').gsub(/[\^\s_\-]/, '').downcase
 
       token = self.class.short_prop_map[name] || self.class.long_prop_map[name]
-      raise UnknownUnicodePropertyError.new(name) unless token
+      validation_error(:property, name) unless token
 
       self.emit(type, token.to_sym, text)
 

data/lib/regexp_parser/scanner/scanner.rl

@@ -28,13 +28,7 @@
 
   comment               = ('#' . [^\n]* . '\n'?);
 
-  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
-                          'cntrl' | 'digit' | 'graph' |
-                          'lower' | 'print' | 'punct' |
-                          'space' | 'upper' | 'xdigit' |
-                          'word'  | 'ascii';
-
-  class_posix           = ('[:' . '^'? . class_name_posix . ':]');
+  class_posix           = ('[:' . '^'? . [^\[\]]* . ':]');
 
 
   # these are not supported in ruby at the moment
@@ -74,8 +68,7 @@
   quantity_maximum      = ',' . (digit+);
   quantity_range        = (digit+) . ',' . (digit+);
   quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
-                          quantity_maximum | quantity_range ) . range_close .
-                          quantifier_mode?;
+                          quantity_maximum | quantity_range ) . range_close;
 
   quantifiers           = quantifier_greedy | quantifier_reluctant |
                           quantifier_possessive | quantifier_interval;
@@ -223,24 +216,28 @@
       fcall character_set;
     };
 
-    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error) {
       text = copy(data, ts, te)
 
       type = :posixclass
       class_name = text[2..-3]
-      if class_name[0].chr == '^'
+      if class_name[0] == '^'
         class_name = class_name[1..-1]
         type = :nonposixclass
       end
 
+      unless self.class.posix_classes.include?(class_name)
+        validation_error(:posix_class, text)
+      end
+
       emit(type, class_name.to_sym, text)
     };
 
     # These are not supported in ruby at the moment. Enable them if they are.
-    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error) {
     #   emit(:set, :collation, copy(data, ts, te))
     # };
-    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error) {
     #   emit(:set, :equivalent, copy(data, ts, te))
     # };
 
@@ -323,7 +320,7 @@
 
     codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
       text = copy(data, ts-1, te)
-      if text[2].chr == '{'
+      if text[2] == '{'
         emit(:escape, :codepoint_list, text)
       else
         emit(:escape, :codepoint,      text)
@@ -419,12 +416,12 @@
 
     backslash . anchor_char > (backslashed, 3) {
       case text = copy(data, ts, te)
-      when '\\A'; emit(:anchor, :bos,                text)
-      when '\\z'; emit(:anchor, :eos,                text)
-      when '\\Z'; emit(:anchor, :eos_ob_eol,         text)
-      when '\\b'; emit(:anchor, :word_boundary,      text)
-      when '\\B'; emit(:anchor, :nonword_boundary,   text)
-      when '\\G'; emit(:anchor, :match_start,        text)
+      when '\A';  emit(:anchor, :bos,                text)
+      when '\z';  emit(:anchor, :eos,                text)
+      when '\Z';  emit(:anchor, :eos_ob_eol,         text)
+      when '\b';  emit(:anchor, :word_boundary,      text)
+      when '\B';  emit(:anchor, :nonword_boundary,   text)
+      when '\G';  emit(:anchor, :match_start,        text)
       end
     };
 
@@ -477,7 +474,7 @@
     group_open . group_options >group_opened {
       text = copy(data, ts, te)
       if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
-        raise InvalidGroupOption.new($1 || "-#{$2}", text)
+        validation_error(:group_option, $1 || "-#{$2}", text)
       end
       emit_options(text)
     };
@@ -605,7 +602,7 @@
       end
     };
 
-    quantifier_interval  {
+    quantifier_interval {
       emit(:quantifier, :interval, copy(data, ts, te))
     };
 
@@ -686,6 +683,7 @@ class Regexp::Scanner
   end
 
   # Invalid groupOption. Used for inline options.
+  # TODO: should become InvalidGroupOptionError in v3.0.0 for consistency
   class InvalidGroupOption < ValidationError
     def initialize(option, text)
       super "Invalid group option #{option} in #{text}"
@@ -706,6 +704,13 @@ class Regexp::Scanner
     end
   end
 
+  # The POSIX class name was not recognized by the scanner.
+  class UnknownPosixClassError < ValidationError
+    def initialize(text)
+      super "Unknown POSIX class #{text}"
+    end
+  end
+
   # Scans the given regular expression text, or Regexp object and collects the
   # emitted token into an array that gets returned at the end. If a block is
   # given, it gets called for each emitted token.
@@ -771,6 +776,11 @@ class Regexp::Scanner
     File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
   end
 
+  def self.posix_classes
+    %w[alnum alpha ascii blank cntrl digit graph
+       lower print punct space upper word xdigit]
+  end
+
   # Emits an array with the details of the scanned pattern
   def emit(type, token, text)
     #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"
@@ -873,15 +883,16 @@ class Regexp::Scanner
 
   # Centralizes and unifies the handling of validation related
   # errors.
-  def validation_error(type, what, reason)
-    case type
-    when :group
-      error = InvalidGroupError.new(what, reason)
-    when :backref
-      error = InvalidBackrefError.new(what, reason)
-    when :sequence
-      error = InvalidSequenceError.new(what, reason)
-    end
+  def validation_error(type, what, reason = nil)
+    error =
+      case type
+      when :backref      then InvalidBackrefError.new(what, reason)
+      when :group        then InvalidGroupError.new(what, reason)
+      when :group_option then InvalidGroupOption.new(what, reason)
+      when :posix_class  then UnknownPosixClassError.new(what)
+      when :property     then UnknownUnicodePropertyError.new(what)
+      when :sequence     then InvalidSequenceError.new(what, reason)
+      end
 
     raise error # unless @@config.validation_ignore
   end