regexp_parser 2.4.0 → 2.7.0

data/lib/regexp_parser/expression/methods/human_name.rb

@@ -0,0 +1,43 @@
+module Regexp::Expression
+  module Shared
+    # default implementation, e.g. "atomic group", "hex escape", "word type", ..
+    def human_name
+      [token, type].compact.join(' ').tr('_', ' ')
+    end
+  end
+
+  Alternation.class_eval                       { def human_name; 'alternation'                 end }
+  Alternative.class_eval                       { def human_name; 'alternative'                 end }
+  Anchor::BOL.class_eval                       { def human_name; 'beginning of line'           end }
+  Anchor::BOS.class_eval                       { def human_name; 'beginning of string'         end }
+  Anchor::EOL.class_eval                       { def human_name; 'end of line'                 end }
+  Anchor::EOS.class_eval                       { def human_name; 'end of string'               end }
+  Anchor::EOSobEOL.class_eval                  { def human_name; 'newline-ready end of string' end }
+  Anchor::MatchStart.class_eval                { def human_name; 'match start'                 end }
+  Anchor::NonWordBoundary.class_eval           { def human_name; 'no word boundary'            end }
+  Anchor::WordBoundary.class_eval              { def human_name; 'word boundary'               end }
+  Assertion::Lookahead.class_eval              { def human_name; 'lookahead'                   end }
+  Assertion::Lookbehind.class_eval             { def human_name; 'lookbehind'                  end }
+  Assertion::NegativeLookahead.class_eval      { def human_name; 'negative lookahead'          end }
+  Assertion::NegativeLookbehind.class_eval     { def human_name; 'negative lookbehind'         end }
+  Backreference::Name.class_eval               { def human_name; 'backreference by name'       end }
+  Backreference::NameCall.class_eval           { def human_name; 'subexpression call by name'  end }
+  Backreference::Number.class_eval             { def human_name; 'backreference'               end }
+  Backreference::NumberRelative.class_eval     { def human_name; 'relative backreference'      end }
+  Backreference::NumberCall.class_eval         { def human_name; 'subexpression call'          end }
+  Backreference::NumberCallRelative.class_eval { def human_name; 'relative subexpression call' end }
+  CharacterSet::IntersectedSequence.class_eval { def human_name; 'intersected sequence'        end }
+  CharacterSet::Intersection.class_eval        { def human_name; 'intersection'                end }
+  CharacterSet::Range.class_eval               { def human_name; 'character range'             end }
+  CharacterType::Any.class_eval                { def human_name; 'match-all'                   end }
+  Comment.class_eval                           { def human_name; 'comment'                     end }
+  Conditional::Branch.class_eval               { def human_name; 'conditional branch'          end }
+  Conditional::Condition.class_eval            { def human_name; 'condition'                   end }
+  Conditional::Expression.class_eval           { def human_name; 'conditional'                 end }
+  Group::Capture.class_eval                    { def human_name; "capture group #{number}"     end }
+  Group::Named.class_eval                      { def human_name; 'named capture group'         end }
+  Keep::Mark.class_eval                        { def human_name; 'keep-mark lookbehind'        end }
+  Literal.class_eval                           { def human_name; 'literal'                     end }
+  Root.class_eval                              { def human_name; 'root'                        end }
+  WhiteSpace.class_eval                        { def human_name; 'free space'                  end }
+end

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -63,16 +63,20 @@ class Regexp::MatchLength
   end
 
   def to_re
-    "(?:#{reify.call}){#{min_rep},#{max_rep unless max_rep == Float::INFINITY}}"
+    /(?:#{reify.call}){#{min_rep},#{max_rep unless max_rep == Float::INFINITY}}/
   end
 
   private
 
   attr_accessor :base_min, :base_max, :min_rep, :max_rep, :exp_class, :reify
 
-  def test_regexp
-    @test_regexp ||= Regexp.new("^#{to_re}$").tap do |regexp|
-      regexp.respond_to?(:match?) || def regexp.match?(str); !!match(str) end
+  if Regexp.method_defined?(:match?) # ruby >= 2.4
+    def test_regexp
+      @test_regexp ||= /^#{to_re}$/
+    end
+  else
+    def test_regexp
+      @test_regexp ||= /^#{to_re}$/.tap { |r| def r.match?(s); !!match(s) end }
     end
   end
 end
@@ -112,7 +116,7 @@ module Regexp::Expression
     end
 
     def inner_match_length
-      dummy = Regexp::Expression::Root.build
+      dummy = Regexp::Expression::Root.construct
       dummy.expressions = expressions.map(&:clone)
       dummy.quantifier = quantifier && quantifier.clone
       dummy.match_length

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -36,11 +36,14 @@ module Regexp::Expression
 
     # Iterates over the expressions of this expression as an array, passing
     # the expression and its index within its parent to the given block.
-    def each_expression(include_self = false)
+    def each_expression(include_self = false, &block)
       return enum_for(__method__, include_self) unless block_given?
 
-      traverse(include_self) do |event, exp, index|
-        yield(exp, index) unless event == :exit
+      block.call(self, 0) if include_self
+
+      each_with_index do |exp, index|
+        block.call(exp, index)
+        exp.each_expression(&block) unless exp.terminal?
       end
     end
 

data/lib/regexp_parser/expression/quantifier.rb

@@ -14,7 +14,7 @@ module Regexp::Expression
       deprecated_old_init(*args) and return if args.count == 4 || args.count == 5
 
       init_from_token_and_options(*args)
-      @mode = (token[/greedy|reluctant|possessive/] || :greedy).to_sym
+      @mode = (token.to_s[/greedy|reluctant|possessive/] || :greedy).to_sym
       @min, @max = minmax
       # TODO: remove in v3.0.0, stop removing parts of #token (?)
       self.token = token.to_s.sub(/_(greedy|possessive|reluctant)/, '').to_sym
@@ -44,10 +44,11 @@ module Regexp::Expression
     def deprecated_old_init(token, text, min, max, mode = :greedy)
       warn "Calling `Expression::Base#quantify` or `#{self.class}.new` with 4+ arguments "\
            "is deprecated.\nIt will no longer be supported in regexp_parser v3.0.0.\n"\
-           "Please pass a Regexp::Token instead, e.g. replace `type, text, min, max, mode` "\
-           "with `::Regexp::Token.new(:quantifier, type, text)`. min, max, and mode "\
-           "will be derived automatically. \nThis is consistent with how Expression::Base "\
-           "instances are created."
+           "Please pass a Regexp::Token instead, e.g. replace `token, text, min, max, mode` "\
+           "with `::Regexp::Token.new(:quantifier, token, text)`. min, max, and mode "\
+           "will be derived automatically.\n"\
+           "Or do `exp.quantifier = #{self.class}.construct(token: token, text: str)`.\n"\
+           "This is consistent with how Expression::Base instances are created. "
       @token = token
       @text  = text
       @min   = min

data/lib/regexp_parser/expression/sequence.rb

@@ -7,31 +7,16 @@ module Regexp::Expression
   # branches, and CharacterSet::Intersection intersected sequences.
   class Sequence < Regexp::Expression::Subexpression
     class << self
-      def add_to(subexpression, params = {}, active_opts = {})
-        sequence = at_levels(
-          subexpression.level,
-          subexpression.set_level,
-          params[:conditional_level] || subexpression.conditional_level
+      def add_to(exp, params = {}, active_opts = {})
+        sequence = construct(
+          level:             exp.level,
+          set_level:         exp.set_level,
+          conditional_level: params[:conditional_level] || exp.conditional_level,
         )
-        sequence.nesting_level = subexpression.nesting_level + 1
         sequence.options = active_opts
-        subexpression.expressions << sequence
+        exp.expressions << sequence
         sequence
       end
-
-      def at_levels(level, set_level, conditional_level)
-        token = Regexp::Token.new(
-          :expression,
-          :sequence,
-          '',
-          nil, # ts
-          nil, # te
-          level,
-          set_level,
-          conditional_level
-        )
-        new(token)
-      end
     end
 
     def starts_at

data/lib/regexp_parser/expression/shared.rb

@@ -1,12 +1,16 @@
 module Regexp::Expression
   module Shared
+    module ClassMethods; end # filled in ./methods/*.rb
+
     def self.included(mod)
       mod.class_eval do
+        extend Shared::ClassMethods
+
         attr_accessor :type, :token, :text, :ts, :te,
                       :level, :set_level, :conditional_level,
-                      :options, :quantifier
+                      :options
 
-        attr_reader   :nesting_level
+        attr_reader   :nesting_level, :quantifier
       end
     end
 
@@ -60,6 +64,10 @@ module Regexp::Expression
       !quantifier.nil?
     end
 
+    def optional?
+      quantified? && quantifier.min == 0
+    end
+
     def offset
       [starts_at, full_length]
     end
@@ -69,7 +77,11 @@ module Regexp::Expression
     end
 
     def terminal?
-      !respond_to?(:expressions)
+      true # overridden to be false in Expression::Subexpression
+    end
+
+    def referential?
+      false # overridden to be true e.g. in Expression::Backreference::Base
     end
 
     def nesting_level=(lvl)
@@ -77,5 +89,10 @@ module Regexp::Expression
       quantifier && quantifier.nesting_level = lvl
       terminal? || each { |subexp| subexp.nesting_level = lvl + 1 }
     end
+
+    def quantifier=(qtf)
+      @quantifier = qtf
+      @repetitions = nil # clear memoized value
+    end
   end
 end

data/lib/regexp_parser/expression/subexpression.rb

@@ -19,7 +19,6 @@ module Regexp::Expression
       if exp.is_a?(WhiteSpace) && last && last.is_a?(WhiteSpace)
         last.merge(exp)
       else
-        exp.nesting_level = nesting_level + 1
         expressions << exp
       end
     end
@@ -53,6 +52,10 @@ module Regexp::Expression
       )
     end
 
+    def terminal?
+      false
+    end
+
     private
 
     def intersperse(expressions, separator)

data/lib/regexp_parser/expression.rb

@@ -13,6 +13,7 @@ require 'regexp_parser/expression/classes/backreference'
 require 'regexp_parser/expression/classes/character_set'
 require 'regexp_parser/expression/classes/character_set/intersection'
 require 'regexp_parser/expression/classes/character_set/range'
+require 'regexp_parser/expression/classes/character_type'
 require 'regexp_parser/expression/classes/conditional'
 require 'regexp_parser/expression/classes/escape_sequence'
 require 'regexp_parser/expression/classes/free_space'
@@ -20,10 +21,11 @@ require 'regexp_parser/expression/classes/group'
 require 'regexp_parser/expression/classes/keep'
 require 'regexp_parser/expression/classes/literal'
 require 'regexp_parser/expression/classes/posix_class'
-require 'regexp_parser/expression/classes/property'
 require 'regexp_parser/expression/classes/root'
-require 'regexp_parser/expression/classes/type'
+require 'regexp_parser/expression/classes/unicode_property'
 
+require 'regexp_parser/expression/methods/construct'
+require 'regexp_parser/expression/methods/human_name'
 require 'regexp_parser/expression/methods/match'
 require 'regexp_parser/expression/methods/match_length'
 require 'regexp_parser/expression/methods/options'

data/lib/regexp_parser/lexer.rb

@@ -13,50 +13,68 @@ class Regexp::Lexer
 
   CONDITION_TOKENS = %i[condition condition_close].freeze
 
-  def self.lex(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
-    new.lex(input, syntax, options: options, &block)
+  def self.lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
+    new.lex(input, syntax, options: options, collect_tokens: collect_tokens, &block)
   end
 
-  def lex(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
-    syntax = Regexp::Syntax.for(syntax)
+  def lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
+    syntax = syntax ? Regexp::Syntax.for(syntax) : Regexp::Syntax::CURRENT
 
+    self.block = block
+    self.collect_tokens = collect_tokens
     self.tokens = []
+    self.prev_token = nil
+    self.preprev_token = nil
     self.nesting = 0
     self.set_nesting = 0
     self.conditional_nesting = 0
     self.shift = 0
 
-    last = nil
-    Regexp::Scanner.scan(input, options: options) do |type, token, text, ts, te|
+    Regexp::Scanner.scan(input, options: options, collect_tokens: false) do |type, token, text, ts, te|
       type, token = *syntax.normalize(type, token)
       syntax.check! type, token
 
       ascend(type, token)
 
-      if type == :quantifier and last
-        break_literal(last)        if last.type == :literal
-        break_codepoint_list(last) if last.token == :codepoint_list
+      if (last = prev_token) &&
+         type == :quantifier &&
+         (
+           (last.type == :literal         && (parts = break_literal(last))) ||
+           (last.token == :codepoint_list && (parts = break_codepoint_list(last)))
+         )
+        emit(parts[0])
+        last = parts[1]
       end
 
       current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
                                   nesting, set_nesting, conditional_nesting)
 
-      current = merge_condition(current) if type == :conditional and
-        CONDITION_TOKENS.include?(token)
-
-      last.next = current if last
-      current.previous = last if last
+      if type == :conditional && CONDITION_TOKENS.include?(token)
+        current = merge_condition(current, last)
+      elsif last
+        last.next = current
+        current.previous = last
+        emit(last)
+      end
 
-      tokens << current
-      last = current
+      self.preprev_token = last
+      self.prev_token = current
 
       descend(type, token)
     end
 
-    if block_given?
-      tokens.map { |t| block.call(t) }
+    emit(prev_token) if prev_token
+
+    collect_tokens ? tokens : nil
+  end
+
+  def emit(token)
+    if block
+      # TODO: in v3.0.0, remove `collect_tokens:` kwarg and only collect w/o block
+      res = block.call(token)
+      tokens << res if collect_tokens
     else
-      tokens
+      tokens << token
     end
   end
 
@@ -66,7 +84,9 @@ class Regexp::Lexer
 
   private
 
-  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting, :shift
+  attr_accessor :block,
+                :collect_tokens, :tokens, :prev_token, :preprev_token,
+                :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
     case type
@@ -96,34 +116,46 @@ class Regexp::Lexer
     lead, last, _ = token.text.partition(/.\z/mu)
     return if lead.empty?
 
-    tokens.pop
-    tokens << Regexp::Token.new(:literal, :literal, lead,
+    token_1 = Regexp::Token.new(:literal, :literal, lead,
               token.ts, (token.te - last.length),
               nesting, set_nesting, conditional_nesting)
-    tokens << Regexp::Token.new(:literal, :literal, last,
+    token_2 = Regexp::Token.new(:literal, :literal, last,
               (token.ts + lead.length), token.te,
               nesting, set_nesting, conditional_nesting)
+
+    token_1.previous = preprev_token
+    token_1.next = token_2
+    token_2.previous = token_1 # .next will be set by #lex
+    [token_1, token_2]
   end
 
+  # if a codepoint list is followed by a quantifier, that quantifier applies
+  # to the last codepoint, e.g. /\u{61 62 63}{3}/ =~ 'abccc'
+  # c.f. #break_literal.
   def break_codepoint_list(token)
     lead, _, tail = token.text.rpartition(' ')
     return if lead.empty?
 
-    tokens.pop
-    tokens << Regexp::Token.new(:escape, :codepoint_list, lead + '}',
+    token_1 = Regexp::Token.new(:escape, :codepoint_list, lead + '}',
               token.ts, (token.te - tail.length),
               nesting, set_nesting, conditional_nesting)
-    tokens << Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
+    token_2 = Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
               (token.ts + lead.length + 1), (token.te + 3),
               nesting, set_nesting, conditional_nesting)
 
     self.shift = shift + 3 # one space less, but extra \, u, {, and }
+
+    token_1.previous = preprev_token
+    token_1.next = token_2
+    token_2.previous = token_1 # .next will be set by #lex
+    [token_1, token_2]
   end
 
-  def merge_condition(current)
-    last = tokens.pop
-    Regexp::Token.new(:conditional, :condition, last.text + current.text,
+  def merge_condition(current, last)
+    token = Regexp::Token.new(:conditional, :condition, last.text + current.text,
       last.ts, current.te, nesting, set_nesting, conditional_nesting)
+    token.previous = preprev_token # .next will be set by #lex
+    token
   end
 
 end # module Regexp::Lexer

data/lib/regexp_parser/parser.rb

@@ -18,12 +18,12 @@ class Regexp::Parser
     end
   end
 
-  def self.parse(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
+  def self.parse(input, syntax = nil, options: nil, &block)
     new.parse(input, syntax, options: options, &block)
   end
 
-  def parse(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
-    root = Root.build(extract_options(input, options))
+  def parse(input, syntax = nil, options: nil, &block)
+    root = Root.construct(options: extract_options(input, options))
 
     self.root = root
     self.node = root
@@ -35,7 +35,7 @@ class Regexp::Parser
 
     self.captured_group_counts = Hash.new(0)
 
-    Regexp::Lexer.scan(input, syntax, options: options) do |token|
+    Regexp::Lexer.scan(input, syntax, options: options, collect_tokens: false) do |token|
       parse_token(token)
     end
 
@@ -200,11 +200,11 @@ class Regexp::Parser
   end
 
   def captured_group_count_at_level
-    captured_group_counts[node.level]
+    captured_group_counts[node]
   end
 
   def count_captured_group
-    captured_group_counts[node.level] += 1
+    captured_group_counts[node] += 1
   end
 
   def close_group
@@ -235,7 +235,15 @@ class Regexp::Parser
     when :number, :number_ref
       node << Backreference::Number.new(token, active_opts)
     when :number_recursion_ref
-      node << Backreference::NumberRecursionLevel.new(token, active_opts)
+      node << Backreference::NumberRecursionLevel.new(token, active_opts).tap do |exp|
+        # TODO: should split off new token number_recursion_rel_ref and new
+        # class NumberRelativeRecursionLevel in v3.0.0 to get rid of this
+        if exp.text =~ /[<'][+-]/
+          assign_effective_number(exp)
+        else
+          exp.effective_number = exp.number
+        end
+      end
     when :number_call
       node << Backreference::NumberCall.new(token, active_opts)
     when :number_rel_ref
@@ -254,6 +262,8 @@ class Regexp::Parser
   def assign_effective_number(exp)
     exp.effective_number =
       exp.number + total_captured_group_count + (exp.number < 0 ? 1 : 0)
+    exp.effective_number > 0 ||
+      raise(ParserError, "Invalid reference: #{exp.reference}")
   end
 
   def conditional(token)
@@ -369,7 +379,7 @@ class Regexp::Parser
   end
 
   def sequence_operation(klass, token)
-    unless node.is_a?(klass)
+    unless node.instance_of?(klass)
       operator = klass.new(token, active_opts)
       sequence = operator.add_sequence(active_opts)
       sequence.expressions = node.expressions
@@ -475,17 +485,14 @@ class Regexp::Parser
     # description of the problem: https://github.com/ammar/regexp_parser/issues/3
     # rationale for this solution: https://github.com/ammar/regexp_parser/pull/69
     if target_node.quantified?
-      new_token = Regexp::Token.new(
-        :group,
-        :passive,
-        '', # text (none because this group is implicit)
-        target_node.ts,
-        nil, # te (unused)
-        target_node.level,
-        target_node.set_level,
-        target_node.conditional_level
+      new_group = Group::Passive.construct(
+        token:             :passive,
+        ts:                target_node.ts,
+        level:             target_node.level,
+        set_level:         target_node.set_level,
+        conditional_level: target_node.conditional_level,
+        options:           active_opts,
       )
-      new_group = Group::Passive.new(new_token, active_opts)
       new_group.implicit = true
       new_group << target_node
       increase_group_level(target_node)
@@ -534,7 +541,7 @@ class Regexp::Parser
 
   def range(token)
     exp = CharacterSet::Range.new(token, active_opts)
-    scope = node.last.is_a?(CharacterSet::IntersectedSequence) ? node.last : node
+    scope = node.last.instance_of?(CharacterSet::IntersectedSequence) ? node.last : node
     exp << scope.expressions.pop
     nest(exp)
   end
@@ -561,7 +568,7 @@ class Regexp::Parser
   end
 
   def close_completed_character_set_range
-    decrease_nesting if node.is_a?(CharacterSet::Range) && node.complete?
+    decrease_nesting if node.instance_of?(CharacterSet::Range) && node.complete?
   end
 
   def active_opts
@@ -572,15 +579,18 @@ class Regexp::Parser
   # an instance of Backreference::Number, its #referenced_expression is set to
   # the instance of Group::Capture that it refers to via its number.
   def assign_referenced_expressions
-    targets = {}
-    # find all referencable expressions
+    # find all referencable and refering expressions
+    targets = { 0 => root }
+    referrers = []
     root.each_expression do |exp|
       exp.is_a?(Group::Capture) && targets[exp.identifier] = exp
+      referrers << exp if exp.referential?
     end
-    # assign them to any refering expressions
-    root.each_expression do |exp|
-      exp.respond_to?(:reference) &&
-        exp.referenced_expression = targets[exp.reference]
+    # assign reference expression to refering expressions
+    # (in a second iteration because there might be forward references)
+    referrers.each do |exp|
+      exp.referenced_expression = targets[exp.reference] ||
+        raise(ParserError, "Invalid reference: #{exp.reference}")
     end
   end
 end # module Regexp::Parser

data/lib/regexp_parser/scanner/property.rl

@@ -17,7 +17,7 @@
       text = copy(data, ts-1, te)
       type = (text[1] == 'P') ^ (text[3] == '^') ? :nonproperty : :property
 
-      name = data[ts+2..te-2].pack('c*').gsub(/[\^\s_\-]/, '').downcase
+      name = text[3..-2].gsub(/[\^\s_\-]/, '').downcase
 
       token = self.class.short_prop_map[name] || self.class.long_prop_map[name]
       validation_error(:property, name) unless token