regexp_parser 1.7.0 → 2.9.0

data/lib/regexp_parser/expression/subexpression.rb

@@ -1,29 +1,25 @@
 module Regexp::Expression
-
   class Subexpression < Regexp::Expression::Base
     include Enumerable
 
     attr_accessor :expressions
 
     def initialize(token, options = {})
-      super
-
       self.expressions = []
+      super
     end
 
     # Override base method to clone the expressions as well.
-    def initialize_clone(orig)
-      self.expressions = orig.expressions.map(&:clone)
+    def initialize_copy(orig)
+      self.expressions = orig.expressions.map do |exp|
+        exp.clone.tap { |copy| copy.parent = self }
+      end
       super
     end
 
     def <<(exp)
-      if exp.is_a?(WhiteSpace) && last && last.is_a?(WhiteSpace)
-        last.merge(exp)
-      else
-        exp.nesting_level = nesting_level + 1
-        expressions << exp
-      end
+      exp.parent = self
+      expressions << exp
     end
 
     %w[[] at each empty? fetch index join last length values_at].each do |method|
@@ -41,19 +37,31 @@ module Regexp::Expression
     end
 
     def te
-      ts + to_s.length
-    end
-
-    def to_s(format = :full)
-      # Note: the format does not get passed down to subexpressions.
-      "#{expressions.join}#{quantifier_affix(format)}"
+      ts + base_length
     end
 
     def to_h
-      attributes.merge({
+      attributes.merge(
         text:        to_s(:base),
         expressions: expressions.map(&:to_h)
-      })
+      )
+    end
+
+    def extract_quantifier_target(quantifier_description)
+      pre_quantifier_decorations = []
+      target = expressions.reverse.find do |exp|
+        if exp.decorative?
+          exp.custom_to_s_handling = true
+          pre_quantifier_decorations << exp.text
+          next
+        end
+        exp
+      end
+      target or raise Regexp::Parser::ParserError,
+        "No valid target found for '#{quantifier_description}' quantifier"
+
+      target.pre_quantifier_decorations = pre_quantifier_decorations
+      target
     end
   end
 end

data/lib/regexp_parser/expression.rb

@@ -1,138 +1,7 @@
-module Regexp::Expression
-
-  class Base
-    attr_accessor :type, :token
-    attr_accessor :text, :ts
-    attr_accessor :level, :set_level, :conditional_level, :nesting_level
-
-    attr_accessor :quantifier
-    attr_accessor :options
-
-    def initialize(token, options = {})
-      self.type              = token.type
-      self.token             = token.token
-      self.text              = token.text
-      self.ts                = token.ts
-      self.level             = token.level
-      self.set_level         = token.set_level
-      self.conditional_level = token.conditional_level
-      self.nesting_level     = 0
-      self.quantifier        = nil
-      self.options           = options
-    end
-
-    def initialize_clone(orig)
-      self.text       = (orig.text       ? orig.text.dup         : nil)
-      self.options    = (orig.options    ? orig.options.dup      : nil)
-      self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
-      super
-    end
-
-    def to_re(format = :full)
-      ::Regexp.new(to_s(format))
-    end
-
-    alias :starts_at :ts
-
-    def full_length
-      to_s.length
-    end
-
-    def offset
-      [starts_at, full_length]
-    end
-
-    def coded_offset
-      '@%d+%d' % offset
-    end
-
-    def to_s(format = :full)
-      "#{text}#{quantifier_affix(format)}"
-    end
-
-    def quantifier_affix(expression_format)
-      quantifier.to_s if quantified? && expression_format != :base
-    end
-
-    def terminal?
-      !respond_to?(:expressions)
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      self.quantifier = Quantifier.new(token, text, min, max, mode)
-    end
-
-    def unquantified_clone
-      clone.tap { |exp| exp.quantifier = nil }
-    end
-
-    def quantified?
-      !quantifier.nil?
-    end
-
-    # Deprecated. Prefer `#repetitions` which has a more uniform interface.
-    def quantity
-      return [nil,nil] unless quantified?
-      [quantifier.min, quantifier.max]
-    end
-
-    def repetitions
-      return 1..1 unless quantified?
-      min = quantifier.min
-      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
-      # fix Range#minmax - https://bugs.ruby-lang.org/issues/15807
-      (min..max).tap { |r| r.define_singleton_method(:minmax) { [min, max] } }
-    end
-
-    def greedy?
-      quantified? and quantifier.greedy?
-    end
-
-    def reluctant?
-      quantified? and quantifier.reluctant?
-    end
-    alias :lazy? :reluctant?
-
-    def possessive?
-      quantified? and quantifier.possessive?
-    end
-
-    def attributes
-      {
-        type:              type,
-        token:             token,
-        text:              to_s(:base),
-        starts_at:         ts,
-        length:            full_length,
-        level:             level,
-        set_level:         set_level,
-        conditional_level: conditional_level,
-        options:           options,
-        quantifier:        quantified? ? quantifier.to_h : nil,
-      }
-    end
-    alias :to_h :attributes
-  end
-
-  def self.parsed(exp)
-    warn('WARNING: Regexp::Expression::Base.parsed is buggy and '\
-         'will be removed in 2.0.0. Use Regexp::Parser.parse instead.')
-    case exp
-    when String
-      Regexp::Parser.parse(exp)
-    when Regexp
-      Regexp::Parser.parse(exp.source) # <- causes loss of root options
-    when Regexp::Expression            # <- never triggers
-      exp
-    else
-      raise ArgumentError, 'Expression.parsed accepts a String, Regexp, or '\
-                           'a Regexp::Expression as a value for exp, but it '\
-                           "was given #{exp.class.name}."
-    end
-  end
-
-end # module Regexp::Expression
+require 'regexp_parser/error'
 
+require 'regexp_parser/expression/shared'
+require 'regexp_parser/expression/base'
 require 'regexp_parser/expression/quantifier'
 require 'regexp_parser/expression/subexpression'
 require 'regexp_parser/expression/sequence'
@@ -140,24 +9,29 @@ require 'regexp_parser/expression/sequence_operation'
 
 require 'regexp_parser/expression/classes/alternation'
 require 'regexp_parser/expression/classes/anchor'
-require 'regexp_parser/expression/classes/backref'
+require 'regexp_parser/expression/classes/backreference'
+require 'regexp_parser/expression/classes/character_set'
+require 'regexp_parser/expression/classes/character_set/intersection'
+require 'regexp_parser/expression/classes/character_set/range'
+require 'regexp_parser/expression/classes/character_type'
 require 'regexp_parser/expression/classes/conditional'
-require 'regexp_parser/expression/classes/escape'
+require 'regexp_parser/expression/classes/escape_sequence'
 require 'regexp_parser/expression/classes/free_space'
 require 'regexp_parser/expression/classes/group'
 require 'regexp_parser/expression/classes/keep'
 require 'regexp_parser/expression/classes/literal'
 require 'regexp_parser/expression/classes/posix_class'
-require 'regexp_parser/expression/classes/property'
 require 'regexp_parser/expression/classes/root'
-require 'regexp_parser/expression/classes/set'
-require 'regexp_parser/expression/classes/set/intersection'
-require 'regexp_parser/expression/classes/set/range'
-require 'regexp_parser/expression/classes/type'
+require 'regexp_parser/expression/classes/unicode_property'
 
+require 'regexp_parser/expression/methods/construct'
+require 'regexp_parser/expression/methods/human_name'
 require 'regexp_parser/expression/methods/match'
 require 'regexp_parser/expression/methods/match_length'
+require 'regexp_parser/expression/methods/negative'
 require 'regexp_parser/expression/methods/options'
+require 'regexp_parser/expression/methods/parts'
+require 'regexp_parser/expression/methods/printing'
 require 'regexp_parser/expression/methods/strfregexp'
 require 'regexp_parser/expression/methods/tests'
 require 'regexp_parser/expression/methods/traverse'

data/lib/regexp_parser/lexer.rb

@@ -4,57 +4,77 @@
 # given syntax flavor.
 class Regexp::Lexer
 
-  OPENING_TOKENS = [
-    :capture, :passive, :lookahead, :nlookahead, :lookbehind, :nlookbehind,
-    :atomic, :options, :options_switch, :named, :absence
+  OPENING_TOKENS = %i[
+    capture passive lookahead nlookahead lookbehind nlookbehind
+    atomic options options_switch named absence open
   ].freeze
 
-  CLOSING_TOKENS = [:close].freeze
+  CLOSING_TOKENS = %i[close].freeze
 
-  def self.lex(input, syntax = "ruby/#{RUBY_VERSION}", &block)
-    new.lex(input, syntax, &block)
+  CONDITION_TOKENS = %i[condition condition_close].freeze
+
+  def self.lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
+    new.lex(input, syntax, options: options, collect_tokens: collect_tokens, &block)
   end
 
-  def lex(input, syntax = "ruby/#{RUBY_VERSION}", &block)
-    syntax = Regexp::Syntax.new(syntax)
+  def lex(input, syntax = nil, options: nil, collect_tokens: true, &block)
+    syntax = syntax ? Regexp::Syntax.for(syntax) : Regexp::Syntax::CURRENT
 
+    self.block = block
+    self.collect_tokens = collect_tokens
     self.tokens = []
+    self.prev_token = nil
+    self.preprev_token = nil
     self.nesting = 0
     self.set_nesting = 0
     self.conditional_nesting = 0
     self.shift = 0
 
-    last = nil
-    Regexp::Scanner.scan(input) do |type, token, text, ts, te|
+    Regexp::Scanner.scan(input, options: options, collect_tokens: false) do |type, token, text, ts, te|
       type, token = *syntax.normalize(type, token)
       syntax.check! type, token
 
       ascend(type, token)
 
-      if type == :quantifier and last
-        break_literal(last)        if last.type == :literal
-        break_codepoint_list(last) if last.token == :codepoint_list
+      if (last = prev_token) &&
+         type == :quantifier &&
+         (
+           (last.type == :literal         && (parts = break_literal(last))) ||
+           (last.token == :codepoint_list && (parts = break_codepoint_list(last)))
+         )
+        emit(parts[0])
+        last = parts[1]
       end
 
       current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
                                   nesting, set_nesting, conditional_nesting)
 
-      current = merge_condition(current) if type == :conditional and
-        [:condition, :condition_close].include?(token)
-
-      last.next = current if last
-      current.previous = last if last
+      if type == :conditional && CONDITION_TOKENS.include?(token)
+        current = merge_condition(current, last)
+      elsif last
+        last.next = current
+        current.previous = last
+        emit(last)
+      end
 
-      tokens << current
-      last = current
+      self.preprev_token = last
+      self.prev_token = current
 
       descend(type, token)
     end
 
-    if block_given?
-      tokens.map { |t| block.call(t) }
+    emit(prev_token) if prev_token
+
+    collect_tokens ? tokens : nil
+  end
+
+  def emit(token)
+    if block
+      # TODO: in v3.0.0, remove `collect_tokens:` kwarg and only collect w/o block
+      res = block.call(token)
+      tokens << res if collect_tokens
     else
-      tokens
+      tokens << token
     end
   end
 
@@ -64,27 +84,37 @@ class Regexp::Lexer
 
   private
 
-  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting, :shift
+  attr_accessor :block,
+                :collect_tokens, :tokens, :prev_token, :preprev_token,
+                :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
+    return unless CLOSING_TOKENS.include?(token)
+
     case type
     when :group, :assertion
-      self.nesting = nesting - 1 if CLOSING_TOKENS.include?(token)
+      self.nesting = nesting - 1
     when :set
-      self.set_nesting = set_nesting - 1 if token == :close
+      self.set_nesting = set_nesting - 1
     when :conditional
-      self.conditional_nesting = conditional_nesting - 1 if token == :close
+      self.conditional_nesting = conditional_nesting - 1
+    else
+      raise "unhandled nesting type #{type}"
     end
   end
 
   def descend(type, token)
+    return unless OPENING_TOKENS.include?(token)
+
     case type
     when :group, :assertion
-      self.nesting = nesting + 1 if OPENING_TOKENS.include?(token)
+      self.nesting = nesting + 1
     when :set
-      self.set_nesting = set_nesting + 1 if token == :open
+      self.set_nesting = set_nesting + 1
     when :conditional
-      self.conditional_nesting = conditional_nesting + 1 if token == :open
+      self.conditional_nesting = conditional_nesting + 1
+    else
+      raise "unhandled nesting type #{type}"
     end
   end
 
@@ -94,34 +124,46 @@ class Regexp::Lexer
     lead, last, _ = token.text.partition(/.\z/mu)
     return if lead.empty?
 
-    tokens.pop
-    tokens << Regexp::Token.new(:literal, :literal, lead,
-              token.ts, (token.te - last.bytesize),
+    token_1 = Regexp::Token.new(:literal, :literal, lead,
+              token.ts, (token.te - last.length),
               nesting, set_nesting, conditional_nesting)
-    tokens << Regexp::Token.new(:literal, :literal, last,
-              (token.ts + lead.bytesize), token.te,
+    token_2 = Regexp::Token.new(:literal, :literal, last,
+              (token.ts + lead.length), token.te,
               nesting, set_nesting, conditional_nesting)
+
+    token_1.previous = preprev_token
+    token_1.next = token_2
+    token_2.previous = token_1 # .next will be set by #lex
+    [token_1, token_2]
   end
 
+  # if a codepoint list is followed by a quantifier, that quantifier applies
+  # to the last codepoint, e.g. /\u{61 62 63}{3}/ =~ 'abccc'
+  # c.f. #break_literal.
   def break_codepoint_list(token)
     lead, _, tail = token.text.rpartition(' ')
     return if lead.empty?
 
-    tokens.pop
-    tokens << Regexp::Token.new(:escape, :codepoint_list, lead + '}',
+    token_1 = Regexp::Token.new(:escape, :codepoint_list, lead + '}',
               token.ts, (token.te - tail.length),
               nesting, set_nesting, conditional_nesting)
-    tokens << Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
+    token_2 = Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
               (token.ts + lead.length + 1), (token.te + 3),
               nesting, set_nesting, conditional_nesting)
 
     self.shift = shift + 3 # one space less, but extra \, u, {, and }
+
+    token_1.previous = preprev_token
+    token_1.next = token_2
+    token_2.previous = token_1 # .next will be set by #lex
+    [token_1, token_2]
   end
 
-  def merge_condition(current)
-    last = tokens.pop
-    Regexp::Token.new(:conditional, :condition, last.text + current.text,
+  def merge_condition(current, last)
+    token = Regexp::Token.new(:conditional, :condition, last.text + current.text,
       last.ts, current.te, nesting, set_nesting, conditional_nesting)
+    token.previous = preprev_token # .next will be set by #lex
+    token
   end
 
 end # module Regexp::Lexer