regexp_parser 2.7.0 → 2.12.0

data/lib/regexp_parser/expression/methods/escape_sequence_codepoint.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Regexp::Expression::EscapeSequence
+  AsciiEscape.class_eval { def codepoint; 0x1B end }
+  Backspace.class_eval   { def codepoint; 0x8  end }
+  Bell.class_eval        { def codepoint; 0x7  end }
+  FormFeed.class_eval    { def codepoint; 0xC  end }
+  Newline.class_eval     { def codepoint; 0xA  end }
+  Return.class_eval      { def codepoint; 0xD  end }
+  Tab.class_eval         { def codepoint; 0x9  end }
+  VerticalTab.class_eval { def codepoint; 0xB  end }
+
+  Literal.class_eval     { def codepoint; text[1].ord end }
+
+  Octal.class_eval       { def codepoint; text[/\d+/].to_i(8) end }
+
+  Hex.class_eval         { def codepoint; text[/\h+/].hex end }
+  Codepoint.class_eval   { def codepoint; text[/\h+/].hex end }
+
+  UTF8Hex.class_eval do
+    def codepoint
+      text.scan(/\h+/).map(&:hex).pack('C*').force_encoding('utf-8').ord
+    end
+  end
+
+  CodepointList.class_eval do
+    # Maybe this should be a unique top-level expression class?
+    def char
+      raise NoMethodError, 'CodepointList responds only to #chars'
+    end
+
+    def codepoint
+      raise NoMethodError, 'CodepointList responds only to #codepoints'
+    end
+
+    def chars
+      codepoints.map { |cp| cp.chr('utf-8') }
+    end
+
+    def codepoints
+      text.scan(/\h+/).map(&:hex)
+    end
+  end
+
+  AbstractMetaControlSequence.class_eval do
+    private
+
+    def control_sequence_to_s(control_sequence)
+      five_lsb = control_sequence.unpack('B*').first[-5..-1]
+      ["000#{five_lsb}"].pack('B*')
+    end
+
+    def meta_char_to_codepoint(meta_char)
+      byte_value = meta_char.ord
+      byte_value < 128 ? byte_value + 128 : byte_value
+    end
+  end
+
+  Control.class_eval do
+    def codepoint
+      control_sequence_to_s(text).ord
+    end
+  end
+
+  Meta.class_eval do
+    def codepoint
+      meta_char_to_codepoint(text[-1])
+    end
+  end
+
+  MetaControl.class_eval do
+    def codepoint
+      meta_char_to_codepoint(control_sequence_to_s(text))
+    end
+  end
+end

data/lib/regexp_parser/expression/methods/human_name.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   module Shared
     # default implementation, e.g. "atomic group", "hex escape", "word type", ..

data/lib/regexp_parser/expression/methods/match.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   class Base
     def match?(string)

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Regexp::MatchLength
   include Enumerable
 

data/lib/regexp_parser/expression/methods/negative.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Regexp::Expression
+  module Shared
+    def negative?
+      false
+    end
+
+    # not an alias so as to respect overrides of #negative?
+    def negated?
+      negative?
+    end
+  end
+
+  Anchor::NonWordBoundary.class_eval       { def negative?; true                          end }
+  Assertion::NegativeLookahead.class_eval  { def negative?; true                          end }
+  Assertion::NegativeLookbehind.class_eval { def negative?; true                          end }
+  CharacterSet.class_eval                  { def negative?; negative                      end }
+  CharacterType::Base.class_eval           { def negative?; token.to_s.start_with?('non') end }
+  PosixClass.class_eval                    { def negative?; type == :nonposixclass        end }
+  UnicodeProperty::Base.class_eval         { def negative?; type == :nonproperty          end }
+end

data/lib/regexp_parser/expression/methods/options.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   class Base
     def multiline?

data/lib/regexp_parser/expression/methods/parts.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Regexp::Expression
+  module Shared
+    # default implementation
+    def parts
+      [text.dup]
+    end
+
+    private
+
+    def intersperse(expressions, separator)
+      expressions.flat_map { |exp| [exp, separator] }.slice(0...-1)
+    end
+  end
+
+  CharacterSet.class_eval            { def parts; ["#{text}#{'^' if negated?}", *expressions, ']']        end }
+  CharacterSet::Range.class_eval     { def parts; intersperse(expressions, text.dup)                      end }
+  Conditional::Expression.class_eval { def parts; [text.dup, condition, *intersperse(branches, '|'), ')'] end }
+  Group::Base.class_eval             { def parts; [text.dup, *expressions, ')']                           end }
+  Group::Passive.class_eval          { def parts; implicit? ? expressions : super                         end }
+  Group::Comment.class_eval          { def parts; [text.dup]                                              end }
+  Subexpression.class_eval           { def parts; expressions                                             end }
+  SequenceOperation.class_eval       { def parts; intersperse(expressions, text.dup)                      end }
+end

data/lib/regexp_parser/expression/methods/printing.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Regexp::Expression
+  module Shared
+    def inspect
+      [
+        "#<#{self.class}",
+        pretty_print_instance_variables.map { |v| " #{v}=#{instance_variable_get(v).inspect}" },
+        ">"
+      ].join
+    end
+
+    # Make pretty-print work despite #inspect implementation.
+    def pretty_print(q)
+      q.pp_object(self)
+    end
+
+    # Called by pretty_print (ruby/pp) and #inspect.
+    def pretty_print_instance_variables
+      [
+        (:@text unless text.to_s.empty?),
+        (:@quantifier if quantified?),
+        (:@options unless options.empty?),
+        (:@expressions unless terminal?),
+      ].compact
+    end
+  end
+end

data/lib/regexp_parser/expression/methods/referenced_expressions.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Regexp::Expression
+  module ReferencedExpressions
+    attr_accessor :referenced_expressions
+
+    def referenced_expression
+      referenced_expressions && referenced_expressions.first
+    end
+
+    def initialize_copy(orig)
+      exp_id = [self.class, self.starts_at]
+
+      # prevent infinite recursion for recursive subexp calls
+      copied = self.class.instance_eval { @copied_ref_exps ||= {} }
+      self.referenced_expressions =
+        if copied[exp_id]
+          orig.referenced_expressions
+        else
+          copied[exp_id] = true
+          orig.referenced_expressions && orig.referenced_expressions.map(&:dup)
+        end
+      copied.clear
+
+      super
+    end
+  end
+
+  Base.include ReferencedExpressions
+end

data/lib/regexp_parser/expression/methods/strfregexp.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   class Base
 

data/lib/regexp_parser/expression/methods/tests.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   module Shared
 
@@ -95,12 +97,49 @@ module Regexp::Expression
     end
 
     # Deep-compare two expressions for equality.
+    #
+    # When changing the conditions, please make sure to update
+    # #pretty_print_instance_variables so that it includes all relevant values.
     def ==(other)
-      other.class == self.class &&
-        other.to_s == to_s &&
-        other.options == options
+      self.class   == other.class &&
+        text       == other.text &&
+        quantifier == other.quantifier &&
+        options    == other.options &&
+        (terminal? || expressions == other.expressions)
     end
     alias :=== :==
     alias :eql? :==
+
+    def optional?
+      quantified? && quantifier.min == 0
+    end
+
+    def quantified?
+      !quantifier.nil?
+    end
   end
+
+  Shared.class_eval                     { def terminal?; self.class.terminal? end }
+  Shared::ClassMethods.class_eval       { def terminal?; true  end }
+  Subexpression.instance_eval           { def terminal?; false end }
+
+  Shared.class_eval                     { def capturing?; self.class.capturing? end }
+  Shared::ClassMethods.class_eval       { def capturing?; false end }
+  Group::Capture.instance_eval          { def capturing?; true  end }
+
+  Shared.class_eval                     { def comment?; self.class.comment? end }
+  Shared::ClassMethods.class_eval       { def comment?; false end }
+  Comment.instance_eval                 { def comment?; true  end }
+  Group::Comment.instance_eval          { def comment?; true  end }
+
+  Shared.class_eval                     { def decorative?; self.class.decorative? end }
+  Shared::ClassMethods.class_eval       { def decorative?; false end }
+  FreeSpace.instance_eval               { def decorative?; true  end }
+  Group::Comment.instance_eval          { def decorative?; true  end }
+
+  Shared.class_eval                     { def referential?; self.class.referential? end }
+  Shared::ClassMethods.class_eval       { def referential?; false end }
+  Backreference::Base.instance_eval     { def referential?; true  end }
+  Conditional::Condition.instance_eval  { def referential?; true  end }
+  Conditional::Expression.instance_eval { def referential?; true  end }
 end

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -1,6 +1,24 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   class Subexpression < Regexp::Expression::Base
 
+    # Traverses the expression, passing each recursive child to the
+    # given block.
+    # If the block takes two arguments, the indices of the children within
+    # their parents are also passed to it.
+    def each_expression(include_self = false, &block)
+      return enum_for(__method__, include_self) unless block
+
+      if block.arity == 1
+        block.call(self) if include_self
+        each_expression_without_index(&block)
+      else
+        block.call(self, 0) if include_self
+        each_expression_with_index(&block)
+      end
+    end
+
     # Traverses the subexpression (depth-first, pre-order) and calls the given
     # block for each expression with three arguments; the traversal event,
     # the expression, and the index of the expression within its parent.
@@ -34,34 +52,31 @@ module Regexp::Expression
     end
     alias :walk :traverse
 
-    # Iterates over the expressions of this expression as an array, passing
-    # the expression and its index within its parent to the given block.
-    def each_expression(include_self = false, &block)
-      return enum_for(__method__, include_self) unless block_given?
+    # Returns a new array with the results of calling the given block once
+    # for every expression. If a block is not given, returns an array with
+    # each expression and its level index as an array.
+    def flat_map(include_self = false, &block)
+      case block && block.arity
+      when nil then each_expression(include_self).to_a
+      when 2   then each_expression(include_self).map(&block)
+      else          each_expression(include_self).map { |exp| block.call(exp) }
+      end
+    end
 
-      block.call(self, 0) if include_self
+    protected
 
+    def each_expression_with_index(&block)
       each_with_index do |exp, index|
         block.call(exp, index)
-        exp.each_expression(&block) unless exp.terminal?
+        exp.each_expression_with_index(&block) unless exp.terminal?
       end
     end
 
-    # Returns a new array with the results of calling the given block once
-    # for every expression. If a block is not given, returns an array with
-    # each expression and its level index as an array.
-    def flat_map(include_self = false)
-      result = []
-
-      each_expression(include_self) do |exp, index|
-        if block_given?
-          result << yield(exp, index)
-        else
-          result << [exp, index]
-        end
+    def each_expression_without_index(&block)
+      each do |exp|
+        block.call(exp)
+        exp.each_expression_without_index(&block) unless exp.terminal?
       end
-
-      result
     end
   end
 end

data/lib/regexp_parser/expression/quantifier.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   # TODO: in v3.0.0, maybe put Shared back into Base, and inherit from Base and
   # call super in #initialize, but raise in #quantifier= and #quantify,
@@ -6,16 +8,12 @@ module Regexp::Expression
   class Quantifier
     include Regexp::Expression::Shared
 
-    MODES = %i[greedy possessive reluctant]
-
-    attr_reader :min, :max, :mode
+    MODES = %i[greedy possessive reluctant].freeze
 
     def initialize(*args)
       deprecated_old_init(*args) and return if args.count == 4 || args.count == 5
 
       init_from_token_and_options(*args)
-      @mode = (token.to_s[/greedy|reluctant|possessive/] || :greedy).to_sym
-      @min, @max = minmax
       # TODO: remove in v3.0.0, stop removing parts of #token (?)
       self.token = token.to_s.sub(/_(greedy|possessive|reluctant)/, '').to_sym
     end
@@ -39,9 +37,21 @@ module Regexp::Expression
     end
     alias :lazy? :reluctant?
 
+    def min
+      derived_data[:min]
+    end
+
+    def max
+      derived_data[:max]
+    end
+
+    def mode
+      derived_data[:mode]
+    end
+
     private
 
-    def deprecated_old_init(token, text, min, max, mode = :greedy)
+    def deprecated_old_init(token, text, _min, _max, _mode = :greedy)
       warn "Calling `Expression::Base#quantify` or `#{self.class}.new` with 4+ arguments "\
            "is deprecated.\nIt will no longer be supported in regexp_parser v3.0.0.\n"\
            "Please pass a Regexp::Token instead, e.g. replace `token, text, min, max, mode` "\
@@ -51,20 +61,25 @@ module Regexp::Expression
            "This is consistent with how Expression::Base instances are created. "
       @token = token
       @text  = text
-      @min   = min
-      @max   = max
-      @mode  = mode
     end
 
-    def minmax
-      case token
-      when /zero_or_one/  then [0, 1]
-      when /zero_or_more/ then [0, -1]
-      when /one_or_more/  then [1, -1]
-      when :interval
-        int_min = text[/\{(\d*)/, 1]
-        int_max = text[/,?(\d*)\}/, 1]
-        [int_min.to_i, (int_max.empty? ? -1 : int_max.to_i)]
+    def derived_data
+      @derived_data ||= begin
+        min, max =
+          case text[0]
+          when '?'; [0, 1]
+          when '*'; [0, -1]
+          when '+'; [1, -1]
+          else
+            int_min = text[/\{(\d*)/, 1]
+            int_max = text[/,?(\d*)\}/, 1]
+            [int_min.to_i, (int_max.empty? ? -1 : int_max.to_i)]
+          end
+
+        mod = text[/.([?+])/, 1]
+        mode = (mod == '?' && :reluctant) || (mod == '+' && :possessive) || :greedy
+
+        { min: min, max: max, mode: mode }
       end
     end
   end

data/lib/regexp_parser/expression/sequence.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   # A sequence of expressions. Differs from a Subexpressions by how it handles
   # quantifiers, as it applies them to its last element instead of itself as
@@ -12,6 +14,7 @@ module Regexp::Expression
           level:             exp.level,
           set_level:         exp.set_level,
           conditional_level: params[:conditional_level] || exp.conditional_level,
+          ts:                params[:ts],
         )
         sequence.options = active_opts
         exp.expressions << sequence
@@ -19,17 +22,12 @@ module Regexp::Expression
       end
     end
 
-    def starts_at
-      expressions.first.starts_at
+    def ts
+      (head = expressions.first) ? head.ts : @ts
     end
-    alias :ts :starts_at
-
-    def quantify(*args)
-      target = expressions.reverse.find { |exp| !exp.is_a?(FreeSpace) }
-      target or raise Regexp::Parser::Error,
-        "No valid target found for '#{text}' quantifier"
 
-      target.quantify(*args)
+    def quantify(token, *args)
+      extract_quantifier_target(token.text).quantify(token, *args)
     end
   end
 end

data/lib/regexp_parser/expression/sequence_operation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   # abstract class
   class SequenceOperation < Regexp::Expression::Subexpression
@@ -5,21 +7,16 @@ module Regexp::Expression
     alias :operands :expressions
     alias :operator :text
 
-    def starts_at
-      expressions.first.starts_at
+    def ts
+      (head = expressions.first) ? head.ts : @ts
     end
-    alias :ts :starts_at
 
     def <<(exp)
       expressions.last << exp
     end
 
-    def add_sequence(active_opts = {})
-      self.class::OPERAND.add_to(self, {}, active_opts)
-    end
-
-    def parts
-      intersperse(expressions, text.dup)
+    def add_sequence(active_opts = {}, params = { ts: 0 })
+      self.class::OPERAND.add_to(self, params, active_opts)
     end
   end
 end

data/lib/regexp_parser/expression/shared.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Regexp::Expression
   module Shared
     module ClassMethods; end # filled in ./methods/*.rb
@@ -8,7 +10,8 @@ module Regexp::Expression
 
         attr_accessor :type, :token, :text, :ts, :te,
                       :level, :set_level, :conditional_level,
-                      :options
+                      :options, :parent,
+                      :custom_to_s_handling, :pre_quantifier_decorations
 
         attr_reader   :nesting_level, :quantifier
       end
@@ -32,6 +35,10 @@ module Regexp::Expression
       self.text       = orig.text.dup         if orig.text
       self.options    = orig.options.dup      if orig.options
       self.quantifier = orig.quantifier.clone if orig.quantifier
+      self.parent     = nil # updated by Subexpression#initialize_copy
+      if orig.pre_quantifier_decorations
+        self.pre_quantifier_decorations = orig.pre_quantifier_decorations.map(&:dup)
+      end
       super
     end
 
@@ -39,35 +46,52 @@ module Regexp::Expression
       ts
     end
 
+    def ends_at(include_quantifier = true)
+      ts + (include_quantifier ? full_length : base_length)
+    end
+
     def base_length
       to_s(:base).length
     end
 
     def full_length
-      to_s.length
-    end
-
+      to_s(:original).length
+    end
+
+    # #to_s reproduces the original source, as an unparser would.
+    #
+    # It takes an optional format argument.
+    #
+    # Example:
+    #
+    # lit = Regexp::Parser.parse(/a +/x)[0]
+    #
+    # lit.to_s            # => 'a+'  # default; with quantifier
+    # lit.to_s(:full)     # => 'a+'  # default; with quantifier
+    # lit.to_s(:base)     # => 'a'   # without quantifier
+    # lit.to_s(:original) # => 'a +' # with quantifier AND intermittent decorations
+    #
     def to_s(format = :full)
-      "#{parts.join}#{quantifier_affix(format)}"
+      base = ''.dup
+      parts.each do |part|
+        if part.instance_of?(String)
+          base << part
+        elsif !part.custom_to_s_handling
+          base << part.to_s(:original)
+        end
+      end
+      "#{base}#{pre_quantifier_decoration(format)}#{quantifier_affix(format)}"
     end
     alias :to_str :to_s
 
-    def parts
-      [text.dup]
+    def pre_quantifier_decoration(expression_format = :original)
+      pre_quantifier_decorations.to_a.join if expression_format == :original
     end
 
-    def quantifier_affix(expression_format)
+    def quantifier_affix(expression_format = :full)
       quantifier.to_s if quantified? && expression_format != :base
     end
 
-    def quantified?
-      !quantifier.nil?
-    end
-
-    def optional?
-      quantified? && quantifier.min == 0
-    end
-
     def offset
       [starts_at, full_length]
     end
@@ -76,14 +100,6 @@ module Regexp::Expression
       '@%d+%d' % offset
     end
 
-    def terminal?
-      true # overridden to be false in Expression::Subexpression
-    end
-
-    def referential?
-      false # overridden to be true e.g. in Expression::Backreference::Base
-    end
-
     def nesting_level=(lvl)
       @nesting_level = lvl
       quantifier && quantifier.nesting_level = lvl