regexp_parser 1.3.0 → 1.6.0

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -0,0 +1,172 @@
+class Regexp::MatchLength
+  include Enumerable
+
+  def self.of(obj)
+    exp = obj.is_a?(Regexp::Expression::Base) ? obj : Regexp::Parser.parse(obj)
+    exp.match_length
+  end
+
+  def initialize(exp, opts = {})
+    self.exp_class = exp.class
+    self.min_rep = exp.repetitions.min
+    self.max_rep = exp.repetitions.max
+    if base = opts[:base]
+      self.base_min = base
+      self.base_max = base
+      self.reify = ->{ '.' * base }
+    else
+      self.base_min = opts.fetch(:base_min)
+      self.base_max = opts.fetch(:base_max)
+      self.reify = opts.fetch(:reify)
+    end
+  end
+
+  def each(opts = {})
+    return enum_for(__method__) unless block_given?
+    limit = opts[:limit] || 1000
+    yielded = 0
+    (min..max).each do |num|
+      next unless include?(num)
+      yield(num)
+      break if (yielded += 1) >= limit
+    end
+  end
+
+  def endless_each(&block)
+    return enum_for(__method__) unless block_given?
+    (min..max).each { |num| yield(num) if include?(num) }
+  end
+
+  def include?(length)
+    test_regexp.match?('X' * length)
+  end
+
+  def fixed?
+    min == max
+  end
+
+  def min
+    min_rep * base_min
+  end
+
+  def max
+    max_rep * base_max
+  end
+
+  def minmax
+    [min, max]
+  end
+
+  def inspect
+    type = exp_class.name.sub('Regexp::Expression::', '')
+    "#<#{self.class}<#{type}> min=#{min} max=#{max}>"
+  end
+
+  def to_re
+    "(?:#{reify.call}){#{min_rep},#{max_rep unless max_rep == Float::INFINITY}}"
+  end
+
+  private
+
+  attr_accessor :base_min, :base_max, :min_rep, :max_rep, :exp_class, :reify
+
+  def test_regexp
+    @test_regexp ||= Regexp.new("^#{to_re}$").tap do |regexp|
+      regexp.respond_to?(:match?) || def regexp.match?(str); !!match(str) end
+    end
+  end
+end
+
+module Regexp::Expression
+  MatchLength = Regexp::MatchLength
+
+  [
+    CharacterSet,
+    CharacterSet::Intersection,
+    CharacterSet::IntersectedSequence,
+    CharacterSet::Range,
+    CharacterType::Base,
+    EscapeSequence::Base,
+    PosixClass,
+    UnicodeProperty::Base,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self, base: 1)
+      end
+    RUBY
+  end
+
+  class Literal
+    def match_length
+      MatchLength.new(self, base: text.length)
+    end
+  end
+
+  class Subexpression
+    def match_length
+      MatchLength.new(self,
+                       base_min: map { |exp| exp.match_length.min }.inject(0, :+),
+                       base_max: map { |exp| exp.match_length.max }.inject(0, :+),
+                       reify: ->{ map { |exp| exp.match_length.to_re }.join })
+    end
+
+    def inner_match_length
+      dummy = Regexp::Expression::Root.build
+      dummy.expressions = expressions.map(&:clone)
+      dummy.quantifier = quantifier && quantifier.clone
+      dummy.match_length
+    end
+  end
+
+  [
+    Alternation,
+    Conditional::Expression,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self,
+                         base_min: map { |exp| exp.match_length.min }.min,
+                         base_max: map { |exp| exp.match_length.max }.max,
+                         reify: ->{ map { |exp| exp.match_length.to_re }.join('|') })
+      end
+    RUBY
+  end
+
+  [
+    Anchor::Base,
+    Assertion::Base,
+    Conditional::Condition,
+    FreeSpace,
+    Keep::Mark,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self, base: 0)
+      end
+    RUBY
+  end
+
+  class Backreference::Base
+    def match_length
+      if referenced_expression.nil?
+        raise ArgumentError, 'Missing referenced_expression - not parsed?'
+      end
+      referenced_expression.unquantified_clone.match_length
+    end
+  end
+
+  class EscapeSequence::CodepointList
+    def match_length
+      MatchLength.new(self, base: codepoints.count)
+    end
+  end
+
+  # Special case. Absence group can match 0.. chars, irrespective of content.
+  # TODO: in theory, they *can* exclude match lengths with `.`: `(?~.{3})`
+  class Group::Absence
+    def match_length
+      MatchLength.new(self, base_min: 0, base_max: Float::INFINITY, reify: ->{ '.*' })
+    end
+  end
+end

data/lib/regexp_parser/expression/methods/options.rb

@@ -0,0 +1,35 @@
+module Regexp::Expression
+  class Base
+    def multiline?
+      options[:m] == true
+    end
+    alias :m? :multiline?
+
+    def case_insensitive?
+      options[:i] == true
+    end
+    alias :i? :case_insensitive?
+    alias :ignore_case? :case_insensitive?
+
+    def free_spacing?
+      options[:x] == true
+    end
+    alias :x? :free_spacing?
+    alias :extended? :free_spacing?
+
+    def default_classes?
+      options[:d] == true
+    end
+    alias :d? :default_classes?
+
+    def ascii_classes?
+      options[:a] == true
+    end
+    alias :a? :ascii_classes?
+
+    def unicode_classes?
+      options[:u] == true
+    end
+    alias :u? :unicode_classes?
+  end
+end

data/lib/regexp_parser/expression/methods/strfregexp.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Base
 
     #   %l  Level (depth) of the expression. Returns 'root' for the root

data/lib/regexp_parser/expression/methods/tests.rb

@@ -75,32 +75,23 @@ module Regexp::Expression
     def one_of?(scope, top = true)
       case scope
       when Array
-        if scope.include?(:*)
-          return (scope.include?(token) or scope.include?(:*))
-        else
-          return scope.include?(token)
-        end
+        scope.include?(:*) || scope.include?(token)
 
       when Hash
         if scope.has_key?(:*)
           test_type = scope.has_key?(type) ? type : :*
-          return one_of?(scope[test_type], false)
+          one_of?(scope[test_type], false)
         else
-          return (scope.has_key?(type) and one_of?(scope[type], false))
+          scope.has_key?(type) && one_of?(scope[type], false)
         end
 
       when Symbol
-        return true if scope == :*
-
-        return is?(scope) unless top
-        return type?(scope) if top
+        scope.equal?(:*) || (top ? type?(scope) : is?(scope))
 
       else
-        raise "Array, Hash, or Symbol expected, #{scope.class.name} given"
+        raise ArgumentError,
+              "Array, Hash, or Symbol expected, #{scope.class.name} given"
       end
-
-      false
     end
-
   end
 end

data/lib/regexp_parser/expression/quantifier.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
       @max   = max
     end
 
-    def initialize_clone(other)
-      other.instance_variable_set(:@text, text.dup)
+    def initialize_clone(orig)
+      @text = orig.text.dup
       super
     end
 

data/lib/regexp_parser/expression/sequence.rb

@@ -18,13 +18,14 @@ module Regexp::Expression
     end
 
     class << self
-      def add_to(subexpression, options = {})
+      def add_to(subexpression, params = {}, active_opts = {})
         sequence = at_levels(
           subexpression.level,
           subexpression.set_level,
-          options[:conditional_level] || subexpression.conditional_level
+          params[:conditional_level] || subexpression.conditional_level
         )
         sequence.nesting_level = subexpression.nesting_level + 1
+        sequence.options = active_opts
         subexpression.expressions << sequence
         sequence
       end
@@ -44,10 +45,6 @@ module Regexp::Expression
       end
     end
 
-    def text
-      to_s
-    end
-
     def starts_at
       expressions.first.starts_at
     end

data/lib/regexp_parser/expression/sequence_operation.rb

@@ -14,12 +14,8 @@ module Regexp::Expression
       expressions.last << exp
     end
 
-    def add_sequence
-      self.class::OPERAND.add_to(self)
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      sequences.last.last.quantify(token, text, min, max, mode)
+    def add_sequence(active_opts = {})
+      self.class::OPERAND.add_to(self, {}, active_opts)
     end
 
     def to_s(format = :full)

data/lib/regexp_parser/expression/subexpression.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
     end
 
     # Override base method to clone the expressions as well.
-    def initialize_clone(other)
-      other.expressions = expressions.map(&:clone)
+    def initialize_clone(orig)
+      self.expressions = orig.expressions.map(&:clone)
       super
     end
 
@@ -46,9 +46,7 @@ module Regexp::Expression
 
     def to_s(format = :full)
       # Note: the format does not get passed down to subexpressions.
-      # Note: cant use #text accessor, b/c it is overriden as def text; to_s end
-      # in Expression::Sequence, causing infinite recursion. Clean-up needed.
-      "#{@text}#{expressions.join}#{quantifier_affix(format)}"
+      "#{expressions.join}#{quantifier_affix(format)}"
     end
 
     def to_h

data/lib/regexp_parser/lexer.rb

@@ -22,6 +22,7 @@ class Regexp::Lexer
     self.nesting = 0
     self.set_nesting = 0
     self.conditional_nesting = 0
+    self.shift = 0
 
     last = nil
     Regexp::Scanner.scan(input) do |type, token, text, ts, te|
@@ -30,15 +31,13 @@ class Regexp::Lexer
 
       ascend(type, token)
 
-      break_literal(last) if type == :quantifier and
-        last and last.type == :literal
-
-      current = Regexp::Token.new(type, token, text, ts, te,
-                nesting, set_nesting, conditional_nesting)
+      if type == :quantifier and last
+        break_literal(last)        if last.type == :literal
+        break_codepoint_list(last) if last.token == :codepoint_list
+      end
 
-      current = merge_literal(current) if type == :literal and
-        set_nesting == 0 and
-        last and last.type == :literal
+      current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
+                                  nesting, set_nesting, conditional_nesting)
 
       current = merge_condition(current) if type == :conditional and
         [:condition, :condition_close].include?(token)
@@ -65,7 +64,7 @@ class Regexp::Lexer
 
   private
 
-  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting
+  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
     case type
@@ -92,44 +91,31 @@ class Regexp::Lexer
   # called by scan to break a literal run that is longer than one character
   # into two separate tokens when it is followed by a quantifier
   def break_literal(token)
-    text = token.text
-    if text.scan(/./mu).length > 1
-      lead = text.sub(/.\z/mu, "")
-      last = text[/.\z/mu] || ''
-
-      if RUBY_VERSION >= '1.9'
-        lead_length = lead.bytesize
-        last_length = last.bytesize
-      else
-        lead_length = lead.length
-        last_length = last.length
-      end
-
-      tokens.pop
-      tokens << Regexp::Token.new(:literal, :literal, lead, token.ts,
-                (token.te - last_length), nesting, set_nesting, conditional_nesting)
-
-      tokens << Regexp::Token.new(:literal, :literal, last,
-                (token.ts + lead_length),
-                token.te, nesting, set_nesting, conditional_nesting)
-    end
+    lead, last, _ = token.text.partition(/.\z/mu)
+    return if lead.empty?
+
+    tokens.pop
+    tokens << Regexp::Token.new(:literal, :literal, lead,
+              token.ts, (token.te - last.bytesize),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:literal, :literal, last,
+              (token.ts + lead.bytesize), token.te,
+              nesting, set_nesting, conditional_nesting)
   end
 
-  # called by scan to merge two consecutive literals. this happens when tokens
-  # get normalized (as in the case of posix/bre) and end up becoming literals.
-  def merge_literal(current)
-    last = tokens.pop
+  def break_codepoint_list(token)
+    lead, _, tail = token.text.rpartition(' ')
+    return if lead.empty?
+
+    tokens.pop
+    tokens << Regexp::Token.new(:escape, :codepoint_list, lead + '}',
+              token.ts, (token.te - tail.length),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
+              (token.ts + lead.length + 1), (token.te + 3),
+              nesting, set_nesting, conditional_nesting)
 
-    Regexp::Token.new(
-      :literal,
-      :literal,
-      last.text + current.text,
-      last.ts,
-      current.te,
-      nesting,
-      set_nesting,
-      conditional_nesting,
-    )
+    self.shift = shift + 3 # one space less, but extra \, u, {, and }
   end
 
   def merge_condition(current)

data/lib/regexp_parser/parser.rb

@@ -39,6 +39,8 @@ class Regexp::Parser
       parse_token(token)
     end
 
+    assign_referenced_expressions
+
     if block_given?
       block.call(root)
     else
@@ -163,14 +165,18 @@ class Regexp::Parser
       node << Backreference::NameCall.new(token, active_opts)
     when :number, :number_ref
       node << Backreference::Number.new(token, active_opts)
-    when :number_rel_ref
-      node << Backreference::NumberRelative.new(token, active_opts)
     when :number_recursion_ref
       node << Backreference::NumberRecursionLevel.new(token, active_opts)
     when :number_call
       node << Backreference::NumberCall.new(token, active_opts)
+    when :number_rel_ref
+      node << Backreference::NumberRelative.new(token, active_opts).tap do |exp|
+        assign_effective_number(exp)
+      end
     when :number_rel_call
-      node << Backreference::NumberCallRelative.new(token, active_opts)
+      node << Backreference::NumberCallRelative.new(token, active_opts).tap do |exp|
+        assign_effective_number(exp)
+      end
     else
       raise UnknownTokenError.new('Backreference', token)
     end
@@ -209,9 +215,9 @@ class Regexp::Parser
       nest_conditional(Conditional::Expression.new(token, active_opts))
     when :condition
       conditional_nesting.last.condition = Conditional::Condition.new(token, active_opts)
-      conditional_nesting.last.branch
+      conditional_nesting.last.add_sequence(active_opts)
     when :separator
-      conditional_nesting.last.branch
+      conditional_nesting.last.add_sequence(active_opts)
       self.node = conditional_nesting.last.branches.last
     when :close
       conditional_nesting.pop
@@ -229,7 +235,7 @@ class Regexp::Parser
   end
 
   def posixclass(token)
-    node << PosixClass.new(token)
+    node << PosixClass.new(token, active_opts)
   end
 
   include Regexp::Expression::UnicodeProperty
@@ -491,6 +497,9 @@ class Regexp::Parser
     end
   end
 
+  MOD_FLAGS = %w[i m x].map(&:to_sym)
+  ENC_FLAGS = %w[a d u].map(&:to_sym)
+
   def options_group(token)
     positive, negative = token.text.split('-', 2)
     negative ||= ''
@@ -499,23 +508,23 @@ class Regexp::Parser
     opt_changes = {}
     new_active_opts = active_opts.dup
 
-    # Negative options have precedence. E.g. /(?i-i)a/ is case-sensitive.
-    %w[i m x].each do |flag|
-      if positive.include?(flag)
-        opt_changes[flag.to_sym] = new_active_opts[flag.to_sym] = true
+    MOD_FLAGS.each do |flag|
+      if positive.include?(flag.to_s)
+        opt_changes[flag] = new_active_opts[flag] = true
       end
-      if negative.include?(flag)
-        opt_changes[flag.to_sym] = false
-        new_active_opts.delete(flag.to_sym)
+      if negative.include?(flag.to_s)
+        opt_changes[flag] = false
+        new_active_opts.delete(flag)
       end
     end
 
-    # Any encoding flag overrides all previous encoding flags. If there are
-    # multiple encoding flags in an options string, the last one wins.
-    # E.g. /(?dau)\w/ matches UTF8 chars but /(?dua)\w/ only ASCII chars.
-    if (flag = positive.reverse[/[adu]/])
-      %w[a d u].each { |key| new_active_opts.delete(key.to_sym) }
-      opt_changes[flag.to_sym] = new_active_opts[flag.to_sym] = true
+    if (enc_flag = positive.reverse[/[adu]/])
+      enc_flag = enc_flag.to_sym
+      (ENC_FLAGS - [enc_flag]).each do |other|
+        opt_changes[other] = false if new_active_opts[other]
+        new_active_opts.delete(other)
+      end
+      opt_changes[enc_flag] = new_active_opts[enc_flag] = true
     end
 
     options_stack << new_active_opts
@@ -600,16 +609,14 @@ class Regexp::Parser
   end
 
   def sequence_operation(klass, token)
-    if node.last.is_a?(klass)
-      self.node = node.last
-    elsif !node.is_a?(klass)
+    unless node.is_a?(klass)
       operator = klass.new(token, active_opts)
-      sequence = operator.add_sequence
+      sequence = operator.add_sequence(active_opts)
       sequence.expressions = node.expressions
       node.expressions = []
       nest(operator)
     end
-    node.add_sequence
+    node.add_sequence(active_opts)
   end
 
   def active_opts
@@ -627,4 +634,20 @@ class Regexp::Parser
   def count_captured_group
     captured_group_counts[node.level] += 1
   end
+
+  def assign_effective_number(exp)
+    exp.effective_number =
+      exp.number + total_captured_group_count + (exp.number < 0 ? 1 : 0)
+  end
+
+  def assign_referenced_expressions
+    targets = {}
+    root.each_expression do |exp|
+      exp.is_a?(Group::Capture) && targets[exp.identifier] = exp
+    end
+    root.each_expression do |exp|
+      exp.respond_to?(:reference) &&
+        exp.referenced_expression = targets[exp.reference]
+    end
+  end
 end # module Regexp::Parser