regexp_parser 1.3.0 → 1.7.1

data/lib/regexp_parser/expression/classes/keep.rb

@@ -1,7 +1,5 @@
 module Regexp::Expression
-
   module Keep
-    class Mark          < Regexp::Expression::Base; end
+    class Mark < Regexp::Expression::Base; end
   end
-
 end

data/lib/regexp_parser/expression/methods/match.rb

@@ -0,0 +1,13 @@
+module Regexp::Expression
+  class Base
+    def match?(string)
+      !!match(string)
+    end
+    alias :matches? :match?
+
+    def match(string, offset = 0)
+      Regexp.new(to_s).match(string, offset)
+    end
+    alias :=~ :match
+  end
+end

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -0,0 +1,172 @@
+class Regexp::MatchLength
+  include Enumerable
+
+  def self.of(obj)
+    exp = obj.is_a?(Regexp::Expression::Base) ? obj : Regexp::Parser.parse(obj)
+    exp.match_length
+  end
+
+  def initialize(exp, opts = {})
+    self.exp_class = exp.class
+    self.min_rep = exp.repetitions.min
+    self.max_rep = exp.repetitions.max
+    if base = opts[:base]
+      self.base_min = base
+      self.base_max = base
+      self.reify = ->{ '.' * base }
+    else
+      self.base_min = opts.fetch(:base_min)
+      self.base_max = opts.fetch(:base_max)
+      self.reify = opts.fetch(:reify)
+    end
+  end
+
+  def each(opts = {})
+    return enum_for(__method__, opts) unless block_given?
+    limit = opts[:limit] || 1000
+    yielded = 0
+    (min..max).each do |num|
+      next unless include?(num)
+      yield(num)
+      break if (yielded += 1) >= limit
+    end
+  end
+
+  def endless_each(&block)
+    return enum_for(__method__) unless block_given?
+    (min..max).each { |num| yield(num) if include?(num) }
+  end
+
+  def include?(length)
+    test_regexp.match?('X' * length)
+  end
+
+  def fixed?
+    min == max
+  end
+
+  def min
+    min_rep * base_min
+  end
+
+  def max
+    max_rep * base_max
+  end
+
+  def minmax
+    [min, max]
+  end
+
+  def inspect
+    type = exp_class.name.sub('Regexp::Expression::', '')
+    "#<#{self.class}<#{type}> min=#{min} max=#{max}>"
+  end
+
+  def to_re
+    "(?:#{reify.call}){#{min_rep},#{max_rep unless max_rep == Float::INFINITY}}"
+  end
+
+  private
+
+  attr_accessor :base_min, :base_max, :min_rep, :max_rep, :exp_class, :reify
+
+  def test_regexp
+    @test_regexp ||= Regexp.new("^#{to_re}$").tap do |regexp|
+      regexp.respond_to?(:match?) || def regexp.match?(str); !!match(str) end
+    end
+  end
+end
+
+module Regexp::Expression
+  MatchLength = Regexp::MatchLength
+
+  [
+    CharacterSet,
+    CharacterSet::Intersection,
+    CharacterSet::IntersectedSequence,
+    CharacterSet::Range,
+    CharacterType::Base,
+    EscapeSequence::Base,
+    PosixClass,
+    UnicodeProperty::Base,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self, base: 1)
+      end
+    RUBY
+  end
+
+  class Literal
+    def match_length
+      MatchLength.new(self, base: text.length)
+    end
+  end
+
+  class Subexpression
+    def match_length
+      MatchLength.new(self,
+                       base_min: map { |exp| exp.match_length.min }.inject(0, :+),
+                       base_max: map { |exp| exp.match_length.max }.inject(0, :+),
+                       reify: ->{ map { |exp| exp.match_length.to_re }.join })
+    end
+
+    def inner_match_length
+      dummy = Regexp::Expression::Root.build
+      dummy.expressions = expressions.map(&:clone)
+      dummy.quantifier = quantifier && quantifier.clone
+      dummy.match_length
+    end
+  end
+
+  [
+    Alternation,
+    Conditional::Expression,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self,
+                         base_min: map { |exp| exp.match_length.min }.min,
+                         base_max: map { |exp| exp.match_length.max }.max,
+                         reify: ->{ map { |exp| exp.match_length.to_re }.join('|') })
+      end
+    RUBY
+  end
+
+  [
+    Anchor::Base,
+    Assertion::Base,
+    Conditional::Condition,
+    FreeSpace,
+    Keep::Mark,
+  ].each do |klass|
+    klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def match_length
+        MatchLength.new(self, base: 0)
+      end
+    RUBY
+  end
+
+  class Backreference::Base
+    def match_length
+      if referenced_expression.nil?
+        raise ArgumentError, 'Missing referenced_expression - not parsed?'
+      end
+      referenced_expression.unquantified_clone.match_length
+    end
+  end
+
+  class EscapeSequence::CodepointList
+    def match_length
+      MatchLength.new(self, base: codepoints.count)
+    end
+  end
+
+  # Special case. Absence group can match 0.. chars, irrespective of content.
+  # TODO: in theory, they *can* exclude match lengths with `.`: `(?~.{3})`
+  class Group::Absence
+    def match_length
+      MatchLength.new(self, base_min: 0, base_max: Float::INFINITY, reify: ->{ '.*' })
+    end
+  end
+end

data/lib/regexp_parser/expression/methods/options.rb

@@ -0,0 +1,35 @@
+module Regexp::Expression
+  class Base
+    def multiline?
+      options[:m] == true
+    end
+    alias :m? :multiline?
+
+    def case_insensitive?
+      options[:i] == true
+    end
+    alias :i? :case_insensitive?
+    alias :ignore_case? :case_insensitive?
+
+    def free_spacing?
+      options[:x] == true
+    end
+    alias :x? :free_spacing?
+    alias :extended? :free_spacing?
+
+    def default_classes?
+      options[:d] == true
+    end
+    alias :d? :default_classes?
+
+    def ascii_classes?
+      options[:a] == true
+    end
+    alias :a? :ascii_classes?
+
+    def unicode_classes?
+      options[:u] == true
+    end
+    alias :u? :unicode_classes?
+  end
+end

data/lib/regexp_parser/expression/methods/strfregexp.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Base
 
     #   %l  Level (depth) of the expression. Returns 'root' for the root

data/lib/regexp_parser/expression/methods/tests.rb

@@ -75,32 +75,23 @@ module Regexp::Expression
     def one_of?(scope, top = true)
       case scope
       when Array
-        if scope.include?(:*)
-          return (scope.include?(token) or scope.include?(:*))
-        else
-          return scope.include?(token)
-        end
+        scope.include?(:*) || scope.include?(token)
 
       when Hash
         if scope.has_key?(:*)
           test_type = scope.has_key?(type) ? type : :*
-          return one_of?(scope[test_type], false)
+          one_of?(scope[test_type], false)
         else
-          return (scope.has_key?(type) and one_of?(scope[type], false))
+          scope.has_key?(type) && one_of?(scope[type], false)
         end
 
       when Symbol
-        return true if scope == :*
-
-        return is?(scope) unless top
-        return type?(scope) if top
+        scope.equal?(:*) || (top ? type?(scope) : is?(scope))
 
       else
-        raise "Array, Hash, or Symbol expected, #{scope.class.name} given"
+        raise ArgumentError,
+              "Array, Hash, or Symbol expected, #{scope.class.name} given"
       end
-
-      false
     end
-
   end
 end

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -14,7 +14,7 @@ module Regexp::Expression
     #
     # Returns self.
     def traverse(include_self = false, &block)
-      raise 'traverse requires a block' unless block_given?
+      return enum_for(__method__, include_self) unless block_given?
 
       block.call(:enter, self, 0) if include_self
 
@@ -37,6 +37,8 @@ module Regexp::Expression
     # Iterates over the expressions of this expression as an array, passing
     # the expression and its index within its parent to the given block.
     def each_expression(include_self = false, &block)
+      return enum_for(__method__, include_self) unless block_given?
+
       traverse(include_self) do |event, exp, index|
         yield(exp, index) unless event == :exit
       end

data/lib/regexp_parser/expression/quantifier.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
       @max   = max
     end
 
-    def initialize_clone(other)
-      other.instance_variable_set(:@text, text.dup)
+    def initialize_clone(orig)
+      @text = orig.text.dup
       super
     end
 

data/lib/regexp_parser/expression/sequence.rb

@@ -18,13 +18,14 @@ module Regexp::Expression
     end
 
     class << self
-      def add_to(subexpression, options = {})
+      def add_to(subexpression, params = {}, active_opts = {})
         sequence = at_levels(
           subexpression.level,
           subexpression.set_level,
-          options[:conditional_level] || subexpression.conditional_level
+          params[:conditional_level] || subexpression.conditional_level
         )
         sequence.nesting_level = subexpression.nesting_level + 1
+        sequence.options = active_opts
         subexpression.expressions << sequence
         sequence
       end
@@ -44,10 +45,6 @@ module Regexp::Expression
       end
     end
 
-    def text
-      to_s
-    end
-
     def starts_at
       expressions.first.starts_at
     end

data/lib/regexp_parser/expression/sequence_operation.rb

@@ -14,12 +14,8 @@ module Regexp::Expression
       expressions.last << exp
     end
 
-    def add_sequence
-      self.class::OPERAND.add_to(self)
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      sequences.last.last.quantify(token, text, min, max, mode)
+    def add_sequence(active_opts = {})
+      self.class::OPERAND.add_to(self, {}, active_opts)
     end
 
     def to_s(format = :full)

data/lib/regexp_parser/expression/subexpression.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
     end
 
     # Override base method to clone the expressions as well.
-    def initialize_clone(other)
-      other.expressions = expressions.map(&:clone)
+    def initialize_clone(orig)
+      self.expressions = orig.expressions.map(&:clone)
       super
     end
 
@@ -46,9 +46,7 @@ module Regexp::Expression
 
     def to_s(format = :full)
       # Note: the format does not get passed down to subexpressions.
-      # Note: cant use #text accessor, b/c it is overriden as def text; to_s end
-      # in Expression::Sequence, causing infinite recursion. Clean-up needed.
-      "#{@text}#{expressions.join}#{quantifier_affix(format)}"
+      "#{expressions.join}#{quantifier_affix(format)}"
     end
 
     def to_h

data/lib/regexp_parser/lexer.rb

@@ -22,6 +22,7 @@ class Regexp::Lexer
     self.nesting = 0
     self.set_nesting = 0
     self.conditional_nesting = 0
+    self.shift = 0
 
     last = nil
     Regexp::Scanner.scan(input) do |type, token, text, ts, te|
@@ -30,15 +31,13 @@ class Regexp::Lexer
 
       ascend(type, token)
 
-      break_literal(last) if type == :quantifier and
-        last and last.type == :literal
-
-      current = Regexp::Token.new(type, token, text, ts, te,
-                nesting, set_nesting, conditional_nesting)
+      if type == :quantifier and last
+        break_literal(last)        if last.type == :literal
+        break_codepoint_list(last) if last.token == :codepoint_list
+      end
 
-      current = merge_literal(current) if type == :literal and
-        set_nesting == 0 and
-        last and last.type == :literal
+      current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
+                                  nesting, set_nesting, conditional_nesting)
 
       current = merge_condition(current) if type == :conditional and
         [:condition, :condition_close].include?(token)
@@ -65,7 +64,7 @@ class Regexp::Lexer
 
   private
 
-  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting
+  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
     case type
@@ -92,44 +91,31 @@ class Regexp::Lexer
   # called by scan to break a literal run that is longer than one character
   # into two separate tokens when it is followed by a quantifier
   def break_literal(token)
-    text = token.text
-    if text.scan(/./mu).length > 1
-      lead = text.sub(/.\z/mu, "")
-      last = text[/.\z/mu] || ''
-
-      if RUBY_VERSION >= '1.9'
-        lead_length = lead.bytesize
-        last_length = last.bytesize
-      else
-        lead_length = lead.length
-        last_length = last.length
-      end
-
-      tokens.pop
-      tokens << Regexp::Token.new(:literal, :literal, lead, token.ts,
-                (token.te - last_length), nesting, set_nesting, conditional_nesting)
-
-      tokens << Regexp::Token.new(:literal, :literal, last,
-                (token.ts + lead_length),
-                token.te, nesting, set_nesting, conditional_nesting)
-    end
+    lead, last, _ = token.text.partition(/.\z/mu)
+    return if lead.empty?
+
+    tokens.pop
+    tokens << Regexp::Token.new(:literal, :literal, lead,
+              token.ts, (token.te - last.bytesize),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:literal, :literal, last,
+              (token.ts + lead.bytesize), token.te,
+              nesting, set_nesting, conditional_nesting)
   end
 
-  # called by scan to merge two consecutive literals. this happens when tokens
-  # get normalized (as in the case of posix/bre) and end up becoming literals.
-  def merge_literal(current)
-    last = tokens.pop
+  def break_codepoint_list(token)
+    lead, _, tail = token.text.rpartition(' ')
+    return if lead.empty?
+
+    tokens.pop
+    tokens << Regexp::Token.new(:escape, :codepoint_list, lead + '}',
+              token.ts, (token.te - tail.length),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
+              (token.ts + lead.length + 1), (token.te + 3),
+              nesting, set_nesting, conditional_nesting)
 
-    Regexp::Token.new(
-      :literal,
-      :literal,
-      last.text + current.text,
-      last.ts,
-      current.te,
-      nesting,
-      set_nesting,
-      conditional_nesting,
-    )
+    self.shift = shift + 3 # one space less, but extra \, u, {, and }
   end
 
   def merge_condition(current)