regexp_parser 1.4.0 → 1.5.0

data/lib/regexp_parser/expression/quantifier.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
       @max   = max
     end
 
-    def initialize_clone(other)
-      other.instance_variable_set(:@text, text.dup)
+    def initialize_clone(orig)
+      @text = orig.text.dup
       super
     end
 

data/lib/regexp_parser/expression/sequence.rb

@@ -44,10 +44,6 @@ module Regexp::Expression
       end
     end
 
-    def text
-      to_s
-    end
-
     def starts_at
       expressions.first.starts_at
     end

data/lib/regexp_parser/expression/subexpression.rb

@@ -12,8 +12,8 @@ module Regexp::Expression
     end
 
     # Override base method to clone the expressions as well.
-    def initialize_clone(other)
-      other.expressions = expressions.map(&:clone)
+    def initialize_clone(orig)
+      self.expressions = orig.expressions.map(&:clone)
       super
     end
 
@@ -46,9 +46,7 @@ module Regexp::Expression
 
     def to_s(format = :full)
       # Note: the format does not get passed down to subexpressions.
-      # Note: cant use #text accessor, b/c it is overriden as def text; to_s end
-      # in Expression::Sequence, causing infinite recursion. Clean-up needed.
-      "#{@text}#{expressions.join}#{quantifier_affix(format)}"
+      "#{expressions.join}#{quantifier_affix(format)}"
     end
 
     def to_h

data/lib/regexp_parser/lexer.rb

@@ -22,6 +22,7 @@ class Regexp::Lexer
     self.nesting = 0
     self.set_nesting = 0
     self.conditional_nesting = 0
+    self.shift = 0
 
     last = nil
     Regexp::Scanner.scan(input) do |type, token, text, ts, te|
@@ -30,11 +31,13 @@ class Regexp::Lexer
 
       ascend(type, token)
 
-      break_literal(last) if type == :quantifier and
-        last and last.type == :literal
+      if type == :quantifier and last
+        break_literal(last)        if last.type == :literal
+        break_codepoint_list(last) if last.token == :codepoint_list
+      end
 
-      current = Regexp::Token.new(type, token, text, ts, te,
-                nesting, set_nesting, conditional_nesting)
+      current = Regexp::Token.new(type, token, text, ts + shift, te + shift,
+                                  nesting, set_nesting, conditional_nesting)
 
       current = merge_literal(current) if type == :literal and
         set_nesting == 0 and
@@ -65,7 +68,7 @@ class Regexp::Lexer
 
   private
 
-  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting
+  attr_accessor :tokens, :nesting, :set_nesting, :conditional_nesting, :shift
 
   def ascend(type, token)
     case type
@@ -92,27 +95,31 @@ class Regexp::Lexer
   # called by scan to break a literal run that is longer than one character
   # into two separate tokens when it is followed by a quantifier
   def break_literal(token)
-    text = token.text
-    if text.scan(/./mu).length > 1
-      lead = text.sub(/.\z/mu, "")
-      last = text[/.\z/mu] || ''
-
-      if RUBY_VERSION >= '1.9'
-        lead_length = lead.bytesize
-        last_length = last.bytesize
-      else
-        lead_length = lead.length
-        last_length = last.length
-      end
+    lead, last, _ = token.text.partition(/.\z/mu)
+    return if lead.empty?
+
+    tokens.pop
+    tokens << Regexp::Token.new(:literal, :literal, lead,
+              token.ts, (token.te - last.bytesize),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:literal, :literal, last,
+              (token.ts + lead.bytesize), token.te,
+              nesting, set_nesting, conditional_nesting)
+  end
 
-      tokens.pop
-      tokens << Regexp::Token.new(:literal, :literal, lead, token.ts,
-                (token.te - last_length), nesting, set_nesting, conditional_nesting)
+  def break_codepoint_list(token)
+    lead, _, tail = token.text.rpartition(' ')
+    return if lead.empty?
 
-      tokens << Regexp::Token.new(:literal, :literal, last,
-                (token.ts + lead_length),
-                token.te, nesting, set_nesting, conditional_nesting)
-    end
+    tokens.pop
+    tokens << Regexp::Token.new(:escape, :codepoint_list, lead + '}',
+              token.ts, (token.te - tail.length),
+              nesting, set_nesting, conditional_nesting)
+    tokens << Regexp::Token.new(:escape, :codepoint_list, '\u{' + tail,
+              (token.ts + lead.length + 1), (token.te + 3),
+              nesting, set_nesting, conditional_nesting)
+
+    self.shift = shift + 3 # one space less, but extra \, u, {, and }
   end
 
   # called by scan to merge two consecutive literals. this happens when tokens

data/lib/regexp_parser/parser.rb

@@ -39,6 +39,8 @@ class Regexp::Parser
       parse_token(token)
     end
 
+    assign_referenced_expressions
+
     if block_given?
       block.call(root)
     else
@@ -163,14 +165,18 @@ class Regexp::Parser
       node << Backreference::NameCall.new(token, active_opts)
     when :number, :number_ref
       node << Backreference::Number.new(token, active_opts)
-    when :number_rel_ref
-      node << Backreference::NumberRelative.new(token, active_opts)
     when :number_recursion_ref
       node << Backreference::NumberRecursionLevel.new(token, active_opts)
     when :number_call
       node << Backreference::NumberCall.new(token, active_opts)
+    when :number_rel_ref
+      node << Backreference::NumberRelative.new(token, active_opts).tap do |exp|
+        assign_effective_number(exp)
+      end
     when :number_rel_call
-      node << Backreference::NumberCallRelative.new(token, active_opts)
+      node << Backreference::NumberCallRelative.new(token, active_opts).tap do |exp|
+        assign_effective_number(exp)
+      end
     else
       raise UnknownTokenError.new('Backreference', token)
     end
@@ -627,4 +633,20 @@ class Regexp::Parser
   def count_captured_group
     captured_group_counts[node.level] += 1
   end
+
+  def assign_effective_number(exp)
+    exp.effective_number =
+      exp.number + total_captured_group_count + (exp.number < 0 ? 1 : 0)
+  end
+
+  def assign_referenced_expressions
+    targets = {}
+    root.each_expression do |exp|
+      exp.is_a?(Group::Capture) && targets[exp.identifier] = exp
+    end
+    root.each_expression do |exp|
+      exp.respond_to?(:reference) &&
+        exp.referenced_expression = targets[exp.reference]
+    end
+  end
 end # module Regexp::Parser

data/lib/regexp_parser/syntax/tokens.rb

@@ -39,15 +39,7 @@ require 'regexp_parser/syntax/tokens/unicode_property'
 # into the All and Types constants.
 module Regexp::Syntax
   module Token
-    if RUBY_VERSION >= '1.9'
-      All = Map.map {|k,v| v}.flatten.uniq.sort
-    else
-      All = Map.map {|k,v| v}.flatten.uniq
-    end
-
-    Types = Map.keys
-
-    All.freeze
-    Types.freeze
+    All   = Map.values.flatten.uniq.sort.freeze
+    Types = Map.keys.freeze
   end
 end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.4.0'
+    VERSION = '1.5.0'
   end
 end

data/regexp_parser.gemspec

@@ -21,12 +21,12 @@ Gem::Specification.new do |gem|
 
   gem.require_paths = ['lib']
 
-  gem.files         = Dir.glob('{lib,test}/**/*.rb') +
+  gem.files         = Dir.glob('{lib,spec}/**/*.rb') +
                       Dir.glob('lib/**/*.rl') +
                       Dir.glob('lib/**/*.yml') +
                       %w(Gemfile Rakefile LICENSE README.md CHANGELOG.md regexp_parser.gemspec)
 
-  gem.test_files    = Dir.glob('test/**/*.rb')
+  gem.test_files    = Dir.glob('spec/**/*.rb')
 
   gem.rdoc_options  = ["--inline-source", "--charset=UTF-8"]
 

data/spec/expression/base_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Expression::Base) do
+  specify('#to_re') do
+    re_text = '^a*(b([cde]+))+f?$'
+
+    re = RP.parse(re_text).to_re
+
+    expect(re).to be_a(::Regexp)
+    expect(re_text).to eq re.source
+  end
+
+  specify('#level') do
+    regexp = /^a(b(c(d)))e$/
+    root = RP.parse(regexp)
+
+    ['^', 'a', '(b(c(d)))', 'e', '$'].each_with_index do |t, i|
+      expect(root[i].to_s).to eq t
+      expect(root[i].level).to eq 0
+    end
+
+    expect(root[2][0].to_s).to eq 'b'
+    expect(root[2][0].level).to eq 1
+
+    expect(root[2][1][0].to_s).to eq 'c'
+    expect(root[2][1][0].level).to eq 2
+
+    expect(root[2][1][1][0].to_s).to eq 'd'
+    expect(root[2][1][1][0].level).to eq 3
+  end
+
+  specify('#terminal?') do
+    root = RP.parse('^a([b]+)c$')
+
+    expect(root).not_to be_terminal
+
+    expect(root[0]).to be_terminal
+    expect(root[1]).to be_terminal
+    expect(root[2]).not_to be_terminal
+    expect(root[2][0]).not_to be_terminal
+    expect(root[2][0][0]).to be_terminal
+    expect(root[3]).to be_terminal
+    expect(root[4]).to be_terminal
+  end
+
+  specify('alt #terminal?') do
+    root = RP.parse('^(ab|cd)$')
+
+    expect(root).not_to be_terminal
+
+    expect(root[0]).to be_terminal
+    expect(root[1]).not_to be_terminal
+    expect(root[1][0]).not_to be_terminal
+    expect(root[1][0][0]).not_to be_terminal
+    expect(root[1][0][0][0]).to be_terminal
+    expect(root[1][0][1]).not_to be_terminal
+    expect(root[1][0][1][0]).to be_terminal
+  end
+
+  specify('#coded_offset') do
+    root = RP.parse('^a*(b+(c?))$')
+
+    expect(root.coded_offset).to eq '@0+12'
+
+    [
+      ['@0+1', '^'],
+      ['@1+2', 'a*'],
+      ['@3+8', '(b+(c?))'],
+      ['@11+1', '$'],
+    ].each_with_index do |check, i|
+      against = [root[i].coded_offset, root[i].to_s]
+
+      expect(against).to eq check
+    end
+
+    expect([root[2][0].coded_offset, root[2][0].to_s]).to eq ['@4+2', 'b+']
+    expect([root[2][1].coded_offset, root[2][1].to_s]).to eq ['@6+4', '(c?)']
+    expect([root[2][1][0].coded_offset, root[2][1][0].to_s]).to eq ['@7+2', 'c?']
+  end
+end

data/spec/expression/clone_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+RSpec.describe('Expression#clone') do
+  specify('Base#clone') do
+    root = RP.parse(/^(?i:a)b+$/i)
+    copy = root.clone
+
+    expect(copy.to_s).to eq root.to_s
+
+    expect(root.object_id).not_to eq copy.object_id
+    expect(root.text).to eq copy.text
+    expect(root.text.object_id).not_to eq copy.text.object_id
+
+    root_1 = root[1]
+    copy_1 = copy[1]
+
+    expect(root_1.options).to eq copy_1.options
+    expect(root_1.options.object_id).not_to eq copy_1.options.object_id
+
+    root_2 = root[2]
+    copy_2 = copy[2]
+
+    expect(root_2).to be_quantified
+    expect(copy_2).to be_quantified
+    expect(root_2.quantifier.text).to eq copy_2.quantifier.text
+    expect(root_2.quantifier.text.object_id).not_to eq copy_2.quantifier.text.object_id
+    expect(root_2.quantifier.object_id).not_to eq copy_2.quantifier.object_id
+
+    # regression test
+    expect { root_2.clone }.not_to change { root_2.quantifier.object_id }
+    expect { root_2.clone }.not_to change { root_2.quantifier.text.object_id }
+  end
+
+  specify('Subexpression#clone') do
+    root = RP.parse(/^a(b([cde])f)g$/)
+    copy = root.clone
+
+    expect(copy.to_s).to eq root.to_s
+
+    expect(root).to respond_to(:expressions)
+    expect(copy).to respond_to(:expressions)
+    expect(root.expressions.object_id).not_to eq copy.expressions.object_id
+    copy.expressions.each_with_index do |exp, index|
+      expect(root[index].object_id).not_to eq exp.object_id
+    end
+    copy[2].each_with_index do |exp, index|
+      expect(root[2][index].object_id).not_to eq exp.object_id
+    end
+
+    # regression test
+    expect { root.clone }.not_to change { root.expressions.object_id }
+  end
+
+  specify('Group::Named#clone') do
+    root = RP.parse('^(?<somename>a)+bc$')
+    copy = root.clone
+
+    expect(copy.to_s).to eq root.to_s
+
+    root_1 = root[1]
+    copy_1 = copy[1]
+
+    expect(root_1.name).to eq copy_1.name
+    expect(root_1.name.object_id).not_to eq copy_1.name.object_id
+    expect(root_1.text).to eq copy_1.text
+    expect(root_1.expressions.object_id).not_to eq copy_1.expressions.object_id
+    copy_1.expressions.each_with_index do |exp, index|
+      expect(root_1[index].object_id).not_to eq exp.object_id
+    end
+
+    # regression test
+    expect { root_1.clone }.not_to change { root_1.name.object_id }
+  end
+
+  specify('Sequence#clone') do
+    root = RP.parse(/(a|b)/)
+    copy = root.clone
+
+    # regression test
+    expect(copy.to_s).to eq root.to_s
+
+    root_seq_op = root[0][0]
+    copy_seq_op = copy[0][0]
+    root_seq_1 = root[0][0][0]
+    copy_seq_1 = copy[0][0][0]
+
+    expect(root_seq_op.object_id).not_to eq copy_seq_op.object_id
+    expect(root_seq_1.object_id).not_to eq copy_seq_1.object_id
+    copy_seq_1.expressions.each_with_index do |exp, index|
+      expect(root_seq_1[index].object_id).not_to eq exp.object_id
+    end
+  end
+
+  describe('Base#unquantified_clone') do
+    it 'produces a clone' do
+      root = RP.parse(/^a(b([cde])f)g$/)
+      copy = root.unquantified_clone
+
+      expect(copy.to_s).to eq root.to_s
+
+      expect(copy.object_id).not_to eq root.object_id
+    end
+
+    it 'does not carry over the callee quantifier' do
+      expect(RP.parse(/a{3}/)[0]).to be_quantified
+      expect(RP.parse(/a{3}/)[0].unquantified_clone).not_to be_quantified
+
+      expect(RP.parse(/[a]{3}/)[0]).to be_quantified
+      expect(RP.parse(/[a]{3}/)[0].unquantified_clone).not_to be_quantified
+
+      expect(RP.parse(/(a|b){3}/)[0]).to be_quantified
+      expect(RP.parse(/(a|b){3}/)[0].unquantified_clone).not_to be_quantified
+    end
+
+    it 'keeps quantifiers of callee children' do
+      expect(RP.parse(/(a{3}){3}/)[0][0]).to be_quantified
+      expect(RP.parse(/(a{3}){3}/)[0].unquantified_clone[0]).to be_quantified
+    end
+  end
+end

data/spec/expression/conditional_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Expression::Conditional) do
+  let(:root)   { RP.parse('^(a(b))(b(?(1)c|(?(2)d|(?(3)e|f)))g)$') }
+  let(:cond_1) { root[2][1] }
+  let(:cond_2) { root[2][1][2][0] }
+  let(:cond_3) { root[2][1][2][0][2][0] }
+
+  specify('root level') do
+    [
+      '^',
+      '(a(b))',
+      '(b(?(1)c|(?(2)d|(?(3)e|f)))g)',
+      '$'
+    ].each_with_index do |t, i|
+      expect(root[i].conditional_level).to eq 0
+      expect(root[i].to_s).to eq t
+    end
+
+    expect(root[2][0].to_s).to eq 'b'
+    expect(root[2][0].conditional_level).to eq 0
+  end
+
+  specify('level one') do
+    condition = cond_1.condition
+    branch_1 = cond_1.branches.first
+
+    expect(condition).to be_a Conditional::Condition
+    expect(condition.to_s).to eq '(1)'
+    expect(condition.conditional_level).to eq 1
+
+    expect(branch_1).to be_a Conditional::Branch
+    expect(branch_1.to_s).to eq 'c'
+    expect(branch_1.conditional_level).to eq 1
+
+    expect(branch_1.first.to_s).to eq 'c'
+    expect(branch_1.first.conditional_level).to eq 1
+  end
+
+  specify('level two') do
+    condition = cond_2.condition
+    branch_1 = cond_2.branches.first
+    branch_2 = cond_2.branches.last
+
+    expect(cond_2.to_s).to start_with '(?'
+    expect(cond_2.conditional_level).to eq 1
+
+    expect(condition).to be_a Conditional::Condition
+    expect(condition.to_s).to eq '(2)'
+    expect(condition.conditional_level).to eq 2
+
+    expect(branch_1).to be_a Conditional::Branch
+    expect(branch_1.to_s).to eq 'd'
+    expect(branch_1.conditional_level).to eq 2
+
+    expect(branch_1.first.to_s).to eq 'd'
+    expect(branch_1.first.conditional_level).to eq 2
+
+    expect(branch_2.first.to_s).to start_with '(?'
+    expect(branch_2.first.conditional_level).to eq 2
+  end
+
+  specify('level three') do
+    condition = cond_3.condition
+    branch_1 = cond_3.branches.first
+    branch_2 = cond_3.branches.last
+
+    expect(condition).to be_a Conditional::Condition
+    expect(condition.to_s).to eq '(3)'
+    expect(condition.conditional_level).to eq 3
+
+    expect(cond_3.to_s).to eq '(?(3)e|f)'
+    expect(cond_3.conditional_level).to eq 2
+
+    expect(branch_1).to be_a Conditional::Branch
+    expect(branch_1.to_s).to eq 'e'
+    expect(branch_1.conditional_level).to eq 3
+
+    expect(branch_1.first.to_s).to eq 'e'
+    expect(branch_1.first.conditional_level).to eq 3
+
+    expect(branch_2).to be_a Conditional::Branch
+    expect(branch_2.to_s).to eq 'f'
+    expect(branch_2.conditional_level).to eq 3
+
+    expect(branch_2.first.to_s).to eq 'f'
+    expect(branch_2.first.conditional_level).to eq 3
+  end
+end