regexp_parser 1.4.0 → 1.5.0

data/spec/lexer/keep_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+RSpec.describe('Keep lexing') do
+  specify('lex keep token') do
+    regexp = /ab\Kcd/
+    tokens = RL.lex(regexp)
+
+    expect(tokens[1].type).to eq :keep
+    expect(tokens[1].token).to eq :mark
+  end
+
+  specify('lex keep nested') do
+    regexp = /(a\Kb)|(c\\\Kd)ef/
+    tokens = RL.lex(regexp)
+
+    expect(tokens[2].type).to eq :keep
+    expect(tokens[2].token).to eq :mark
+
+    expect(tokens[9].type).to eq :keep
+    expect(tokens[9].token).to eq :mark
+  end
+end

data/{test/lexer/test_literals.rb → spec/lexer/literals_spec.rb}

@@ -1,9 +1,6 @@
-# -*- encoding: utf-8 -*-
-
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerLiterals < Test::Unit::TestCase
+require 'spec_helper'
 
+RSpec.describe('Literal lexing') do
   tests = {
     # ascii, single byte characters
     'a' => {
@@ -91,40 +88,39 @@ class LexerLiterals < Test::Unit::TestCase
   }
 
   tests.each_with_index do |(pattern, checks), count|
-    define_method "test_lex_literal_runs_#{count}" do
+    specify("lex_literal_runs_#{count}") do
       tokens = RL.lex(pattern)
 
       checks.each do |index, (type, token, text, ts, te, level, set_level, conditional_level)|
         struct = tokens.at(index)
 
-        assert_equal type,              struct.type
-        assert_equal token,             struct.token
-        assert_equal text,              struct.text
-        assert_equal ts,                struct.ts
-        assert_equal te,                struct.te
-        assert_equal level,             struct.level
-        assert_equal set_level,         struct.set_level
-        assert_equal conditional_level, struct.conditional_level
+        expect(struct.type).to eq type
+        expect(struct.token).to eq token
+        expect(struct.text).to eq text
+        expect(struct.ts).to eq ts
+        expect(struct.te).to eq te
+        expect(struct.level).to eq level
+        expect(struct.set_level).to eq set_level
+        expect(struct.conditional_level).to eq conditional_level
       end
     end
   end
 
-  def test_lex_single_2_byte_char
-    tokens = RL.lex('ا+')
+  specify('lex single 2 byte char') do
+    tokens = RL.lex("\u0627+")
 
-    assert_equal 2, tokens.length
+    expect(tokens.length).to eq 2
   end
 
-  def test_lex_single_3_byte_char
-    tokens = RL.lex('れ+')
+  specify('lex single 3 byte char') do
+    tokens = RL.lex("\u308C+")
 
-    assert_equal 2, tokens.length
+    expect(tokens.length).to eq 2
   end
 
-  def test_lex_single_4_byte_char
-    tokens = RL.lex('𝄞+')
+  specify('lex single 4 byte char') do
+    tokens = RL.lex("\u{1D11E}+")
 
-    assert_equal 2, tokens.length
+    expect(tokens.length).to eq 2
   end
-
 end

data/{test/lexer/test_nesting.rb → spec/lexer/nesting_spec.rb}

@@ -1,7 +1,6 @@
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerNesting < Test::Unit::TestCase
+require 'spec_helper'
 
+RSpec.describe('Nesting lexing') do
   tests = {
     '(((b)))' => {
       0     => [:group,       :capture,       '(',      0,  1, 0, 0, 0],
@@ -111,22 +110,21 @@ class LexerNesting < Test::Unit::TestCase
   }
 
   tests.each_with_index do |(pattern, checks), count|
-    define_method "test_lex_nesting_in_'#{pattern}'_#{count}" do
+    specify("lex_nesting_in_'#{pattern}'_#{count}") do
       tokens = RL.lex(pattern, 'ruby/1.9')
 
       checks.each do |offset, (type, token, text, ts, te, level, set_level, conditional_level)|
         struct = tokens.at(offset)
 
-        assert_equal type,              struct.type
-        assert_equal token,             struct.token
-        assert_equal text,              struct.text
-        assert_equal ts,                struct.ts
-        assert_equal te,                struct.te
-        assert_equal level,             struct.level
-        assert_equal set_level,         struct.set_level
-        assert_equal conditional_level, struct.conditional_level
+        expect(struct.type).to eq type
+        expect(struct.token).to eq token
+        expect(struct.text).to eq text
+        expect(struct.ts).to eq ts
+        expect(struct.te).to eq te
+        expect(struct.level).to eq level
+        expect(struct.set_level).to eq set_level
+        expect(struct.conditional_level).to eq conditional_level
       end
     end
   end
-
 end

data/spec/lexer/refcalls_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe('RefCall lexing') do
+  tests = {
+    # Traditional numerical group back-reference
+    '(abc)\1'           => [3, :backref, :number,                '\1',         5,  7, 0, 0, 0],
+
+    # Group back-references, named, numbered, and relative
+    '(?<X>abc)\k<X>'    => [3, :backref, :name_ref,              '\k<X>',      9, 14, 0, 0, 0],
+    "(?<X>abc)\\k'X'"   => [3, :backref, :name_ref,              "\\k'X'",     9, 14, 0, 0, 0],
+
+    '(abc)\k<1>'        => [3, :backref, :number_ref,            '\k<1>',      5, 10, 0, 0, 0],
+    "(abc)\\k'1'"       => [3, :backref, :number_ref,            "\\k'1'",     5, 10, 0, 0, 0],
+
+    '(abc)\k<-1>'       => [3, :backref, :number_rel_ref,        '\k<-1>',     5, 11, 0, 0, 0],
+    "(abc)\\k'-1'"      => [3, :backref, :number_rel_ref,        "\\k'-1'",    5, 11, 0, 0, 0],
+
+    # Sub-expression invocation, named, numbered, and relative
+    '(?<X>abc)\g<X>'    => [3, :backref, :name_call,             '\g<X>',      9, 14, 0, 0, 0],
+    "(?<X>abc)\\g'X'"   => [3, :backref, :name_call,             "\\g'X'",     9, 14, 0, 0, 0],
+
+    '(abc)\g<1>'        => [3, :backref, :number_call,           '\g<1>',      5, 10, 0, 0, 0],
+    "(abc)\\g'1'"       => [3, :backref, :number_call,           "\\g'1'",     5, 10, 0, 0, 0],
+
+    '(abc)\g<-1>'       => [3, :backref, :number_rel_call,       '\g<-1>',     5, 11, 0, 0, 0],
+    "(abc)\\g'-1'"      => [3, :backref, :number_rel_call,       "\\g'-1'",    5, 11, 0, 0, 0],
+
+    '(abc)\g<+1>'       => [3, :backref, :number_rel_call,       '\g<+1>',     5, 11, 0, 0, 0],
+    "(abc)\\g'+1'"      => [3, :backref, :number_rel_call,       "\\g'+1'",    5, 11, 0, 0, 0],
+
+    # Group back-references, with nesting level
+    '(?<X>abc)\k<X-0>'  => [3, :backref, :name_recursion_ref,    '\k<X-0>',    9, 16, 0, 0, 0],
+    "(?<X>abc)\\k'X-0'" => [3, :backref, :name_recursion_ref,    "\\k'X-0'",   9, 16, 0, 0, 0],
+
+    '(abc)\k<1-0>'      => [3, :backref, :number_recursion_ref,  '\k<1-0>',    5, 12, 0, 0, 0],
+    "(abc)\\k'1-0'"     => [3, :backref, :number_recursion_ref,  "\\k'1-0'",   5, 12, 0, 0, 0],
+  }
+
+  tests.each_with_index do |(pattern, (index, type, token, text, ts, te, level, set_level, conditional_level)), count|
+    specify("lexer_#{type}_#{token}_#{count}") do
+      tokens = RL.lex(pattern, 'ruby/1.9')
+      struct = tokens.at(index)
+
+      expect(struct.type).to eq type
+      expect(struct.token).to eq token
+      expect(struct.text).to eq text
+      expect(struct.ts).to eq ts
+      expect(struct.te).to eq te
+      expect(struct.level).to eq level
+      expect(struct.set_level).to eq set_level
+      expect(struct.conditional_level).to eq conditional_level
+    end
+  end
+end

data/spec/parser/all_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Parser) do
+  specify('parse returns a root expression') do
+    expect(RP.parse('abc')).to be_instance_of(Regexp::Expression::Root)
+  end
+
+  specify('parse root contains expressions') do
+    root = RP.parse(/^a.c+[^one]{2,3}\b\d\\\C-C$/)
+    expect(root.expressions).to all(be_a Regexp::Expression::Base)
+  end
+
+  specify('parse node types') do
+    root = RP.parse('^(one){2,3}([^d\\]efm-qz\\,\\-]*)(ghi)+$')
+
+    expect(root[1][0]).to be_a(Literal)
+    expect(root[1]).to be_quantified
+    expect(root[2][0]).to be_a(CharacterSet)
+    expect(root[2]).not_to be_quantified
+    expect(root[3]).to be_a(Group::Capture)
+    expect(root[3]).to be_quantified
+  end
+
+  specify('parse no quantifier target raises error') do
+    expect { RP.parse('?abc') }.to raise_error(ArgumentError)
+  end
+
+  specify('parse sequence no quantifier target raises error') do
+    expect { RP.parse('abc|?def') }.to raise_error(ArgumentError)
+  end
+end

data/spec/parser/alternation_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+
+RSpec.describe('Alternation parsing') do
+  let(:root) { RP.parse('(ab??|cd*|ef+)*|(gh|ij|kl)?') }
+
+  specify('parse alternation root') do
+    e = root[0]
+    expect(e).to be_a(Alternation)
+  end
+
+  specify('parse alternation alts') do
+    alts = root[0].alternatives
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+
+    expect(alts[0][0]).to be_a(Group::Capture)
+    expect(alts[1][0]).to be_a(Group::Capture)
+
+    expect(alts.length).to eq 2
+  end
+
+  specify('parse alternation nested') do
+    e = root[0].alternatives[0][0][0]
+
+    expect(e).to be_a(Alternation)
+  end
+
+  specify('parse alternation nested sequence') do
+    alts = root[0][0]
+    nested = alts[0][0][0]
+
+    expect(nested).to be_a(Alternative)
+
+    expect(nested[0]).to be_a(Literal)
+    expect(nested[1]).to be_a(Literal)
+    expect(nested.expressions.length).to eq 2
+  end
+
+  specify('parse alternation nested groups') do
+    root = RP.parse('(i|ey|([ougfd]+)|(ney))')
+
+    alts = root[0][0].alternatives
+    expect(alts.length).to eq 4
+  end
+
+  specify('parse alternation grouped alts') do
+    root = RP.parse('ca((n)|(t)|(ll)|(b))')
+
+    alts = root[1][0].alternatives
+
+    expect(alts.length).to eq 4
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+    expect(alts[2]).to be_a(Alternative)
+    expect(alts[3]).to be_a(Alternative)
+  end
+
+  specify('parse alternation nested grouped alts') do
+    root = RP.parse('ca((n|t)|(ll|b))')
+
+    alts = root[1][0].alternatives
+
+    expect(alts.length).to eq 2
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+
+    subalts = root[1][0][0][0][0].alternatives
+
+    expect(alts.length).to eq 2
+
+    expect(subalts[0]).to be_a(Alternative)
+    expect(subalts[1]).to be_a(Alternative)
+  end
+
+  specify('parse alternation continues after nesting') do
+    root = RP.parse(/a|(b)c/)
+
+    seq = root[0][1].expressions
+
+    expect(seq.length).to eq 2
+
+    expect(seq[0]).to be_a(Group::Capture)
+    expect(seq[1]).to be_a(Literal)
+  end
+end

data/{test/parser/test_anchors.rb → spec/parser/anchors_spec.rb}

@@ -1,7 +1,6 @@
-require File.expand_path("../../helpers", __FILE__)
-
-class TestParserAnchors < Test::Unit::TestCase
+require 'spec_helper'
 
+RSpec.describe('Anchor parsing') do
   tests = {
     '^a'      => [0, :anchor,   :bol,                 Anchor::BOL],
     'a$'      => [1, :anchor,   :eol,                 Anchor::EOL],
@@ -19,16 +18,14 @@ class TestParserAnchors < Test::Unit::TestCase
   }
 
   tests.each_with_index do |(pattern, (index, type, token, klass)), count|
-    define_method "test_parse_anchor_#{token}_#{count}" do
+    specify("parse_anchor_#{token}_#{count}") do
       root = RP.parse(pattern, 'ruby/1.9')
-      exp  = root.expressions.at(index)
+      exp = root.expressions.at(index)
 
-      assert exp.is_a?(klass),
-             "Expected #{klass}, but got #{exp.class.name}"
+      expect(exp).to be_a(klass)
 
-      assert_equal type,  exp.type
-      assert_equal token, exp.token
+      expect(exp.type).to eq type
+      expect(exp.token).to eq token
     end
   end
-
 end

data/spec/parser/conditionals_spec.rb

@@ -0,0 +1,179 @@
+require 'spec_helper'
+
+RSpec.describe('Conditional parsing') do
+  specify('parse conditional') do
+    regexp = Regexp.new('(?<A>a)(?(<A>)T|F)/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1]
+
+    expect(exp).to be_a(Conditional::Expression)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :open
+    expect(exp.to_s).to eq '(?(<A>)T|F)'
+    expect(exp.reference).to eq 'A'
+  end
+
+  specify('parse conditional condition') do
+    regexp = Regexp.new('(?<A>a)(?(<A>)T|F)/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1].condition
+
+    expect(exp).to be_a(Conditional::Condition)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :condition
+    expect(exp.to_s).to eq '(<A>)'
+    expect(exp.reference).to eq 'A'
+    expect(exp.referenced_expression.to_s).to eq '(?<A>a)'
+  end
+
+  specify('parse conditional condition with number ref') do
+    regexp = Regexp.new('(a)(?(1)T|F)/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1].condition
+
+    expect(exp).to be_a(Conditional::Condition)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :condition
+    expect(exp.to_s).to eq '(1)'
+    expect(exp.reference).to eq 1
+    expect(exp.referenced_expression.to_s).to eq '(a)'
+  end
+
+  specify('parse conditional nested groups') do
+    regexp = Regexp.new('((a)|(b)|((?(2)(c(d|e)+)?|(?(3)f|(?(4)(g|(h)(i)))))))/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    group = root.first
+    expect(group).to be_instance_of(Group::Capture)
+
+    alt = group.first
+    expect(alt).to be_instance_of(Alternation)
+    expect(alt.length).to eq 3
+
+    expect(alt.map(&:first)).to all(be_a Group::Capture)
+
+    subgroup = alt[2].first
+    conditional = subgroup.first
+
+    expect(conditional).to be_instance_of(Conditional::Expression)
+    expect(conditional.length).to eq 3
+
+    expect(conditional[0]).to be_instance_of(Conditional::Condition)
+    expect(conditional[0].to_s).to eq '(2)'
+
+    condition = conditional.condition
+    expect(condition).to be_instance_of(Conditional::Condition)
+    expect(condition.to_s).to eq '(2)'
+
+    branches = conditional.branches
+    expect(branches.length).to eq 2
+    expect(branches).to be_instance_of(Array)
+  end
+
+  specify('parse conditional nested') do
+    regexp = Regexp.new('(a(b(c(d)(e))))(?(1)(?(2)d|(?(3)e|f))|(?(4)(?(5)g|h)))/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    {
+      1 => [2, root[1]],
+      2 => [2, root[1][1][0]],
+      3 => [2, root[1][1][0][2][0]],
+      4 => [1, root[1][2][0]],
+      5 => [2, root[1][2][0][1][0]]
+    }.each do |index, example|
+      branch_count, exp = example
+
+      expect(exp).to be_instance_of(Conditional::Expression)
+      expect(exp.condition.to_s).to eq "(#{index})"
+      expect(exp.branches.length).to eq branch_count
+    end
+  end
+
+  specify('parse conditional nested alternation') do
+    regexp = Regexp.new('(a)(?(1)(b|c|d)|(e|f|g))(h)(?(2)(i|j|k)|(l|m|n))|o|p/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    expect(root.first).to be_instance_of(Alternation)
+
+    [
+      [3, 'b|c|d', root[0][0][1][1][0][0]],
+      [3, 'e|f|g', root[0][0][1][2][0][0]],
+      [3, 'i|j|k', root[0][0][3][1][0][0]],
+      [3, 'l|m|n', root[0][0][3][2][0][0]]
+    ].each do |example|
+      alt_count, alt_text, exp = example
+
+      expect(exp).to be_instance_of(Alternation)
+      expect(exp.to_s).to eq alt_text
+      expect(exp.alternatives.length).to eq alt_count
+    end
+  end
+
+  specify('parse conditional extra separator') do
+    regexp = Regexp.new('(?<A>a)(?(<A>)T|)/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    branches = root[1].branches
+
+    expect(branches.length).to eq 2
+
+    seq_1, seq_2 = branches
+
+    [seq_1, seq_2].each do |seq|
+      expect(seq).to be_a(Sequence)
+
+      expect(seq.type).to eq :expression
+      expect(seq.token).to eq :sequence
+    end
+
+    expect(seq_1.to_s).to eq 'T'
+    expect(seq_2.to_s).to eq ''
+  end
+
+  specify('parse conditional quantified') do
+    regexp = Regexp.new('(foo)(?(1)\d|(\w)){42}/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    conditional = root[1]
+
+    expect(conditional).to be_quantified
+    expect(conditional.quantifier.to_s).to eq '{42}'
+    expect(conditional.to_s).to eq '(?(1)\\d|(\\w)){42}'
+    expect(conditional.branches.any?(&:quantified?)).to be false
+  end
+
+  specify('parse conditional branch content quantified') do
+    regexp = Regexp.new('(foo)(?(1)\d{23}|(\w){42})/')
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    conditional = root[1]
+
+    expect(conditional).not_to be_quantified
+    expect(conditional.branches.any?(&:quantified?)).to be false
+    expect(conditional.branches[0][0]).to be_quantified
+    expect(conditional.branches[0][0].quantifier.to_s).to eq '{23}'
+    expect(conditional.branches[1][0]).to be_quantified
+    expect(conditional.branches[1][0].quantifier.to_s).to eq '{42}'
+  end
+
+  specify('parse conditional excessive branches') do
+    regexp = '(?<A>a)(?(<A>)T|F|X)'
+
+    expect { RP.parse(regexp, 'ruby/2.0') }.to raise_error(Conditional::TooManyBranches)
+  end
+end