regexp_parser 1.4.0 → 1.7.1

data/spec/parser/alternation_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+
+RSpec.describe('Alternation parsing') do
+  let(:root) { RP.parse('(ab??|cd*|ef+)*|(gh|ij|kl)?') }
+
+  specify('parse alternation root') do
+    e = root[0]
+    expect(e).to be_a(Alternation)
+  end
+
+  specify('parse alternation alts') do
+    alts = root[0].alternatives
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+
+    expect(alts[0][0]).to be_a(Group::Capture)
+    expect(alts[1][0]).to be_a(Group::Capture)
+
+    expect(alts.length).to eq 2
+  end
+
+  specify('parse alternation nested') do
+    e = root[0].alternatives[0][0][0]
+
+    expect(e).to be_a(Alternation)
+  end
+
+  specify('parse alternation nested sequence') do
+    alts = root[0][0]
+    nested = alts[0][0][0]
+
+    expect(nested).to be_a(Alternative)
+
+    expect(nested[0]).to be_a(Literal)
+    expect(nested[1]).to be_a(Literal)
+    expect(nested.expressions.length).to eq 2
+  end
+
+  specify('parse alternation nested groups') do
+    root = RP.parse('(i|ey|([ougfd]+)|(ney))')
+
+    alts = root[0][0].alternatives
+    expect(alts.length).to eq 4
+  end
+
+  specify('parse alternation grouped alts') do
+    root = RP.parse('ca((n)|(t)|(ll)|(b))')
+
+    alts = root[1][0].alternatives
+
+    expect(alts.length).to eq 4
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+    expect(alts[2]).to be_a(Alternative)
+    expect(alts[3]).to be_a(Alternative)
+  end
+
+  specify('parse alternation nested grouped alts') do
+    root = RP.parse('ca((n|t)|(ll|b))')
+
+    alts = root[1][0].alternatives
+
+    expect(alts.length).to eq 2
+
+    expect(alts[0]).to be_a(Alternative)
+    expect(alts[1]).to be_a(Alternative)
+
+    subalts = root[1][0][0][0][0].alternatives
+
+    expect(alts.length).to eq 2
+
+    expect(subalts[0]).to be_a(Alternative)
+    expect(subalts[1]).to be_a(Alternative)
+  end
+
+  specify('parse alternation continues after nesting') do
+    root = RP.parse(/a|(b)c/)
+
+    seq = root[0][1].expressions
+
+    expect(seq.length).to eq 2
+
+    expect(seq[0]).to be_a(Group::Capture)
+    expect(seq[1]).to be_a(Literal)
+  end
+end

data/spec/parser/anchors_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+RSpec.describe('Anchor parsing') do
+  include_examples 'parse', /^a/,   0 =>  [:anchor,   :bol,               Anchor::BOL]
+  include_examples 'parse', /a$/,   1 =>  [:anchor,   :eol,               Anchor::EOL]
+
+  include_examples 'parse', /\Aa/,  0 =>  [:anchor,   :bos,               Anchor::BOS]
+  include_examples 'parse', /a\z/,  1 =>  [:anchor,   :eos,               Anchor::EOS]
+  include_examples 'parse', /a\Z/,  1 =>  [:anchor,   :eos_ob_eol,        Anchor::EOSobEOL]
+
+  include_examples 'parse', /a\b/,  1 =>  [:anchor,   :word_boundary,     Anchor::WordBoundary]
+  include_examples 'parse', /a\B/,  1 =>  [:anchor,   :nonword_boundary,  Anchor::NonWordBoundary]
+
+  include_examples 'parse', /a\G/,  1 =>  [:anchor,   :match_start,       Anchor::MatchStart]
+
+  include_examples 'parse', /\\A/,  0 =>  [:escape,   :backslash,         EscapeSequence::Literal]
+end

data/spec/parser/conditionals_spec.rb

@@ -0,0 +1,179 @@
+require 'spec_helper'
+
+RSpec.describe('Conditional parsing') do
+  specify('parse conditional') do
+    regexp = /(?<A>a)(?(<A>)T|F)/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1]
+
+    expect(exp).to be_a(Conditional::Expression)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :open
+    expect(exp.to_s).to eq '(?(<A>)T|F)'
+    expect(exp.reference).to eq 'A'
+  end
+
+  specify('parse conditional condition') do
+    regexp = /(?<A>a)(?(<A>)T|F)/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1].condition
+
+    expect(exp).to be_a(Conditional::Condition)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :condition
+    expect(exp.to_s).to eq '(<A>)'
+    expect(exp.reference).to eq 'A'
+    expect(exp.referenced_expression.to_s).to eq '(?<A>a)'
+  end
+
+  specify('parse conditional condition with number ref') do
+    regexp = /(a)(?(1)T|F)/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    exp = root[1].condition
+
+    expect(exp).to be_a(Conditional::Condition)
+
+    expect(exp.type).to eq :conditional
+    expect(exp.token).to eq :condition
+    expect(exp.to_s).to eq '(1)'
+    expect(exp.reference).to eq 1
+    expect(exp.referenced_expression.to_s).to eq '(a)'
+  end
+
+  specify('parse conditional nested groups') do
+    regexp = /((a)|(b)|((?(2)(c(d|e)+)?|(?(3)f|(?(4)(g|(h)(i)))))))/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    group = root.first
+    expect(group).to be_instance_of(Group::Capture)
+
+    alt = group.first
+    expect(alt).to be_instance_of(Alternation)
+    expect(alt.length).to eq 3
+
+    expect(alt.map(&:first)).to all(be_a Group::Capture)
+
+    subgroup = alt[2].first
+    conditional = subgroup.first
+
+    expect(conditional).to be_instance_of(Conditional::Expression)
+    expect(conditional.length).to eq 3
+
+    expect(conditional[0]).to be_instance_of(Conditional::Condition)
+    expect(conditional[0].to_s).to eq '(2)'
+
+    condition = conditional.condition
+    expect(condition).to be_instance_of(Conditional::Condition)
+    expect(condition.to_s).to eq '(2)'
+
+    branches = conditional.branches
+    expect(branches.length).to eq 2
+    expect(branches).to be_instance_of(Array)
+  end
+
+  specify('parse conditional nested') do
+    regexp = /(a(b(c(d)(e))))(?(1)(?(2)d|(?(3)e|f))|(?(4)(?(5)g|h)))/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    {
+      1 => [2, root[1]],
+      2 => [2, root[1][1][0]],
+      3 => [2, root[1][1][0][2][0]],
+      4 => [1, root[1][2][0]],
+      5 => [2, root[1][2][0][1][0]]
+    }.each do |index, example|
+      branch_count, exp = example
+
+      expect(exp).to be_instance_of(Conditional::Expression)
+      expect(exp.condition.to_s).to eq "(#{index})"
+      expect(exp.branches.length).to eq branch_count
+    end
+  end
+
+  specify('parse conditional nested alternation') do
+    regexp = /(a)(?(1)(b|c|d)|(e|f|g))(h)(?(2)(i|j|k)|(l|m|n))|o|p/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+
+    expect(root.to_s).to eq regexp.source
+
+    expect(root.first).to be_instance_of(Alternation)
+
+    [
+      [3, 'b|c|d', root[0][0][1][1][0][0]],
+      [3, 'e|f|g', root[0][0][1][2][0][0]],
+      [3, 'i|j|k', root[0][0][3][1][0][0]],
+      [3, 'l|m|n', root[0][0][3][2][0][0]]
+    ].each do |example|
+      alt_count, alt_text, exp = example
+
+      expect(exp).to be_instance_of(Alternation)
+      expect(exp.to_s).to eq alt_text
+      expect(exp.alternatives.length).to eq alt_count
+    end
+  end
+
+  specify('parse conditional extra separator') do
+    regexp = /(?<A>a)(?(<A>)T|)/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    branches = root[1].branches
+
+    expect(branches.length).to eq 2
+
+    seq_1, seq_2 = branches
+
+    [seq_1, seq_2].each do |seq|
+      expect(seq).to be_a(Sequence)
+
+      expect(seq.type).to eq :expression
+      expect(seq.token).to eq :sequence
+    end
+
+    expect(seq_1.to_s).to eq 'T'
+    expect(seq_2.to_s).to eq ''
+  end
+
+  specify('parse conditional quantified') do
+    regexp = /(foo)(?(1)\d|(\w)){42}/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    conditional = root[1]
+
+    expect(conditional).to be_quantified
+    expect(conditional.quantifier.to_s).to eq '{42}'
+    expect(conditional.to_s).to eq '(?(1)\\d|(\\w)){42}'
+    expect(conditional.branches.any?(&:quantified?)).to be false
+  end
+
+  specify('parse conditional branch content quantified') do
+    regexp = /(foo)(?(1)\d{23}|(\w){42})/
+
+    root = RP.parse(regexp, 'ruby/2.0')
+    conditional = root[1]
+
+    expect(conditional).not_to be_quantified
+    expect(conditional.branches.any?(&:quantified?)).to be false
+    expect(conditional.branches[0][0]).to be_quantified
+    expect(conditional.branches[0][0].quantifier.to_s).to eq '{23}'
+    expect(conditional.branches[1][0]).to be_quantified
+    expect(conditional.branches[1][0].quantifier.to_s).to eq '{42}'
+  end
+
+  specify('parse conditional excessive branches') do
+    regexp = '(?<A>a)(?(<A>)T|F|X)'
+
+    expect { RP.parse(regexp, 'ruby/2.0') }.to raise_error(Conditional::TooManyBranches)
+  end
+end

data/spec/parser/errors_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe('Parsing errors') do
+  let(:parser) { Regexp::Parser.new }
+  before { parser.parse(/foo/) } # initializes ivars
+
+  it('raises UnknownTokenTypeError for unknown token types') do
+    expect { parser.send(:parse_token, Regexp::Token.new(:foo, :bar)) }
+      .to raise_error(Regexp::Parser::UnknownTokenTypeError)
+  end
+
+  RSpec.shared_examples 'UnknownTokenError' do |type, token|
+    it "raises for unkown tokens of type #{type}" do
+      expect { parser.send(:parse_token, Regexp::Token.new(type, :foo)) }
+        .to raise_error(Regexp::Parser::UnknownTokenError)
+    end
+  end
+
+  include_examples 'UnknownTokenError', :anchor
+  include_examples 'UnknownTokenError', :backref
+  include_examples 'UnknownTokenError', :conditional
+  include_examples 'UnknownTokenError', :free_space
+  include_examples 'UnknownTokenError', :group
+  include_examples 'UnknownTokenError', :meta
+  include_examples 'UnknownTokenError', :nonproperty
+  include_examples 'UnknownTokenError', :property
+  include_examples 'UnknownTokenError', :quantifier
+  include_examples 'UnknownTokenError', :set
+  include_examples 'UnknownTokenError', :type
+end

data/spec/parser/escapes_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+RSpec.describe('EscapeSequence parsing') do
+  include_examples 'parse', /a\ac/,          1 => [:escape, :bell,              EscapeSequence::Bell]
+  include_examples 'parse', /a\ec/,          1 => [:escape, :escape,            EscapeSequence::AsciiEscape]
+  include_examples 'parse', /a\fc/,          1 => [:escape, :form_feed,         EscapeSequence::FormFeed]
+  include_examples 'parse', /a\nc/,          1 => [:escape, :newline,           EscapeSequence::Newline]
+  include_examples 'parse', /a\rc/,          1 => [:escape, :carriage,          EscapeSequence::Return]
+  include_examples 'parse', /a\tc/,          1 => [:escape, :tab,               EscapeSequence::Tab]
+  include_examples 'parse', /a\vc/,          1 => [:escape, :vertical_tab,      EscapeSequence::VerticalTab]
+
+  # meta character escapes
+  include_examples 'parse', /a\.c/,          1 => [:escape, :dot,               EscapeSequence::Literal]
+  include_examples 'parse', /a\?c/,          1 => [:escape, :zero_or_one,       EscapeSequence::Literal]
+  include_examples 'parse', /a\*c/,          1 => [:escape, :zero_or_more,      EscapeSequence::Literal]
+  include_examples 'parse', /a\+c/,          1 => [:escape, :one_or_more,       EscapeSequence::Literal]
+  include_examples 'parse', /a\|c/,          1 => [:escape, :alternation,       EscapeSequence::Literal]
+  include_examples 'parse', /a\(c/,          1 => [:escape, :group_open,        EscapeSequence::Literal]
+  include_examples 'parse', /a\)c/,          1 => [:escape, :group_close,       EscapeSequence::Literal]
+  include_examples 'parse', /a\{c/,          1 => [:escape, :interval_open,     EscapeSequence::Literal]
+  include_examples 'parse', /a\}c/,          1 => [:escape, :interval_close,    EscapeSequence::Literal]
+
+  # unicode escapes
+  include_examples 'parse', /a\u0640/,       1 => [:escape, :codepoint,         EscapeSequence::Codepoint]
+  include_examples 'parse', /a\u{41 1F60D}/, 1 => [:escape, :codepoint_list,    EscapeSequence::CodepointList]
+  include_examples 'parse', /a\u{10FFFF}/,   1 => [:escape, :codepoint_list,    EscapeSequence::CodepointList]
+
+    # hex escapes
+  include_examples 'parse', /a\xFF/n,        1 => [:escape, :hex,               EscapeSequence::Hex]
+
+  # octal escapes
+  include_examples 'parse', /a\177/n,        1 => [:escape, :octal,             EscapeSequence::Octal]
+
+  specify('parse chars and codepoints') do
+    root = RP.parse(/\n\?\101\x42\u0043\u{44 45}/)
+
+    expect(root[0].char).to eq "\n"
+    expect(root[0].codepoint).to eq 10
+
+    expect(root[1].char).to eq '?'
+    expect(root[1].codepoint).to eq 63
+
+    expect(root[2].char).to eq 'A'
+    expect(root[2].codepoint).to eq 65
+
+    expect(root[3].char).to eq 'B'
+    expect(root[3].codepoint).to eq 66
+
+    expect(root[4].char).to eq 'C'
+    expect(root[4].codepoint).to eq 67
+
+    expect(root[5].chars).to eq %w[D E]
+    expect(root[5].codepoints).to eq [68, 69]
+
+    expect { root[5].char }.to raise_error(/#chars/)
+    expect { root[5].codepoint }.to raise_error(/#codepoints/)
+  end
+
+  specify('parse escape control sequence lower') do
+    root = RP.parse(/a\\\c2b/)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Control)
+    expect(root[2].text).to eq '\\c2'
+    expect(root[2].char).to eq "\x12"
+    expect(root[2].codepoint).to eq 18
+  end
+
+  specify('parse escape control sequence upper') do
+    root = RP.parse(/\d\\\C-C\w/)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Control)
+    expect(root[2].text).to eq '\\C-C'
+    expect(root[2].char).to eq "\x03"
+    expect(root[2].codepoint).to eq 3
+  end
+
+  specify('parse escape meta sequence') do
+    root = RP.parse(/\Z\\\M-Z/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Meta)
+    expect(root[2].text).to eq '\\M-Z'
+    expect(root[2].char).to eq "\u00DA"
+    expect(root[2].codepoint).to eq 218
+  end
+
+  specify('parse escape meta control sequence') do
+    root = RP.parse(/\A\\\M-\C-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\M-\\C-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse lower c meta control sequence') do
+    root = RP.parse(/\A\\\M-\cX/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\M-\\cX'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse escape reverse meta control sequence') do
+    root = RP.parse(/\A\\\C-\M-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\C-\\M-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse escape reverse lower c meta control sequence') do
+    root = RP.parse(/\A\\\c\M-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\c\\M-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+end

data/spec/parser/free_space_spec.rb

@@ -0,0 +1,130 @@
+require 'spec_helper'
+
+RSpec.describe('FreeSpace parsing') do
+  specify('parse free space spaces') do
+    regexp = /a ? b * c + d{2,4}/x
+    root = RP.parse(regexp)
+
+    0.upto(6) do |i|
+      if i.odd?
+        expect(root[i]).to be_instance_of(WhiteSpace)
+        expect(root[i].text).to eq '  '
+      else
+        expect(root[i]).to be_instance_of(Literal)
+        expect(root[i]).to be_quantified
+      end
+    end
+  end
+
+  specify('parse non free space literals') do
+    regexp = /a b c d/
+    root = RP.parse(regexp)
+
+    expect(root.first).to be_instance_of(Literal)
+    expect(root.first.text).to eq 'a b c d'
+  end
+
+  specify('parse free space comments') do
+    regexp = /
+      a   ?     # One letter
+      b {2,5}   # Another one
+      [c-g]  +  # A set
+      (h|i|j) | # A group
+      klm *
+      nop +
+    /x
+
+    root = RP.parse(regexp)
+
+    alt = root.first
+    expect(alt).to be_instance_of(Alternation)
+
+    alt_1 = alt.alternatives.first
+    expect(alt_1).to be_instance_of(Alternative)
+    expect(alt_1.length).to eq 15
+
+    [0, 2, 4, 6, 8, 12, 14].each do |i|
+      expect(alt_1[i]).to be_instance_of(WhiteSpace)
+    end
+
+    [3, 7, 11].each { |i| expect(alt_1[i].class).to eq Comment }
+
+    alt_2 = alt.alternatives.last
+    expect(alt_2).to be_instance_of(Alternative)
+    expect(alt_2.length).to eq 7
+
+    [0, 2, 4, 6].each { |i| expect(alt_2[i].class).to eq WhiteSpace }
+
+    expect(alt_2[1]).to be_instance_of(Comment)
+  end
+
+  specify('parse free space nested comments') do
+    regexp = /
+      # Group one
+      (
+       abc  # Comment one
+       \d?  # Optional \d
+      )+
+
+      # Group two
+      (
+       def  # Comment two
+       \s?  # Optional \s
+      )?
+    /x
+
+    root = RP.parse(regexp)
+
+    top_comment_1 = root[1]
+    expect(top_comment_1).to be_instance_of(Comment)
+    expect(top_comment_1.text).to eq "# Group one\n"
+    expect(top_comment_1.starts_at).to eq 7
+
+    top_comment_2 = root[5]
+    expect(top_comment_2).to be_instance_of(Comment)
+    expect(top_comment_2.text).to eq "# Group two\n"
+    expect(top_comment_2.starts_at).to eq 95
+
+    [3, 7].each do |g,|
+      group = root[g]
+
+      [3, 7].each do |c|
+        comment = group[c]
+        expect(comment).to be_instance_of(Comment)
+        expect(comment.text.length).to eq 14
+      end
+    end
+  end
+
+  specify('parse free space quantifiers') do
+    regexp = /
+      a
+      # comment 1
+      ?
+      (
+       b # comment 2
+       # comment 3
+       +
+      )
+      # comment 4
+      *
+    /x
+
+    root = RP.parse(regexp)
+
+    literal_1 = root[1]
+    expect(literal_1).to be_instance_of(Literal)
+    expect(literal_1).to be_quantified
+    expect(literal_1.quantifier.token).to eq :zero_or_one
+
+    group = root[5]
+    expect(group).to be_instance_of(Group::Capture)
+    expect(group).to be_quantified
+    expect(group.quantifier.token).to eq :zero_or_more
+
+    literal_2 = group[1]
+    expect(literal_2).to be_instance_of(Literal)
+    expect(literal_2).to be_quantified
+    expect(literal_2.quantifier.token).to eq :one_or_more
+  end
+end