regexp_parser 1.4.0 → 1.5.0

data/spec/parser/errors_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+RSpec.describe('Parsing errors') do
+  let(:rp) { Regexp::Parser.new }
+  before { rp.parse(/foo/) }
+
+  specify('parser unknown token type') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:foo, :bar)) }
+      .to raise_error(Regexp::Parser::UnknownTokenTypeError)
+  end
+
+  specify('parser unknown set token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:set, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown meta token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:meta, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown character type token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:type, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown unicode property token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:property, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown unicode nonproperty token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:nonproperty, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown anchor token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:anchor, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown quantifier token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:quantifier, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+
+  specify('parser unknown group open token') do
+    expect { rp.send(:parse_token, Regexp::Token.new(:group, :foo)) }
+      .to raise_error(Regexp::Parser::UnknownTokenError)
+  end
+end

data/spec/parser/escapes_spec.rb

@@ -0,0 +1,132 @@
+require 'spec_helper'
+
+RSpec.describe('Escape parsing') do
+  tests = {
+    /a\ac/          => [1, :escape, :bell,              EscapeSequence::Bell],
+    /a\ec/          => [1, :escape, :escape,            EscapeSequence::AsciiEscape],
+    /a\fc/          => [1, :escape, :form_feed,         EscapeSequence::FormFeed],
+    /a\nc/          => [1, :escape, :newline,           EscapeSequence::Newline],
+    /a\rc/          => [1, :escape, :carriage,          EscapeSequence::Return],
+    /a\tc/          => [1, :escape, :tab,               EscapeSequence::Tab],
+    /a\vc/          => [1, :escape, :vertical_tab,      EscapeSequence::VerticalTab],
+
+    # meta character escapes
+    /a\.c/          => [1, :escape, :dot,               EscapeSequence::Literal],
+    /a\?c/          => [1, :escape, :zero_or_one,       EscapeSequence::Literal],
+    /a\*c/          => [1, :escape, :zero_or_more,      EscapeSequence::Literal],
+    /a\+c/          => [1, :escape, :one_or_more,       EscapeSequence::Literal],
+    /a\|c/          => [1, :escape, :alternation,       EscapeSequence::Literal],
+    /a\(c/          => [1, :escape, :group_open,        EscapeSequence::Literal],
+    /a\)c/          => [1, :escape, :group_close,       EscapeSequence::Literal],
+    /a\{c/          => [1, :escape, :interval_open,     EscapeSequence::Literal],
+    /a\}c/          => [1, :escape, :interval_close,    EscapeSequence::Literal],
+
+    # unicode escapes
+    /a\u0640/       => [1, :escape, :codepoint,         EscapeSequence::Codepoint],
+    /a\u{41 1F60D}/ => [1, :escape, :codepoint_list,    EscapeSequence::CodepointList],
+    /a\u{10FFFF}/   => [1, :escape, :codepoint_list,    EscapeSequence::CodepointList],
+
+     # hex escapes
+    /a\xFF/n        => [1, :escape, :hex,               EscapeSequence::Hex],
+
+    # octal escapes
+    /a\177/n        => [1, :escape, :octal,             EscapeSequence::Octal],
+  }
+
+  tests.each_with_index do |(pattern, (index, type, token, klass)), count|
+    specify("parse_escape_#{token}_#{count = (count + 1)}") do
+      root = RP.parse(pattern, 'ruby/1.9')
+      exp = root.expressions.at(index)
+
+      expect(exp).to be_a(klass)
+
+      expect(exp.type).to eq type
+      expect(exp.token).to eq token
+    end
+  end
+
+  specify('parse chars and codepoints') do
+    root = RP.parse(/\n\?\101\x42\u0043\u{44 45}/)
+
+    expect(root[0].char).to eq "\n"
+    expect(root[0].codepoint).to eq 10
+
+    expect(root[1].char).to eq '?'
+    expect(root[1].codepoint).to eq 63
+
+    expect(root[2].char).to eq 'A'
+    expect(root[2].codepoint).to eq 65
+
+    expect(root[3].char).to eq 'B'
+    expect(root[3].codepoint).to eq 66
+
+    expect(root[4].char).to eq 'C'
+    expect(root[4].codepoint).to eq 67
+
+    expect(root[5].chars).to eq %w[D E]
+    expect(root[5].codepoints).to eq [68, 69]
+  end
+
+  specify('parse escape control sequence lower') do
+    root = RP.parse(/a\\\c2b/)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Control)
+    expect(root[2].text).to eq '\\c2'
+    expect(root[2].char).to eq "\x12"
+    expect(root[2].codepoint).to eq 18
+  end
+
+  specify('parse escape control sequence upper') do
+    root = RP.parse(/\d\\\C-C\w/)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Control)
+    expect(root[2].text).to eq '\\C-C'
+    expect(root[2].char).to eq "\x03"
+    expect(root[2].codepoint).to eq 3
+  end
+
+  specify('parse escape meta sequence') do
+    root = RP.parse(/\Z\\\M-Z/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::Meta)
+    expect(root[2].text).to eq '\\M-Z'
+    expect(root[2].char).to eq "\u00DA"
+    expect(root[2].codepoint).to eq 218
+  end
+
+  specify('parse escape meta control sequence') do
+    root = RP.parse(/\A\\\M-\C-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\M-\\C-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse lower c meta control sequence') do
+    root = RP.parse(/\A\\\M-\cX/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\M-\\cX'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse escape reverse meta control sequence') do
+    root = RP.parse(/\A\\\C-\M-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\C-\\M-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+
+  specify('parse escape reverse lower c meta control sequence') do
+    root = RP.parse(/\A\\\c\M-X/n)
+
+    expect(root[2]).to be_instance_of(EscapeSequence::MetaControl)
+    expect(root[2].text).to eq '\\c\\M-X'
+    expect(root[2].char).to eq "\u0098"
+    expect(root[2].codepoint).to eq 152
+  end
+end

data/spec/parser/free_space_spec.rb

@@ -0,0 +1,130 @@
+require 'spec_helper'
+
+RSpec.describe('FreeSpace parsing') do
+  specify('parse free space spaces') do
+    regexp = /a ? b * c + d{2,4}/x
+    root = RP.parse(regexp)
+
+    0.upto(6) do |i|
+      if i.odd?
+        expect(root[i]).to be_instance_of(WhiteSpace)
+        expect(root[i].text).to eq '  '
+      else
+        expect(root[i]).to be_instance_of(Literal)
+        expect(root[i]).to be_quantified
+      end
+    end
+  end
+
+  specify('parse non free space literals') do
+    regexp = /a b c d/
+    root = RP.parse(regexp)
+
+    expect(root.first).to be_instance_of(Literal)
+    expect(root.first.text).to eq 'a b c d'
+  end
+
+  specify('parse free space comments') do
+    regexp = /
+      a   ?     # One letter
+      b {2,5}   # Another one
+      [c-g]  +  # A set
+      (h|i|j) | # A group
+      klm *
+      nop +
+    /x
+
+    root = RP.parse(regexp)
+
+    alt = root.first
+    expect(alt).to be_instance_of(Alternation)
+
+    alt_1 = alt.alternatives.first
+    expect(alt_1).to be_instance_of(Alternative)
+    expect(alt_1.length).to eq 15
+
+    [0, 2, 4, 6, 8, 12, 14].each do |i|
+      expect(alt_1[i]).to be_instance_of(WhiteSpace)
+    end
+
+    [3, 7, 11].each { |i| expect(alt_1[i].class).to eq Comment }
+
+    alt_2 = alt.alternatives.last
+    expect(alt_2).to be_instance_of(Alternative)
+    expect(alt_2.length).to eq 7
+
+    [0, 2, 4, 6].each { |i| expect(alt_2[i].class).to eq WhiteSpace }
+
+    expect(alt_2[1]).to be_instance_of(Comment)
+  end
+
+  specify('parse free space nested comments') do
+    regexp = /
+      # Group one
+      (
+       abc  # Comment one
+       \d?  # Optional \d
+      )+
+
+      # Group two
+      (
+       def  # Comment two
+       \s?  # Optional \s
+      )?
+    /x
+
+    root = RP.parse(regexp)
+
+    top_comment_1 = root[1]
+    expect(top_comment_1).to be_instance_of(Comment)
+    expect(top_comment_1.text).to eq "# Group one\n"
+    expect(top_comment_1.starts_at).to eq 7
+
+    top_comment_2 = root[5]
+    expect(top_comment_2).to be_instance_of(Comment)
+    expect(top_comment_2.text).to eq "# Group two\n"
+    expect(top_comment_2.starts_at).to eq 95
+
+    [3, 7].each do |g,|
+      group = root[g]
+
+      [3, 7].each do |c|
+        comment = group[c]
+        expect(comment).to be_instance_of(Comment)
+        expect(comment.text.length).to eq 14
+      end
+    end
+  end
+
+  specify('parse free space quantifiers') do
+    regexp = /
+      a
+      # comment 1
+      ?
+      (
+       b # comment 2
+       # comment 3
+       +
+      )
+      # comment 4
+      *
+    /x
+
+    root = RP.parse(regexp)
+
+    literal_1 = root[1]
+    expect(literal_1).to be_instance_of(Literal)
+    expect(literal_1).to be_quantified
+    expect(literal_1.quantifier.token).to eq :zero_or_one
+
+    group = root[5]
+    expect(group).to be_instance_of(Group::Capture)
+    expect(group).to be_quantified
+    expect(group.quantifier.token).to eq :zero_or_more
+
+    literal_2 = group[1]
+    expect(literal_2).to be_instance_of(Literal)
+    expect(literal_2).to be_quantified
+    expect(literal_2.quantifier.token).to eq :one_or_more
+  end
+end

data/spec/parser/groups_spec.rb

@@ -0,0 +1,267 @@
+require 'spec_helper'
+
+RSpec.describe('Group parsing') do
+  specify('parse root options mi') do
+    t = RP.parse(/[abc]/mi, 'ruby/1.8')
+
+    expect(t.m?).to be true
+    expect(t.i?).to be true
+    expect(t.x?).to be false
+  end
+
+  specify('parse option group') do
+    t = RP.parse(/(?m:a)/, 'ruby/1.8')
+
+    expect(t[0]).to be_instance_of(Group::Options)
+    expect(t[0].token).to eq :options
+
+    expect(t[0].m?).to be true
+    expect(t[0].i?).to be false
+    expect(t[0].x?).to be false
+
+    expect(t[0].option_changes[:m]).to be true
+    expect(t[0].option_changes[:i]).to be_nil
+  end
+
+  specify('parse self defeating option group') do
+    t = RP.parse(/(?m-m:a)/, 'ruby/1.8')
+
+    expect(t[0].m?).to be false
+    expect(t[0].i?).to be false
+    expect(t[0].x?).to be false
+
+    expect(t[0].option_changes[:m]).to be false
+    expect(t[0].option_changes[:i]).to be_nil
+  end
+
+  specify('parse nested options activate one') do
+    t = RP.parse(/(?x-mi:a(?m:b))/, 'ruby/1.8')
+
+    expect(t[0].m?).to be false
+    expect(t[0].i?).to be false
+    expect(t[0].x?).to be true
+
+    expect(t[0][1].m?).to be true
+    expect(t[0][1].i?).to be false
+    expect(t[0][1].x?).to be true
+
+    expect(t[0][1].option_changes[:m]).to be true
+    expect(t[0][1].option_changes[:i]).to be_nil
+    expect(t[0][1].option_changes[:x]).to be_nil
+  end
+
+  specify('parse nested options deactivate one') do
+    t = RP.parse(/(?ix-m:a(?-i:b))/, 'ruby/1.8')
+
+    expect(t[0].m?).to be false
+    expect(t[0].i?).to be true
+    expect(t[0].x?).to be true
+
+    expect(t[0][1].m?).to be false
+    expect(t[0][1].i?).to be false
+    expect(t[0][1].x?).to be true
+
+    expect(t[0][1].option_changes[:i]).to be false
+    expect(t[0][1].option_changes[:m]).to be_nil
+    expect(t[0][1].option_changes[:x]).to be_nil
+  end
+
+  specify('parse nested options invert all') do
+    t = RP.parse('(?xi-m:a(?m-ix:b))', 'ruby/1.8')
+
+    expect(t[0].m?).to be false
+    expect(t[0].i?).to be true
+    expect(t[0].x?).to be true
+
+    expect(t[0][1].m?).to be true
+    expect(t[0][1].i?).to be false
+    expect(t[0][1].x?).to be false
+
+    expect(t[0][1].option_changes[:m]).to be true
+    expect(t[0][1].option_changes[:i]).to be false
+    expect(t[0][1].option_changes[:x]).to be false
+  end
+
+  specify('parse nested options affect literal subexpressions') do
+    t = RP.parse(/(?x-mi:a(?m:b))/, 'ruby/1.8')
+
+    expect(t[0][0].m?).to be false
+    expect(t[0][0].i?).to be false
+    expect(t[0][0].x?).to be true
+
+    expect(t[0][1][0].m?).to be true
+    expect(t[0][1][0].i?).to be false
+    expect(t[0][1][0].x?).to be true
+  end
+
+  specify('parse option switch group') do
+    t = RP.parse(/a(?i-m)b/m, 'ruby/1.8')
+
+    expect(t[1]).to be_instance_of(Group::Options)
+    expect(t[1].token).to eq :options_switch
+
+    expect(t[1].m?).to be false
+    expect(t[1].i?).to be true
+    expect(t[1].x?).to be false
+
+    expect(t[1].option_changes[:i]).to be true
+    expect(t[1].option_changes[:m]).to be false
+    expect(t[1].option_changes[:x]).to be_nil
+  end
+
+  specify('parse option switch affects following expressions') do
+    t = RP.parse(/a(?i-m)b/m, 'ruby/1.8')
+
+    expect(t[0].m?).to be true
+    expect(t[0].i?).to be false
+    expect(t[0].x?).to be false
+
+    expect(t[2].m?).to be false
+    expect(t[2].i?).to be true
+    expect(t[2].x?).to be false
+  end
+
+  specify('parse option switch in group') do
+    t = RP.parse(/(a(?i-m)b)c/m, 'ruby/1.8')
+
+    group1 = t[0]
+
+    expect(group1.m?).to be true
+    expect(group1.i?).to be false
+    expect(group1.x?).to be false
+
+    expect(group1[0].m?).to be true
+    expect(group1[0].i?).to be false
+    expect(group1[0].x?).to be false
+
+    expect(group1[1].m?).to be false
+    expect(group1[1].i?).to be true
+    expect(group1[1].x?).to be false
+
+    expect(group1[2].m?).to be false
+    expect(group1[2].i?).to be true
+    expect(group1[2].x?).to be false
+
+    expect(t[1].m?).to be true
+    expect(t[1].i?).to be false
+    expect(t[1].x?).to be false
+  end
+
+  specify('parse nested option switch in group') do
+    t = RP.parse(/((?i-m)(a(?-i)b))/m, 'ruby/1.8')
+
+    group2 = t[0][1]
+
+    expect(group2.m?).to be false
+    expect(group2.i?).to be true
+    expect(group2.x?).to be false
+
+    expect(group2[0].m?).to be false
+    expect(group2[0].i?).to be true
+    expect(group2[0].x?).to be false
+
+    expect(group2[1].m?).to be false
+    expect(group2[1].i?).to be false
+    expect(group2[1].x?).to be false
+
+    expect(group2[2].m?).to be false
+    expect(group2[2].i?).to be false
+    expect(group2[2].x?).to be false
+  end
+
+  specify('parse options dau') do
+    t = RP.parse('(?dua:abc)')
+
+    expect(t[0].d?).to be false
+    expect(t[0].a?).to be true
+    expect(t[0].u?).to be false
+  end
+
+  specify('parse nested options dau') do
+    t = RP.parse('(?u:a(?d:b))')
+
+    expect(t[0].u?).to be true
+    expect(t[0].d?).to be false
+    expect(t[0].a?).to be false
+
+    expect(t[0][1].d?).to be true
+    expect(t[0][1].a?).to be false
+    expect(t[0][1].u?).to be false
+  end
+
+  specify('parse nested options da') do
+    t = RP.parse('(?di-xm:a(?da-x:b))')
+
+    expect(t[0].d?).to be true
+    expect(t[0].i?).to be true
+    expect(t[0].m?).to be false
+    expect(t[0].x?).to be false
+    expect(t[0].a?).to be false
+    expect(t[0].u?).to be false
+
+    expect(t[0][1].d?).to be false
+    expect(t[0][1].a?).to be true
+    expect(t[0][1].u?).to be false
+    expect(t[0][1].x?).to be false
+    expect(t[0][1].m?).to be false
+    expect(t[0][1].i?).to be true
+  end
+
+  specify('parse lookahead') do
+    t = RP.parse('(?=abc)(?!def)', 'ruby/1.8')
+
+    expect(t[0]).to be_a(Assertion::Lookahead)
+
+    expect(t[1]).to be_a(Assertion::NegativeLookahead)
+  end
+
+  specify('parse lookbehind') do
+    t = RP.parse('(?<=abc)(?<!def)', 'ruby/1.9')
+
+    expect(t[0]).to be_a(Assertion::Lookbehind)
+
+    expect(t[1]).to be_a(Assertion::NegativeLookbehind)
+  end
+
+  specify('parse comment') do
+    t = RP.parse('a(?# is for apple)b(?# for boy)c(?# cat)')
+
+    [1, 3, 5].each do |i|
+      expect(t[i]).to be_a(Group::Comment)
+
+      expect(t[i].type).to eq :group
+      expect(t[i].token).to eq :comment
+    end
+  end
+
+  specify('parse absence group', if: ruby_version_at_least('2.4.1')) do
+    t = RP.parse('a(?~b)c(?~d)e')
+
+    [1, 3].each do |i|
+      expect(t[i]).to be_a(Group::Absence)
+
+      expect(t[i].type).to eq :group
+      expect(t[i].token).to eq :absence
+    end
+  end
+
+  specify('parse group number') do
+    t = RP.parse('(a)(?=b)((?:c)(d|(e)))')
+    expect(t[0].number).to eq 1
+    expect(t[1]).not_to respond_to(:number)
+    expect(t[2].number).to eq 2
+    expect(t[2][0]).not_to respond_to(:number)
+    expect(t[2][1].number).to eq 3
+    expect(t[2][1][0][1][0].number).to eq 4
+  end
+
+  specify('parse group number at level') do
+    t = RP.parse('(a)(?=b)((?:c)(d|(e)))')
+    expect(t[0].number_at_level).to eq 1
+    expect(t[1]).not_to respond_to(:number_at_level)
+    expect(t[2].number_at_level).to eq 2
+    expect(t[2][0]).not_to respond_to(:number_at_level)
+    expect(t[2][1].number_at_level).to eq 1
+    expect(t[2][1][0][1][0].number_at_level).to eq 1
+  end
+end