regexp_parser 1.3.0 → 1.6.0

data/spec/parser/set/ranges_spec.rb

@@ -0,0 +1,111 @@
+require 'spec_helper'
+
+RSpec.describe('CharacterSet::Range parsing') do
+  specify('parse set range') do
+    root = RP.parse('[a-z]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range).to be_instance_of(CharacterSet::Range)
+    expect(range.count).to eq 2
+    expect(range.first.to_s).to eq 'a'
+    expect(range.first).to be_instance_of(Literal)
+    expect(range.last.to_s).to eq 'z'
+    expect(range.last).to be_instance_of(Literal)
+    expect(set).to match 'm'
+  end
+
+  specify('parse set range hex') do
+    root = RP.parse('[\\x00-\\x99]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range).to be_instance_of(CharacterSet::Range)
+    expect(range.count).to eq 2
+    expect(range.first.to_s).to eq '\\x00'
+    expect(range.first).to be_instance_of(EscapeSequence::Hex)
+    expect(range.last.to_s).to eq '\\x99'
+    expect(range.last).to be_instance_of(EscapeSequence::Hex)
+    expect(set).to match '\\x50'
+  end
+
+  specify('parse set range unicode') do
+    root = RP.parse('[\\u{40 42}-\\u1234]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range).to be_instance_of(CharacterSet::Range)
+    expect(range.count).to eq 2
+    expect(range.first.to_s).to eq '\\u{40 42}'
+    expect(range.first).to be_instance_of(EscapeSequence::CodepointList)
+    expect(range.last.to_s).to eq '\\u1234'
+    expect(range.last).to be_instance_of(EscapeSequence::Codepoint)
+    expect(set).to match '\\u600'
+  end
+
+  specify('parse set range edge case leading dash') do
+    root = RP.parse('[--z]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range.count).to eq 2
+    expect(set).to match 'a'
+  end
+
+  specify('parse set range edge case trailing dash') do
+    root = RP.parse('[!--]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range.count).to eq 2
+    expect(set).to match '$'
+  end
+
+  specify('parse set range edge case leading negate') do
+    root = RP.parse('[^-z]')
+    set = root[0]
+
+    expect(set.count).to eq 2
+    expect(set).to     match 'a'
+    expect(set).not_to match 'z'
+  end
+
+  specify('parse set range edge case trailing negate') do
+    root = RP.parse('[!-^]')
+    set = root[0]
+    range = set[0]
+
+    expect(set.count).to eq 1
+    expect(range.count).to eq 2
+    expect(set).to match '$'
+  end
+
+  specify('parse set range edge case leading intersection') do
+    root = RP.parse('[[\\-ab]&&-bc]')
+    set = root[0]
+
+    expect(set.count).to eq 1
+    expect(set.first.last.to_s).to eq '-bc'
+    expect(set).to     match '-'
+    expect(set).to     match 'b'
+    expect(set).not_to match 'a'
+    expect(set).not_to match 'c'
+  end
+
+  specify('parse set range edge case trailing intersection') do
+    root = RP.parse('[bc-&&[\\-ab]]')
+    set = root[0]
+
+    expect(set.count).to eq 1
+    expect(set.first.first.to_s).to eq 'bc-'
+    expect(set).to     match '-'
+    expect(set).to     match 'b'
+    expect(set).not_to match 'a'
+    expect(set).not_to match 'c'
+  end
+end

data/spec/parser/sets_spec.rb

@@ -0,0 +1,178 @@
+require 'spec_helper'
+
+RSpec.describe('CharacterSet parsing') do
+  specify('parse set basic') do
+    root = RP.parse('[ab]+')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 2
+
+    expect(exp[0]).to be_instance_of(Literal)
+    expect(exp[0].text).to eq 'a'
+    expect(exp[1]).to be_instance_of(Literal)
+    expect(exp[1].text).to eq 'b'
+
+    expect(exp).to be_quantified
+    expect(exp.quantifier.min).to eq 1
+    expect(exp.quantifier.max).to eq(-1)
+  end
+
+  specify('parse set char type') do
+    root = RP.parse('[a\\dc]')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 3
+
+    expect(exp[1]).to be_instance_of(CharacterType::Digit)
+    expect(exp[1].text).to eq '\\d'
+  end
+
+  specify('parse set escape sequence backspace') do
+    root = RP.parse('[a\\bc]')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 3
+
+    expect(exp[1]).to be_instance_of(EscapeSequence::Backspace)
+    expect(exp[1].text).to eq '\\b'
+
+    expect(exp).to     match 'a'
+    expect(exp).to     match "\b"
+    expect(exp).not_to match 'b'
+    expect(exp).to     match 'c'
+  end
+
+  specify('parse set escape sequence hex') do
+    root = RP.parse('[a\\x20c]', :any)
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 3
+
+    expect(exp[1]).to be_instance_of(EscapeSequence::Hex)
+    expect(exp[1].text).to eq '\\x20'
+  end
+
+  specify('parse set escape sequence codepoint') do
+    root = RP.parse('[a\\u0640]')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 2
+
+    expect(exp[1]).to be_instance_of(EscapeSequence::Codepoint)
+    expect(exp[1].text).to eq '\\u0640'
+  end
+
+  specify('parse set escape sequence codepoint list') do
+    root = RP.parse('[a\\u{41 1F60D}]')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 2
+
+    expect(exp[1]).to be_instance_of(EscapeSequence::CodepointList)
+    expect(exp[1].text).to eq '\\u{41 1F60D}'
+  end
+
+  specify('parse set posix class') do
+    root = RP.parse('[[:digit:][:^lower:]]+')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 2
+
+    expect(exp[0]).to be_instance_of(PosixClass)
+    expect(exp[0].text).to eq '[:digit:]'
+    expect(exp[1]).to be_instance_of(PosixClass)
+    expect(exp[1].text).to eq '[:^lower:]'
+  end
+
+  specify('parse set nesting') do
+    root = RP.parse('[a[b[c]d]e]')
+
+    exp = root[0]
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 3
+    expect(exp[0]).to be_instance_of(Literal)
+    expect(exp[2]).to be_instance_of(Literal)
+
+    subset1 = exp[1]
+    expect(subset1).to be_instance_of(CharacterSet)
+    expect(subset1.count).to eq 3
+    expect(subset1[0]).to be_instance_of(Literal)
+    expect(subset1[2]).to be_instance_of(Literal)
+
+    subset2 = subset1[1]
+    expect(subset2).to be_instance_of(CharacterSet)
+    expect(subset2.count).to eq 1
+    expect(subset2[0]).to be_instance_of(Literal)
+  end
+
+  specify('parse set nesting negative') do
+    root = RP.parse('[a[^b[c]]]')
+    exp = root[0]
+
+    expect(exp).to be_instance_of(CharacterSet)
+    expect(exp.count).to eq 2
+    expect(exp[0]).to be_instance_of(Literal)
+    expect(exp).not_to be_negative
+
+    subset1 = exp[1]
+    expect(subset1).to be_instance_of(CharacterSet)
+    expect(subset1.count).to eq 2
+    expect(subset1[0]).to be_instance_of(Literal)
+    expect(subset1).to be_negative
+
+    subset2 = subset1[1]
+    expect(subset2).to be_instance_of(CharacterSet)
+    expect(subset2.count).to eq 1
+    expect(subset2[0]).to be_instance_of(Literal)
+    expect(subset2).not_to be_negative
+  end
+
+  specify('parse set nesting #to_s') do
+    pattern = '[a[b[^c]]]'
+    root = RP.parse(pattern)
+
+    expect(root.to_s).to eq pattern
+  end
+
+  specify('parse set literals are not merged') do
+    root = RP.parse("[#{('a' * 10)}]")
+    exp = root[0]
+
+    expect(exp.count).to eq 10
+  end
+
+  specify('parse set whitespace is not merged') do
+    root = RP.parse("[#{(' ' * 10)}]")
+    exp = root[0]
+
+    expect(exp.count).to eq 10
+  end
+
+  specify('parse set whitespace is not merged in x mode') do
+    root = RP.parse("(?x)[#{(' ' * 10)}]")
+    exp = root[1]
+
+    expect(exp.count).to eq 10
+  end
+
+  specify('parse set collating sequence') do
+    root = RP.parse('[a[.span-ll.]h]', :any)
+    exp = root[0]
+
+    expect(exp[1].to_s).to eq '[.span-ll.]'
+  end
+
+  specify('parse set character equivalents') do
+    root = RP.parse('[a[=e=]h]', :any)
+    exp = root[0]
+
+    expect(exp[1].to_s).to eq '[=e=]'
+  end
+end

data/spec/parser/types_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+RSpec.describe('CharacterType parsing') do
+  include_examples 'parse', /a\dc/,   1 =>  [:type,   :digit,     CharacterType::Digit]
+  include_examples 'parse', /a\Dc/,   1 =>  [:type,   :nondigit,  CharacterType::NonDigit]
+
+  include_examples 'parse', /a\sc/,   1 =>  [:type,   :space,     CharacterType::Space]
+  include_examples 'parse', /a\Sc/,   1 =>  [:type,   :nonspace,  CharacterType::NonSpace]
+
+  include_examples 'parse', /a\hc/,   1 =>  [:type,   :hex,       CharacterType::Hex]
+  include_examples 'parse', /a\Hc/,   1 =>  [:type,   :nonhex,    CharacterType::NonHex]
+
+  include_examples 'parse', /a\wc/,   1 =>  [:type,   :word,      CharacterType::Word]
+  include_examples 'parse', /a\Wc/,   1 =>  [:type,   :nonword,   CharacterType::NonWord]
+
+  include_examples 'parse', 'a\\Rc',  1 =>  [:type,   :linebreak, CharacterType::Linebreak]
+  include_examples 'parse', 'a\\Xc',  1 =>  [:type,   :xgrapheme, CharacterType::ExtendedGrapheme]
+end

data/spec/scanner/all_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Scanner) do
+  specify('scanner returns an array') do
+    expect(RS.scan('abc')).to be_instance_of(Array)
+  end
+
+  specify('scanner returns tokens as arrays') do
+    tokens = RS.scan('^abc+[^one]{2,3}\\b\\d\\\\C-C$')
+    expect(tokens).to all(be_a Array)
+    expect(tokens.map(&:length)).to all(eq 5)
+  end
+
+  specify('scanner token count') do
+    re = /^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i
+    expect(RS.scan(re).length).to eq 28
+  end
+end

data/spec/scanner/anchors_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+RSpec.describe('Anchor scanning') do
+  include_examples 'scan', '^abc',    0 => [:anchor,  :bol,              '^',    0, 1]
+  include_examples 'scan', 'abc$',    1 => [:anchor,  :eol,              '$',    3, 4]
+
+  include_examples 'scan', '\Aabc',   0 => [:anchor,  :bos,              '\A',   0, 2]
+  include_examples 'scan', 'abc\z',   1 => [:anchor,  :eos,              '\z',   3, 5]
+  include_examples 'scan', 'abc\Z',   1 => [:anchor,  :eos_ob_eol,       '\Z',   3, 5]
+
+  include_examples 'scan', 'a\bc',    1 => [:anchor,  :word_boundary,    '\b',   1, 3]
+  include_examples 'scan', 'a\Bc',    1 => [:anchor,  :nonword_boundary, '\B',   1, 3]
+
+  include_examples 'scan', 'a\Gc',    1 => [:anchor,  :match_start,      '\G',   1, 3]
+
+  include_examples 'scan', "\\\\Ac",  0 => [:escape, :backslash,         '\\\\', 0, 2]
+  include_examples 'scan', "a\\\\z",  1 => [:escape, :backslash,         '\\\\', 1, 3]
+  include_examples 'scan', "a\\\\Z",  1 => [:escape, :backslash,         '\\\\', 1, 3]
+  include_examples 'scan', "a\\\\bc", 1 => [:escape, :backslash,         '\\\\', 1, 3]
+  include_examples 'scan', "a\\\\Bc", 1 => [:escape, :backslash,         '\\\\', 1, 3]
+end

data/spec/scanner/conditionals_spec.rb

@@ -0,0 +1,128 @@
+require 'spec_helper'
+
+RSpec.describe('Conditional scanning') do
+  include_examples 'scan', /(a)(?(1)T|F)1/,       3  => [:conditional,  :open,            '(?',  3,  5]
+  include_examples 'scan', /(a)(?(1)T|F)2/,       4  => [:conditional,  :condition_open,  '(',   5,  6]
+  include_examples 'scan', /(a)(?(1)T|F)3/,       5  => [:conditional,  :condition,       '1',   6,  7]
+  include_examples 'scan', /(a)(?(1)T|F)4/,       6  => [:conditional,  :condition_close, ')',   7,  8]
+  include_examples 'scan', /(a)(?(1)T|F)5/,       7  => [:literal,      :literal,         'T',   8,  9]
+  include_examples 'scan', /(a)(?(1)T|F)6/,       8  => [:conditional,  :separator,       '|',   9,  10]
+  include_examples 'scan', /(a)(?(1)T|F)7/,       9  => [:literal,      :literal,         'F',   10, 11]
+  include_examples 'scan', /(a)(?(1)T|F)8/,       10 => [:conditional,  :close,           ')',   11, 12]
+  include_examples 'scan', /(a)(?(1)TRUE)9/,      8  => [:conditional,  :close,           ')',   12, 13]
+  include_examples 'scan', /(a)(?(1)TRUE|)10/,    8  => [:conditional,  :separator,       '|',   12, 13]
+  include_examples 'scan', /(a)(?(1)TRUE|)11/,    9  => [:conditional,  :close,           ')',   13, 14]
+  include_examples 'scan', /(?<N>A)(?(<N>)T|F)1/, 5  => [:conditional,  :condition,       '<N>', 10, 13]
+  include_examples 'scan', /(?'N'A)(?('N')T|F)2/, 5  => [:conditional,  :condition,       "'N'", 10, 13]
+
+  include_examples 'scan', /(a(b(c)))(?(1)(?(2)d|(?(3)e|f))|(?(2)(?(1)g|h)))/,
+    0  => [:group,        :capture,         '(',   0,  1],
+    1  => [:literal,      :literal,         'a',   1,  2],
+    2  => [:group,        :capture,         '(',   2,  3],
+    3  => [:literal,      :literal,         'b',   3,  4],
+    4  => [:group,        :capture,         '(',   4,  5],
+    5  => [:literal,      :literal,         'c',   5,  6],
+    6  => [:group,        :close,           ')',   6,  7],
+    7  => [:group,        :close,           ')',   7,  8],
+    8  => [:group,        :close,           ')',   8,  9],
+    9  => [:conditional,  :open,            '(?',  9, 11],
+    10 => [:conditional,  :condition_open,  '(',  11, 12],
+    11 => [:conditional,  :condition,       '1',  12, 13],
+    12 => [:conditional,  :condition_close, ')',  13, 14],
+    13 => [:conditional,  :open,            '(?', 14, 16],
+    14 => [:conditional,  :condition_open,  '(',  16, 17],
+    15 => [:conditional,  :condition,       '2',  17, 18],
+    16 => [:conditional,  :condition_close, ')',  18, 19],
+    17 => [:literal,      :literal,         'd',  19, 20],
+    18 => [:conditional,  :separator,       '|',  20, 21],
+    19 => [:conditional,  :open,            '(?', 21, 23],
+    20 => [:conditional,  :condition_open,  '(',  23, 24],
+    21 => [:conditional,  :condition,       '3',  24, 25],
+    22 => [:conditional,  :condition_close, ')',  25, 26],
+    23 => [:literal,      :literal,         'e',  26, 27],
+    24 => [:conditional,  :separator,       '|',  27, 28],
+    25 => [:literal,      :literal,         'f',  28, 29],
+    26 => [:conditional,  :close,           ')',  29, 30],
+    27 => [:conditional,  :close,           ')',  30, 31],
+    28 => [:conditional,  :separator,       '|',  31, 32],
+    29 => [:conditional,  :open,            '(?', 32, 34],
+    30 => [:conditional,  :condition_open,  '(',  34, 35],
+    31 => [:conditional,  :condition,       '2',  35, 36],
+    32 => [:conditional,  :condition_close, ')',  36, 37],
+    33 => [:conditional,  :open,            '(?', 37, 39],
+    34 => [:conditional,  :condition_open,  '(',  39, 40],
+    35 => [:conditional,  :condition,       '1',  40, 41],
+    36 => [:conditional,  :condition_close, ')',  41, 42],
+    37 => [:literal,      :literal,         'g',  42, 43],
+    38 => [:conditional,  :separator,       '|',  43, 44],
+    39 => [:literal,      :literal,         'h',  44, 45],
+    40 => [:conditional,  :close,           ')',  45, 46],
+    41 => [:conditional,  :close,           ')',  46, 47],
+    42 => [:conditional,  :close,           ')',  47, 48]
+
+  include_examples 'scan', /((a)|(b)|((?(2)(c(d|e)+)?|(?(3)f|(?(4)(g|(h)(i)))))))/,
+    0  => [:group,        :capture,         '(',   0,  1],
+    1  => [:group,        :capture,         '(',   1,  2],
+    2  => [:literal,      :literal,         'a',   2,  3],
+    3  => [:group,        :close,           ')',   3,  4],
+    4  => [:meta,         :alternation,     '|',   4,  5],
+    5  => [:group,        :capture,         '(',   5,  6],
+    6  => [:literal,      :literal,         'b',   6,  7],
+    7  => [:group,        :close,           ')',   7,  8],
+    8  => [:meta,         :alternation,     '|',   8,  9],
+    9  => [:group,        :capture,         '(',   9, 10],
+    10 => [:conditional,  :open,            '(?', 10, 12],
+    11 => [:conditional,  :condition_open,  '(',  12, 13],
+    12 => [:conditional,  :condition,       '2',  13, 14],
+    13 => [:conditional,  :condition_close, ')',  14, 15],
+    14 => [:group,        :capture,         '(',  15, 16],
+    15 => [:literal,      :literal,         'c',  16, 17],
+    16 => [:group,        :capture,         '(',  17, 18],
+    17 => [:literal,      :literal,         'd',  18, 19],
+    18 => [:meta,         :alternation,     '|',  19, 20],
+    19 => [:literal,      :literal,         'e',  20, 21],
+    20 => [:group,        :close,           ')',  21, 22],
+    21 => [:quantifier,   :one_or_more,     '+',  22, 23],
+    22 => [:group,        :close,           ')',  23, 24],
+    23 => [:quantifier,   :zero_or_one,     '?',  24, 25],
+    24 => [:conditional,  :separator,       '|',  25, 26],
+    25 => [:conditional,  :open,            '(?', 26, 28],
+    26 => [:conditional,  :condition_open,  '(',  28, 29],
+    27 => [:conditional,  :condition,       '3',  29, 30],
+    28 => [:conditional,  :condition_close, ')',  30, 31],
+    29 => [:literal,      :literal,         'f',  31, 32],
+    30 => [:conditional,  :separator,       '|',  32, 33],
+    31 => [:conditional,  :open,            '(?', 33, 35],
+    32 => [:conditional,  :condition_open,  '(',  35, 36],
+    33 => [:conditional,  :condition,       '4',  36, 37],
+    34 => [:conditional,  :condition_close, ')',  37, 38],
+    35 => [:group,        :capture,         '(',  38, 39],
+    36 => [:literal,      :literal,         'g',  39, 40],
+    37 => [:meta,         :alternation,     '|',  40, 41],
+    38 => [:group,        :capture,         '(',  41, 42],
+    39 => [:literal,      :literal,         'h',  42, 43],
+    40 => [:group,        :close,           ')',  43, 44],
+    41 => [:group,        :capture,         '(',  44, 45],
+    42 => [:literal,      :literal,         'i',  45, 46],
+    43 => [:group,        :close,           ')',  46, 47],
+    44 => [:group,        :close,           ')',  47, 48],
+    45 => [:conditional,  :close,           ')',  48, 49],
+    46 => [:conditional,  :close,           ')',  49, 50],
+    47 => [:conditional,  :close,           ')',  50, 51],
+    48 => [:group,        :close,           ')',  51, 52],
+    49 => [:group,        :close,           ')',  52, 53]
+
+  include_examples 'scan', /(a)(?(1)(b|c|d)|(e|f|g))(h)(?(2)(i|j|k)|(l|m|n))|o|p/,
+    9  => [:meta,        :alternation, '|', 10, 11],
+    11 => [:meta,        :alternation, '|', 12, 13],
+    14 => [:conditional, :separator,   '|', 15, 16],
+    17 => [:meta,        :alternation, '|', 18, 19],
+    19 => [:meta,        :alternation, '|', 20, 21],
+    32 => [:meta,        :alternation, '|', 34, 35],
+    34 => [:meta,        :alternation, '|', 36, 37],
+    37 => [:conditional, :separator,   '|', 39, 40],
+    40 => [:meta,        :alternation, '|', 42, 43],
+    42 => [:meta,        :alternation, '|', 44, 45],
+    46 => [:meta,        :alternation, '|', 48, 49],
+    48 => [:meta,        :alternation, '|', 50, 51]
+end