regexp_parser 1.7.0 → 2.9.0

data/spec/parser/types_spec.rb

@@ -1,18 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('CharacterType parsing') do
-  include_examples 'parse', /a\dc/,   1 =>  [:type,   :digit,     CharacterType::Digit]
-  include_examples 'parse', /a\Dc/,   1 =>  [:type,   :nondigit,  CharacterType::NonDigit]
-
-  include_examples 'parse', /a\sc/,   1 =>  [:type,   :space,     CharacterType::Space]
-  include_examples 'parse', /a\Sc/,   1 =>  [:type,   :nonspace,  CharacterType::NonSpace]
-
-  include_examples 'parse', /a\hc/,   1 =>  [:type,   :hex,       CharacterType::Hex]
-  include_examples 'parse', /a\Hc/,   1 =>  [:type,   :nonhex,    CharacterType::NonHex]
-
-  include_examples 'parse', /a\wc/,   1 =>  [:type,   :word,      CharacterType::Word]
-  include_examples 'parse', /a\Wc/,   1 =>  [:type,   :nonword,   CharacterType::NonWord]
-
-  include_examples 'parse', 'a\\Rc',  1 =>  [:type,   :linebreak, CharacterType::Linebreak]
-  include_examples 'parse', 'a\\Xc',  1 =>  [:type,   :xgrapheme, CharacterType::ExtendedGrapheme]
-end

data/spec/scanner/all_spec.rb

@@ -1,18 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe(Regexp::Scanner) do
-  specify('scanner returns an array') do
-    expect(RS.scan('abc')).to be_instance_of(Array)
-  end
-
-  specify('scanner returns tokens as arrays') do
-    tokens = RS.scan('^abc+[^one]{2,3}\\b\\d\\\\C-C$')
-    expect(tokens).to all(be_a Array)
-    expect(tokens.map(&:length)).to all(eq 5)
-  end
-
-  specify('scanner token count') do
-    re = /^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i
-    expect(RS.scan(re).length).to eq 28
-  end
-end

data/spec/scanner/anchors_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Anchor scanning') do
-  include_examples 'scan', '^abc',    0 => [:anchor,  :bol,              '^',    0, 1]
-  include_examples 'scan', 'abc$',    1 => [:anchor,  :eol,              '$',    3, 4]
-
-  include_examples 'scan', '\Aabc',   0 => [:anchor,  :bos,              '\A',   0, 2]
-  include_examples 'scan', 'abc\z',   1 => [:anchor,  :eos,              '\z',   3, 5]
-  include_examples 'scan', 'abc\Z',   1 => [:anchor,  :eos_ob_eol,       '\Z',   3, 5]
-
-  include_examples 'scan', 'a\bc',    1 => [:anchor,  :word_boundary,    '\b',   1, 3]
-  include_examples 'scan', 'a\Bc',    1 => [:anchor,  :nonword_boundary, '\B',   1, 3]
-
-  include_examples 'scan', 'a\Gc',    1 => [:anchor,  :match_start,      '\G',   1, 3]
-
-  include_examples 'scan', "\\\\Ac",  0 => [:escape, :backslash,         '\\\\', 0, 2]
-  include_examples 'scan', "a\\\\z",  1 => [:escape, :backslash,         '\\\\', 1, 3]
-  include_examples 'scan', "a\\\\Z",  1 => [:escape, :backslash,         '\\\\', 1, 3]
-  include_examples 'scan', "a\\\\bc", 1 => [:escape, :backslash,         '\\\\', 1, 3]
-  include_examples 'scan', "a\\\\Bc", 1 => [:escape, :backslash,         '\\\\', 1, 3]
-end

data/spec/scanner/conditionals_spec.rb

@@ -1,128 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Conditional scanning') do
-  include_examples 'scan', /(a)(?(1)T|F)1/,       3  => [:conditional,  :open,            '(?',  3,  5]
-  include_examples 'scan', /(a)(?(1)T|F)2/,       4  => [:conditional,  :condition_open,  '(',   5,  6]
-  include_examples 'scan', /(a)(?(1)T|F)3/,       5  => [:conditional,  :condition,       '1',   6,  7]
-  include_examples 'scan', /(a)(?(1)T|F)4/,       6  => [:conditional,  :condition_close, ')',   7,  8]
-  include_examples 'scan', /(a)(?(1)T|F)5/,       7  => [:literal,      :literal,         'T',   8,  9]
-  include_examples 'scan', /(a)(?(1)T|F)6/,       8  => [:conditional,  :separator,       '|',   9,  10]
-  include_examples 'scan', /(a)(?(1)T|F)7/,       9  => [:literal,      :literal,         'F',   10, 11]
-  include_examples 'scan', /(a)(?(1)T|F)8/,       10 => [:conditional,  :close,           ')',   11, 12]
-  include_examples 'scan', /(a)(?(1)TRUE)9/,      8  => [:conditional,  :close,           ')',   12, 13]
-  include_examples 'scan', /(a)(?(1)TRUE|)10/,    8  => [:conditional,  :separator,       '|',   12, 13]
-  include_examples 'scan', /(a)(?(1)TRUE|)11/,    9  => [:conditional,  :close,           ')',   13, 14]
-  include_examples 'scan', /(?<N>A)(?(<N>)T|F)1/, 5  => [:conditional,  :condition,       '<N>', 10, 13]
-  include_examples 'scan', /(?'N'A)(?('N')T|F)2/, 5  => [:conditional,  :condition,       "'N'", 10, 13]
-
-  include_examples 'scan', /(a(b(c)))(?(1)(?(2)d|(?(3)e|f))|(?(2)(?(1)g|h)))/,
-    0  => [:group,        :capture,         '(',   0,  1],
-    1  => [:literal,      :literal,         'a',   1,  2],
-    2  => [:group,        :capture,         '(',   2,  3],
-    3  => [:literal,      :literal,         'b',   3,  4],
-    4  => [:group,        :capture,         '(',   4,  5],
-    5  => [:literal,      :literal,         'c',   5,  6],
-    6  => [:group,        :close,           ')',   6,  7],
-    7  => [:group,        :close,           ')',   7,  8],
-    8  => [:group,        :close,           ')',   8,  9],
-    9  => [:conditional,  :open,            '(?',  9, 11],
-    10 => [:conditional,  :condition_open,  '(',  11, 12],
-    11 => [:conditional,  :condition,       '1',  12, 13],
-    12 => [:conditional,  :condition_close, ')',  13, 14],
-    13 => [:conditional,  :open,            '(?', 14, 16],
-    14 => [:conditional,  :condition_open,  '(',  16, 17],
-    15 => [:conditional,  :condition,       '2',  17, 18],
-    16 => [:conditional,  :condition_close, ')',  18, 19],
-    17 => [:literal,      :literal,         'd',  19, 20],
-    18 => [:conditional,  :separator,       '|',  20, 21],
-    19 => [:conditional,  :open,            '(?', 21, 23],
-    20 => [:conditional,  :condition_open,  '(',  23, 24],
-    21 => [:conditional,  :condition,       '3',  24, 25],
-    22 => [:conditional,  :condition_close, ')',  25, 26],
-    23 => [:literal,      :literal,         'e',  26, 27],
-    24 => [:conditional,  :separator,       '|',  27, 28],
-    25 => [:literal,      :literal,         'f',  28, 29],
-    26 => [:conditional,  :close,           ')',  29, 30],
-    27 => [:conditional,  :close,           ')',  30, 31],
-    28 => [:conditional,  :separator,       '|',  31, 32],
-    29 => [:conditional,  :open,            '(?', 32, 34],
-    30 => [:conditional,  :condition_open,  '(',  34, 35],
-    31 => [:conditional,  :condition,       '2',  35, 36],
-    32 => [:conditional,  :condition_close, ')',  36, 37],
-    33 => [:conditional,  :open,            '(?', 37, 39],
-    34 => [:conditional,  :condition_open,  '(',  39, 40],
-    35 => [:conditional,  :condition,       '1',  40, 41],
-    36 => [:conditional,  :condition_close, ')',  41, 42],
-    37 => [:literal,      :literal,         'g',  42, 43],
-    38 => [:conditional,  :separator,       '|',  43, 44],
-    39 => [:literal,      :literal,         'h',  44, 45],
-    40 => [:conditional,  :close,           ')',  45, 46],
-    41 => [:conditional,  :close,           ')',  46, 47],
-    42 => [:conditional,  :close,           ')',  47, 48]
-
-  include_examples 'scan', /((a)|(b)|((?(2)(c(d|e)+)?|(?(3)f|(?(4)(g|(h)(i)))))))/,
-    0  => [:group,        :capture,         '(',   0,  1],
-    1  => [:group,        :capture,         '(',   1,  2],
-    2  => [:literal,      :literal,         'a',   2,  3],
-    3  => [:group,        :close,           ')',   3,  4],
-    4  => [:meta,         :alternation,     '|',   4,  5],
-    5  => [:group,        :capture,         '(',   5,  6],
-    6  => [:literal,      :literal,         'b',   6,  7],
-    7  => [:group,        :close,           ')',   7,  8],
-    8  => [:meta,         :alternation,     '|',   8,  9],
-    9  => [:group,        :capture,         '(',   9, 10],
-    10 => [:conditional,  :open,            '(?', 10, 12],
-    11 => [:conditional,  :condition_open,  '(',  12, 13],
-    12 => [:conditional,  :condition,       '2',  13, 14],
-    13 => [:conditional,  :condition_close, ')',  14, 15],
-    14 => [:group,        :capture,         '(',  15, 16],
-    15 => [:literal,      :literal,         'c',  16, 17],
-    16 => [:group,        :capture,         '(',  17, 18],
-    17 => [:literal,      :literal,         'd',  18, 19],
-    18 => [:meta,         :alternation,     '|',  19, 20],
-    19 => [:literal,      :literal,         'e',  20, 21],
-    20 => [:group,        :close,           ')',  21, 22],
-    21 => [:quantifier,   :one_or_more,     '+',  22, 23],
-    22 => [:group,        :close,           ')',  23, 24],
-    23 => [:quantifier,   :zero_or_one,     '?',  24, 25],
-    24 => [:conditional,  :separator,       '|',  25, 26],
-    25 => [:conditional,  :open,            '(?', 26, 28],
-    26 => [:conditional,  :condition_open,  '(',  28, 29],
-    27 => [:conditional,  :condition,       '3',  29, 30],
-    28 => [:conditional,  :condition_close, ')',  30, 31],
-    29 => [:literal,      :literal,         'f',  31, 32],
-    30 => [:conditional,  :separator,       '|',  32, 33],
-    31 => [:conditional,  :open,            '(?', 33, 35],
-    32 => [:conditional,  :condition_open,  '(',  35, 36],
-    33 => [:conditional,  :condition,       '4',  36, 37],
-    34 => [:conditional,  :condition_close, ')',  37, 38],
-    35 => [:group,        :capture,         '(',  38, 39],
-    36 => [:literal,      :literal,         'g',  39, 40],
-    37 => [:meta,         :alternation,     '|',  40, 41],
-    38 => [:group,        :capture,         '(',  41, 42],
-    39 => [:literal,      :literal,         'h',  42, 43],
-    40 => [:group,        :close,           ')',  43, 44],
-    41 => [:group,        :capture,         '(',  44, 45],
-    42 => [:literal,      :literal,         'i',  45, 46],
-    43 => [:group,        :close,           ')',  46, 47],
-    44 => [:group,        :close,           ')',  47, 48],
-    45 => [:conditional,  :close,           ')',  48, 49],
-    46 => [:conditional,  :close,           ')',  49, 50],
-    47 => [:conditional,  :close,           ')',  50, 51],
-    48 => [:group,        :close,           ')',  51, 52],
-    49 => [:group,        :close,           ')',  52, 53]
-
-  include_examples 'scan', /(a)(?(1)(b|c|d)|(e|f|g))(h)(?(2)(i|j|k)|(l|m|n))|o|p/,
-    9  => [:meta,        :alternation, '|', 10, 11],
-    11 => [:meta,        :alternation, '|', 12, 13],
-    14 => [:conditional, :separator,   '|', 15, 16],
-    17 => [:meta,        :alternation, '|', 18, 19],
-    19 => [:meta,        :alternation, '|', 20, 21],
-    32 => [:meta,        :alternation, '|', 34, 35],
-    34 => [:meta,        :alternation, '|', 36, 37],
-    37 => [:conditional, :separator,   '|', 39, 40],
-    40 => [:meta,        :alternation, '|', 42, 43],
-    42 => [:meta,        :alternation, '|', 44, 45],
-    46 => [:meta,        :alternation, '|', 48, 49],
-    48 => [:meta,        :alternation, '|', 50, 51]
-end

data/spec/scanner/errors_spec.rb

@@ -1,68 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe(Regexp::Scanner) do
-  RSpec.shared_examples 'scan error' do |error, issue, source|
-    it "raises #{error} for #{issue} `#{source}`" do
-      expect { RS.scan(source) }.to raise_error(error)
-    end
-  end
-
-  include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[a'
-  include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[[:alpha:]'
-  include_examples 'scan error', RS::PrematureEndError, 'unbalanced group', '(abc'
-  include_examples 'scan error', RS::PrematureEndError, 'unbalanced interval', 'a{1,2'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in property', '\p{asci'
-  include_examples 'scan error', RS::PrematureEndError, 'incomplete property', '\p{ascii abc'
-  include_examples 'scan error', RS::PrematureEndError, 'eof options', '(?mix'
-  include_examples 'scan error', RS::PrematureEndError, 'eof escape', '\\'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in hex escape', '\x'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u0'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u00'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u000'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{00'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000 '
-  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000 0000'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c\M'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c\M-'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-\M'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-\M-'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\\'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\c'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\C'
-  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\C-'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid hex', '\xZ'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid hex', '\xZ0'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\cü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\c\M-ü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\C-ü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\C-\M-ü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-ü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-\cü'
-  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-\C-ü'
-  include_examples 'scan error', RS::ScannerError, 'invalid c-seq', '\Ca'
-  include_examples 'scan error', RS::ScannerError, 'invalid m-seq', '\Ma'
-  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', "(?'')"
-  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', "(?''empty-name)"
-  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', '(?<>)'
-  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', '(?<>empty-name)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?foo)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?mix abc)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?mix^bc'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-foo)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-u)'
-  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-mixu)'
-  include_examples 'scan error', RS::InvalidBackrefError, 'empty backref', '\k<>'
-  include_examples 'scan error', RS::InvalidBackrefError, 'empty backref', '\k\'\''
-  include_examples 'scan error', RS::InvalidBackrefError, 'empty refcall', '\g<>'
-  include_examples 'scan error', RS::InvalidBackrefError, 'empty refcall', '\g\'\''
-  include_examples 'scan error', RS::UnknownUnicodePropertyError, 'unknown property', '\p{foobar}'
-end

data/spec/scanner/escapes_spec.rb

@@ -1,53 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Escape scanning') do
-  include_examples 'scan', /c\at/,            1 => [:escape,  :bell,             '\a',             1,  3]
-
-  # not an escape outside a character set
-  include_examples 'scan', /c\bt/,            1 => [:anchor,  :word_boundary,    '\b',             1, 3]
-
-  include_examples 'scan', /c\ft/,            1 => [:escape,  :form_feed,        '\f',             1,  3]
-  include_examples 'scan', /c\nt/,            1 => [:escape,  :newline,          '\n',             1,  3]
-  include_examples 'scan', /c\tt/,            1 => [:escape,  :tab,              '\t',             1,  3]
-  include_examples 'scan', /c\vt/,            1 => [:escape,  :vertical_tab,     '\v',             1,  3]
-
-  include_examples 'scan', 'c\qt',            1 => [:escape,  :literal,          '\q',             1,  3]
-
-  include_examples 'scan', 'a\012c',          1 => [:escape,  :octal,            '\012',           1,  5]
-  include_examples 'scan', 'a\0124',          1 => [:escape,  :octal,            '\012',           1,  5]
-  include_examples 'scan', '\712+7',          0 => [:escape,  :octal,            '\712',           0,  4]
-
-  include_examples 'scan', 'a\x24c',          1 => [:escape,  :hex,              '\x24',           1,  5]
-  include_examples 'scan', 'a\x0640c',        1 => [:escape,  :hex,              '\x06',           1,  5]
-
-  include_examples 'scan', 'a\u0640c',        1 => [:escape,  :codepoint,        '\u0640',         1,  7]
-  include_examples 'scan', 'a\u{640 0641}c',  1 => [:escape,  :codepoint_list,   '\u{640 0641}',   1,  13]
-  include_examples 'scan', 'a\u{10FFFF}c',    1 => [:escape,  :codepoint_list,   '\u{10FFFF}',     1,  11]
-
-  include_examples 'scan', /a\cBc/,           1 => [:escape,  :control,          '\cB',            1,  4]
-  include_examples 'scan', /a\c^c/,           1 => [:escape,  :control,          '\c^',            1,  4]
-  include_examples 'scan', /a\c\n/,           1 => [:escape,  :control,          '\c\n',           1,  5]
-  include_examples 'scan', /a\c\\b/,          1 => [:escape,  :control,          '\c\\\\',         1,  5]
-  include_examples 'scan', /a\C-bc/,          1 => [:escape,  :control,          '\C-b',           1,  5]
-  include_examples 'scan', /a\C-^b/,          1 => [:escape,  :control,          '\C-^',           1,  5]
-  include_examples 'scan', /a\C-\nb/,         1 => [:escape,  :control,          '\C-\n',          1,  6]
-  include_examples 'scan', /a\C-\\b/,         1 => [:escape,  :control,          '\C-\\\\',        1,  6]
-  include_examples 'scan', /a\c\M-Bc/n,       1 => [:escape,  :control,          '\c\M-B',         1,  7]
-  include_examples 'scan', /a\C-\M-Bc/n,      1 => [:escape,  :control,          '\C-\M-B',        1,  8]
-
-  include_examples 'scan', /a\M-Bc/n,         1 => [:escape,  :meta_sequence,    '\M-B',           1,  5]
-  include_examples 'scan', /a\M-\cBc/n,       1 => [:escape,  :meta_sequence,    '\M-\cB',         1,  7]
-  include_examples 'scan', /a\M-\c^/n,        1 => [:escape,  :meta_sequence,    '\M-\c^',         1,  7]
-  include_examples 'scan', /a\M-\c\n/n,       1 => [:escape,  :meta_sequence,    '\M-\c\n',        1,  8]
-  include_examples 'scan', /a\M-\c\\/n,       1 => [:escape,  :meta_sequence,    '\M-\c\\\\',      1,  8]
-  include_examples 'scan', /a\M-\C-Bc/n,      1 => [:escape,  :meta_sequence,    '\M-\C-B',        1,  8]
-  include_examples 'scan', /a\M-\C-\\/n,      1 => [:escape,  :meta_sequence,    '\M-\C-\\\\',     1,  9]
-
-  include_examples 'scan', 'ab\\\xcd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
-  include_examples 'scan', 'ab\\\0cd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
-  include_examples 'scan', 'ab\\\Kcd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
-
-  include_examples 'scan', 'ab\^cd',          1 => [:escape,  :bol,              '\^',             2,  4]
-  include_examples 'scan', 'ab\$cd',          1 => [:escape,  :eol,              '\$',             2,  4]
-  include_examples 'scan', 'ab\[cd',          1 => [:escape,  :set_open,         '\[',             2,  4]
-end

data/spec/scanner/free_space_spec.rb

@@ -1,133 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('FreeSpace scanning') do
-  describe('scan free space tokens') do
-    let(:tokens) { RS.scan(/
-      a
-      b  ?  c  *
-      d  {2,3}
-      e + | f +
-    /x) }
-
-    0.upto(24).select(&:even?).each do |i|
-      it "scans #{i} as free space" do
-        expect(tokens[i][0]).to eq :free_space
-        expect(tokens[i][1]).to eq :whitespace
-      end
-    end
-    0.upto(24).reject(&:even?).each do |i|
-      it "does not scan #{i} as free space" do
-        expect(tokens[i][0]).not_to eq :free_space
-        expect(tokens[i][1]).not_to eq :whitespace
-      end
-    end
-
-    it 'sets the correct text' do
-      [0, 2, 10, 14].each { |i| expect(tokens[i][2]).to eq "\n      " }
-      [4, 6, 8, 12].each { |i| expect(tokens[i][2]).to eq '  ' }
-    end
-  end
-
-  describe('scan free space comments') do
-    include_examples 'scan', /
-      a + # A + comment
-      b  ?  # B ? comment
-      c  {2,3} # C {2,3} comment
-      d + | e + # D|E comment
-    /x,
-      5  => [:free_space, :comment,  "# A + comment\n",      11,  25],
-      11 => [:free_space, :comment,  "# B ? comment\n",      37,  51],
-      17 => [:free_space, :comment,  "# C {2,3} comment\n",  66,  84],
-      29 => [:free_space, :comment,  "# D|E comment\n",     100, 114]
-  end
-
-  describe('scan free space inlined') do
-    include_examples 'scan', /a b(?x:c d e)f g/,
-      0 => [:literal,    :literal,     'a b',   0,  3],
-      1 => [:group,      :options,     '(?x:',  3,  7],
-      2 => [:literal,    :literal,     'c',     7,  8],
-      3 => [:free_space, :whitespace,  ' ',     8,  9],
-      4 => [:literal,    :literal,     'd',     9, 10],
-      5 => [:free_space, :whitespace,  ' ',    10, 11],
-      6 => [:literal,    :literal,     'e',    11, 12],
-      7 => [:group,      :close,       ')',    12, 13],
-      8 => [:literal,    :literal,     'f g',  13, 16]
-  end
-
-  describe('scan free space nested') do
-    include_examples 'scan', /a b(?x:c d(?-x:e f)g h)i j/,
-      0  => [:literal,     :literal,     'a b',     0,  3],
-      1  => [:group,       :options,     '(?x:',    3,  7],
-      2  => [:literal,     :literal,     'c',       7,  8],
-      3  => [:free_space,  :whitespace,  ' ',       8,  9],
-      4  => [:literal,     :literal,     'd',       9, 10],
-      5  => [:group,       :options,     '(?-x:',  10, 15],
-      6  => [:literal,     :literal,     'e f',    15, 18],
-      7  => [:group,       :close,       ')',      18, 19],
-      8  => [:literal,     :literal,     'g',      19, 20],
-      9  => [:free_space,  :whitespace,  ' ',      20, 21],
-      10 => [:literal,     :literal,     'h',      21, 22],
-      11 => [:group,       :close,       ')',      22, 23],
-      12 => [:literal,     :literal,     'i j',    23, 26]
-  end
-
-  describe('scan free space nested groups') do
-    include_examples 'scan', /(a (b(?x: (c d) (?-x:(e f) )g) h)i j)/,
-      0  => [:group,       :capture,     '(',       0,  1],
-      1  => [:literal,     :literal,     'a ',      1,  3],
-      2  => [:group,       :capture,     '(',       3,  4],
-      3  => [:literal,     :literal,     'b',       4,  5],
-      4  => [:group,       :options,     '(?x:',    5,  9],
-      5  => [:free_space,  :whitespace,  ' ',       9, 10],
-      6  => [:group,       :capture,     '(',      10, 11],
-      7  => [:literal,     :literal,     'c',      11, 12],
-      8  => [:free_space,  :whitespace,  ' ',      12, 13],
-      9  => [:literal,     :literal,     'd',      13, 14],
-      10 => [:group,       :close,       ')',      14, 15],
-      11 => [:free_space,  :whitespace,  ' ',      15, 16],
-      12 => [:group,       :options,     '(?-x:',  16, 21],
-      13 => [:group,       :capture,     '(',      21, 22],
-      14 => [:literal,     :literal,     'e f',    22, 25],
-      15 => [:group,       :close,       ')',      25, 26],
-      16 => [:literal,     :literal,     ' ',      26, 27],
-      17 => [:group,       :close,       ')',      27, 28],
-      18 => [:literal,     :literal,     'g',      28, 29],
-      19 => [:group,       :close,       ')',      29, 30],
-      20 => [:literal,     :literal,     ' h',     30, 32],
-      21 => [:group,       :close,       ')',      32, 33],
-      22 => [:literal,     :literal,     'i j',    33, 36],
-      23 => [:group,       :close,       ')',      36, 37]
-  end
-
-  describe('scan free space switch groups') do
-    include_examples 'scan', /(a (b((?x) (c d) ((?-x)(e f) )g) h)i j)/,
-      0  => [:group,      :capture,        '(',     0,  1],
-      1  => [:literal,    :literal,        'a ',    1,  3],
-      2  => [:group,      :capture,        '(',     3,  4],
-      3  => [:literal,    :literal,        'b',     4,  5],
-      4  => [:group,      :capture,        '(',     5,  6],
-      5  => [:group,      :options_switch, '(?x',   6,  9],
-      6  => [:group,      :close,           ')',    9,  10],
-      7  => [:free_space, :whitespace,      ' ',    10, 11],
-      8  => [:group,      :capture,         '(',    11, 12],
-      9  => [:literal,    :literal,         'c',    12, 13],
-      10 => [:free_space, :whitespace,      ' ',    13, 14],
-      11 => [:literal,    :literal,         'd',    14, 15],
-      12 => [:group,      :close,           ')',    15, 16],
-      13 => [:free_space, :whitespace,      ' ',    16, 17],
-      14 => [:group,      :capture,         '(',    17, 18],
-      15 => [:group,      :options_switch, '(?-x',  18, 22],
-      16 => [:group,      :close,          ')',     22, 23],
-      17 => [:group,      :capture,        '(',     23, 24],
-      18 => [:literal,    :literal,        'e f',   24, 27],
-      19 => [:group,      :close,          ')',     27, 28],
-      20 => [:literal,    :literal,        ' ',     28, 29],
-      21 => [:group,      :close,          ')',     29, 30],
-      22 => [:literal,    :literal,        'g',     30, 31],
-      23 => [:group,      :close,          ')',     31, 32],
-      24 => [:literal,    :literal,        ' h',    32, 34],
-      25 => [:group,      :close,          ')',     34, 35],
-      26 => [:literal,    :literal,        'i j',   35, 38],
-      27 => [:group,      :close,          ')',     38, 39]
-  end
-end

data/spec/scanner/groups_spec.rb

@@ -1,52 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Group scanning') do
-  # Group types
-  include_examples 'scan', '(?>abc)',         0 => [:group,     :atomic,         '(?>',        0, 3]
-  include_examples 'scan', '(abc)',           0 => [:group,     :capture,        '(',          0, 1]
-
-  include_examples 'scan', '(?<name>abc)',    0 => [:group,     :named_ab,       '(?<name>',   0, 8]
-  include_examples 'scan', "(?'name'abc)",    0 => [:group,     :named_sq,       "(?'name'",   0, 8]
-
-  include_examples 'scan', '(?<name_1>abc)',  0 => [:group,     :named_ab,       '(?<name_1>', 0,10]
-  include_examples 'scan', "(?'name_1'abc)",  0 => [:group,     :named_sq,       "(?'name_1'", 0,10]
-
-  include_examples 'scan', '(?:abc)',         0 => [:group,     :passive,        '(?:',        0, 3]
-  include_examples 'scan', '(?:)',            0 => [:group,     :passive,        '(?:',        0, 3]
-  include_examples 'scan', '(?::)',           0 => [:group,     :passive,        '(?:',        0, 3]
-
-  # Comments
-  include_examples 'scan', '(?#abc)',         0 => [:group,     :comment,        '(?#abc)',    0, 7]
-  include_examples 'scan', '(?#)',            0 => [:group,     :comment,        '(?#)',       0, 4]
-
-  # Assertions
-  include_examples 'scan', '(?=abc)',         0 => [:assertion, :lookahead,      '(?=',        0, 3]
-  include_examples 'scan', '(?!abc)',         0 => [:assertion, :nlookahead,     '(?!',        0, 3]
-  include_examples 'scan', '(?<=abc)',        0 => [:assertion, :lookbehind,     '(?<=',       0, 4]
-  include_examples 'scan', '(?<!abc)',        0 => [:assertion, :nlookbehind,    '(?<!',       0, 4]
-
-  # Options
-  include_examples 'scan', '(?-mix:abc)',     0 => [:group,     :options,        '(?-mix:',    0, 7]
-  include_examples 'scan', '(?m-ix:abc)',     0 => [:group,     :options,        '(?m-ix:',    0, 7]
-  include_examples 'scan', '(?mi-x:abc)',     0 => [:group,     :options,        '(?mi-x:',    0, 7]
-  include_examples 'scan', '(?mix:abc)',      0 => [:group,     :options,        '(?mix:',     0, 6]
-  include_examples 'scan', '(?m:)',           0 => [:group,     :options,        '(?m:',       0, 4]
-  include_examples 'scan', '(?i:)',           0 => [:group,     :options,        '(?i:',       0, 4]
-  include_examples 'scan', '(?x:)',           0 => [:group,     :options,        '(?x:',       0, 4]
-  include_examples 'scan', '(?mix)',          0 => [:group,     :options_switch, '(?mix',      0, 5]
-  include_examples 'scan', '(?d-mix:abc)',    0 => [:group,     :options,        '(?d-mix:',   0, 8]
-  include_examples 'scan', '(?a-mix:abc)',    0 => [:group,     :options,        '(?a-mix:',   0, 8]
-  include_examples 'scan', '(?u-mix:abc)',    0 => [:group,     :options,        '(?u-mix:',   0, 8]
-  include_examples 'scan', '(?da-m:abc)',     0 => [:group,     :options,        '(?da-m:',    0, 7]
-  include_examples 'scan', '(?du-x:abc)',     0 => [:group,     :options,        '(?du-x:',    0, 7]
-  include_examples 'scan', '(?dau-i:abc)',    0 => [:group,     :options,        '(?dau-i:',   0, 8]
-  include_examples 'scan', '(?dau:abc)',      0 => [:group,     :options,        '(?dau:',     0, 6]
-  include_examples 'scan', '(?d:)',           0 => [:group,     :options,        '(?d:',       0, 4]
-  include_examples 'scan', '(?a:)',           0 => [:group,     :options,        '(?a:',       0, 4]
-  include_examples 'scan', '(?u:)',           0 => [:group,     :options,        '(?u:',       0, 4]
-  include_examples 'scan', '(?dau)',          0 => [:group,     :options_switch, '(?dau',      0, 5]
-
-  if ruby_version_at_least('2.4.1')
-    include_examples 'scan', '(?~abc)', 0 => [:group, :absence, '(?~', 0, 3]
-  end
-end

data/spec/scanner/keep_spec.rb

@@ -1,10 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Keep scanning') do
-  include_examples 'scan', /ab\Kcd/,
-    1 => [:keep, :mark, '\K', 2,  4]
-
-  include_examples 'scan', /(a\Kb)|(c\\\Kd)ef/,
-    2 => [:keep, :mark, '\K', 2,  4],
-    9 => [:keep, :mark, '\K', 11, 13]
-end

data/spec/scanner/literals_spec.rb

@@ -1,49 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('UTF8 scanning') do
-  # ascii, single byte characters
-  include_examples 'scan', 'a', 0              => [:literal,     :literal,       'a',        0, 1]
-
-  include_examples 'scan', 'ab+', 0            => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab+', 1            => [:quantifier,  :one_or_more,   '+',        2, 3]
-
-  # 2 byte wide characters, Arabic
-  include_examples 'scan', 'aاbبcت', 0         => [:literal,     :literal,       'aاbبcت',   0, 9]
-
-  include_examples 'scan', 'aاbبت?', 0         => [:literal,     :literal,       'aاbبت',    0, 8]
-  include_examples 'scan', 'aاbبت?', 1         => [:quantifier,  :zero_or_one,   '?',        8, 9]
-
-  include_examples 'scan', 'aا?bبcت+', 0       => [:literal,     :literal,       'aا',       0, 3]
-  include_examples 'scan', 'aا?bبcت+', 1       => [:quantifier,  :zero_or_one,   '?',        3, 4]
-  include_examples 'scan', 'aا?bبcت+', 2       => [:literal,     :literal,       'bبcت',     4, 10]
-  include_examples 'scan', 'aا?bبcت+', 3       => [:quantifier,  :one_or_more,   '+',        10, 11]
-
-  include_examples 'scan', 'a(اbب+)cت?', 0     => [:literal,     :literal,       'a',        0, 1]
-  include_examples 'scan', 'a(اbب+)cت?', 1     => [:group,       :capture,       '(',        1, 2]
-  include_examples 'scan', 'a(اbب+)cت?', 2     => [:literal,     :literal,       'اbب',      2, 7]
-  include_examples 'scan', 'a(اbب+)cت?', 3     => [:quantifier,  :one_or_more,   '+',        7, 8]
-  include_examples 'scan', 'a(اbب+)cت?', 4     => [:group,       :close,         ')',        8, 9]
-  include_examples 'scan', 'a(اbب+)cت?', 5     => [:literal,     :literal,       'cت',       9, 12]
-  include_examples 'scan', 'a(اbب+)cت?', 6     => [:quantifier,  :zero_or_one,   '?',        12, 13]
-
-  # 3 byte wide characters, Japanese
-  include_examples 'scan', 'ab?れます+cd', 0    => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab?れます+cd', 1    => [:quantifier,  :zero_or_one,   '?',        2, 3]
-  include_examples 'scan', 'ab?れます+cd', 2    => [:literal,     :literal,       'れます',    3, 12]
-  include_examples 'scan', 'ab?れます+cd', 3    => [:quantifier,  :one_or_more,   '+',        12, 13]
-  include_examples 'scan', 'ab?れます+cd', 4    => [:literal,     :literal,       'cd',       13, 15]
-
-  # 4 byte wide characters, Osmanya
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 0      => [:literal,     :literal,       '𐒀𐒁',       0, 8]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 1      => [:quantifier,  :zero_or_one,   '?',        8, 9]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 2      => [:literal,     :literal,       '𐒂ab',      9, 15]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 3      => [:quantifier,  :one_or_more,   '+',        15, 16]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 4      => [:literal,     :literal,       '𐒃',        16, 20]
-
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 0      => [:literal,     :literal,       'mu𝄞',       0, 6]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 1      => [:quantifier,  :zero_or_one,   '?',        6, 7]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 2      => [:literal,     :literal,       'si',       7, 9]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 3      => [:quantifier,  :zero_or_more,  '*',        9, 10]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 4      => [:literal,     :literal,       '𝄫c',       10, 15]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 5      => [:quantifier,  :one_or_more,   '+',        15, 16]
-end

data/spec/scanner/meta_spec.rb

@@ -1,18 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Meta scanning') do
-  include_examples 'scan', /abc??|def*+|ghi+/,
-    0  => [:literal,     :literal,                 'abc',  0,  3],
-    1  => [:quantifier,  :zero_or_one_reluctant,   '??',   3,  5],
-    2  => [:meta,        :alternation,             '|',    5,  6],
-    3  => [:literal,     :literal,                 'def',  6,  9],
-    4  => [:quantifier,  :zero_or_more_possessive, '*+',   9,  11],
-    5  => [:meta,        :alternation,             '|',    11, 12]
-
-  include_examples 'scan', /(a\|b)|(c|d)\|(e[|]f)/,
-    2  => [:escape,      :alternation,             '\|',   2,  4],
-    5  => [:meta,        :alternation,             '|',    6,  7],
-    8  => [:meta,        :alternation,             '|',    9,  10],
-    11 => [:escape,      :alternation,             '\|',   12, 14],
-    15 => [:literal,     :literal,                 '|',    17, 18]
-end