regexp_parser 2.1.1 → 2.5.0

data/spec/scanner/free_space_spec.rb

@@ -1,165 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('FreeSpace scanning') do
-  describe('scan free space tokens') do
-    let(:tokens) { RS.scan(/
-      a
-      b  ?  c  *
-      d  {2,3}
-      e + | f +
-    /x) }
-
-    0.upto(24).select(&:even?).each do |i|
-      it "scans #{i} as free space" do
-        expect(tokens[i][0]).to eq :free_space
-        expect(tokens[i][1]).to eq :whitespace
-      end
-    end
-    0.upto(24).reject(&:even?).each do |i|
-      it "does not scan #{i} as free space" do
-        expect(tokens[i][0]).not_to eq :free_space
-        expect(tokens[i][1]).not_to eq :whitespace
-      end
-    end
-
-    it 'sets the correct text' do
-      [0, 2, 10, 14].each { |i| expect(tokens[i][2]).to eq "\n      " }
-      [4, 6, 8, 12].each { |i| expect(tokens[i][2]).to eq '  ' }
-    end
-  end
-
-  describe('scan free space comments') do
-    include_examples 'scan', /
-      a + # A + comment
-      b  ?  # B ? comment
-      c  {2,3} # C {2,3} comment
-      d + | e + # D|E comment
-    /x,
-      5  => [:free_space, :comment,  "# A + comment\n",      11,  25],
-      11 => [:free_space, :comment,  "# B ? comment\n",      37,  51],
-      17 => [:free_space, :comment,  "# C {2,3} comment\n",  66,  84],
-      29 => [:free_space, :comment,  "# D|E comment\n",     100, 114]
-
-    # single line / no trailing newline (c.f. issue #66)
-    include_examples 'scan', /a # b/x,
-      0 => [:literal,    :literal,    'a',   0,  1],
-      1 => [:free_space, :whitespace, ' ',   1,  2],
-      2 => [:free_space, :comment,    "# b", 2,  5]
-
-    # without spaces (c.f. issue #66)
-    include_examples 'scan', /a#b/x,
-      0 => [:literal,    :literal,  'a',   0,  1],
-      1 => [:free_space, :comment,  "#b", 1,  3]
-  end
-
-  describe('scan free space inlined') do
-    include_examples 'scan', /a b(?x:c d e)f g/,
-      0 => [:literal,    :literal,     'a b',   0,  3],
-      1 => [:group,      :options,     '(?x:',  3,  7],
-      2 => [:literal,    :literal,     'c',     7,  8],
-      3 => [:free_space, :whitespace,  ' ',     8,  9],
-      4 => [:literal,    :literal,     'd',     9, 10],
-      5 => [:free_space, :whitespace,  ' ',    10, 11],
-      6 => [:literal,    :literal,     'e',    11, 12],
-      7 => [:group,      :close,       ')',    12, 13],
-      8 => [:literal,    :literal,     'f g',  13, 16]
-  end
-
-  describe('scan free space nested') do
-    include_examples 'scan', /a b(?x:c d(?-x:e f)g h)i j/,
-      0  => [:literal,     :literal,     'a b',     0,  3],
-      1  => [:group,       :options,     '(?x:',    3,  7],
-      2  => [:literal,     :literal,     'c',       7,  8],
-      3  => [:free_space,  :whitespace,  ' ',       8,  9],
-      4  => [:literal,     :literal,     'd',       9, 10],
-      5  => [:group,       :options,     '(?-x:',  10, 15],
-      6  => [:literal,     :literal,     'e f',    15, 18],
-      7  => [:group,       :close,       ')',      18, 19],
-      8  => [:literal,     :literal,     'g',      19, 20],
-      9  => [:free_space,  :whitespace,  ' ',      20, 21],
-      10 => [:literal,     :literal,     'h',      21, 22],
-      11 => [:group,       :close,       ')',      22, 23],
-      12 => [:literal,     :literal,     'i j',    23, 26]
-  end
-
-  describe('scan free space nested groups') do
-    include_examples 'scan', /(a (b(?x: (c d) (?-x:(e f) )g) h)i j)/,
-      0  => [:group,       :capture,     '(',       0,  1],
-      1  => [:literal,     :literal,     'a ',      1,  3],
-      2  => [:group,       :capture,     '(',       3,  4],
-      3  => [:literal,     :literal,     'b',       4,  5],
-      4  => [:group,       :options,     '(?x:',    5,  9],
-      5  => [:free_space,  :whitespace,  ' ',       9, 10],
-      6  => [:group,       :capture,     '(',      10, 11],
-      7  => [:literal,     :literal,     'c',      11, 12],
-      8  => [:free_space,  :whitespace,  ' ',      12, 13],
-      9  => [:literal,     :literal,     'd',      13, 14],
-      10 => [:group,       :close,       ')',      14, 15],
-      11 => [:free_space,  :whitespace,  ' ',      15, 16],
-      12 => [:group,       :options,     '(?-x:',  16, 21],
-      13 => [:group,       :capture,     '(',      21, 22],
-      14 => [:literal,     :literal,     'e f',    22, 25],
-      15 => [:group,       :close,       ')',      25, 26],
-      16 => [:literal,     :literal,     ' ',      26, 27],
-      17 => [:group,       :close,       ')',      27, 28],
-      18 => [:literal,     :literal,     'g',      28, 29],
-      19 => [:group,       :close,       ')',      29, 30],
-      20 => [:literal,     :literal,     ' h',     30, 32],
-      21 => [:group,       :close,       ')',      32, 33],
-      22 => [:literal,     :literal,     'i j',    33, 36],
-      23 => [:group,       :close,       ')',      36, 37]
-  end
-
-  describe('scan free space switch groups') do
-    include_examples 'scan', /(a (b((?x) (c d) ((?-x)(e f) )g) h)i j)/,
-      0  => [:group,      :capture,        '(',     0,  1],
-      1  => [:literal,    :literal,        'a ',    1,  3],
-      2  => [:group,      :capture,        '(',     3,  4],
-      3  => [:literal,    :literal,        'b',     4,  5],
-      4  => [:group,      :capture,        '(',     5,  6],
-      5  => [:group,      :options_switch, '(?x',   6,  9],
-      6  => [:group,      :close,           ')',    9,  10],
-      7  => [:free_space, :whitespace,      ' ',    10, 11],
-      8  => [:group,      :capture,         '(',    11, 12],
-      9  => [:literal,    :literal,         'c',    12, 13],
-      10 => [:free_space, :whitespace,      ' ',    13, 14],
-      11 => [:literal,    :literal,         'd',    14, 15],
-      12 => [:group,      :close,           ')',    15, 16],
-      13 => [:free_space, :whitespace,      ' ',    16, 17],
-      14 => [:group,      :capture,         '(',    17, 18],
-      15 => [:group,      :options_switch, '(?-x',  18, 22],
-      16 => [:group,      :close,          ')',     22, 23],
-      17 => [:group,      :capture,        '(',     23, 24],
-      18 => [:literal,    :literal,        'e f',   24, 27],
-      19 => [:group,      :close,          ')',     27, 28],
-      20 => [:literal,    :literal,        ' ',     28, 29],
-      21 => [:group,      :close,          ')',     29, 30],
-      22 => [:literal,    :literal,        'g',     30, 31],
-      23 => [:group,      :close,          ')',     31, 32],
-      24 => [:literal,    :literal,        ' h',    32, 34],
-      25 => [:group,      :close,          ')',     34, 35],
-      26 => [:literal,    :literal,        'i j',   35, 38],
-      27 => [:group,      :close,          ')',     38, 39]
-  end
-
-  describe('scanning `#` in regular (non-x mode)') do # c.f. issue 70
-    include_examples 'scan', /a#bcd/,
-      0 => [:literal, :literal, 'a#bcd', 0, 5]
-    include_examples 'scan', /a # bcd/,
-      0 => [:literal, :literal, 'a # bcd', 0, 7]
-
-    include_examples 'scan', /a#\d/,
-      0 => [:literal, :literal, 'a#', 0, 2],
-      1 => [:type,    :digit,   '\d', 2, 4]
-    include_examples 'scan', /a # \d/,
-      0 => [:literal, :literal, 'a # ', 0, 4],
-      1 => [:type,    :digit,   '\d',   4, 6]
-
-    include_examples 'scan', /a#()/,
-      0 => [:literal, :literal, 'a#', 0, 2],
-      1 => [:group,   :capture, '(',  2, 3]
-    include_examples 'scan', /a # ()/,
-      0 => [:literal, :literal, 'a # ', 0, 4],
-      1 => [:group,   :capture, '(',    4, 5]
-  end
-end

data/spec/scanner/groups_spec.rb

@@ -1,61 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Group scanning') do
-  # Group types
-  include_examples 'scan', '(?>abc)',         0 => [:group,     :atomic,         '(?>',        0, 3]
-  include_examples 'scan', '(abc)',           0 => [:group,     :capture,        '(',          0, 1]
-
-  # Named groups
-  # only names that start with a hyphen or digit (ascii or other) are invalid
-  include_examples 'scan', '(?<name>abc)',    0 => [:group,     :named_ab,       '(?<name>',   0, 8]
-  include_examples 'scan', "(?'name'abc)",    0 => [:group,     :named_sq,       "(?'name'",   0, 8]
-  include_examples 'scan', '(?<name_1>abc)',  0 => [:group,     :named_ab,       '(?<name_1>', 0,10]
-  include_examples 'scan', "(?'name_1'abc)",  0 => [:group,     :named_sq,       "(?'name_1'", 0,10]
-  include_examples 'scan', '(?<name-1>abc)',  0 => [:group,     :named_ab,       '(?<name-1>', 0,10]
-  include_examples 'scan', "(?'name-1'abc)",  0 => [:group,     :named_sq,       "(?'name-1'", 0,10]
-  include_examples 'scan', "(?<name'1>abc)",  0 => [:group,     :named_ab,       "(?<name'1>", 0,10]
-  include_examples 'scan', "(?'name>1'abc)",  0 => [:group,     :named_sq,       "(?'name>1'", 0,10]
-  include_examples 'scan', '(?<üüuuüü>abc)',  0 => [:group,     :named_ab,       '(?<üüuuüü>', 0,10]
-  include_examples 'scan', "(?'üüuuüü'abc)",  0 => [:group,     :named_sq,       "(?'üüuuüü'", 0,10]
-  include_examples 'scan', "(?<😋1234😋>abc)",  0 => [:group,     :named_ab,       "(?<😋1234😋>", 0,10]
-  include_examples 'scan', "(?'😋1234😋'abc)",  0 => [:group,     :named_sq,       "(?'😋1234😋'", 0,10]
-
-  include_examples 'scan', '(?:abc)',         0 => [:group,     :passive,        '(?:',        0, 3]
-  include_examples 'scan', '(?:)',            0 => [:group,     :passive,        '(?:',        0, 3]
-  include_examples 'scan', '(?::)',           0 => [:group,     :passive,        '(?:',        0, 3]
-
-  # Comments
-  include_examples 'scan', '(?#abc)',         0 => [:group,     :comment,        '(?#abc)',    0, 7]
-  include_examples 'scan', '(?#)',            0 => [:group,     :comment,        '(?#)',       0, 4]
-
-  # Assertions
-  include_examples 'scan', '(?=abc)',         0 => [:assertion, :lookahead,      '(?=',        0, 3]
-  include_examples 'scan', '(?!abc)',         0 => [:assertion, :nlookahead,     '(?!',        0, 3]
-  include_examples 'scan', '(?<=abc)',        0 => [:assertion, :lookbehind,     '(?<=',       0, 4]
-  include_examples 'scan', '(?<!abc)',        0 => [:assertion, :nlookbehind,    '(?<!',       0, 4]
-
-  # Options
-  include_examples 'scan', '(?-mix:abc)',     0 => [:group,     :options,        '(?-mix:',    0, 7]
-  include_examples 'scan', '(?m-ix:abc)',     0 => [:group,     :options,        '(?m-ix:',    0, 7]
-  include_examples 'scan', '(?mi-x:abc)',     0 => [:group,     :options,        '(?mi-x:',    0, 7]
-  include_examples 'scan', '(?mix:abc)',      0 => [:group,     :options,        '(?mix:',     0, 6]
-  include_examples 'scan', '(?m:)',           0 => [:group,     :options,        '(?m:',       0, 4]
-  include_examples 'scan', '(?i:)',           0 => [:group,     :options,        '(?i:',       0, 4]
-  include_examples 'scan', '(?x:)',           0 => [:group,     :options,        '(?x:',       0, 4]
-  include_examples 'scan', '(?mix)',          0 => [:group,     :options_switch, '(?mix',      0, 5]
-  include_examples 'scan', '(?d-mix:abc)',    0 => [:group,     :options,        '(?d-mix:',   0, 8]
-  include_examples 'scan', '(?a-mix:abc)',    0 => [:group,     :options,        '(?a-mix:',   0, 8]
-  include_examples 'scan', '(?u-mix:abc)',    0 => [:group,     :options,        '(?u-mix:',   0, 8]
-  include_examples 'scan', '(?da-m:abc)',     0 => [:group,     :options,        '(?da-m:',    0, 7]
-  include_examples 'scan', '(?du-x:abc)',     0 => [:group,     :options,        '(?du-x:',    0, 7]
-  include_examples 'scan', '(?dau-i:abc)',    0 => [:group,     :options,        '(?dau-i:',   0, 8]
-  include_examples 'scan', '(?dau:abc)',      0 => [:group,     :options,        '(?dau:',     0, 6]
-  include_examples 'scan', '(?d:)',           0 => [:group,     :options,        '(?d:',       0, 4]
-  include_examples 'scan', '(?a:)',           0 => [:group,     :options,        '(?a:',       0, 4]
-  include_examples 'scan', '(?u:)',           0 => [:group,     :options,        '(?u:',       0, 4]
-  include_examples 'scan', '(?dau)',          0 => [:group,     :options_switch, '(?dau',      0, 5]
-
-  if ruby_version_at_least('2.4.1')
-    include_examples 'scan', '(?~abc)', 0 => [:group, :absence, '(?~', 0, 3]
-  end
-end

data/spec/scanner/keep_spec.rb

@@ -1,10 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Keep scanning') do
-  include_examples 'scan', /ab\Kcd/,
-    1 => [:keep, :mark, '\K', 2,  4]
-
-  include_examples 'scan', /(a\Kb)|(c\\\Kd)ef/,
-    2 => [:keep, :mark, '\K', 2,  4],
-    9 => [:keep, :mark, '\K', 11, 13]
-end

data/spec/scanner/literals_spec.rb

@@ -1,39 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('UTF8 scanning') do
-  # ascii, single byte characters
-  include_examples 'scan', 'a',
-    0 => [:literal,     :literal,       'a',        0, 1]
-
-  include_examples 'scan', 'ab+',
-    0 => [:literal,     :literal,       'ab',       0, 2],
-    1 => [:quantifier,  :one_or_more,   '+',        2, 3]
-
-  # 2 byte wide characters
-  include_examples 'scan', 'äöü',
-    0 => [:literal,     :literal,        'äöü',     0, 3]
-
-  # 3 byte wide characters, Japanese
-  include_examples 'scan', 'ab?れます+cd',
-    0 => [:literal,     :literal,       'ab',       0, 2],
-    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
-    2 => [:literal,     :literal,       'れます',    3, 6],
-    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
-    4 => [:literal,     :literal,       'cd',       7, 9]
-
-  # 4 byte wide characters, Osmanya
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃',
-    0 => [:literal,     :literal,       '𐒀𐒁',       0, 2],
-    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
-    2 => [:literal,     :literal,       '𐒂ab',      3, 6],
-    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
-    4 => [:literal,     :literal,       '𐒃',        7, 8]
-
-  include_examples 'scan', 'mu𝄞?si*𝄫c+',
-    0 => [:literal,     :literal,       'mu𝄞',       0, 3],
-    1 => [:quantifier,  :zero_or_one,   '?',        3, 4],
-    2 => [:literal,     :literal,       'si',       4, 6],
-    3 => [:quantifier,  :zero_or_more,  '*',        6, 7],
-    4 => [:literal,     :literal,       '𝄫c',       7, 9],
-    5 => [:quantifier,  :one_or_more,   '+',        9, 10]
-end

data/spec/scanner/meta_spec.rb

@@ -1,18 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Meta scanning') do
-  include_examples 'scan', /abc??|def*+|ghi+/,
-    0  => [:literal,     :literal,                 'abc',  0,  3],
-    1  => [:quantifier,  :zero_or_one_reluctant,   '??',   3,  5],
-    2  => [:meta,        :alternation,             '|',    5,  6],
-    3  => [:literal,     :literal,                 'def',  6,  9],
-    4  => [:quantifier,  :zero_or_more_possessive, '*+',   9,  11],
-    5  => [:meta,        :alternation,             '|',    11, 12]
-
-  include_examples 'scan', /(a\|b)|(c|d)\|(e[|]f)/,
-    2  => [:escape,      :alternation,             '\|',   2,  4],
-    5  => [:meta,        :alternation,             '|',    6,  7],
-    8  => [:meta,        :alternation,             '|',    9,  10],
-    11 => [:escape,      :alternation,             '\|',   12, 14],
-    15 => [:literal,     :literal,                 '|',    17, 18]
-end

data/spec/scanner/options_spec.rb

@@ -1,36 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('passing options to scan') do
-  def expect_type_tokens(tokens, type_tokens)
-    expect(tokens.map { |type, token, *| [type, token] }).to eq(type_tokens)
-  end
-
-  it 'raises if if scanning from a Regexp and options are passed' do
-    expect { RS.scan(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
-      ArgumentError,
-      'options cannot be supplied unless scanning a String'
-    )
-  end
-
-  it 'sets free_spacing based on options if scanning from a String' do
-    expect_type_tokens(
-      RS.scan('a+#c', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED),
-      [
-        %i[literal literal],
-        %i[quantifier one_or_more],
-        %i[free_space comment]
-      ]
-    )
-  end
-
-  it 'does not set free_spacing if scanning from a String and passing no options' do
-    expect_type_tokens(
-      RS.scan('a+#c'),
-      [
-        %i[literal literal],
-        %i[quantifier one_or_more],
-        %i[literal literal]
-      ]
-    )
-  end
-end

data/spec/scanner/properties_spec.rb

@@ -1,64 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Property scanning') do
-  RSpec.shared_examples 'scan property' do |text, token|
-    it("scans \\p{#{text}} as property #{token}") do
-      result = RS.scan("\\p{#{text}}")[0]
-      expect(result[0..1]).to eq [:property, token]
-    end
-
-    it("scans \\P{#{text}} as nonproperty #{token}") do
-      result = RS.scan("\\P{#{text}}")[0]
-      expect(result[0..1]).to eq [:nonproperty, token]
-    end
-
-    it("scans \\p{^#{text}} as nonproperty #{token}") do
-      result = RS.scan("\\p{^#{text}}")[0]
-      expect(result[0..1]).to eq [:nonproperty, token]
-    end
-
-    it("scans double-negated \\P{^#{text}} as property #{token}") do
-      result = RS.scan("\\P{^#{text}}")[0]
-      expect(result[0..1]).to eq [:property, token]
-    end
-  end
-
-  include_examples 'scan property', 'Alnum',                :alnum
-
-  include_examples 'scan property', 'XPosixPunct',          :xposixpunct
-
-  include_examples 'scan property', 'Newline',              :newline
-
-  include_examples 'scan property', 'Any',                  :any
-
-  include_examples 'scan property', 'Assigned',             :assigned
-
-  include_examples 'scan property', 'Age=1.1',              :'age=1.1'
-  include_examples 'scan property', 'Age=10.0',             :'age=10.0'
-
-  include_examples 'scan property', 'ahex',                 :ascii_hex_digit
-  include_examples 'scan property', 'ASCII_Hex_Digit',      :ascii_hex_digit # test underscore
-
-  include_examples 'scan property', 'sd',                   :soft_dotted
-  include_examples 'scan property', 'Soft-Dotted',          :soft_dotted # test dash
-
-  include_examples 'scan property', 'Egyp',                 :egyptian_hieroglyphs
-  include_examples 'scan property', 'Egyptian Hieroglyphs', :egyptian_hieroglyphs # test whitespace
-
-  include_examples 'scan property', 'Linb',                 :linear_b
-  include_examples 'scan property', 'Linear-B',             :linear_b # test dash
-
-  include_examples 'scan property', 'InArabic',             :in_arabic # test block
-  include_examples 'scan property', 'in Arabic',            :in_arabic # test block w. whitespace
-  include_examples 'scan property', 'In_Arabic',            :in_arabic # test block w. underscore
-
-  include_examples 'scan property', 'Yiii',                 :yi
-  include_examples 'scan property', 'Yi',                   :yi
-
-  include_examples 'scan property', 'Zinh',                 :inherited
-  include_examples 'scan property', 'Inherited',            :inherited
-  include_examples 'scan property', 'Qaai',                 :inherited
-
-  include_examples 'scan property', 'Zzzz',                 :unknown
-  include_examples 'scan property', 'Unknown',              :unknown
-end

data/spec/scanner/quantifiers_spec.rb

@@ -1,25 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Quantifier scanning') do
-  include_examples 'scan', 'a?',       1 => [:quantifier,  :zero_or_one,             '?',     1, 2]
-  include_examples 'scan', 'a??',      1 => [:quantifier,  :zero_or_one_reluctant,   '??',    1, 3]
-  include_examples 'scan', 'a?+',      1 => [:quantifier,  :zero_or_one_possessive,  '?+',    1, 3]
-
-  include_examples 'scan', 'a*',       1 => [:quantifier,  :zero_or_more,            '*',     1, 2]
-  include_examples 'scan', 'a*?',      1 => [:quantifier,  :zero_or_more_reluctant,  '*?',    1, 3]
-  include_examples 'scan', 'a*+',      1 => [:quantifier,  :zero_or_more_possessive, '*+',    1, 3]
-
-  include_examples 'scan', 'a+',       1 => [:quantifier,  :one_or_more,             '+',     1, 2]
-  include_examples 'scan', 'a+?',      1 => [:quantifier,  :one_or_more_reluctant,   '+?',    1, 3]
-  include_examples 'scan', 'a++',      1 => [:quantifier,  :one_or_more_possessive,  '++',    1, 3]
-
-  include_examples 'scan', 'a{2}',     1 => [:quantifier,  :interval,                '{2}',   1, 4]
-  include_examples 'scan', 'a{2,}',    1 => [:quantifier,  :interval,                '{2,}',  1, 5]
-  include_examples 'scan', 'a{,2}',    1 => [:quantifier,  :interval,                '{,2}',  1, 5]
-  include_examples 'scan', 'a{2,4}',   1 => [:quantifier,  :interval,                '{2,4}', 1, 6]
-
-  # special case: chained quantifiers
-  include_examples 'scan', 'a+{2}{3}', 1 => [:quantifier,  :one_or_more,             '+',     1, 2]
-  include_examples 'scan', 'a+{2}{3}', 2 => [:quantifier,  :interval,                '{2}',   2, 5]
-  include_examples 'scan', 'a+{2}{3}', 3 => [:quantifier,  :interval,                '{3}',   5, 8]
-end

data/spec/scanner/refcalls_spec.rb

@@ -1,55 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('RefCall scanning') do
-  # Traditional numerical group back-reference
-  include_examples 'scan', '(abc)\1' ,          3 => [:backref, :number,                  '\1',         5, 7]
-
-  # Group back-references, named, numbered, and relative
-  #
-  # NOTE: only \g supports forward-looking references using '+', e.g. \g<+1>
-  # refers to the next group, but \k<+1> refers to a group named '+1'.
-  # Inversely, only \k supports addition or substraction of a recursion level.
-  # E.g. \k<x+0> refers to a group named 'x' at the current recursion level,
-  # but \g<x+0> refers to a a group named 'x+0'.
-  #
-  include_examples 'scan', '(?<X>abc)\k<X>',    3 => [:backref, :name_ref_ab,             '\k<X>',      9, 14]
-  include_examples 'scan', "(?<X>abc)\\k'X'",   3 => [:backref, :name_ref_sq,             "\\k'X'",     9, 14]
-
-  include_examples 'scan', '(?<+1>abc)\k<+1>',  3 => [:backref, :name_ref_ab,             '\k<+1>',    10, 16]
-  include_examples 'scan', "(?<+1>abc)\\k'+1'", 3 => [:backref, :name_ref_sq,             "\\k'+1'",   10, 16]
-
-  include_examples 'scan', '(abc)\k<1>',        3 => [:backref, :number_ref_ab,           '\k<1>',      5, 10]
-  include_examples 'scan', "(abc)\\k'1'",       3 => [:backref, :number_ref_sq,           "\\k'1'",     5, 10]
-
-  include_examples 'scan', '(abc)\k<-1>',       3 => [:backref, :number_rel_ref_ab,       '\k<-1>',     5, 11]
-  include_examples 'scan', "(abc)\\k'-1'",      3 => [:backref, :number_rel_ref_sq,       "\\k'-1'",    5, 11]
-
-  # Sub-expression invocation, named, numbered, and relative
-  include_examples 'scan', '(?<X>abc)\g<X>',    3 => [:backref, :name_call_ab,            '\g<X>',      9, 14]
-  include_examples 'scan', "(?<X>abc)\\g'X'",   3 => [:backref, :name_call_sq,            "\\g'X'",     9, 14]
-
-  include_examples 'scan', '(?<X>abc)\g<X-1>',  3 => [:backref, :name_call_ab,            '\g<X-1>',    9, 16]
-  include_examples 'scan', "(?<X>abc)\\g'X-1'", 3 => [:backref, :name_call_sq,            "\\g'X-1'",   9, 16]
-
-  include_examples 'scan', '(abc)\g<1>',        3 => [:backref, :number_call_ab,          '\g<1>',      5, 10]
-  include_examples 'scan', "(abc)\\g'1'",       3 => [:backref, :number_call_sq,          "\\g'1'",     5, 10]
-
-  include_examples 'scan', 'a(b|\g<0>)',        4 => [:backref, :number_call_ab,          '\g<0>',      4, 9]
-  include_examples 'scan', "a(b|\\g'0')",       4 => [:backref, :number_call_sq,          "\\g'0'",     4, 9]
-
-  include_examples 'scan', '(abc)\g<-1>',       3 => [:backref, :number_rel_call_ab,      '\g<-1>',     5, 11]
-  include_examples 'scan', "(abc)\\g'-1'",      3 => [:backref, :number_rel_call_sq,      "\\g'-1'",    5, 11]
-
-  include_examples 'scan', '\g<+1>(abc)',       0 => [:backref, :number_rel_call_ab,      '\g<+1>',     0, 6]
-  include_examples 'scan', "\\g'+1'(abc)",      0 => [:backref, :number_rel_call_sq,      "\\g'+1'",    0, 6]
-
-  # Group back-references, with recursion level
-  include_examples 'scan', '(?<X>abc)\k<X-0>',  3 => [:backref, :name_recursion_ref_ab,   '\k<X-0>',    9, 16]
-  include_examples 'scan', "(?<X>abc)\\k'X-0'", 3 => [:backref, :name_recursion_ref_sq,   "\\k'X-0'",   9, 16]
-
-  include_examples 'scan', '(abc)\k<1-0>',      3 => [:backref, :number_recursion_ref_ab, '\k<1-0>',    5, 12]
-  include_examples 'scan', "(abc)\\k'1-0'",     3 => [:backref, :number_recursion_ref_sq, "\\k'1-0'",   5, 12]
-
-  include_examples 'scan', '(abc)\k<+1-0>',     3 => [:backref, :name_recursion_ref_ab,   '\k<+1-0>',   5, 13]
-  include_examples 'scan', "(abc)\\k'+1-0'",    3 => [:backref, :name_recursion_ref_sq,   "\\k'+1-0'",  5, 13]
-end

data/spec/scanner/sets_spec.rb

@@ -1,151 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Set scanning') do
-  include_examples 'scan', /[a]/,                   0 => [:set,    :open,            '[',          0, 1]
-  include_examples 'scan', /[b]/,                   2 => [:set,    :close,           ']',          2, 3]
-  include_examples 'scan', /[^n]/,                  1 => [:set,    :negate,          '^',          1, 2]
-
-  include_examples 'scan', /[c]/,                   1 => [:literal, :literal,        'c',          1, 2]
-  include_examples 'scan', /[^d]/,                  2 => [:literal, :literal,        'd',          2, 3]
-
-  include_examples 'scan', /[\b]/,                  1 => [:escape, :backspace,       '\b',         1, 3]
-  include_examples 'scan', /[A\bX]/,                2 => [:escape, :backspace,       '\b',         2, 4]
-
-  include_examples 'scan', /[\a]/,                  1 => [:escape, :bell,            '\a',         1, 3]
-  include_examples 'scan', /[\e]/,                  1 => [:escape, :escape,          '\e',         1, 3]
-  include_examples 'scan', /[\f]/,                  1 => [:escape, :form_feed,       '\f',         1, 3]
-  include_examples 'scan', /[\n]/,                  1 => [:escape, :newline,         '\n',         1, 3]
-  include_examples 'scan', /[\r]/,                  1 => [:escape, :carriage,        '\r',         1, 3]
-  include_examples 'scan', /[\t]/,                  1 => [:escape, :tab,             '\t',         1, 3]
-  include_examples 'scan', /[\v]/,                  1 => [:escape, :vertical_tab,    '\v',         1, 3]
-
-  include_examples 'scan', /[.]/,                   1 => [:literal, :literal,        '.',          1, 2]
-  include_examples 'scan', /[?]/,                   1 => [:literal, :literal,        '?',          1, 2]
-  include_examples 'scan', /[*]/,                   1 => [:literal, :literal,        '*',          1, 2]
-  include_examples 'scan', /[+]/,                   1 => [:literal, :literal,        '+',          1, 2]
-  include_examples 'scan', /[{]/,                   1 => [:literal, :literal,        '{',          1, 2]
-  include_examples 'scan', /[}]/,                   1 => [:literal, :literal,        '}',          1, 2]
-  include_examples 'scan', /[<]/,                   1 => [:literal, :literal,        '<',          1, 2]
-  include_examples 'scan', /[>]/,                   1 => [:literal, :literal,        '>',          1, 2]
-
-  include_examples 'scan', '[\\\\]',                1 => [:escape, :backslash,       '\\\\',       1, 3]
-  include_examples 'scan', '[\u0040]',              1 => [:escape, :codepoint,       '\u0040',     1, 7]
-  include_examples 'scan', '[\u{40}]',              1 => [:escape, :codepoint_list,  '\u{40}',     1, 7]
-  include_examples 'scan', '[\c2]',                 1 => [:escape, :control,         '\c2',        1, 4]
-  include_examples 'scan', '[\C-C]',                1 => [:escape, :control,         '\C-C',       1, 5]
-  include_examples 'scan', '[\x20]',                1 => [:escape, :hex,             '\x20',       1, 5]
-  include_examples 'scan', '[\M-Z]',                1 => [:escape, :meta_sequence,   '\M-Z',       1, 5]
-  include_examples 'scan', '[\M-\C-X]',             1 => [:escape, :meta_sequence,   '\M-\C-X',    1, 8]
-  include_examples 'scan', '[\\[]',                 1 => [:escape, :set_open,        '\[',         1, 3]
-  include_examples 'scan', '[\\]]',                 1 => [:escape, :set_close,       '\]',         1, 3]
-  include_examples 'scan', '[a\-]',                 2 => [:escape, :literal,         '\-',         2, 4]
-  include_examples 'scan', '[\-c]',                 1 => [:escape, :literal,         '\-',         1, 3]
-  include_examples 'scan', '[\.]',                  1 => [:escape, :literal,         '\.',         1, 3]
-  include_examples 'scan', '[\?]',                  1 => [:escape, :literal,         '\?',         1, 3]
-  include_examples 'scan', '[\*]',                  1 => [:escape, :literal,         '\*',         1, 3]
-  include_examples 'scan', '[\+]',                  1 => [:escape, :literal,         '\+',         1, 3]
-  include_examples 'scan', '[\|]',                  1 => [:escape, :literal,         '\|',         1, 3]
-  include_examples 'scan', '[\{]',                  1 => [:escape, :literal,         '\{',         1, 3]
-  include_examples 'scan', '[\}]',                  1 => [:escape, :literal,         '\}',         1, 3]
-  include_examples 'scan', '[\(]',                  1 => [:escape, :literal,         '\(',         1, 3]
-  include_examples 'scan', '[\)]',                  1 => [:escape, :literal,         '\)',         1, 3]
-  include_examples 'scan', '[\!]',                  1 => [:escape, :literal,         '\!',         1, 3]
-  include_examples 'scan', '[\#]',                  1 => [:escape, :literal,         '\#',         1, 3]
-  include_examples 'scan', '[\A]',                  1 => [:escape, :literal,         '\A',         1, 3]
-  include_examples 'scan', '[\z]',                  1 => [:escape, :literal,         '\z',         1, 3]
-  include_examples 'scan', '[\g]',                  1 => [:escape, :literal,         '\g',         1, 3]
-  include_examples 'scan', '[\K]',                  1 => [:escape, :literal,         '\K',         1, 3]
-  include_examples 'scan', '[\R]',                  1 => [:escape, :literal,         '\R',         1, 3]
-  include_examples 'scan', '[\X]',                  1 => [:escape, :literal,         '\X',         1, 3]
-  include_examples 'scan', '[\B]',                  1 => [:escape, :literal,         '\B',         1, 3]
-
-  include_examples 'scan', /[\d]/,                  1 => [:type,   :digit,           '\d',         1, 3]
-  include_examples 'scan', /[\da-z]/,               1 => [:type,   :digit,           '\d',         1, 3]
-  include_examples 'scan', /[\D]/,                  1 => [:type,   :nondigit,        '\D',         1, 3]
-
-  include_examples 'scan', /[\h]/,                  1 => [:type,   :hex,             '\h',         1, 3]
-  include_examples 'scan', /[\H]/,                  1 => [:type,   :nonhex,          '\H',         1, 3]
-
-  include_examples 'scan', /[\s]/,                  1 => [:type,   :space,           '\s',         1, 3]
-  include_examples 'scan', /[\S]/,                  1 => [:type,   :nonspace,        '\S',         1, 3]
-
-  include_examples 'scan', /[\w]/,                  1 => [:type,   :word,            '\w',         1, 3]
-  include_examples 'scan', /[\W]/,                  1 => [:type,   :nonword,         '\W',         1, 3]
-
-  include_examples 'scan', /[a-b]/,                 1 => [:literal, :literal,        'a',          1, 2]
-  include_examples 'scan', /[a-c]/,                 2 => [:set,     :range,          '-',          2, 3]
-  include_examples 'scan', /[a-d]/,                 3 => [:literal, :literal,        'd',          3, 4]
-  include_examples 'scan', /[a-b-]/,                4 => [:literal, :literal,        '-',          4, 5]
-  include_examples 'scan', /[-a]/,                  1 => [:literal, :literal,        '-',          1, 2]
-  include_examples 'scan', /[a-c^]/,                4 => [:literal, :literal,        '^',          4, 5]
-  include_examples 'scan', /[a-bd-f]/,              2 => [:set,     :range,          '-',          2, 3]
-  include_examples 'scan', /[a-cd-f]/,              5 => [:set,     :range,          '-',          5, 6]
-  # this is a buggy range, it matches only `c`, but not `a`, `b` or `-`
-  include_examples 'scan', /[a-[c]]/,               2 => [:set,     :range,          '-',          2, 3]
-  # these are not ranges, they match `a`, `c` and `-` (or non-`-` if negated)
-  include_examples 'scan', /[[a]-[c]]/,             4 => [:literal, :literal,        '-',          4, 5]
-  include_examples 'scan', /[[a]-c]/,               4 => [:literal, :literal,        '-',          4, 5]
-  include_examples 'scan', /[^-c]/,                 2 => [:literal, :literal,        '-',          2, 3]
-
-  include_examples 'scan', /[a[:digit:]c]/,         2 => [:posixclass,    :digit,    '[:digit:]',  2, 11]
-  include_examples 'scan', /[[:digit:][:space:]]/,  2 => [:posixclass,    :space,    '[:space:]', 10, 19]
-  include_examples 'scan', /[[:^digit:]]/,          1 => [:nonposixclass, :digit,    '[:^digit:]', 1, 11]
-
-  include_examples 'scan', /[a-d&&g-h]/,            4 => [:set,    :intersection,    '&&',         4, 6]
-  include_examples 'scan', /[a&&]/,                 2 => [:set,    :intersection,    '&&',         2, 4]
-  include_examples 'scan', /[&&z]/,                 1 => [:set,    :intersection,    '&&',         1, 3]
-  include_examples 'scan', /[&&]/,                  1 => [:set,    :intersection,    '&&',         1, 3]
-
-  include_examples 'scan', /[a\p{digit}c]/,         2 => [:property,    :digit,      '\p{digit}',  2, 11]
-  include_examples 'scan', /[a\P{digit}c]/,         2 => [:nonproperty, :digit,      '\P{digit}',  2, 11]
-  include_examples 'scan', /[a\p{^digit}c]/,        2 => [:nonproperty, :digit,      '\p{^digit}', 2, 12]
-  include_examples 'scan', /[a\P{^digit}c]/,        2 => [:property,    :digit,      '\P{^digit}', 2, 12]
-
-  include_examples 'scan', /[a\p{ALPHA}c]/,         2 => [:property,    :alpha,      '\p{ALPHA}',  2, 11]
-  include_examples 'scan', /[a\p{P}c]/,             2 => [:property,    :punctuation,'\p{P}',      2, 7]
-  include_examples 'scan', /[a\p{P}\P{P}c]/,        3 => [:nonproperty, :punctuation,'\P{P}',      7, 12]
-
-  include_examples 'scan', /[\x20-\x27]/,
-    1 => [:escape, :hex,             '\x20',       1, 5],
-    2 => [:set,    :range,           '-',          5, 6],
-    3 => [:escape, :hex,             '\x27',       6, 10]
-
-  include_examples 'scan', /[a-w&&[^c-g]z]/,
-    5 => [:set,    :open,            '[',          6, 7],
-    6 => [:set,    :negate,          '^',          7, 8],
-    8 => [:set,    :range,           '-',          9, 10],
-    10=> [:set,    :close,           ']',          11, 12]
-
-  # Collations/collating sequences and character equivalents are not enabled
-  # in Ruby at the moment. If they ever are, enable them in the scanner,
-  # add them to a new syntax version, and handle them in the parser. Until then,
-  # expect them to be scanned as regular subsets containing literals.
-  # include_examples 'scan', /[a[.a-b.]c]/,           2 => [:set,    :collation,       '[.a-b.]',    2,  9]
-  # include_examples 'scan', /[a[=e=]c]/,             2 => [:set,    :equivalent,      '[=e=]',      2,  7]
-  include_examples 'scan', '[a[.a-b.]c]',
-    2 => [:set,     :open,           '[',          2,  3],
-    3 => [:literal, :literal,        '.',          3,  4],
-    4 => [:literal, :literal,        'a',          4,  5]
-  include_examples 'scan', '[a[=e=]c]',
-    2 => [:set,     :open,           '[',          2,  3],
-    3 => [:literal, :literal,        '=',          3,  4],
-    4 => [:literal, :literal,        'e',          4,  5]
-
-  # multi-byte characters should not affect indices
-  include_examples 'scan', /[れます]/,
-    0 => [:set,     :open,           '[',          0, 1],
-    1 => [:literal, :literal,        'れ',          1, 2],
-    2 => [:literal, :literal,        'ま',          2, 3],
-    3 => [:literal, :literal,        'す',          3, 4],
-    4 => [:set,     :close,          ']',          4, 5]
-
-  specify('set literal encoding') do
-    text = RS.scan('[a]')[1][2].to_s
-    expect(text).to eq 'a'
-    expect(text.encoding.to_s).to eq 'UTF-8'
-
-    text = RS.scan("[\u{1F632}]")[1][2].to_s
-    expect(text).to eq "\u{1F632}"
-    expect(text.encoding.to_s).to eq 'UTF-8'
-  end
-end