regexp_parser 1.3.0 → 1.7.1

data/spec/scanner/delimiters_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+RSpec.describe('Literal delimiter scanning') do
+  include_examples 'scan', '}',
+    0 => [:literal,     :literal,       '}',        0,  1]
+
+  include_examples 'scan', '}}',
+    0 => [:literal,     :literal,       '}}',       0,  2]
+
+  include_examples 'scan', '{',
+    0 => [:literal,     :literal,       '{',        0,  1]
+
+  include_examples 'scan', '{{',
+    0 => [:literal,     :literal,       '{{',       0,  2]
+
+  include_examples 'scan', '{}',
+    0 => [:literal,     :literal,       '{}',       0,  2]
+
+  include_examples 'scan', '}{',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '}{+',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '{{var}}',
+    0 => [:literal,     :literal,       '{{var}}',  0,  7]
+
+  include_examples 'scan', 'a{1,2',
+    0 => [:literal,     :literal,       'a{1,2',    0,  5]
+
+  include_examples 'scan', '({.+})',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:literal,     :literal,       '{',        1,  2],
+    2 => [:meta,        :dot,           '.',        2,  3],
+    3 => [:quantifier,  :one_or_more,   '+',        3,  4],
+    4 => [:literal,     :literal,       '}',        4,  5],
+    5 => [:group,       :close,         ')',        5,  6]
+
+  include_examples 'scan', ']',
+    0 => [:literal,     :literal,       ']',        0,  1]
+
+  include_examples 'scan', ']]',
+    0 => [:literal,     :literal,       ']]',       0,  2]
+
+  include_examples 'scan', ']\[',
+    0 => [:literal,     :literal,       ']',        0,  1],
+    1 => [:escape,      :set_open,      '\[',       1,  3]
+
+  include_examples 'scan', '()',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:group,       :close,         ')',        1,  2]
+end

data/spec/scanner/errors_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Scanner) do
+  RSpec.shared_examples 'scan error' do |error, issue, source|
+    it "raises #{error} for #{issue} `#{source}`" do
+      expect { RS.scan(source) }.to raise_error(error)
+    end
+  end
+
+  include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[a'
+  include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[[:alpha:]'
+  include_examples 'scan error', RS::PrematureEndError, 'unbalanced group', '(abc'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in property', '\p{asci'
+  include_examples 'scan error', RS::PrematureEndError, 'incomplete property', '\p{ascii abc'
+  include_examples 'scan error', RS::PrematureEndError, 'eof options', '(?mix'
+  include_examples 'scan error', RS::PrematureEndError, 'eof escape', '\\'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in hex escape', '\x'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u0'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u00'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u000'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{00'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000 '
+  include_examples 'scan error', RS::PrematureEndError, 'eof in cp escape', '\u{0000 0000'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c\M'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\c\M-'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-\M'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in c-seq', '\C-\M-'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\\'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\c'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\C'
+  include_examples 'scan error', RS::PrematureEndError, 'eof in m-seq', '\M-\C-'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid hex', '\xZ'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid hex', '\xZ0'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\cü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\c\M-ü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\C-ü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid c-seq', '\C-\M-ü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-ü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-\cü'
+  include_examples 'scan error', RS::InvalidSequenceError, 'invalid m-seq', '\M-\C-ü'
+  include_examples 'scan error', RS::ScannerError, 'invalid c-seq', '\Ca'
+  include_examples 'scan error', RS::ScannerError, 'invalid m-seq', '\Ma'
+  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', "(?'')"
+  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', "(?''empty-name)"
+  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', '(?<>)'
+  include_examples 'scan error', RS::InvalidGroupError, 'invalid group', '(?<>empty-name)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?foo)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?mix abc)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?mix^bc'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid option', '(?)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-foo)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-u)'
+  include_examples 'scan error', RS::InvalidGroupOption, 'invalid neg option', '(?-mixu)'
+  include_examples 'scan error', RS::InvalidBackrefError, 'empty backref', '\k<>'
+  include_examples 'scan error', RS::InvalidBackrefError, 'empty backref', '\k\'\''
+  include_examples 'scan error', RS::InvalidBackrefError, 'empty refcall', '\g<>'
+  include_examples 'scan error', RS::InvalidBackrefError, 'empty refcall', '\g\'\''
+  include_examples 'scan error', RS::UnknownUnicodePropertyError, 'unknown property', '\p{foobar}'
+end

data/spec/scanner/escapes_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+RSpec.describe('Escape scanning') do
+  include_examples 'scan', /c\at/,            1 => [:escape,  :bell,             '\a',             1,  3]
+
+  # not an escape outside a character set
+  include_examples 'scan', /c\bt/,            1 => [:anchor,  :word_boundary,    '\b',             1, 3]
+
+  include_examples 'scan', /c\ft/,            1 => [:escape,  :form_feed,        '\f',             1,  3]
+  include_examples 'scan', /c\nt/,            1 => [:escape,  :newline,          '\n',             1,  3]
+  include_examples 'scan', /c\tt/,            1 => [:escape,  :tab,              '\t',             1,  3]
+  include_examples 'scan', /c\vt/,            1 => [:escape,  :vertical_tab,     '\v',             1,  3]
+
+  include_examples 'scan', 'c\qt',            1 => [:escape,  :literal,          '\q',             1,  3]
+
+  include_examples 'scan', 'a\012c',          1 => [:escape,  :octal,            '\012',           1,  5]
+  include_examples 'scan', 'a\0124',          1 => [:escape,  :octal,            '\012',           1,  5]
+  include_examples 'scan', '\712+7',          0 => [:escape,  :octal,            '\712',           0,  4]
+
+  include_examples 'scan', 'a\x24c',          1 => [:escape,  :hex,              '\x24',           1,  5]
+  include_examples 'scan', 'a\x0640c',        1 => [:escape,  :hex,              '\x06',           1,  5]
+
+  include_examples 'scan', 'a\u0640c',        1 => [:escape,  :codepoint,        '\u0640',         1,  7]
+  include_examples 'scan', 'a\u{640 0641}c',  1 => [:escape,  :codepoint_list,   '\u{640 0641}',   1,  13]
+  include_examples 'scan', 'a\u{10FFFF}c',    1 => [:escape,  :codepoint_list,   '\u{10FFFF}',     1,  11]
+
+  include_examples 'scan', /a\cBc/,           1 => [:escape,  :control,          '\cB',            1,  4]
+  include_examples 'scan', /a\c^c/,           1 => [:escape,  :control,          '\c^',            1,  4]
+  include_examples 'scan', /a\c\n/,           1 => [:escape,  :control,          '\c\n',           1,  5]
+  include_examples 'scan', /a\c\\b/,          1 => [:escape,  :control,          '\c\\\\',         1,  5]
+  include_examples 'scan', /a\C-bc/,          1 => [:escape,  :control,          '\C-b',           1,  5]
+  include_examples 'scan', /a\C-^b/,          1 => [:escape,  :control,          '\C-^',           1,  5]
+  include_examples 'scan', /a\C-\nb/,         1 => [:escape,  :control,          '\C-\n',          1,  6]
+  include_examples 'scan', /a\C-\\b/,         1 => [:escape,  :control,          '\C-\\\\',        1,  6]
+  include_examples 'scan', /a\c\M-Bc/n,       1 => [:escape,  :control,          '\c\M-B',         1,  7]
+  include_examples 'scan', /a\C-\M-Bc/n,      1 => [:escape,  :control,          '\C-\M-B',        1,  8]
+
+  include_examples 'scan', /a\M-Bc/n,         1 => [:escape,  :meta_sequence,    '\M-B',           1,  5]
+  include_examples 'scan', /a\M-\cBc/n,       1 => [:escape,  :meta_sequence,    '\M-\cB',         1,  7]
+  include_examples 'scan', /a\M-\c^/n,        1 => [:escape,  :meta_sequence,    '\M-\c^',         1,  7]
+  include_examples 'scan', /a\M-\c\n/n,       1 => [:escape,  :meta_sequence,    '\M-\c\n',        1,  8]
+  include_examples 'scan', /a\M-\c\\/n,       1 => [:escape,  :meta_sequence,    '\M-\c\\\\',      1,  8]
+  include_examples 'scan', /a\M-\C-Bc/n,      1 => [:escape,  :meta_sequence,    '\M-\C-B',        1,  8]
+  include_examples 'scan', /a\M-\C-\\/n,      1 => [:escape,  :meta_sequence,    '\M-\C-\\\\',     1,  9]
+
+  include_examples 'scan', 'ab\\\xcd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
+  include_examples 'scan', 'ab\\\0cd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
+  include_examples 'scan', 'ab\\\Kcd',        1 => [:escape,  :backslash,        '\\\\',           2,  4]
+
+  include_examples 'scan', 'ab\^cd',          1 => [:escape,  :bol,              '\^',             2,  4]
+  include_examples 'scan', 'ab\$cd',          1 => [:escape,  :eol,              '\$',             2,  4]
+  include_examples 'scan', 'ab\[cd',          1 => [:escape,  :set_open,         '\[',             2,  4]
+end

data/spec/scanner/free_space_spec.rb

@@ -0,0 +1,133 @@
+require 'spec_helper'
+
+RSpec.describe('FreeSpace scanning') do
+  describe('scan free space tokens') do
+    let(:tokens) { RS.scan(/
+      a
+      b  ?  c  *
+      d  {2,3}
+      e + | f +
+    /x) }
+
+    0.upto(24).select(&:even?).each do |i|
+      it "scans #{i} as free space" do
+        expect(tokens[i][0]).to eq :free_space
+        expect(tokens[i][1]).to eq :whitespace
+      end
+    end
+    0.upto(24).reject(&:even?).each do |i|
+      it "does not scan #{i} as free space" do
+        expect(tokens[i][0]).not_to eq :free_space
+        expect(tokens[i][1]).not_to eq :whitespace
+      end
+    end
+
+    it 'sets the correct text' do
+      [0, 2, 10, 14].each { |i| expect(tokens[i][2]).to eq "\n      " }
+      [4, 6, 8, 12].each { |i| expect(tokens[i][2]).to eq '  ' }
+    end
+  end
+
+  describe('scan free space comments') do
+    include_examples 'scan', /
+      a + # A + comment
+      b  ?  # B ? comment
+      c  {2,3} # C {2,3} comment
+      d + | e + # D|E comment
+    /x,
+      5  => [:free_space, :comment,  "# A + comment\n",      11,  25],
+      11 => [:free_space, :comment,  "# B ? comment\n",      37,  51],
+      17 => [:free_space, :comment,  "# C {2,3} comment\n",  66,  84],
+      29 => [:free_space, :comment,  "# D|E comment\n",     100, 114]
+  end
+
+  describe('scan free space inlined') do
+    include_examples 'scan', /a b(?x:c d e)f g/,
+      0 => [:literal,    :literal,     'a b',   0,  3],
+      1 => [:group,      :options,     '(?x:',  3,  7],
+      2 => [:literal,    :literal,     'c',     7,  8],
+      3 => [:free_space, :whitespace,  ' ',     8,  9],
+      4 => [:literal,    :literal,     'd',     9, 10],
+      5 => [:free_space, :whitespace,  ' ',    10, 11],
+      6 => [:literal,    :literal,     'e',    11, 12],
+      7 => [:group,      :close,       ')',    12, 13],
+      8 => [:literal,    :literal,     'f g',  13, 16]
+  end
+
+  describe('scan free space nested') do
+    include_examples 'scan', /a b(?x:c d(?-x:e f)g h)i j/,
+      0  => [:literal,     :literal,     'a b',     0,  3],
+      1  => [:group,       :options,     '(?x:',    3,  7],
+      2  => [:literal,     :literal,     'c',       7,  8],
+      3  => [:free_space,  :whitespace,  ' ',       8,  9],
+      4  => [:literal,     :literal,     'd',       9, 10],
+      5  => [:group,       :options,     '(?-x:',  10, 15],
+      6  => [:literal,     :literal,     'e f',    15, 18],
+      7  => [:group,       :close,       ')',      18, 19],
+      8  => [:literal,     :literal,     'g',      19, 20],
+      9  => [:free_space,  :whitespace,  ' ',      20, 21],
+      10 => [:literal,     :literal,     'h',      21, 22],
+      11 => [:group,       :close,       ')',      22, 23],
+      12 => [:literal,     :literal,     'i j',    23, 26]
+  end
+
+  describe('scan free space nested groups') do
+    include_examples 'scan', /(a (b(?x: (c d) (?-x:(e f) )g) h)i j)/,
+      0  => [:group,       :capture,     '(',       0,  1],
+      1  => [:literal,     :literal,     'a ',      1,  3],
+      2  => [:group,       :capture,     '(',       3,  4],
+      3  => [:literal,     :literal,     'b',       4,  5],
+      4  => [:group,       :options,     '(?x:',    5,  9],
+      5  => [:free_space,  :whitespace,  ' ',       9, 10],
+      6  => [:group,       :capture,     '(',      10, 11],
+      7  => [:literal,     :literal,     'c',      11, 12],
+      8  => [:free_space,  :whitespace,  ' ',      12, 13],
+      9  => [:literal,     :literal,     'd',      13, 14],
+      10 => [:group,       :close,       ')',      14, 15],
+      11 => [:free_space,  :whitespace,  ' ',      15, 16],
+      12 => [:group,       :options,     '(?-x:',  16, 21],
+      13 => [:group,       :capture,     '(',      21, 22],
+      14 => [:literal,     :literal,     'e f',    22, 25],
+      15 => [:group,       :close,       ')',      25, 26],
+      16 => [:literal,     :literal,     ' ',      26, 27],
+      17 => [:group,       :close,       ')',      27, 28],
+      18 => [:literal,     :literal,     'g',      28, 29],
+      19 => [:group,       :close,       ')',      29, 30],
+      20 => [:literal,     :literal,     ' h',     30, 32],
+      21 => [:group,       :close,       ')',      32, 33],
+      22 => [:literal,     :literal,     'i j',    33, 36],
+      23 => [:group,       :close,       ')',      36, 37]
+  end
+
+  describe('scan free space switch groups') do
+    include_examples 'scan', /(a (b((?x) (c d) ((?-x)(e f) )g) h)i j)/,
+      0  => [:group,      :capture,        '(',     0,  1],
+      1  => [:literal,    :literal,        'a ',    1,  3],
+      2  => [:group,      :capture,        '(',     3,  4],
+      3  => [:literal,    :literal,        'b',     4,  5],
+      4  => [:group,      :capture,        '(',     5,  6],
+      5  => [:group,      :options_switch, '(?x',   6,  9],
+      6  => [:group,      :close,           ')',    9,  10],
+      7  => [:free_space, :whitespace,      ' ',    10, 11],
+      8  => [:group,      :capture,         '(',    11, 12],
+      9  => [:literal,    :literal,         'c',    12, 13],
+      10 => [:free_space, :whitespace,      ' ',    13, 14],
+      11 => [:literal,    :literal,         'd',    14, 15],
+      12 => [:group,      :close,           ')',    15, 16],
+      13 => [:free_space, :whitespace,      ' ',    16, 17],
+      14 => [:group,      :capture,         '(',    17, 18],
+      15 => [:group,      :options_switch, '(?-x',  18, 22],
+      16 => [:group,      :close,          ')',     22, 23],
+      17 => [:group,      :capture,        '(',     23, 24],
+      18 => [:literal,    :literal,        'e f',   24, 27],
+      19 => [:group,      :close,          ')',     27, 28],
+      20 => [:literal,    :literal,        ' ',     28, 29],
+      21 => [:group,      :close,          ')',     29, 30],
+      22 => [:literal,    :literal,        'g',     30, 31],
+      23 => [:group,      :close,          ')',     31, 32],
+      24 => [:literal,    :literal,        ' h',    32, 34],
+      25 => [:group,      :close,          ')',     34, 35],
+      26 => [:literal,    :literal,        'i j',   35, 38],
+      27 => [:group,      :close,          ')',     38, 39]
+  end
+end

data/spec/scanner/groups_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+RSpec.describe('Group scanning') do
+  # Group types
+  include_examples 'scan', '(?>abc)',         0 => [:group,     :atomic,         '(?>',        0, 3]
+  include_examples 'scan', '(abc)',           0 => [:group,     :capture,        '(',          0, 1]
+
+  include_examples 'scan', '(?<name>abc)',    0 => [:group,     :named_ab,       '(?<name>',   0, 8]
+  include_examples 'scan', "(?'name'abc)",    0 => [:group,     :named_sq,       "(?'name'",   0, 8]
+
+  include_examples 'scan', '(?<name_1>abc)',  0 => [:group,     :named_ab,       '(?<name_1>', 0,10]
+  include_examples 'scan', "(?'name_1'abc)",  0 => [:group,     :named_sq,       "(?'name_1'", 0,10]
+
+  include_examples 'scan', '(?:abc)',         0 => [:group,     :passive,        '(?:',        0, 3]
+  include_examples 'scan', '(?:)',            0 => [:group,     :passive,        '(?:',        0, 3]
+  include_examples 'scan', '(?::)',           0 => [:group,     :passive,        '(?:',        0, 3]
+
+  # Comments
+  include_examples 'scan', '(?#abc)',         0 => [:group,     :comment,        '(?#abc)',    0, 7]
+  include_examples 'scan', '(?#)',            0 => [:group,     :comment,        '(?#)',       0, 4]
+
+  # Assertions
+  include_examples 'scan', '(?=abc)',         0 => [:assertion, :lookahead,      '(?=',        0, 3]
+  include_examples 'scan', '(?!abc)',         0 => [:assertion, :nlookahead,     '(?!',        0, 3]
+  include_examples 'scan', '(?<=abc)',        0 => [:assertion, :lookbehind,     '(?<=',       0, 4]
+  include_examples 'scan', '(?<!abc)',        0 => [:assertion, :nlookbehind,    '(?<!',       0, 4]
+
+  # Options
+  include_examples 'scan', '(?-mix:abc)',     0 => [:group,     :options,        '(?-mix:',    0, 7]
+  include_examples 'scan', '(?m-ix:abc)',     0 => [:group,     :options,        '(?m-ix:',    0, 7]
+  include_examples 'scan', '(?mi-x:abc)',     0 => [:group,     :options,        '(?mi-x:',    0, 7]
+  include_examples 'scan', '(?mix:abc)',      0 => [:group,     :options,        '(?mix:',     0, 6]
+  include_examples 'scan', '(?m:)',           0 => [:group,     :options,        '(?m:',       0, 4]
+  include_examples 'scan', '(?i:)',           0 => [:group,     :options,        '(?i:',       0, 4]
+  include_examples 'scan', '(?x:)',           0 => [:group,     :options,        '(?x:',       0, 4]
+  include_examples 'scan', '(?mix)',          0 => [:group,     :options_switch, '(?mix',      0, 5]
+  include_examples 'scan', '(?d-mix:abc)',    0 => [:group,     :options,        '(?d-mix:',   0, 8]
+  include_examples 'scan', '(?a-mix:abc)',    0 => [:group,     :options,        '(?a-mix:',   0, 8]
+  include_examples 'scan', '(?u-mix:abc)',    0 => [:group,     :options,        '(?u-mix:',   0, 8]
+  include_examples 'scan', '(?da-m:abc)',     0 => [:group,     :options,        '(?da-m:',    0, 7]
+  include_examples 'scan', '(?du-x:abc)',     0 => [:group,     :options,        '(?du-x:',    0, 7]
+  include_examples 'scan', '(?dau-i:abc)',    0 => [:group,     :options,        '(?dau-i:',   0, 8]
+  include_examples 'scan', '(?dau:abc)',      0 => [:group,     :options,        '(?dau:',     0, 6]
+  include_examples 'scan', '(?d:)',           0 => [:group,     :options,        '(?d:',       0, 4]
+  include_examples 'scan', '(?a:)',           0 => [:group,     :options,        '(?a:',       0, 4]
+  include_examples 'scan', '(?u:)',           0 => [:group,     :options,        '(?u:',       0, 4]
+  include_examples 'scan', '(?dau)',          0 => [:group,     :options_switch, '(?dau',      0, 5]
+
+  if ruby_version_at_least('2.4.1')
+    include_examples 'scan', '(?~abc)', 0 => [:group, :absence, '(?~', 0, 3]
+  end
+end

data/spec/scanner/keep_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+RSpec.describe('Keep scanning') do
+  include_examples 'scan', /ab\Kcd/,
+    1 => [:keep, :mark, '\K', 2,  4]
+
+  include_examples 'scan', /(a\Kb)|(c\\\Kd)ef/,
+    2 => [:keep, :mark, '\K', 2,  4],
+    9 => [:keep, :mark, '\K', 11, 13]
+end

data/spec/scanner/literals_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+RSpec.describe('UTF8 scanning') do
+  # ascii, single byte characters
+  include_examples 'scan', 'a', 0              => [:literal,     :literal,       'a',        0, 1]
+
+  include_examples 'scan', 'ab+', 0            => [:literal,     :literal,       'ab',       0, 2]
+  include_examples 'scan', 'ab+', 1            => [:quantifier,  :one_or_more,   '+',        2, 3]
+
+  # 2 byte wide characters, Arabic
+  include_examples 'scan', 'aاbبcت', 0         => [:literal,     :literal,       'aاbبcت',   0, 9]
+
+  include_examples 'scan', 'aاbبت?', 0         => [:literal,     :literal,       'aاbبت',    0, 8]
+  include_examples 'scan', 'aاbبت?', 1         => [:quantifier,  :zero_or_one,   '?',        8, 9]
+
+  include_examples 'scan', 'aا?bبcت+', 0       => [:literal,     :literal,       'aا',       0, 3]
+  include_examples 'scan', 'aا?bبcت+', 1       => [:quantifier,  :zero_or_one,   '?',        3, 4]
+  include_examples 'scan', 'aا?bبcت+', 2       => [:literal,     :literal,       'bبcت',     4, 10]
+  include_examples 'scan', 'aا?bبcت+', 3       => [:quantifier,  :one_or_more,   '+',        10, 11]
+
+  include_examples 'scan', 'a(اbب+)cت?', 0     => [:literal,     :literal,       'a',        0, 1]
+  include_examples 'scan', 'a(اbب+)cت?', 1     => [:group,       :capture,       '(',        1, 2]
+  include_examples 'scan', 'a(اbب+)cت?', 2     => [:literal,     :literal,       'اbب',      2, 7]
+  include_examples 'scan', 'a(اbب+)cت?', 3     => [:quantifier,  :one_or_more,   '+',        7, 8]
+  include_examples 'scan', 'a(اbب+)cت?', 4     => [:group,       :close,         ')',        8, 9]
+  include_examples 'scan', 'a(اbب+)cت?', 5     => [:literal,     :literal,       'cت',       9, 12]
+  include_examples 'scan', 'a(اbب+)cت?', 6     => [:quantifier,  :zero_or_one,   '?',        12, 13]
+
+  # 3 byte wide characters, Japanese
+  include_examples 'scan', 'ab?れます+cd', 0    => [:literal,     :literal,       'ab',       0, 2]
+  include_examples 'scan', 'ab?れます+cd', 1    => [:quantifier,  :zero_or_one,   '?',        2, 3]
+  include_examples 'scan', 'ab?れます+cd', 2    => [:literal,     :literal,       'れます',    3, 12]
+  include_examples 'scan', 'ab?れます+cd', 3    => [:quantifier,  :one_or_more,   '+',        12, 13]
+  include_examples 'scan', 'ab?れます+cd', 4    => [:literal,     :literal,       'cd',       13, 15]
+
+  # 4 byte wide characters, Osmanya
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 0      => [:literal,     :literal,       '𐒀𐒁',       0, 8]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 1      => [:quantifier,  :zero_or_one,   '?',        8, 9]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 2      => [:literal,     :literal,       '𐒂ab',      9, 15]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 3      => [:quantifier,  :one_or_more,   '+',        15, 16]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 4      => [:literal,     :literal,       '𐒃',        16, 20]
+
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 0      => [:literal,     :literal,       'mu𝄞',       0, 6]
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 1      => [:quantifier,  :zero_or_one,   '?',        6, 7]
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 2      => [:literal,     :literal,       'si',       7, 9]
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 3      => [:quantifier,  :zero_or_more,  '*',        9, 10]
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 4      => [:literal,     :literal,       '𝄫c',       10, 15]
+  include_examples 'scan', 'mu𝄞?si*𝄫c+', 5      => [:quantifier,  :one_or_more,   '+',        15, 16]
+end

data/spec/scanner/meta_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+RSpec.describe('Meta scanning') do
+  include_examples 'scan', /abc??|def*+|ghi+/,
+    0  => [:literal,     :literal,                 'abc',  0,  3],
+    1  => [:quantifier,  :zero_or_one_reluctant,   '??',   3,  5],
+    2  => [:meta,        :alternation,             '|',    5,  6],
+    3  => [:literal,     :literal,                 'def',  6,  9],
+    4  => [:quantifier,  :zero_or_more_possessive, '*+',   9,  11],
+    5  => [:meta,        :alternation,             '|',    11, 12]
+
+  include_examples 'scan', /(a\|b)|(c|d)\|(e[|]f)/,
+    2  => [:escape,      :alternation,             '\|',   2,  4],
+    5  => [:meta,        :alternation,             '|',    6,  7],
+    8  => [:meta,        :alternation,             '|',    9,  10],
+    11 => [:escape,      :alternation,             '\|',   12, 14],
+    15 => [:literal,     :literal,                 '|',    17, 18]
+end

data/spec/scanner/properties_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+RSpec.describe('Property scanning') do
+  RSpec.shared_examples 'scan property' do |text, token|
+    it("scans \\p{#{text}} as property #{token}") do
+      result = RS.scan("\\p{#{text}}")[0]
+      expect(result[0..1]).to eq [:property, token]
+    end
+
+    it("scans \\P{#{text}} as nonproperty #{token}") do
+      result = RS.scan("\\P{#{text}}")[0]
+      expect(result[0..1]).to eq [:nonproperty, token]
+    end
+
+    it("scans \\p{^#{text}} as nonproperty #{token}") do
+      result = RS.scan("\\p{^#{text}}")[0]
+      expect(result[0..1]).to eq [:nonproperty, token]
+    end
+
+    it("scans double-negated \\P{^#{text}} as property #{token}") do
+      result = RS.scan("\\P{^#{text}}")[0]
+      expect(result[0..1]).to eq [:property, token]
+    end
+  end
+
+  include_examples 'scan property', 'Alnum',                :alnum
+
+  include_examples 'scan property', 'XPosixPunct',          :xposixpunct
+
+  include_examples 'scan property', 'Newline',              :newline
+
+  include_examples 'scan property', 'Any',                  :any
+
+  include_examples 'scan property', 'Assigned',             :assigned
+
+  include_examples 'scan property', 'Age=1.1',              :'age=1.1'
+  include_examples 'scan property', 'Age=10.0',             :'age=10.0'
+
+  include_examples 'scan property', 'ahex',                 :ascii_hex_digit
+  include_examples 'scan property', 'ASCII_Hex_Digit',      :ascii_hex_digit # test underscore
+
+  include_examples 'scan property', 'sd',                   :soft_dotted
+  include_examples 'scan property', 'Soft-Dotted',          :soft_dotted # test dash
+
+  include_examples 'scan property', 'Egyp',                 :egyptian_hieroglyphs
+  include_examples 'scan property', 'Egyptian Hieroglyphs', :egyptian_hieroglyphs # test whitespace
+
+  include_examples 'scan property', 'Linb',                 :linear_b
+  include_examples 'scan property', 'Linear-B',             :linear_b # test dash
+
+  include_examples 'scan property', 'InArabic',             :in_arabic # test block
+  include_examples 'scan property', 'in Arabic',            :in_arabic # test block w. whitespace
+  include_examples 'scan property', 'In_Arabic',            :in_arabic # test block w. underscore
+
+  include_examples 'scan property', 'Yiii',                 :yi
+  include_examples 'scan property', 'Yi',                   :yi
+
+  include_examples 'scan property', 'Zinh',                 :inherited
+  include_examples 'scan property', 'Inherited',            :inherited
+  include_examples 'scan property', 'Qaai',                 :inherited
+
+  include_examples 'scan property', 'Zzzz',                 :unknown
+  include_examples 'scan property', 'Unknown',              :unknown
+end