regexp_parser 1.8.2 → 2.1.0

data/spec/parser/quantifiers_spec.rb

@@ -11,6 +11,7 @@ RSpec.describe('Quantifier parsing') do
       expect(exp.quantifier.min).to eq min
       expect(exp.quantifier.max).to eq max
       expect(exp.quantifier.mode).to eq mode
+      expect(exp.quantifier.text).to eq text
     end
   end
 
@@ -37,6 +38,21 @@ RSpec.describe('Quantifier parsing') do
   include_examples 'quantifier', /a{4}+b/,   '{4}+',   :possessive, :interval,     4, 4
   include_examples 'quantifier', /a{004}+b/, '{004}+', :possessive, :interval,     4, 4
 
+  # special case: exps with chained quantifiers are wrapped in implicit passive groups
+  include_examples 'parse', /a+{2}{3}/,
+    0 => [
+      :group, :passive, Group::Passive, implicit?: true, level: 0,
+      quantifier: Quantifier.new(:interval, '{3}', 3, 3, :greedy)
+    ],
+    [0, 0] => [
+      :group, :passive, Group::Passive, implicit?: true, level: 1,
+      quantifier: Quantifier.new(:interval, '{2}', 2, 2, :greedy)
+    ],
+    [0, 0, 0] => [
+      :literal, :literal, Literal, text: 'a', level: 2,
+      quantifier: Quantifier.new(:one_or_more, '+', 1, -1, :greedy)
+    ]
+
   specify('mode-checking methods') do
     exp = RP.parse(/a??/).first
 

data/spec/parser/refcalls_spec.rb

@@ -29,6 +29,11 @@ RSpec.describe('Refcall parsing') do
   include_examples 'parse', /(abc)\g'1'/,
     1 => [:backref, :number_call, Backreference::NumberCall, number: 1]
 
+  include_examples 'parse', '\g<0>',
+    0 => [:backref, :number_call, Backreference::NumberCall, number: 0]
+  include_examples 'parse', "\\g'0'",
+    0 => [:backref, :number_call, Backreference::NumberCall, number: 0]
+
   include_examples 'parse', /(abc)\g<-1>/,
     1 => [:backref, :number_rel_call, Backreference::NumberCallRelative, number: -1]
   include_examples 'parse', /(abc)\g'-1'/,

data/spec/parser/set/ranges_spec.rb

@@ -17,7 +17,7 @@ RSpec.describe('CharacterSet::Range parsing') do
   end
 
   specify('parse set range hex') do
-    root = RP.parse('[\\x00-\\x99]')
+    root = RP.parse('[\\x00-\\x22]')
     set = root[0]
     range = set[0]
 
@@ -26,9 +26,9 @@ RSpec.describe('CharacterSet::Range parsing') do
     expect(range.count).to eq 2
     expect(range.first.to_s).to eq '\\x00'
     expect(range.first).to be_instance_of(EscapeSequence::Hex)
-    expect(range.last.to_s).to eq '\\x99'
+    expect(range.last.to_s).to eq '\\x22'
     expect(range.last).to be_instance_of(EscapeSequence::Hex)
-    expect(set).to match '\\x50'
+    expect(set).to match "\x11"
   end
 
   specify('parse set range unicode') do

data/spec/scanner/escapes_spec.rb

@@ -4,14 +4,20 @@ RSpec.describe('Escape scanning') do
   include_examples 'scan', /c\at/,            1 => [:escape,  :bell,             '\a',             1,  3]
 
   # not an escape outside a character set
-  include_examples 'scan', /c\bt/,            1 => [:anchor,  :word_boundary,    '\b',             1, 3]
+  include_examples 'scan', /c\bt/,            1 => [:anchor,  :word_boundary,    '\b',             1,  3]
 
   include_examples 'scan', /c\ft/,            1 => [:escape,  :form_feed,        '\f',             1,  3]
   include_examples 'scan', /c\nt/,            1 => [:escape,  :newline,          '\n',             1,  3]
   include_examples 'scan', /c\tt/,            1 => [:escape,  :tab,              '\t',             1,  3]
   include_examples 'scan', /c\vt/,            1 => [:escape,  :vertical_tab,     '\v',             1,  3]
 
+  # ineffectual literal escapes
+  # these cause "Unknown escape" warnings in Ruby for ascii chars,
+  # and simply drop the backslash for non-ascii chars (/\ü/.inspect == '/ü/').
+  # In terms of matching, Ruby treats them both like non-escaped literals.
   include_examples 'scan', 'c\qt',            1 => [:escape,  :literal,          '\q',             1,  3]
+  include_examples 'scan', 'a\üc',            1 => [:escape, :literal,           '\ü',             1,  3]
+  include_examples 'scan', 'a\😋c',           1 => [:escape, :literal,            '\😋',            1,  3]
 
   # these incomplete ref/call sequences are treated as literal escapes by Ruby
   include_examples 'scan', 'c\gt',            1 => [:escape,  :literal,          '\g',             1,  3]
@@ -21,6 +27,7 @@ RSpec.describe('Escape scanning') do
   include_examples 'scan', 'a\0124',          1 => [:escape,  :octal,            '\012',           1,  5]
   include_examples 'scan', '\712+7',          0 => [:escape,  :octal,            '\712',           0,  4]
 
+  include_examples 'scan', 'a\xA',            1 => [:escape,  :hex,              '\xA',            1,  4]
   include_examples 'scan', 'a\x24c',          1 => [:escape,  :hex,              '\x24',           1,  5]
   include_examples 'scan', 'a\x0640c',        1 => [:escape,  :hex,              '\x06',           1,  5]
 

data/spec/scanner/groups_spec.rb

@@ -5,11 +5,20 @@ RSpec.describe('Group scanning') do
   include_examples 'scan', '(?>abc)',         0 => [:group,     :atomic,         '(?>',        0, 3]
   include_examples 'scan', '(abc)',           0 => [:group,     :capture,        '(',          0, 1]
 
+  # Named groups
+  # only names that start with a hyphen or digit (ascii or other) are invalid
   include_examples 'scan', '(?<name>abc)',    0 => [:group,     :named_ab,       '(?<name>',   0, 8]
   include_examples 'scan', "(?'name'abc)",    0 => [:group,     :named_sq,       "(?'name'",   0, 8]
-
   include_examples 'scan', '(?<name_1>abc)',  0 => [:group,     :named_ab,       '(?<name_1>', 0,10]
   include_examples 'scan', "(?'name_1'abc)",  0 => [:group,     :named_sq,       "(?'name_1'", 0,10]
+  include_examples 'scan', '(?<name-1>abc)',  0 => [:group,     :named_ab,       '(?<name-1>', 0,10]
+  include_examples 'scan', "(?'name-1'abc)",  0 => [:group,     :named_sq,       "(?'name-1'", 0,10]
+  include_examples 'scan', "(?<name'1>abc)",  0 => [:group,     :named_ab,       "(?<name'1>", 0,10]
+  include_examples 'scan', "(?'name>1'abc)",  0 => [:group,     :named_sq,       "(?'name>1'", 0,10]
+  include_examples 'scan', '(?<üüuuüü>abc)',  0 => [:group,     :named_ab,       '(?<üüuuüü>', 0,10]
+  include_examples 'scan', "(?'üüuuüü'abc)",  0 => [:group,     :named_sq,       "(?'üüuuüü'", 0,10]
+  include_examples 'scan', "(?<😋1234😋>abc)",  0 => [:group,     :named_ab,       "(?<😋1234😋>", 0,10]
+  include_examples 'scan', "(?'😋1234😋'abc)",  0 => [:group,     :named_sq,       "(?'😋1234😋'", 0,10]
 
   include_examples 'scan', '(?:abc)',         0 => [:group,     :passive,        '(?:',        0, 3]
   include_examples 'scan', '(?:)',            0 => [:group,     :passive,        '(?:',        0, 3]

data/spec/scanner/literals_spec.rb

@@ -2,48 +2,38 @@ require 'spec_helper'
 
 RSpec.describe('UTF8 scanning') do
   # ascii, single byte characters
-  include_examples 'scan', 'a', 0              => [:literal,     :literal,       'a',        0, 1]
+  include_examples 'scan', 'a',
+    0 => [:literal,     :literal,       'a',        0, 1]
 
-  include_examples 'scan', 'ab+', 0            => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab+', 1            => [:quantifier,  :one_or_more,   '+',        2, 3]
+  include_examples 'scan', 'ab+',
+    0 => [:literal,     :literal,       'ab',       0, 2],
+    1 => [:quantifier,  :one_or_more,   '+',        2, 3]
 
-  # 2 byte wide characters, Arabic
-  include_examples 'scan', 'aاbبcت', 0         => [:literal,     :literal,       'aاbبcت',   0, 9]
-
-  include_examples 'scan', 'aاbبت?', 0         => [:literal,     :literal,       'aاbبت',    0, 8]
-  include_examples 'scan', 'aاbبت?', 1         => [:quantifier,  :zero_or_one,   '?',        8, 9]
-
-  include_examples 'scan', 'aا?bبcت+', 0       => [:literal,     :literal,       'aا',       0, 3]
-  include_examples 'scan', 'aا?bبcت+', 1       => [:quantifier,  :zero_or_one,   '?',        3, 4]
-  include_examples 'scan', 'aا?bبcت+', 2       => [:literal,     :literal,       'bبcت',     4, 10]
-  include_examples 'scan', 'aا?bبcت+', 3       => [:quantifier,  :one_or_more,   '+',        10, 11]
-
-  include_examples 'scan', 'a(اbب+)cت?', 0     => [:literal,     :literal,       'a',        0, 1]
-  include_examples 'scan', 'a(اbب+)cت?', 1     => [:group,       :capture,       '(',        1, 2]
-  include_examples 'scan', 'a(اbب+)cت?', 2     => [:literal,     :literal,       'اbب',      2, 7]
-  include_examples 'scan', 'a(اbب+)cت?', 3     => [:quantifier,  :one_or_more,   '+',        7, 8]
-  include_examples 'scan', 'a(اbب+)cت?', 4     => [:group,       :close,         ')',        8, 9]
-  include_examples 'scan', 'a(اbب+)cت?', 5     => [:literal,     :literal,       'cت',       9, 12]
-  include_examples 'scan', 'a(اbب+)cت?', 6     => [:quantifier,  :zero_or_one,   '?',        12, 13]
+  # 2 byte wide characters
+  include_examples 'scan', 'äöü',
+    0 => [:literal,     :literal,        'äöü',     0, 3]
 
   # 3 byte wide characters, Japanese
-  include_examples 'scan', 'ab?れます+cd', 0    => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab?れます+cd', 1    => [:quantifier,  :zero_or_one,   '?',        2, 3]
-  include_examples 'scan', 'ab?れます+cd', 2    => [:literal,     :literal,       'れます',    3, 12]
-  include_examples 'scan', 'ab?れます+cd', 3    => [:quantifier,  :one_or_more,   '+',        12, 13]
-  include_examples 'scan', 'ab?れます+cd', 4    => [:literal,     :literal,       'cd',       13, 15]
+  include_examples 'scan', 'ab?れます+cd',
+    0 => [:literal,     :literal,       'ab',       0, 2],
+    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
+    2 => [:literal,     :literal,       'れます',    3, 6],
+    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
+    4 => [:literal,     :literal,       'cd',       7, 9]
 
   # 4 byte wide characters, Osmanya
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 0      => [:literal,     :literal,       '𐒀𐒁',       0, 8]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 1      => [:quantifier,  :zero_or_one,   '?',        8, 9]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 2      => [:literal,     :literal,       '𐒂ab',      9, 15]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 3      => [:quantifier,  :one_or_more,   '+',        15, 16]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 4      => [:literal,     :literal,       '𐒃',        16, 20]
-
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 0      => [:literal,     :literal,       'mu𝄞',       0, 6]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 1      => [:quantifier,  :zero_or_one,   '?',        6, 7]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 2      => [:literal,     :literal,       'si',       7, 9]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 3      => [:quantifier,  :zero_or_more,  '*',        9, 10]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 4      => [:literal,     :literal,       '𝄫c',       10, 15]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 5      => [:quantifier,  :one_or_more,   '+',        15, 16]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃',
+    0 => [:literal,     :literal,       '𐒀𐒁',       0, 2],
+    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
+    2 => [:literal,     :literal,       '𐒂ab',      3, 6],
+    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
+    4 => [:literal,     :literal,       '𐒃',        7, 8]
+
+  include_examples 'scan', 'mu𝄞?si*𝄫c+',
+    0 => [:literal,     :literal,       'mu𝄞',       0, 3],
+    1 => [:quantifier,  :zero_or_one,   '?',        3, 4],
+    2 => [:literal,     :literal,       'si',       4, 6],
+    3 => [:quantifier,  :zero_or_more,  '*',        6, 7],
+    4 => [:literal,     :literal,       '𝄫c',       7, 9],
+    5 => [:quantifier,  :one_or_more,   '+',        9, 10]
 end

data/spec/scanner/quantifiers_spec.rb

@@ -1,20 +1,25 @@
 require 'spec_helper'
 
 RSpec.describe('Quantifier scanning') do
-  include_examples 'scan', 'a?',     1 => [:quantifier,  :zero_or_one,             '?',     1, 2]
-  include_examples 'scan', 'a??',    1 => [:quantifier,  :zero_or_one_reluctant,   '??',    1, 3]
-  include_examples 'scan', 'a?+',    1 => [:quantifier,  :zero_or_one_possessive,  '?+',    1, 3]
+  include_examples 'scan', 'a?',       1 => [:quantifier,  :zero_or_one,             '?',     1, 2]
+  include_examples 'scan', 'a??',      1 => [:quantifier,  :zero_or_one_reluctant,   '??',    1, 3]
+  include_examples 'scan', 'a?+',      1 => [:quantifier,  :zero_or_one_possessive,  '?+',    1, 3]
 
-  include_examples 'scan', 'a*',     1 => [:quantifier,  :zero_or_more,            '*',     1, 2]
-  include_examples 'scan', 'a*?',    1 => [:quantifier,  :zero_or_more_reluctant,  '*?',    1, 3]
-  include_examples 'scan', 'a*+',    1 => [:quantifier,  :zero_or_more_possessive, '*+',    1, 3]
+  include_examples 'scan', 'a*',       1 => [:quantifier,  :zero_or_more,            '*',     1, 2]
+  include_examples 'scan', 'a*?',      1 => [:quantifier,  :zero_or_more_reluctant,  '*?',    1, 3]
+  include_examples 'scan', 'a*+',      1 => [:quantifier,  :zero_or_more_possessive, '*+',    1, 3]
 
-  include_examples 'scan', 'a+',     1 => [:quantifier,  :one_or_more,             '+',     1, 2]
-  include_examples 'scan', 'a+?',    1 => [:quantifier,  :one_or_more_reluctant,   '+?',    1, 3]
-  include_examples 'scan', 'a++',    1 => [:quantifier,  :one_or_more_possessive,  '++',    1, 3]
+  include_examples 'scan', 'a+',       1 => [:quantifier,  :one_or_more,             '+',     1, 2]
+  include_examples 'scan', 'a+?',      1 => [:quantifier,  :one_or_more_reluctant,   '+?',    1, 3]
+  include_examples 'scan', 'a++',      1 => [:quantifier,  :one_or_more_possessive,  '++',    1, 3]
 
-  include_examples 'scan', 'a{2}',   1 => [:quantifier,  :interval,                '{2}',   1, 4]
-  include_examples 'scan', 'a{2,}',  1 => [:quantifier,  :interval,                '{2,}',  1, 5]
-  include_examples 'scan', 'a{,2}',  1 => [:quantifier,  :interval,                '{,2}',  1, 5]
-  include_examples 'scan', 'a{2,4}', 1 => [:quantifier,  :interval,                '{2,4}', 1, 6]
+  include_examples 'scan', 'a{2}',     1 => [:quantifier,  :interval,                '{2}',   1, 4]
+  include_examples 'scan', 'a{2,}',    1 => [:quantifier,  :interval,                '{2,}',  1, 5]
+  include_examples 'scan', 'a{,2}',    1 => [:quantifier,  :interval,                '{,2}',  1, 5]
+  include_examples 'scan', 'a{2,4}',   1 => [:quantifier,  :interval,                '{2,4}', 1, 6]
+
+  # special case: chained quantifiers
+  include_examples 'scan', 'a+{2}{3}', 1 => [:quantifier,  :one_or_more,             '+',     1, 2]
+  include_examples 'scan', 'a+{2}{3}', 2 => [:quantifier,  :interval,                '{2}',   2, 5]
+  include_examples 'scan', 'a+{2}{3}', 3 => [:quantifier,  :interval,                '{3}',   5, 8]
 end

data/spec/scanner/refcalls_spec.rb

@@ -5,9 +5,19 @@ RSpec.describe('RefCall scanning') do
   include_examples 'scan', '(abc)\1' ,          3 => [:backref, :number,                  '\1',         5, 7]
 
   # Group back-references, named, numbered, and relative
+  #
+  # NOTE: only \g supports forward-looking references using '+', e.g. \g<+1>
+  # refers to the next group, but \k<+1> refers to a group named '+1'.
+  # Inversely, only \k supports addition or substraction of a recursion level.
+  # E.g. \k<x+0> refers to a group named 'x' at the current recursion level,
+  # but \g<x+0> refers to a a group named 'x+0'.
+  #
   include_examples 'scan', '(?<X>abc)\k<X>',    3 => [:backref, :name_ref_ab,             '\k<X>',      9, 14]
   include_examples 'scan', "(?<X>abc)\\k'X'",   3 => [:backref, :name_ref_sq,             "\\k'X'",     9, 14]
 
+  include_examples 'scan', '(?<+1>abc)\k<+1>',  3 => [:backref, :name_ref_ab,             '\k<+1>',    10, 16]
+  include_examples 'scan', "(?<+1>abc)\\k'+1'", 3 => [:backref, :name_ref_sq,             "\\k'+1'",   10, 16]
+
   include_examples 'scan', '(abc)\k<1>',        3 => [:backref, :number_ref_ab,           '\k<1>',      5, 10]
   include_examples 'scan', "(abc)\\k'1'",       3 => [:backref, :number_ref_sq,           "\\k'1'",     5, 10]
 
@@ -18,9 +28,15 @@ RSpec.describe('RefCall scanning') do
   include_examples 'scan', '(?<X>abc)\g<X>',    3 => [:backref, :name_call_ab,            '\g<X>',      9, 14]
   include_examples 'scan', "(?<X>abc)\\g'X'",   3 => [:backref, :name_call_sq,            "\\g'X'",     9, 14]
 
+  include_examples 'scan', '(?<X>abc)\g<X-1>',  3 => [:backref, :name_call_ab,            '\g<X-1>',    9, 16]
+  include_examples 'scan', "(?<X>abc)\\g'X-1'", 3 => [:backref, :name_call_sq,            "\\g'X-1'",   9, 16]
+
   include_examples 'scan', '(abc)\g<1>',        3 => [:backref, :number_call_ab,          '\g<1>',      5, 10]
   include_examples 'scan', "(abc)\\g'1'",       3 => [:backref, :number_call_sq,          "\\g'1'",     5, 10]
 
+  include_examples 'scan', 'a(b|\g<0>)',        4 => [:backref, :number_call_ab,          '\g<0>',      4, 9]
+  include_examples 'scan', "a(b|\\g'0')",       4 => [:backref, :number_call_sq,          "\\g'0'",     4, 9]
+
   include_examples 'scan', '(abc)\g<-1>',       3 => [:backref, :number_rel_call_ab,      '\g<-1>',     5, 11]
   include_examples 'scan', "(abc)\\g'-1'",      3 => [:backref, :number_rel_call_sq,      "\\g'-1'",    5, 11]
 
@@ -33,4 +49,7 @@ RSpec.describe('RefCall scanning') do
 
   include_examples 'scan', '(abc)\k<1-0>',      3 => [:backref, :number_recursion_ref_ab, '\k<1-0>',    5, 12]
   include_examples 'scan', "(abc)\\k'1-0'",     3 => [:backref, :number_recursion_ref_sq, "\\k'1-0'",   5, 12]
+
+  include_examples 'scan', '(abc)\k<+1-0>',     3 => [:backref, :name_recursion_ref_ab,   '\k<+1-0>',   5, 13]
+  include_examples 'scan', "(abc)\\k'+1-0'",    3 => [:backref, :name_recursion_ref_sq,   "\\k'+1-0'",  5, 13]
 end

data/spec/scanner/sets_spec.rb

@@ -6,8 +6,18 @@ RSpec.describe('Set scanning') do
   include_examples 'scan', /[^n]/,                  1 => [:set,    :negate,          '^',          1, 2]
 
   include_examples 'scan', /[c]/,                   1 => [:literal, :literal,        'c',          1, 2]
-  include_examples 'scan', /[\b]/,                  1 => [:escape,  :backspace,      '\b',         1, 3]
-  include_examples 'scan', /[A\bX]/,                2 => [:escape,  :backspace,      '\b',         2, 4]
+  include_examples 'scan', /[^d]/,                  2 => [:literal, :literal,        'd',          2, 3]
+
+  include_examples 'scan', /[\b]/,                  1 => [:escape, :backspace,       '\b',         1, 3]
+  include_examples 'scan', /[A\bX]/,                2 => [:escape, :backspace,       '\b',         2, 4]
+
+  include_examples 'scan', /[\a]/,                  1 => [:escape, :bell,            '\a',         1, 3]
+  include_examples 'scan', /[\e]/,                  1 => [:escape, :escape,          '\e',         1, 3]
+  include_examples 'scan', /[\f]/,                  1 => [:escape, :form_feed,       '\f',         1, 3]
+  include_examples 'scan', /[\n]/,                  1 => [:escape, :newline,         '\n',         1, 3]
+  include_examples 'scan', /[\r]/,                  1 => [:escape, :carriage,        '\r',         1, 3]
+  include_examples 'scan', /[\t]/,                  1 => [:escape, :tab,             '\t',         1, 3]
+  include_examples 'scan', /[\v]/,                  1 => [:escape, :vertical_tab,    '\v',         1, 3]
 
   include_examples 'scan', /[.]/,                   1 => [:literal, :literal,        '.',          1, 2]
   include_examples 'scan', /[?]/,                   1 => [:literal, :literal,        '?',          1, 2]
@@ -18,24 +28,36 @@ RSpec.describe('Set scanning') do
   include_examples 'scan', /[<]/,                   1 => [:literal, :literal,        '<',          1, 2]
   include_examples 'scan', /[>]/,                   1 => [:literal, :literal,        '>',          1, 2]
 
-  include_examples 'scan', /[äöü]/,                 2 => [:literal, :literal,        'ö',          3, 5]
-
-  include_examples 'scan', /[\x20]/,                1 => [:escape, :hex,             '\x20',       1, 5]
-
-  include_examples 'scan', '[\.]',                  1 => [:escape, :dot,             '\.',         1, 3]
+  include_examples 'scan', '[\\\\]',                1 => [:escape, :backslash,       '\\\\',       1, 3]
+  include_examples 'scan', '[\u0040]',              1 => [:escape, :codepoint,       '\u0040',     1, 7]
+  include_examples 'scan', '[\u{40}]',              1 => [:escape, :codepoint_list,  '\u{40}',     1, 7]
+  include_examples 'scan', '[\c2]',                 1 => [:escape, :control,         '\c2',        1, 4]
+  include_examples 'scan', '[\C-C]',                1 => [:escape, :control,         '\C-C',       1, 5]
+  include_examples 'scan', '[\x20]',                1 => [:escape, :hex,             '\x20',       1, 5]
+  include_examples 'scan', '[\M-Z]',                1 => [:escape, :meta_sequence,   '\M-Z',       1, 5]
+  include_examples 'scan', '[\M-\C-X]',             1 => [:escape, :meta_sequence,   '\M-\C-X',    1, 8]
+  include_examples 'scan', '[\\[]',                 1 => [:escape, :set_open,        '\[',         1, 3]
+  include_examples 'scan', '[\\]]',                 1 => [:escape, :set_close,       '\]',         1, 3]
+  include_examples 'scan', '[a\-]',                 2 => [:escape, :literal,         '\-',         2, 4]
+  include_examples 'scan', '[\-c]',                 1 => [:escape, :literal,         '\-',         1, 3]
+  include_examples 'scan', '[\.]',                  1 => [:escape, :literal,         '\.',         1, 3]
+  include_examples 'scan', '[\?]',                  1 => [:escape, :literal,         '\?',         1, 3]
+  include_examples 'scan', '[\*]',                  1 => [:escape, :literal,         '\*',         1, 3]
+  include_examples 'scan', '[\+]',                  1 => [:escape, :literal,         '\+',         1, 3]
+  include_examples 'scan', '[\|]',                  1 => [:escape, :literal,         '\|',         1, 3]
+  include_examples 'scan', '[\{]',                  1 => [:escape, :literal,         '\{',         1, 3]
+  include_examples 'scan', '[\}]',                  1 => [:escape, :literal,         '\}',         1, 3]
+  include_examples 'scan', '[\(]',                  1 => [:escape, :literal,         '\(',         1, 3]
+  include_examples 'scan', '[\)]',                  1 => [:escape, :literal,         '\)',         1, 3]
   include_examples 'scan', '[\!]',                  1 => [:escape, :literal,         '\!',         1, 3]
   include_examples 'scan', '[\#]',                  1 => [:escape, :literal,         '\#',         1, 3]
-  include_examples 'scan', '[\\]]',                 1 => [:escape, :set_close,       '\]',         1, 3]
-  include_examples 'scan', '[\\\\]',                1 => [:escape, :backslash,       '\\\\',       1, 3]
   include_examples 'scan', '[\A]',                  1 => [:escape, :literal,         '\A',         1, 3]
   include_examples 'scan', '[\z]',                  1 => [:escape, :literal,         '\z',         1, 3]
   include_examples 'scan', '[\g]',                  1 => [:escape, :literal,         '\g',         1, 3]
   include_examples 'scan', '[\K]',                  1 => [:escape, :literal,         '\K',         1, 3]
   include_examples 'scan', '[\R]',                  1 => [:escape, :literal,         '\R',         1, 3]
   include_examples 'scan', '[\X]',                  1 => [:escape, :literal,         '\X',         1, 3]
-  include_examples 'scan', '[\c2]',                 1 => [:escape, :literal,         '\c',         1, 3]
   include_examples 'scan', '[\B]',                  1 => [:escape, :literal,         '\B',         1, 3]
-  include_examples 'scan', '[a\-c]',                2 => [:escape, :literal,         '\-',         2, 4]
 
   include_examples 'scan', /[\d]/,                  1 => [:type,   :digit,           '\d',         1, 3]
   include_examples 'scan', /[\da-z]/,               1 => [:type,   :digit,           '\d',         1, 3]
@@ -56,19 +78,23 @@ RSpec.describe('Set scanning') do
   include_examples 'scan', /[a-b-]/,                4 => [:literal, :literal,        '-',          4, 5]
   include_examples 'scan', /[-a]/,                  1 => [:literal, :literal,        '-',          1, 2]
   include_examples 'scan', /[a-c^]/,                4 => [:literal, :literal,        '^',          4, 5]
-  include_examples 'scan', /[a-bd-f]/,              2 => [:set,    :range,           '-',          2, 3]
-  include_examples 'scan', /[a-cd-f]/,              5 => [:set,    :range,           '-',          5, 6]
+  include_examples 'scan', /[a-bd-f]/,              2 => [:set,     :range,          '-',          2, 3]
+  include_examples 'scan', /[a-cd-f]/,              5 => [:set,     :range,          '-',          5, 6]
+  # this is a buggy range, it matches only `c`, but not `a`, `b` or `-`
+  include_examples 'scan', /[a-[c]]/,               2 => [:set,     :range,          '-',          2, 3]
+  # these are not ranges, they match `a`, `c` and `-` (or non-`-` if negated)
+  include_examples 'scan', /[[a]-[c]]/,             4 => [:literal, :literal,        '-',          4, 5]
+  include_examples 'scan', /[[a]-c]/,               4 => [:literal, :literal,        '-',          4, 5]
+  include_examples 'scan', /[^-c]/,                 2 => [:literal, :literal,        '-',          2, 3]
 
   include_examples 'scan', /[a[:digit:]c]/,         2 => [:posixclass,    :digit,    '[:digit:]',  2, 11]
   include_examples 'scan', /[[:digit:][:space:]]/,  2 => [:posixclass,    :space,    '[:space:]', 10, 19]
   include_examples 'scan', /[[:^digit:]]/,          1 => [:nonposixclass, :digit,    '[:^digit:]', 1, 11]
 
-  include_examples 'scan', /[a[.a-b.]c]/,           2 => [:set,    :collation,       '[.a-b.]',    2,  9]
-  include_examples 'scan', /[a[=e=]c]/,             2 => [:set,    :equivalent,      '[=e=]',      2,  7]
-
   include_examples 'scan', /[a-d&&g-h]/,            4 => [:set,    :intersection,    '&&',         4, 6]
   include_examples 'scan', /[a&&]/,                 2 => [:set,    :intersection,    '&&',         2, 4]
   include_examples 'scan', /[&&z]/,                 1 => [:set,    :intersection,    '&&',         1, 3]
+  include_examples 'scan', /[&&]/,                  1 => [:set,    :intersection,    '&&',         1, 3]
 
   include_examples 'scan', /[a\p{digit}c]/,         2 => [:property,    :digit,      '\p{digit}',  2, 11]
   include_examples 'scan', /[a\P{digit}c]/,         2 => [:nonproperty, :digit,      '\P{digit}',  2, 11]
@@ -90,6 +116,29 @@ RSpec.describe('Set scanning') do
     8 => [:set,    :range,           '-',          9, 10],
     10=> [:set,    :close,           ']',          11, 12]
 
+  # Collations/collating sequences and character equivalents are not enabled
+  # in Ruby at the moment. If they ever are, enable them in the scanner,
+  # add them to a new syntax version, and handle them in the parser. Until then,
+  # expect them to be scanned as regular subsets containing literals.
+  # include_examples 'scan', /[a[.a-b.]c]/,           2 => [:set,    :collation,       '[.a-b.]',    2,  9]
+  # include_examples 'scan', /[a[=e=]c]/,             2 => [:set,    :equivalent,      '[=e=]',      2,  7]
+  include_examples 'scan', '[a[.a-b.]c]',
+    2 => [:set,     :open,           '[',          2,  3],
+    3 => [:literal, :literal,        '.',          3,  4],
+    4 => [:literal, :literal,        'a',          4,  5]
+  include_examples 'scan', '[a[=e=]c]',
+    2 => [:set,     :open,           '[',          2,  3],
+    3 => [:literal, :literal,        '=',          3,  4],
+    4 => [:literal, :literal,        'e',          4,  5]
+
+  # multi-byte characters should not affect indices
+  include_examples 'scan', /[れます]/,
+    0 => [:set,     :open,           '[',          0, 1],
+    1 => [:literal, :literal,        'れ',          1, 2],
+    2 => [:literal, :literal,        'ま',          2, 3],
+    3 => [:literal, :literal,        'す',          3, 4],
+    4 => [:set,     :close,          ']',          4, 5]
+
   specify('set literal encoding') do
     text = RS.scan('[a]')[1][2].to_s
     expect(text).to eq 'a'

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+require 'ice_nine'
 require 'regexp_parser'
 require 'regexp_property_values'
 require_relative 'support/shared_examples'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: regexp_parser
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Ammar Ali
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-11 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies: []
 description: A library for tokenizing, lexing, and parsing Ruby regular expressions.
 email:
@@ -23,6 +23,7 @@ files:
 - README.md
 - Rakefile
 - lib/regexp_parser.rb
+- lib/regexp_parser/error.rb
 - lib/regexp_parser/expression.rb
 - lib/regexp_parser/expression/classes/alternation.rb
 - lib/regexp_parser/expression/classes/anchor.rb
@@ -102,8 +103,6 @@ files:
 - spec/expression/methods/tests_spec.rb
 - spec/expression/methods/traverse_spec.rb
 - spec/expression/options_spec.rb
-- spec/expression/root_spec.rb
-- spec/expression/sequence_spec.rb
 - spec/expression/subexpression_spec.rb
 - spec/expression/to_h_spec.rb
 - spec/expression/to_s_spec.rb
@@ -185,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0.rc.1
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Scanner, lexer, parser for ruby's regular expressions
@@ -200,8 +199,6 @@ test_files:
 - spec/expression/methods/tests_spec.rb
 - spec/expression/methods/traverse_spec.rb
 - spec/expression/options_spec.rb
-- spec/expression/root_spec.rb
-- spec/expression/sequence_spec.rb
 - spec/expression/subexpression_spec.rb
 - spec/expression/to_h_spec.rb
 - spec/expression/to_s_spec.rb