regexp_parser 1.7.0 → 2.0.0

data/lib/regexp_parser/syntax/version_lookup.rb

@@ -74,9 +74,9 @@ module Regexp::Syntax
   end
 
   def warn_if_future_version(const_name)
-    return if comparable_version(const_name) < comparable_version('3.0.0')
+    return if comparable_version(const_name) < comparable_version('4.0.0')
 
-    warn('This library has only been tested up to Ruby 2.x, '\
+    warn('This library has only been tested up to Ruby 3.x, '\
          "but you are running with #{const_get(const_name).inspect}")
   end
 end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.7.0'
+    VERSION = '2.0.0'
   end
 end

data/regexp_parser.gemspec

@@ -32,5 +32,5 @@ Gem::Specification.new do |gem|
 
   gem.platform      = Gem::Platform::RUBY
 
-  gem.required_ruby_version = '>= 1.9.1'
+  gem.required_ruby_version = '>= 2.0.0'
 end

data/spec/expression/base_spec.rb

@@ -91,4 +91,14 @@ RSpec.describe(Regexp::Expression::Base) do
     expect(RP.parse(/a*/)[0].repetitions).to eq 0..(Float::INFINITY)
     expect(RP.parse(/a+/)[0].repetitions).to eq 1..(Float::INFINITY)
   end
+
+  specify('#base_length') do
+    expect(RP.parse(/(aa)/)[0].base_length).to eq 4
+    expect(RP.parse(/(aa){42}/)[0].base_length).to eq 4
+  end
+
+  specify('#full_length') do
+    expect(RP.parse(/(aa)/)[0].full_length).to eq 4
+    expect(RP.parse(/(aa){42}/)[0].full_length).to eq 8
+  end
 end

data/spec/expression/to_s_spec.rb

@@ -97,4 +97,20 @@ RSpec.describe('Expression#to_s') do
 
     expect(Regexp.new(root.to_s, Regexp::EXTENDED).match(str)[0]).to eq multiline.match(str)[0]
   end
+
+  # special case: implicit groups used for chained quantifiers produce no parens
+  specify 'chained quantifiers #to_s' do
+    pattern = /a+{1}{2}/
+    root = RP.parse(pattern)
+    expect(root.to_s).to eq 'a+{1}{2}'
+  end
+
+  # regression test for https://github.com/ammar/regexp_parser/issues/74
+  specify('non-ascii comment') do
+    pattern = '(?x) 😋 # 😋'
+    root = RP.parse(pattern)
+    expect(root.last).to be_a Regexp::Expression::Comment
+    expect(root.last.to_s).to eq '# 😋'
+    expect(root.to_s).to eq pattern
+  end
 end

data/spec/lexer/delimiters_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+RSpec.describe('Literal delimiter lexing') do
+  include_examples 'lex', '}',
+    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0]
+
+  include_examples 'lex', '}}',
+    0 => [:literal,     :literal,       '}}',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '{',
+    0 => [:literal,     :literal,       '{',       0,  1,  0, 0, 0]
+
+  include_examples 'lex', '{{',
+    0 => [:literal,     :literal,       '{{',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '{}',
+    0 => [:literal,     :literal,       '{}',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '}{',
+    0 => [:literal,     :literal,       '}{',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '}{+',
+    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0],
+    1 => [:literal,     :literal,       '{',       1,  2,  0, 0, 0],
+    2 => [:quantifier,  :one_or_more,   '+',       2,  3,  0, 0, 0]
+
+  include_examples 'lex', '{{var}}',
+    0 => [:literal,     :literal,       '{{var}}',  0,  7,  0, 0, 0]
+
+  include_examples 'lex', 'a{b}c',
+    0 => [:literal,     :literal,       'a{b}c',    0,  5,  0, 0, 0]
+
+  include_examples 'lex', 'a{1,2',
+    0 => [:literal,     :literal,       'a{1,2',    0,  5,  0, 0, 0]
+
+  include_examples 'lex', '({.+})',
+    0 => [:group,       :capture,       '(',    0,  1,  0, 0, 0],
+    1 => [:literal,     :literal,       '{',    1,  2,  1, 0, 0],
+    2 => [:meta,        :dot,           '.',    2,  3,  1, 0, 0],
+    3 => [:quantifier,  :one_or_more,   '+',    3,  4,  1, 0, 0],
+    4 => [:literal,     :literal,       '}',    4,  5,  1, 0, 0],
+    5 => [:group,       :close,         ')',    5,  6,  0, 0, 0]
+
+  include_examples 'lex', ']',
+    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0]
+
+  include_examples 'lex', ']]',
+    0 => [:literal,     :literal,       ']]',       0,  2,  0, 0, 0]
+
+  include_examples 'lex', ']\[',
+    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0],
+    1 => [:escape,      :set_open,      '\[',       1,  3,  0, 0, 0]
+
+  include_examples 'lex', '()',
+    0 => [:group,       :capture,       '(',        0,  1,  0, 0, 0],
+    1 => [:group,       :close,         ')',        1,  2,  0, 0, 0]
+
+  include_examples 'lex', '{abc:.+}}}[^}]]}',
+    0 => [:literal,     :literal,       '{abc:',    0,  5,  0, 0, 0],
+    1 => [:meta,        :dot,           '.',        5,  6,  0, 0, 0],
+    2 => [:quantifier,  :one_or_more,   '+',        6,  7,  0, 0, 0],
+    3 => [:literal,     :literal,       '}}}',      7,  10, 0, 0, 0],
+    4 => [:set,         :open,          '[',        10, 11, 0, 0, 0],
+    5 => [:set,         :negate,        '^',        11, 12, 0, 1, 0],
+    6 => [:literal,     :literal,       '}',        12, 13, 0, 1, 0],
+    7 => [:set,         :close,         ']',        13, 14, 0, 0, 0],
+    8 => [:literal,     :literal,       ']}',       14, 16, 0, 0, 0]
+end

data/spec/lexer/literals_spec.rb

@@ -10,67 +10,42 @@ RSpec.describe('Literal lexing') do
     1 => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
     2 => [:quantifier,  :one_or_more,   '+',        2, 3, 0, 0, 0]
 
-  # 2 byte wide characters, Arabic
-  include_examples 'lex', 'ا',
-    0 => [:literal,     :literal,       'ا',        0, 2, 0, 0, 0]
-
-  include_examples 'lex', 'aاbبcت',
-    0 => [:literal,     :literal,       'aاbبcت',   0, 9, 0, 0, 0]
-
-  include_examples 'lex', 'aاbبت?',
-    0 => [:literal,     :literal,       'aاbب',     0, 6, 0, 0, 0],
-    1 => [:literal,     :literal,       'ت',        6, 8, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0, 0]
-
-  include_examples 'lex', 'aا?bبcت+',
-    0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-    1 => [:literal,     :literal,       'ا',        1, 3, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        3, 4, 0, 0, 0],
-    3 => [:literal,     :literal,       'bبc',      4, 8, 0, 0, 0],
-    4 => [:literal,     :literal,       'ت',        8, 10, 0, 0, 0],
-    5 => [:quantifier,  :one_or_more,   '+',        10, 11, 0, 0, 0]
-
-  include_examples 'lex', 'a(اbب+)cت?',
-    0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-    1 => [:group,       :capture,       '(',        1, 2, 0, 0, 0],
-    2 => [:literal,     :literal,       'اb',       2, 5, 1, 0, 0],
-    3 => [:literal,     :literal,       'ب',        5, 7, 1, 0, 0],
-    4 => [:quantifier,  :one_or_more,   '+',        7, 8, 1, 0, 0],
-    5 => [:group,       :close,         ')',        8, 9, 0, 0, 0],
-    6 => [:literal,     :literal,       'c',        9, 10, 0, 0, 0],
-    7 => [:literal,     :literal,       'ت',        10, 12, 0, 0, 0],
-    8 => [:quantifier,  :zero_or_one,   '?',        12, 13, 0, 0, 0]
+  # 2 byte wide characters
+  include_examples 'lex', 'äöü+',
+    0 => [:literal,     :literal,       'äö',       0, 2, 0, 0, 0],
+    1 => [:literal,     :literal,       'ü',        2, 3, 0, 0, 0],
+    2 => [:quantifier,  :one_or_more,   '+',        3, 4, 0, 0, 0]
 
   # 3 byte wide characters, Japanese
   include_examples 'lex', 'ab?れます+cd',
     0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
     1 => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
     2 => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0, 0],
-    3 => [:literal,     :literal,       'れま',     3, 9, 0, 0, 0],
-    4 => [:literal,     :literal,       'す',       9, 12, 0, 0, 0],
-    5 => [:quantifier,  :one_or_more,   '+',        12, 13, 0, 0, 0],
-    6 => [:literal,     :literal,       'cd',       13, 15, 0, 0, 0]
+    3 => [:literal,     :literal,       'れま',     3, 5, 0, 0, 0],
+    4 => [:literal,     :literal,       'す',       5, 6, 0, 0, 0],
+    5 => [:quantifier,  :one_or_more,   '+',        6, 7, 0, 0, 0],
+    6 => [:literal,     :literal,       'cd',       7, 9, 0, 0, 0]
 
   # 4 byte wide characters, Osmanya
   include_examples 'lex', '𐒀𐒁?𐒂ab+𐒃',
-    0 => [:literal,     :literal,       '𐒀',        0, 4, 0, 0, 0],
-    1 => [:literal,     :literal,       '𐒁',        4, 8, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0, 0],
-    3 => [:literal,     :literal,       '𐒂a',       9, 14, 0, 0, 0],
-    4 => [:literal,     :literal,       'b',        14, 15, 0, 0, 0],
-    5 => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0, 0],
-    6 => [:literal,     :literal,       '𐒃',        16, 20, 0, 0, 0]
+    0 => [:literal,     :literal,       '𐒀',        0, 1, 0, 0, 0],
+    1 => [:literal,     :literal,       '𐒁',        1, 2, 0, 0, 0],
+    2 => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0, 0],
+    3 => [:literal,     :literal,       '𐒂a',       3, 5, 0, 0, 0],
+    4 => [:literal,     :literal,       'b',        5, 6, 0, 0, 0],
+    5 => [:quantifier,  :one_or_more,   '+',        6, 7, 0, 0, 0],
+    6 => [:literal,     :literal,       '𐒃',        7, 8, 0, 0, 0]
 
   include_examples 'lex', 'mu𝄞?si*𝄫c+',
     0 => [:literal,     :literal,       'mu',       0, 2, 0, 0, 0],
-    1 => [:literal,     :literal,       '𝄞',        2, 6, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        6, 7, 0, 0, 0],
-    3 => [:literal,     :literal,       's',        7, 8, 0, 0, 0],
-    4 => [:literal,     :literal,       'i',        8, 9, 0, 0, 0],
-    5 => [:quantifier,  :zero_or_more,  '*',        9, 10, 0, 0, 0],
-    6 => [:literal,     :literal,       '𝄫',        10, 14, 0, 0, 0],
-    7 => [:literal,     :literal,       'c',        14, 15, 0, 0, 0],
-    8 => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0, 0]
+    1 => [:literal,     :literal,       '𝄞',        2, 3, 0, 0, 0],
+    2 => [:quantifier,  :zero_or_one,   '?',        3, 4, 0, 0, 0],
+    3 => [:literal,     :literal,       's',        4, 5, 0, 0, 0],
+    4 => [:literal,     :literal,       'i',        5, 6, 0, 0, 0],
+    5 => [:quantifier,  :zero_or_more,  '*',        6, 7, 0, 0, 0],
+    6 => [:literal,     :literal,       '𝄫',        7, 8, 0, 0, 0],
+    7 => [:literal,     :literal,       'c',        8, 9, 0, 0, 0],
+    8 => [:quantifier,  :one_or_more,   '+',        9, 10, 0, 0, 0]
 
   specify('lex single 2 byte char') do
     tokens = RL.lex("\u0627+")

data/spec/parser/escapes_spec.rb

@@ -25,7 +25,7 @@ RSpec.describe('EscapeSequence parsing') do
   include_examples 'parse', /a\u{41 1F60D}/, 1 => [:escape, :codepoint_list,    EscapeSequence::CodepointList]
   include_examples 'parse', /a\u{10FFFF}/,   1 => [:escape, :codepoint_list,    EscapeSequence::CodepointList]
 
-    # hex escapes
+  # hex escapes
   include_examples 'parse', /a\xFF/n,        1 => [:escape, :hex,               EscapeSequence::Hex]
 
   # octal escapes

data/spec/parser/options_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe('passing options to parse') do
+  it 'raises if if parsing from a Regexp and options are passed' do
+    expect { RP.parse(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
+      ArgumentError,
+      'options cannot be supplied unless parsing a String'
+    )
+  end
+
+  it 'sets options if parsing from a String' do
+    root = RP.parse('a+', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED)
+
+    expect(root.options).to eq(m: true, x: true)
+  end
+
+  it 'allows options to not be supplied when parsing from a Regexp' do
+    root = RP.parse(/a+/ix)
+
+    expect(root.options).to eq(i: true, x: true)
+  end
+
+  it 'has an empty option-hash when parsing from a String and passing no options' do
+    root = RP.parse('a+')
+
+    expect(root.options).to be_empty
+  end
+end

data/spec/parser/quantifiers_spec.rb

@@ -35,6 +35,22 @@ RSpec.describe('Quantifier parsing') do
   include_examples 'quantifier', /a{4}b/,    '{4}',    :greedy,     :interval,     4, 4
   include_examples 'quantifier', /a{4}?b/,   '{4}?',   :reluctant,  :interval,     4, 4
   include_examples 'quantifier', /a{4}+b/,   '{4}+',   :possessive, :interval,     4, 4
+  include_examples 'quantifier', /a{004}+b/, '{004}+', :possessive, :interval,     4, 4
+
+  # special case: exps with chained quantifiers are wrapped in implicit passive groups
+  include_examples 'parse', /a+{2}{3}/,
+    0 => [
+      :group, :passive, Group::Passive, implicit?: true, level: 0,
+      quantifier: Quantifier.new(:interval, '{3}', 3, 3, :greedy)
+    ],
+    [0, 0] => [
+      :group, :passive, Group::Passive, implicit?: true, level: 1,
+      quantifier: Quantifier.new(:interval, '{2}', 2, 2, :greedy)
+    ],
+    [0, 0, 0] => [
+      :literal, :literal, Literal, text: 'a', level: 2,
+      quantifier: Quantifier.new(:one_or_more, '+', 1, -1, :greedy)
+    ]
 
   specify('mode-checking methods') do
     exp = RP.parse(/a??/).first

data/spec/parser/set/ranges_spec.rb

@@ -17,7 +17,7 @@ RSpec.describe('CharacterSet::Range parsing') do
   end
 
   specify('parse set range hex') do
-    root = RP.parse('[\\x00-\\x99]')
+    root = RP.parse('[\\x00-\\x22]')
     set = root[0]
     range = set[0]
 
@@ -26,9 +26,9 @@ RSpec.describe('CharacterSet::Range parsing') do
     expect(range.count).to eq 2
     expect(range.first.to_s).to eq '\\x00'
     expect(range.first).to be_instance_of(EscapeSequence::Hex)
-    expect(range.last.to_s).to eq '\\x99'
+    expect(range.last.to_s).to eq '\\x22'
     expect(range.last).to be_instance_of(EscapeSequence::Hex)
-    expect(set).to match '\\x50'
+    expect(set).to match "\x11"
   end
 
   specify('parse set range unicode') do

data/spec/scanner/delimiters_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+RSpec.describe('Literal delimiter scanning') do
+  include_examples 'scan', '}',
+    0 => [:literal,     :literal,       '}',        0,  1]
+
+  include_examples 'scan', '}}',
+    0 => [:literal,     :literal,       '}}',       0,  2]
+
+  include_examples 'scan', '{',
+    0 => [:literal,     :literal,       '{',        0,  1]
+
+  include_examples 'scan', '{{',
+    0 => [:literal,     :literal,       '{{',       0,  2]
+
+  include_examples 'scan', '{}',
+    0 => [:literal,     :literal,       '{}',       0,  2]
+
+  include_examples 'scan', '}{',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '}{+',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '{{var}}',
+    0 => [:literal,     :literal,       '{{var}}',  0,  7]
+
+  include_examples 'scan', 'a{1,2',
+    0 => [:literal,     :literal,       'a{1,2',    0,  5]
+
+  include_examples 'scan', '({.+})',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:literal,     :literal,       '{',        1,  2],
+    2 => [:meta,        :dot,           '.',        2,  3],
+    3 => [:quantifier,  :one_or_more,   '+',        3,  4],
+    4 => [:literal,     :literal,       '}',        4,  5],
+    5 => [:group,       :close,         ')',        5,  6]
+
+  include_examples 'scan', ']',
+    0 => [:literal,     :literal,       ']',        0,  1]
+
+  include_examples 'scan', ']]',
+    0 => [:literal,     :literal,       ']]',       0,  2]
+
+  include_examples 'scan', ']\[',
+    0 => [:literal,     :literal,       ']',        0,  1],
+    1 => [:escape,      :set_open,      '\[',       1,  3]
+
+  include_examples 'scan', '()',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:group,       :close,         ')',        1,  2]
+end

data/spec/scanner/errors_spec.rb

@@ -10,7 +10,6 @@ RSpec.describe(Regexp::Scanner) do
   include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[a'
   include_examples 'scan error', RS::PrematureEndError, 'unbalanced set', '[[:alpha:]'
   include_examples 'scan error', RS::PrematureEndError, 'unbalanced group', '(abc'
-  include_examples 'scan error', RS::PrematureEndError, 'unbalanced interval', 'a{1,2'
   include_examples 'scan error', RS::PrematureEndError, 'eof in property', '\p{asci'
   include_examples 'scan error', RS::PrematureEndError, 'incomplete property', '\p{ascii abc'
   include_examples 'scan error', RS::PrematureEndError, 'eof options', '(?mix'

data/spec/scanner/escapes_spec.rb

@@ -11,7 +11,17 @@ RSpec.describe('Escape scanning') do
   include_examples 'scan', /c\tt/,            1 => [:escape,  :tab,              '\t',             1,  3]
   include_examples 'scan', /c\vt/,            1 => [:escape,  :vertical_tab,     '\v',             1,  3]
 
+  # ineffectual literal escapes
+  # these cause "Unknown escape" warnings in Ruby for ascii chars,
+  # and simply drop the backslash for non-ascii chars (/\ü/.inspect == '/ü/').
+  # In terms of matching, Ruby treats them both like non-escaped literals.
   include_examples 'scan', 'c\qt',            1 => [:escape,  :literal,          '\q',             1,  3]
+  include_examples 'scan', 'a\üc',            1 => [:escape, :literal,           '\ü',             1,  3]
+  include_examples 'scan', 'a\😋c',           1 => [:escape, :literal,            '\😋',            1,  3]
+
+  # these incomplete ref/call sequences are treated as literal escapes by Ruby
+  include_examples 'scan', 'c\gt',            1 => [:escape,  :literal,          '\g',             1,  3]
+  include_examples 'scan', 'c\kt',            1 => [:escape,  :literal,          '\k',             1,  3]
 
   include_examples 'scan', 'a\012c',          1 => [:escape,  :octal,            '\012',           1,  5]
   include_examples 'scan', 'a\0124',          1 => [:escape,  :octal,            '\012',           1,  5]

data/spec/scanner/free_space_spec.rb

@@ -39,6 +39,17 @@ RSpec.describe('FreeSpace scanning') do
       11 => [:free_space, :comment,  "# B ? comment\n",      37,  51],
       17 => [:free_space, :comment,  "# C {2,3} comment\n",  66,  84],
       29 => [:free_space, :comment,  "# D|E comment\n",     100, 114]
+
+    # single line / no trailing newline (c.f. issue #66)
+    include_examples 'scan', /a # b/x,
+      0 => [:literal,    :literal,    'a',   0,  1],
+      1 => [:free_space, :whitespace, ' ',   1,  2],
+      2 => [:free_space, :comment,    "# b", 2,  5]
+
+    # without spaces (c.f. issue #66)
+    include_examples 'scan', /a#b/x,
+      0 => [:literal,    :literal,  'a',   0,  1],
+      1 => [:free_space, :comment,  "#b", 1,  3]
   end
 
   describe('scan free space inlined') do
@@ -130,4 +141,25 @@ RSpec.describe('FreeSpace scanning') do
       26 => [:literal,    :literal,        'i j',   35, 38],
       27 => [:group,      :close,          ')',     38, 39]
   end
+
+  describe('scanning `#` in regular (non-x mode)') do # c.f. issue 70
+    include_examples 'scan', /a#bcd/,
+      0 => [:literal, :literal, 'a#bcd', 0, 5]
+    include_examples 'scan', /a # bcd/,
+      0 => [:literal, :literal, 'a # bcd', 0, 7]
+
+    include_examples 'scan', /a#\d/,
+      0 => [:literal, :literal, 'a#', 0, 2],
+      1 => [:type,    :digit,   '\d', 2, 4]
+    include_examples 'scan', /a # \d/,
+      0 => [:literal, :literal, 'a # ', 0, 4],
+      1 => [:type,    :digit,   '\d',   4, 6]
+
+    include_examples 'scan', /a#()/,
+      0 => [:literal, :literal, 'a#', 0, 2],
+      1 => [:group,   :capture, '(',  2, 3]
+    include_examples 'scan', /a # ()/,
+      0 => [:literal, :literal, 'a # ', 0, 4],
+      1 => [:group,   :capture, '(',    4, 5]
+  end
 end

data/spec/scanner/literals_spec.rb

@@ -2,48 +2,38 @@ require 'spec_helper'
 
 RSpec.describe('UTF8 scanning') do
   # ascii, single byte characters
-  include_examples 'scan', 'a', 0              => [:literal,     :literal,       'a',        0, 1]
+  include_examples 'scan', 'a',
+    0 => [:literal,     :literal,       'a',        0, 1]
 
-  include_examples 'scan', 'ab+', 0            => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab+', 1            => [:quantifier,  :one_or_more,   '+',        2, 3]
+  include_examples 'scan', 'ab+',
+    0 => [:literal,     :literal,       'ab',       0, 2],
+    1 => [:quantifier,  :one_or_more,   '+',        2, 3]
 
-  # 2 byte wide characters, Arabic
-  include_examples 'scan', 'aاbبcت', 0         => [:literal,     :literal,       'aاbبcت',   0, 9]
-
-  include_examples 'scan', 'aاbبت?', 0         => [:literal,     :literal,       'aاbبت',    0, 8]
-  include_examples 'scan', 'aاbبت?', 1         => [:quantifier,  :zero_or_one,   '?',        8, 9]
-
-  include_examples 'scan', 'aا?bبcت+', 0       => [:literal,     :literal,       'aا',       0, 3]
-  include_examples 'scan', 'aا?bبcت+', 1       => [:quantifier,  :zero_or_one,   '?',        3, 4]
-  include_examples 'scan', 'aا?bبcت+', 2       => [:literal,     :literal,       'bبcت',     4, 10]
-  include_examples 'scan', 'aا?bبcت+', 3       => [:quantifier,  :one_or_more,   '+',        10, 11]
-
-  include_examples 'scan', 'a(اbب+)cت?', 0     => [:literal,     :literal,       'a',        0, 1]
-  include_examples 'scan', 'a(اbب+)cت?', 1     => [:group,       :capture,       '(',        1, 2]
-  include_examples 'scan', 'a(اbب+)cت?', 2     => [:literal,     :literal,       'اbب',      2, 7]
-  include_examples 'scan', 'a(اbب+)cت?', 3     => [:quantifier,  :one_or_more,   '+',        7, 8]
-  include_examples 'scan', 'a(اbب+)cت?', 4     => [:group,       :close,         ')',        8, 9]
-  include_examples 'scan', 'a(اbب+)cت?', 5     => [:literal,     :literal,       'cت',       9, 12]
-  include_examples 'scan', 'a(اbب+)cت?', 6     => [:quantifier,  :zero_or_one,   '?',        12, 13]
+  # 2 byte wide characters
+  include_examples 'scan', 'äöü',
+    0 => [:literal,     :literal,        'äöü',     0, 3]
 
   # 3 byte wide characters, Japanese
-  include_examples 'scan', 'ab?れます+cd', 0    => [:literal,     :literal,       'ab',       0, 2]
-  include_examples 'scan', 'ab?れます+cd', 1    => [:quantifier,  :zero_or_one,   '?',        2, 3]
-  include_examples 'scan', 'ab?れます+cd', 2    => [:literal,     :literal,       'れます',    3, 12]
-  include_examples 'scan', 'ab?れます+cd', 3    => [:quantifier,  :one_or_more,   '+',        12, 13]
-  include_examples 'scan', 'ab?れます+cd', 4    => [:literal,     :literal,       'cd',       13, 15]
+  include_examples 'scan', 'ab?れます+cd',
+    0 => [:literal,     :literal,       'ab',       0, 2],
+    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
+    2 => [:literal,     :literal,       'れます',    3, 6],
+    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
+    4 => [:literal,     :literal,       'cd',       7, 9]
 
   # 4 byte wide characters, Osmanya
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 0      => [:literal,     :literal,       '𐒀𐒁',       0, 8]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 1      => [:quantifier,  :zero_or_one,   '?',        8, 9]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 2      => [:literal,     :literal,       '𐒂ab',      9, 15]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 3      => [:quantifier,  :one_or_more,   '+',        15, 16]
-  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 4      => [:literal,     :literal,       '𐒃',        16, 20]
-
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 0      => [:literal,     :literal,       'mu𝄞',       0, 6]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 1      => [:quantifier,  :zero_or_one,   '?',        6, 7]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 2      => [:literal,     :literal,       'si',       7, 9]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 3      => [:quantifier,  :zero_or_more,  '*',        9, 10]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 4      => [:literal,     :literal,       '𝄫c',       10, 15]
-  include_examples 'scan', 'mu𝄞?si*𝄫c+', 5      => [:quantifier,  :one_or_more,   '+',        15, 16]
+  include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃',
+    0 => [:literal,     :literal,       '𐒀𐒁',       0, 2],
+    1 => [:quantifier,  :zero_or_one,   '?',        2, 3],
+    2 => [:literal,     :literal,       '𐒂ab',      3, 6],
+    3 => [:quantifier,  :one_or_more,   '+',        6, 7],
+    4 => [:literal,     :literal,       '𐒃',        7, 8]
+
+  include_examples 'scan', 'mu𝄞?si*𝄫c+',
+    0 => [:literal,     :literal,       'mu𝄞',       0, 3],
+    1 => [:quantifier,  :zero_or_one,   '?',        3, 4],
+    2 => [:literal,     :literal,       'si',       4, 6],
+    3 => [:quantifier,  :zero_or_more,  '*',        6, 7],
+    4 => [:literal,     :literal,       '𝄫c',       7, 9],
+    5 => [:quantifier,  :one_or_more,   '+',        9, 10]
 end