regexp_parser 2.2.0 → 2.2.1

data/spec/expression/options_spec.rb

@@ -1,128 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Expression#options') do
-  it 'returns a hash of options/flags that affect the expression' do
-    exp = RP.parse(/a/ix)[0]
-    expect(exp).to be_a Literal
-    expect(exp.options).to eq(i: true, x: true)
-  end
-
-  it 'includes options that are locally enabled via special groups' do
-    exp = RP.parse(/(?x)(?m:a)/i)[1][0]
-    expect(exp).to be_a Literal
-    expect(exp.options).to eq(i: true, m: true, x: true)
-  end
-
-  it 'excludes locally disabled options' do
-    exp = RP.parse(/(?x)(?-im:a)/i)[1][0]
-    expect(exp).to be_a Literal
-    expect(exp.options).to eq(x: true)
-  end
-
-  it 'gives correct precedence to negative options' do
-    # Negative options have precedence. E.g. /(?i-i)a/ is case-sensitive.
-    regexp = /(?i-i:a)/
-    expect(regexp).to match 'a'
-    expect(regexp).not_to match 'A'
-
-    exp = RP.parse(regexp)[0][0]
-    expect(exp).to be_a Literal
-    expect(exp.options).to eq({})
-  end
-
-  it 'correctly handles multiple negative option parts' do
-    regexp = /(?--m--mx--) . /mx
-    expect(regexp).to match ' . '
-    expect(regexp).not_to match '.'
-    expect(regexp).not_to match "\n"
-
-    exp = RP.parse(regexp)[2]
-    expect(exp.options).to eq({})
-  end
-
-  it 'gives correct precedence when encountering multiple encoding flags' do
-    # Any encoding flag overrides all previous encoding flags. If there are
-    # multiple encoding flags in an options string, the last one wins.
-    # E.g. /(?dau)\w/ matches UTF8 chars but /(?dua)\w/ only ASCII chars.
-    regexp1 = /(?dau)\w/
-    regexp2 = /(?dua)\w/
-    expect(regexp1).to match 'ü'
-    expect(regexp2).not_to match 'ü'
-
-    exp1 = RP.parse(regexp1)[1]
-    exp2 = RP.parse(regexp2)[1]
-    expect(exp1.options).to eq(u: true)
-    expect(exp2.options).to eq(a: true)
-  end
-
-  it 'is accessible via shortcuts' do
-    exp = Root.build
-
-    expect { exp.options[:i] = true }
-      .to  change { exp.i? }.from(false).to(true)
-      .and change { exp.ignore_case? }.from(false).to(true)
-      .and change { exp.case_insensitive? }.from(false).to(true)
-
-    expect { exp.options[:m] = true }
-      .to  change { exp.m? }.from(false).to(true)
-      .and change { exp.multiline? }.from(false).to(true)
-
-    expect { exp.options[:x] = true }
-      .to  change { exp.x? }.from(false).to(true)
-      .and change { exp.extended? }.from(false).to(true)
-      .and change { exp.free_spacing? }.from(false).to(true)
-
-    expect { exp.options[:a] = true }
-      .to  change { exp.a? }.from(false).to(true)
-      .and change { exp.ascii_classes? }.from(false).to(true)
-
-    expect { exp.options[:d] = true }
-      .to  change { exp.d? }.from(false).to(true)
-      .and change { exp.default_classes? }.from(false).to(true)
-
-    expect { exp.options[:u] = true }
-      .to  change { exp.u? }.from(false).to(true)
-      .and change { exp.unicode_classes? }.from(false).to(true)
-  end
-
-  RSpec.shared_examples '#options' do |regexp, path, klass|
-    it "works for expression class #{klass}" do
-      exp = RP.parse(/#{regexp.source}/i).dig(*path)
-      expect(exp).to be_a(klass)
-      expect(exp).to be_i
-      expect(exp).not_to be_x
-    end
-  end
-
-  include_examples '#options', //, [], Root
-  include_examples '#options', /a/, [0], Literal
-  include_examples '#options', /\A/, [0], Anchor::Base
-  include_examples '#options', /\d/, [0], CharacterType::Base
-  include_examples '#options', /\n/, [0], EscapeSequence::Base
-  include_examples '#options', /\K/, [0], Keep::Mark
-  include_examples '#options', /./, [0], CharacterType::Any
-  include_examples '#options', /(a)/, [0], Group::Base
-  include_examples '#options', /(a)/, [0, 0], Literal
-  include_examples '#options', /(?=a)/, [0], Assertion::Base
-  include_examples '#options', /(?=a)/, [0, 0], Literal
-  include_examples '#options', /(a|b)/, [0], Group::Base
-  include_examples '#options', /(a|b)/, [0, 0], Alternation
-  include_examples '#options', /(a|b)/, [0, 0, 0], Alternative
-  include_examples '#options', /(a|b)/, [0, 0, 0, 0], Literal
-  include_examples '#options', /(a)\1/, [1], Backreference::Base
-  include_examples '#options', /(a)\k<1>/, [1], Backreference::Number
-  include_examples '#options', /(a)\g<1>/, [1], Backreference::NumberCall
-  include_examples '#options', /[a]/, [0], CharacterSet
-  include_examples '#options', /[a]/, [0, 0], Literal
-  include_examples '#options', /[a-z]/, [0, 0], CharacterSet::Range
-  include_examples '#options', /[a-z]/, [0, 0, 0], Literal
-  include_examples '#options', /[a&&z]/, [0, 0], CharacterSet::Intersection
-  include_examples '#options', /[a&&z]/, [0, 0, 0], CharacterSet::IntersectedSequence
-  include_examples '#options', /[a&&z]/, [0, 0, 0, 0], Literal
-  include_examples '#options', /[[:ascii:]]/, [0, 0], PosixClass
-  include_examples '#options', /\p{word}/, [0], UnicodeProperty::Base
-  include_examples '#options', /(a)(?(1)b|c)/, [1], Conditional::Expression
-  include_examples '#options', /(a)(?(1)b|c)/, [1, 0], Conditional::Condition
-  include_examples '#options', /(a)(?(1)b|c)/, [1, 1], Conditional::Branch
-  include_examples '#options', /(a)(?(1)b|c)/, [1, 1, 0], Literal
-end

data/spec/expression/subexpression_spec.rb

@@ -1,50 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe(Regexp::Expression::Subexpression) do
-  specify('#ts, #te') do
-    regx = /abcd|ghij|klmn|pqur/
-    root = RP.parse(regx)
-
-    alt = root.first
-
-    { 0 => [0, 4], 1 => [5, 9], 2 => [10, 14], 3 => [15, 19] }.each do |index, span|
-      sequence = alt[index]
-
-      expect(sequence.ts).to eq span[0]
-      expect(sequence.te).to eq span[1]
-    end
-  end
-
-  specify('#nesting_level') do
-    root = RP.parse(/a(b(\d|[ef-g[h]]))/)
-
-    tests = {
-      'a'            => 1,
-      'b'            => 2,
-      '\d|[ef-g[h]]' => 3, # alternation
-      '\d'           => 4, # first alternative
-      '[ef-g[h]]'    => 4, # second alternative
-      'e'            => 5,
-      'f-g'          => 5,
-      'f'            => 6,
-      'g'            => 6,
-      'h'            => 6,
-    }
-
-    root.each_expression do |exp|
-      next unless (expected_nesting_level = tests.delete(exp.to_s))
-      expect(expected_nesting_level).to eq exp.nesting_level
-    end
-
-    expect(tests).to be_empty
-  end
-
-  specify('#dig') do
-    root = RP.parse(/(((a)))/)
-
-    expect(root.dig(0).to_s).to eq '(((a)))'
-    expect(root.dig(0, 0, 0, 0).to_s).to eq 'a'
-    expect(root.dig(0, 0, 0, 0, 0)).to be_nil
-    expect(root.dig(3, 7)).to be_nil
-  end
-end

data/spec/expression/to_h_spec.rb

@@ -1,26 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Expression#to_h') do
-  specify('Root#to_h') do
-    root = RP.parse('abc')
-
-    hash = root.to_h
-
-    expect(token: :root, type: :expression, text: 'abc', starts_at: 0, length: 3, quantifier: nil, options: {}, level: nil, set_level: nil, conditional_level: nil, expressions: [{ token: :literal, type: :literal, text: 'abc', starts_at: 0, length: 3, quantifier: nil, options: {}, level: 0, set_level: 0, conditional_level: 0 }]).to eq hash
-  end
-
-  specify('Quantifier#to_h') do
-    root = RP.parse('a{2,4}')
-    exp = root.expressions.at(0)
-
-    hash = exp.quantifier.to_h
-
-    expect(max: 4, min: 2, mode: :greedy, text: '{2,4}', token: :interval).to eq hash
-  end
-
-  specify('Conditional#to_h') do
-    root = RP.parse('(?<A>a)(?(<A>)b|c)', 'ruby/2.0')
-
-    expect { root.to_h }.not_to(raise_error)
-  end
-end

data/spec/expression/to_s_spec.rb

@@ -1,108 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Expression#to_s') do
-  def parse_frozen(pattern, ruby_version = nil)
-    IceNine.deep_freeze(RP.parse(pattern, *ruby_version))
-  end
-
-  def expect_round_trip(pattern, ruby_version = nil)
-    parsed = parse_frozen(pattern, ruby_version)
-
-    expect(parsed.to_s).to eql(pattern)
-  end
-
-  specify('literal alternation') do
-    expect_round_trip('abcd|ghij|klmn|pqur')
-  end
-
-  specify('quantified alternations') do
-    expect_round_trip('(?:a?[b]+(c){2}|d+[e]*(f)?)|(?:g+[h]?(i){2,3}|j*[k]{3,5}(l)?)')
-  end
-
-  specify('quantified sets') do
-    expect_round_trip('[abc]+|[^def]{3,6}')
-  end
-
-  specify('property sets') do
-    expect_round_trip('[\\a\\b\\p{Lu}\\P{Z}\\c\\d]+', 'ruby/1.9')
-  end
-
-  specify('groups') do
-    expect_round_trip("(a(?>b(?:c(?<n>d(?'N'e)??f)+g)*+h)*i)++", 'ruby/1.9')
-  end
-
-  specify('assertions') do
-    expect_round_trip('(a+(?=b+(?!c+(?<=d+(?<!e+)?f+)?g+)?h+)?i+)?', 'ruby/1.9')
-  end
-
-  specify('comments') do
-    expect_round_trip('(?#start)a(?#middle)b(?#end)')
-  end
-
-  specify('options') do
-    expect_round_trip('(?mix:start)a(?-mix:middle)b(?i-mx:end)')
-  end
-
-  specify('url') do
-    expect_round_trip('(^$)|(^(http|https):\\/\\/[a-z0-9]+([\\-\\.]{1}[a-z0-9]+)*' + '\\.[a-z]{2,5}(([0-9]{1,5})?\\/.*)?$)')
-  end
-
-  specify('multiline source') do
-    multiline = /
-          \A
-          a?      # One letter
-          b{2,5}  # Another one
-          [c-g]+  # A set
-          \z
-        /x
-
-    expect(parse_frozen(multiline).to_s).to eql(multiline.source)
-  end
-
-  specify('multiline #to_s') do
-    multiline = /
-          \A
-          a?      # One letter
-          b{2,5}  # Another one
-          [c-g]+  # A set
-          \z
-        /x
-
-    expect_round_trip(multiline.to_s)
-  end
-
-  # Free spacing expressions that use spaces between quantifiers and their
-  # targets do not produce identical results due to the way quantifiers are
-  # applied to expressions (members, not nodes) and the merging of consecutive
-  # space nodes. This tests that they produce equivalent results.
-  specify('multiline equivalence') do
-    multiline = /
-          \A
-          a   ?             # One letter
-          b {2,5}           # Another one
-          [c-g]  +          # A set
-          \z
-        /x
-
-    str = 'bbbcged'
-    root = parse_frozen(multiline)
-
-    expect(Regexp.new(root.to_s, Regexp::EXTENDED).match(str)[0]).to eql(multiline.match(str)[0])
-  end
-
-  # special case: implicit groups used for chained quantifiers produce no parens
-  specify 'chained quantifiers #to_s' do
-    pattern = /a+{1}{2}/
-    root = parse_frozen(pattern)
-    expect(root.to_s).to eql('a+{1}{2}')
-  end
-
-  # regression test for https://github.com/ammar/regexp_parser/issues/74
-  specify('non-ascii comment') do
-    pattern = '(?x) 😋 # 😋'
-    root = RP.parse(pattern)
-    expect(root.last).to be_a(Regexp::Expression::Comment)
-    expect(root.last.to_s).to eql('# 😋')
-    expect(root.to_s).to eql(pattern)
-  end
-end

data/spec/lexer/all_spec.rb

@@ -1,22 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe(Regexp::Lexer) do
-  specify('lexer returns an array') do
-    expect(RL.lex('abc')).to be_instance_of(Array)
-  end
-
-  specify('lexer returns tokens') do
-    tokens = RL.lex('^abc+[^one]{2,3}\\b\\d\\\\C-C$')
-    expect(tokens).to all(be_a Regexp::Token)
-    expect(tokens.map { |token| token.to_a.length }).to all(eq 8)
-  end
-
-  specify('lexer token count') do
-    tokens = RL.lex(/^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i)
-    expect(tokens.length).to eq 28
-  end
-
-  specify('lexer scan alias') do
-    expect(RL.scan(/a|b|c/)).to eq RL.lex(/a|b|c/)
-  end
-end

data/spec/lexer/conditionals_spec.rb

@@ -1,53 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Conditional lexing') do
-  include_examples 'lex', /(?<A>a)(?(<A>)b|c)/,
-    3 => [:conditional, :open,       '(?',     7,  9, 0, 0, 0],
-    4 => [:conditional, :condition,  '(<A>)',  9, 14, 0, 0, 1],
-    6 => [:conditional, :separator,  '|',     15, 16, 0, 0, 1],
-    8 => [:conditional, :close,      ')',     17, 18, 0, 0, 0]
-
-  include_examples 'lex', /((?<A>a)(?<B>(?(<A>)b|((?(<B>)[e-g]|[h-j])))))/,
-     0 => [:group,       :capture,     '(',       0,  1, 0, 0, 0],
-     1 => [:group,       :named,       '(?<A>',   1,  6, 1, 0, 0],
-     5 => [:conditional, :open,        '(?',     13, 15, 2, 0, 0],
-     6 => [:conditional, :condition,   '(<A>)',  15, 20, 2, 0, 1],
-     8 => [:conditional, :separator,   '|',      21, 22, 2, 0, 1],
-    10 => [:conditional, :open,        '(?',     23, 25, 3, 0, 1],
-    11 => [:conditional, :condition,   '(<B>)',  25, 30, 3, 0, 2],
-    12 => [:set,         :open,        '[',      30, 31, 3, 0, 2],
-    13 => [:literal,     :literal,     'e',      31, 32, 3, 1, 2],
-    14 => [:set,         :range,       '-',      32, 33, 3, 1, 2],
-    15 => [:literal,     :literal,     'g',      33, 34, 3, 1, 2],
-    16 => [:set,         :close,       ']',      34, 35, 3, 0, 2],
-    17 => [:conditional, :separator,   '|',      35, 36, 3, 0, 2],
-    23 => [:conditional, :close,       ')',      41, 42, 3, 0, 1],
-    25 => [:conditional, :close,       ')',      43, 44, 2, 0, 0],
-    26 => [:group,       :close,       ')',      44, 45, 1, 0, 0],
-    27 => [:group,       :close,       ')',      45, 46, 0, 0, 0]
-
-  include_examples 'lex', /(a(b(c)))(?(1)(?(2)(?(3)d|e))|(?(3)(?(2)f|g)|(?(1)f|g)))/,
-     9 => [:conditional, :open,       '(?',    9, 11, 0, 0, 0],
-    10 => [:conditional, :condition,  '(1)',  11, 14, 0, 0, 1],
-    11 => [:conditional, :open,       '(?',   14, 16, 0, 0, 1],
-    12 => [:conditional, :condition,  '(2)',  16, 19, 0, 0, 2],
-    13 => [:conditional, :open,       '(?',   19, 21, 0, 0, 2],
-    14 => [:conditional, :condition,  '(3)',  21, 24, 0, 0, 3],
-    16 => [:conditional, :separator,  '|',    25, 26, 0, 0, 3],
-    18 => [:conditional, :close,      ')',    27, 28, 0, 0, 2],
-    19 => [:conditional, :close,      ')',    28, 29, 0, 0, 1],
-    20 => [:conditional, :separator,  '|',    29, 30, 0, 0, 1],
-    21 => [:conditional, :open,       '(?',   30, 32, 0, 0, 1],
-    22 => [:conditional, :condition,  '(3)',  32, 35, 0, 0, 2],
-    23 => [:conditional, :open,       '(?',   35, 37, 0, 0, 2],
-    24 => [:conditional, :condition,  '(2)',  37, 40, 0, 0, 3],
-    26 => [:conditional, :separator,  '|',    41, 42, 0, 0, 3],
-    28 => [:conditional, :close,      ')',    43, 44, 0, 0, 2],
-    29 => [:conditional, :separator,  '|',    44, 45, 0, 0, 2],
-    30 => [:conditional, :open,       '(?',   45, 47, 0, 0, 2],
-    31 => [:conditional, :condition,  '(1)',  47, 50, 0, 0, 3],
-    33 => [:conditional, :separator,  '|',    51, 52, 0, 0, 3],
-    35 => [:conditional, :close,      ')',    53, 54, 0, 0, 2],
-    36 => [:conditional, :close,      ')',    54, 55, 0, 0, 1],
-    37 => [:conditional, :close,      ')',    55, 56, 0, 0, 0]
-end

data/spec/lexer/delimiters_spec.rb

@@ -1,68 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Literal delimiter lexing') do
-  include_examples 'lex', '}',
-    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0]
-
-  include_examples 'lex', '}}',
-    0 => [:literal,     :literal,       '}}',      0,  2,  0, 0, 0]
-
-  include_examples 'lex', '{',
-    0 => [:literal,     :literal,       '{',       0,  1,  0, 0, 0]
-
-  include_examples 'lex', '{{',
-    0 => [:literal,     :literal,       '{{',      0,  2,  0, 0, 0]
-
-  include_examples 'lex', '{}',
-    0 => [:literal,     :literal,       '{}',      0,  2,  0, 0, 0]
-
-  include_examples 'lex', '}{',
-    0 => [:literal,     :literal,       '}{',      0,  2,  0, 0, 0]
-
-  include_examples 'lex', '}{+',
-    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0],
-    1 => [:literal,     :literal,       '{',       1,  2,  0, 0, 0],
-    2 => [:quantifier,  :one_or_more,   '+',       2,  3,  0, 0, 0]
-
-  include_examples 'lex', '{{var}}',
-    0 => [:literal,     :literal,       '{{var}}',  0,  7,  0, 0, 0]
-
-  include_examples 'lex', 'a{b}c',
-    0 => [:literal,     :literal,       'a{b}c',    0,  5,  0, 0, 0]
-
-  include_examples 'lex', 'a{1,2',
-    0 => [:literal,     :literal,       'a{1,2',    0,  5,  0, 0, 0]
-
-  include_examples 'lex', '({.+})',
-    0 => [:group,       :capture,       '(',    0,  1,  0, 0, 0],
-    1 => [:literal,     :literal,       '{',    1,  2,  1, 0, 0],
-    2 => [:meta,        :dot,           '.',    2,  3,  1, 0, 0],
-    3 => [:quantifier,  :one_or_more,   '+',    3,  4,  1, 0, 0],
-    4 => [:literal,     :literal,       '}',    4,  5,  1, 0, 0],
-    5 => [:group,       :close,         ')',    5,  6,  0, 0, 0]
-
-  include_examples 'lex', ']',
-    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0]
-
-  include_examples 'lex', ']]',
-    0 => [:literal,     :literal,       ']]',       0,  2,  0, 0, 0]
-
-  include_examples 'lex', ']\[',
-    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0],
-    1 => [:escape,      :set_open,      '\[',       1,  3,  0, 0, 0]
-
-  include_examples 'lex', '()',
-    0 => [:group,       :capture,       '(',        0,  1,  0, 0, 0],
-    1 => [:group,       :close,         ')',        1,  2,  0, 0, 0]
-
-  include_examples 'lex', '{abc:.+}}}[^}]]}',
-    0 => [:literal,     :literal,       '{abc:',    0,  5,  0, 0, 0],
-    1 => [:meta,        :dot,           '.',        5,  6,  0, 0, 0],
-    2 => [:quantifier,  :one_or_more,   '+',        6,  7,  0, 0, 0],
-    3 => [:literal,     :literal,       '}}}',      7,  10, 0, 0, 0],
-    4 => [:set,         :open,          '[',        10, 11, 0, 0, 0],
-    5 => [:set,         :negate,        '^',        11, 12, 0, 1, 0],
-    6 => [:literal,     :literal,       '}',        12, 13, 0, 1, 0],
-    7 => [:set,         :close,         ']',        13, 14, 0, 0, 0],
-    8 => [:literal,     :literal,       ']}',       14, 16, 0, 0, 0]
-end

data/spec/lexer/escapes_spec.rb

@@ -1,14 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Escape lexing') do
-  include_examples 'lex', '\u{62}',
-    0 => [:escape,  :codepoint_list, '\u{62}',       0, 6,  0, 0, 0]
-
-  include_examples 'lex', '\u{62 63 64}',
-    0 => [:escape,  :codepoint_list, '\u{62 63 64}', 0, 12, 0, 0, 0]
-
-  include_examples 'lex', '\u{62 63 64}+',
-    0 => [:escape,     :codepoint_list, '\u{62 63}',  0,  9,  0, 0, 0],
-    1 => [:escape,     :codepoint_list, '\u{64}',     9,  15, 0, 0, 0],
-    2 => [:quantifier, :one_or_more,    '+',          15, 16, 0, 0, 0]
-end

data/spec/lexer/keep_spec.rb

@@ -1,10 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Keep lexing') do
-  include_examples 'lex', /ab\Kcd/,
-    1 => [:keep, :mark, '\K', 2,  4,  0, 0, 0]
-
-  include_examples 'lex', /(a\Kb)|(c\\\Kd)ef/,
-    2 => [:keep, :mark, '\K', 2,  4,  1, 0, 0],
-    9 => [:keep, :mark, '\K', 11, 13, 1, 0, 0]
-end

data/spec/lexer/literals_spec.rb

@@ -1,64 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Literal lexing') do
-  # ascii, single byte characters
-  include_examples 'lex', 'a',
-    0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0]
-
-  include_examples 'lex', 'ab+',
-    0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-    1 => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
-    2 => [:quantifier,  :one_or_more,   '+',        2, 3, 0, 0, 0]
-
-  # 2 byte wide characters
-  include_examples 'lex', 'äöü+',
-    0 => [:literal,     :literal,       'äö',       0, 2, 0, 0, 0],
-    1 => [:literal,     :literal,       'ü',        2, 3, 0, 0, 0],
-    2 => [:quantifier,  :one_or_more,   '+',        3, 4, 0, 0, 0]
-
-  # 3 byte wide characters, Japanese
-  include_examples 'lex', 'ab?れます+cd',
-    0 => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-    1 => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0, 0],
-    3 => [:literal,     :literal,       'れま',     3, 5, 0, 0, 0],
-    4 => [:literal,     :literal,       'す',       5, 6, 0, 0, 0],
-    5 => [:quantifier,  :one_or_more,   '+',        6, 7, 0, 0, 0],
-    6 => [:literal,     :literal,       'cd',       7, 9, 0, 0, 0]
-
-  # 4 byte wide characters, Osmanya
-  include_examples 'lex', '𐒀𐒁?𐒂ab+𐒃',
-    0 => [:literal,     :literal,       '𐒀',        0, 1, 0, 0, 0],
-    1 => [:literal,     :literal,       '𐒁',        1, 2, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0, 0],
-    3 => [:literal,     :literal,       '𐒂a',       3, 5, 0, 0, 0],
-    4 => [:literal,     :literal,       'b',        5, 6, 0, 0, 0],
-    5 => [:quantifier,  :one_or_more,   '+',        6, 7, 0, 0, 0],
-    6 => [:literal,     :literal,       '𐒃',        7, 8, 0, 0, 0]
-
-  include_examples 'lex', 'mu𝄞?si*𝄫c+',
-    0 => [:literal,     :literal,       'mu',       0, 2, 0, 0, 0],
-    1 => [:literal,     :literal,       '𝄞',        2, 3, 0, 0, 0],
-    2 => [:quantifier,  :zero_or_one,   '?',        3, 4, 0, 0, 0],
-    3 => [:literal,     :literal,       's',        4, 5, 0, 0, 0],
-    4 => [:literal,     :literal,       'i',        5, 6, 0, 0, 0],
-    5 => [:quantifier,  :zero_or_more,  '*',        6, 7, 0, 0, 0],
-    6 => [:literal,     :literal,       '𝄫',        7, 8, 0, 0, 0],
-    7 => [:literal,     :literal,       'c',        8, 9, 0, 0, 0],
-    8 => [:quantifier,  :one_or_more,   '+',        9, 10, 0, 0, 0]
-
-  specify('lex single 2 byte char') do
-    tokens = RL.lex("\u0627+")
-    expect(tokens.count).to eq 2
-  end
-
-  specify('lex single 3 byte char') do
-    tokens = RL.lex("\u308C+")
-    expect(tokens.count).to eq 2
-  end
-
-  specify('lex single 4 byte char') do
-    tokens = RL.lex("\u{1D11E}+")
-    expect(tokens.count).to eq 2
-  end
-end

data/spec/lexer/nesting_spec.rb

@@ -1,99 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe('Nesting lexing') do
-  include_examples 'lex', /(((b)))/,
-    0 => [:group,    :capture,   '(',  0,  1, 0, 0, 0],
-    1 => [:group,    :capture,   '(',  1,  2, 1, 0, 0],
-    2 => [:group,    :capture,   '(',  2,  3, 2, 0, 0],
-    3 => [:literal,  :literal,   'b',  3,  4, 3, 0, 0],
-    4 => [:group,    :close,     ')',  4,  5, 2, 0, 0],
-    5 => [:group,    :close,     ')',  5,  6, 1, 0, 0],
-    6 => [:group,    :close,     ')',  6,  7, 0, 0, 0]
-
-  include_examples 'lex', /(\((b)\))/,
-    0 => [:group,    :capture,      '(',   0,  1, 0, 0, 0],
-    1 => [:escape,   :group_open,   '\(',  1,  3, 1, 0, 0],
-    2 => [:group,    :capture,      '(',   3,  4, 1, 0, 0],
-    3 => [:literal,  :literal,      'b',   4,  5, 2, 0, 0],
-    4 => [:group,    :close,        ')',   5,  6, 1, 0, 0],
-    5 => [:escape,   :group_close,  '\)',  6,  8, 1, 0, 0],
-    6 => [:group,    :close,        ')',   8,  9, 0, 0, 0]
-
-  include_examples 'lex', /(?>a(?>b(?>c)))/,
-    0 => [:group,    :atomic,       '(?>',  0,  3, 0, 0, 0],
-    2 => [:group,    :atomic,       '(?>',  4,  7, 1, 0, 0],
-    4 => [:group,    :atomic,       '(?>',  8, 11, 2, 0, 0],
-    6 => [:group,    :close,        ')',   12, 13, 2, 0, 0],
-    7 => [:group,    :close,        ')',   13, 14, 1, 0, 0],
-    8 => [:group,    :close,        ')',   14, 15, 0, 0, 0]
-
-  include_examples 'lex', /(?:a(?:b(?:c)))/,
-    0 => [:group,       :passive,       '(?:',    0,  3, 0, 0, 0],
-    2 => [:group,       :passive,       '(?:',    4,  7, 1, 0, 0],
-    4 => [:group,       :passive,       '(?:',    8, 11, 2, 0, 0],
-    6 => [:group,       :close,         ')',     12, 13, 2, 0, 0],
-    7 => [:group,       :close,         ')',     13, 14, 1, 0, 0],
-    8 => [:group,       :close,         ')',     14, 15, 0, 0, 0]
-
-  include_examples 'lex', /(?=a(?!b(?<=c(?<!d))))/,
-    0  => [:assertion,   :lookahead,     '(?=',    0,  3, 0, 0, 0],
-    2  => [:assertion,   :nlookahead,    '(?!',    4,  7, 1, 0, 0],
-    4  => [:assertion,   :lookbehind,    '(?<=',   8, 12, 2, 0, 0],
-    6  => [:assertion,   :nlookbehind,   '(?<!',  13, 17, 3, 0, 0],
-    8  => [:group,       :close,         ')',     18, 19, 3, 0, 0],
-    9  => [:group,       :close,         ')',     19, 20, 2, 0, 0],
-    10 => [:group,       :close,         ')',     20, 21, 1, 0, 0],
-    11 => [:group,       :close,         ')',     21, 22, 0, 0, 0]
-
-  include_examples 'lex', /((?#a)b(?#c)d(?#e))/,
-    0 => [:group,       :capture,       '(',      0,  1, 0, 0, 0],
-    1 => [:group,       :comment,       '(?#a)',  1,  6, 1, 0, 0],
-    3 => [:group,       :comment,       '(?#c)',  7, 12, 1, 0, 0],
-    5 => [:group,       :comment,       '(?#e)', 13, 18, 1, 0, 0],
-    6 => [:group,       :close,         ')',     18, 19, 0, 0, 0]
-
-  include_examples 'lex', /a[b-e]f/,
-    1 => [:set,         :open,          '[',      1,  2, 0, 0, 0],
-    2 => [:literal,     :literal,       'b',      2,  3, 0, 1, 0],
-    3 => [:set,         :range,         '-',      3,  4, 0, 1, 0],
-    4 => [:literal,     :literal,       'e',      4,  5, 0, 1, 0],
-    5 => [:set,         :close,         ']',      5,  6, 0, 0, 0]
-
-  include_examples 'lex', '[[:word:]&&[^c]z]',
-    0 => [:set,         :open,          '[',          0,  1, 0, 0, 0],
-    1 => [:posixclass,  :word,          '[:word:]',   1,  9, 0, 1, 0],
-    2 => [:set,         :intersection,  '&&',         9, 11, 0, 1, 0],
-    3 => [:set,         :open,          '[',         11, 12, 0, 1, 0],
-    4 => [:set,         :negate,        '^',         12, 13, 0, 2, 0],
-    5 => [:literal,     :literal,       'c',         13, 14, 0, 2, 0],
-    6 => [:set,         :close,         ']',         14, 15, 0, 1, 0],
-    7 => [:literal,     :literal,       'z',         15, 16, 0, 1, 0],
-    8 => [:set,         :close,         ']',         16, 17, 0, 0, 0]
-
-  include_examples 'lex', '[\p{word}&&[^c]z]',
-    0 => [:set,         :open,          '[',          0,  1, 0, 0, 0],
-    1 => [:property,    :word,          '\p{word}',   1,  9, 0, 1, 0],
-    2 => [:set,         :intersection,  '&&',         9, 11, 0, 1, 0],
-    3 => [:set,         :open,          '[',         11, 12, 0, 1, 0],
-    4 => [:set,         :negate,        '^',         12, 13, 0, 2, 0],
-    5 => [:literal,     :literal,       'c',         13, 14, 0, 2, 0],
-    6 => [:set,         :close,         ']',         14, 15, 0, 1, 0],
-    7 => [:literal,     :literal,       'z',         15, 16, 0, 1, 0],
-    8 => [:set,         :close,         ']',         16, 17, 0, 0, 0]
-
-  include_examples 'lex', /[a[b[c[d-g]]]]/,
-    0  => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-    1  => [:literal,     :literal,       'a',      1,  2, 0, 1, 0],
-    2  => [:set,         :open,          '[',      2,  3, 0, 1, 0],
-    3  => [:literal,     :literal,       'b',      3,  4, 0, 2, 0],
-    4  => [:set,         :open,          '[',      4,  5, 0, 2, 0],
-    5  => [:literal,     :literal,       'c',      5,  6, 0, 3, 0],
-    6  => [:set,         :open,          '[',      6,  7, 0, 3, 0],
-    7  => [:literal,     :literal,       'd',      7,  8, 0, 4, 0],
-    8  => [:set,         :range,         '-',      8,  9, 0, 4, 0],
-    9  => [:literal,     :literal,       'g',      9, 10, 0, 4, 0],
-    10 => [:set,         :close,         ']',     10, 11, 0, 3, 0],
-    11 => [:set,         :close,         ']',     11, 12, 0, 2, 0],
-    12 => [:set,         :close,         ']',     12, 13, 0, 1, 0],
-    13 => [:set,         :close,         ']',     13, 14, 0, 0, 0]
-end