regexp_parser 1.3.0 → 1.6.0

data/spec/scanner/quantifiers_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+RSpec.describe('Quantifier scanning') do
+  include_examples 'scan', 'a?',     1 => [:quantifier,  :zero_or_one,             '?',     1, 2]
+  include_examples 'scan', 'a??',    1 => [:quantifier,  :zero_or_one_reluctant,   '??',    1, 3]
+  include_examples 'scan', 'a?+',    1 => [:quantifier,  :zero_or_one_possessive,  '?+',    1, 3]
+
+  include_examples 'scan', 'a*',     1 => [:quantifier,  :zero_or_more,            '*',     1, 2]
+  include_examples 'scan', 'a*?',    1 => [:quantifier,  :zero_or_more_reluctant,  '*?',    1, 3]
+  include_examples 'scan', 'a*+',    1 => [:quantifier,  :zero_or_more_possessive, '*+',    1, 3]
+
+  include_examples 'scan', 'a+',     1 => [:quantifier,  :one_or_more,             '+',     1, 2]
+  include_examples 'scan', 'a+?',    1 => [:quantifier,  :one_or_more_reluctant,   '+?',    1, 3]
+  include_examples 'scan', 'a++',    1 => [:quantifier,  :one_or_more_possessive,  '++',    1, 3]
+
+  include_examples 'scan', 'a{2}',   1 => [:quantifier,  :interval,                '{2}',   1, 4]
+  include_examples 'scan', 'a{2,}',  1 => [:quantifier,  :interval,                '{2,}',  1, 5]
+  include_examples 'scan', 'a{,2}',  1 => [:quantifier,  :interval,                '{,2}',  1, 5]
+  include_examples 'scan', 'a{2,4}', 1 => [:quantifier,  :interval,                '{2,4}', 1, 6]
+end

data/spec/scanner/refcalls_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+RSpec.describe('RefCall scanning') do
+  # Traditional numerical group back-reference
+  include_examples 'scan', '(abc)\1' ,          3 => [:backref, :number,                  '\1',         5, 7]
+
+  # Group back-references, named, numbered, and relative
+  include_examples 'scan', '(?<X>abc)\k<X>',    3 => [:backref, :name_ref_ab,             '\k<X>',      9, 14]
+  include_examples 'scan', "(?<X>abc)\\k'X'",   3 => [:backref, :name_ref_sq,             "\\k'X'",     9, 14]
+
+  include_examples 'scan', '(abc)\k<1>',        3 => [:backref, :number_ref_ab,           '\k<1>',      5, 10]
+  include_examples 'scan', "(abc)\\k'1'",       3 => [:backref, :number_ref_sq,           "\\k'1'",     5, 10]
+
+  include_examples 'scan', '(abc)\k<-1>',       3 => [:backref, :number_rel_ref_ab,       '\k<-1>',     5, 11]
+  include_examples 'scan', "(abc)\\k'-1'",      3 => [:backref, :number_rel_ref_sq,       "\\k'-1'",    5, 11]
+
+  # Sub-expression invocation, named, numbered, and relative
+  include_examples 'scan', '(?<X>abc)\g<X>',    3 => [:backref, :name_call_ab,            '\g<X>',      9, 14]
+  include_examples 'scan', "(?<X>abc)\\g'X'",   3 => [:backref, :name_call_sq,            "\\g'X'",     9, 14]
+
+  include_examples 'scan', '(abc)\g<1>',        3 => [:backref, :number_call_ab,          '\g<1>',      5, 10]
+  include_examples 'scan', "(abc)\\g'1'",       3 => [:backref, :number_call_sq,          "\\g'1'",     5, 10]
+
+  include_examples 'scan', '(abc)\g<-1>',       3 => [:backref, :number_rel_call_ab,      '\g<-1>',     5, 11]
+  include_examples 'scan', "(abc)\\g'-1'",      3 => [:backref, :number_rel_call_sq,      "\\g'-1'",    5, 11]
+
+  include_examples 'scan', '\g<+1>(abc)',       0 => [:backref, :number_rel_call_ab,      '\g<+1>',     0, 6]
+  include_examples 'scan', "\\g'+1'(abc)",      0 => [:backref, :number_rel_call_sq,      "\\g'+1'",    0, 6]
+
+  # Group back-references, with recursion level
+  include_examples 'scan', '(?<X>abc)\k<X-0>',  3 => [:backref, :name_recursion_ref_ab,   '\k<X-0>',    9, 16]
+  include_examples 'scan', "(?<X>abc)\\k'X-0'", 3 => [:backref, :name_recursion_ref_sq,   "\\k'X-0'",   9, 16]
+
+  include_examples 'scan', '(abc)\k<1-0>',      3 => [:backref, :number_recursion_ref_ab, '\k<1-0>',    5, 12]
+  include_examples 'scan', "(abc)\\k'1-0'",     3 => [:backref, :number_recursion_ref_sq, "\\k'1-0'",   5, 12]
+end

data/spec/scanner/sets_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+RSpec.describe('Set scanning') do
+  include_examples 'scan', /[a]/,                   0 => [:set,    :open,            '[',          0, 1]
+  include_examples 'scan', /[b]/,                   2 => [:set,    :close,           ']',          2, 3]
+  include_examples 'scan', /[^n]/,                  1 => [:set,    :negate,          '^',          1, 2]
+
+  include_examples 'scan', /[c]/,                   1 => [:literal, :literal,        'c',          1, 2]
+  include_examples 'scan', /[\b]/,                  1 => [:escape,  :backspace,      '\b',         1, 3]
+  include_examples 'scan', /[A\bX]/,                2 => [:escape,  :backspace,      '\b',         2, 4]
+
+  include_examples 'scan', /[.]/,                   1 => [:literal, :literal,        '.',          1, 2]
+  include_examples 'scan', /[?]/,                   1 => [:literal, :literal,        '?',          1, 2]
+  include_examples 'scan', /[*]/,                   1 => [:literal, :literal,        '*',          1, 2]
+  include_examples 'scan', /[+]/,                   1 => [:literal, :literal,        '+',          1, 2]
+  include_examples 'scan', /[{]/,                   1 => [:literal, :literal,        '{',          1, 2]
+  include_examples 'scan', /[}]/,                   1 => [:literal, :literal,        '}',          1, 2]
+  include_examples 'scan', /[<]/,                   1 => [:literal, :literal,        '<',          1, 2]
+  include_examples 'scan', /[>]/,                   1 => [:literal, :literal,        '>',          1, 2]
+
+  include_examples 'scan', /[äöü]/,                 2 => [:literal, :literal,        'ö',          3, 5]
+
+  include_examples 'scan', /[\x20]/,                1 => [:escape, :hex,             '\x20',       1, 5]
+
+  include_examples 'scan', '[\.]',                  1 => [:escape, :dot,             '\.',         1, 3]
+  include_examples 'scan', '[\!]',                  1 => [:escape, :literal,         '\!',         1, 3]
+  include_examples 'scan', '[\#]',                  1 => [:escape, :literal,         '\#',         1, 3]
+  include_examples 'scan', '[\\]]',                 1 => [:escape, :set_close,       '\]',         1, 3]
+  include_examples 'scan', '[\\\\]',                1 => [:escape, :backslash,       '\\\\',       1, 3]
+  include_examples 'scan', '[\A]',                  1 => [:escape, :literal,         '\A',         1, 3]
+  include_examples 'scan', '[\z]',                  1 => [:escape, :literal,         '\z',         1, 3]
+  include_examples 'scan', '[\g]',                  1 => [:escape, :literal,         '\g',         1, 3]
+  include_examples 'scan', '[\K]',                  1 => [:escape, :literal,         '\K',         1, 3]
+  include_examples 'scan', '[\R]',                  1 => [:escape, :literal,         '\R',         1, 3]
+  include_examples 'scan', '[\X]',                  1 => [:escape, :literal,         '\X',         1, 3]
+  include_examples 'scan', '[\c2]',                 1 => [:escape, :literal,         '\c',         1, 3]
+  include_examples 'scan', '[\B]',                  1 => [:escape, :literal,         '\B',         1, 3]
+  include_examples 'scan', '[a\-c]',                2 => [:escape, :literal,         '\-',         2, 4]
+
+  include_examples 'scan', /[\d]/,                  1 => [:type,   :digit,           '\d',         1, 3]
+  include_examples 'scan', /[\da-z]/,               1 => [:type,   :digit,           '\d',         1, 3]
+  include_examples 'scan', /[\D]/,                  1 => [:type,   :nondigit,        '\D',         1, 3]
+
+  include_examples 'scan', /[\h]/,                  1 => [:type,   :hex,             '\h',         1, 3]
+  include_examples 'scan', /[\H]/,                  1 => [:type,   :nonhex,          '\H',         1, 3]
+
+  include_examples 'scan', /[\s]/,                  1 => [:type,   :space,           '\s',         1, 3]
+  include_examples 'scan', /[\S]/,                  1 => [:type,   :nonspace,        '\S',         1, 3]
+
+  include_examples 'scan', /[\w]/,                  1 => [:type,   :word,            '\w',         1, 3]
+  include_examples 'scan', /[\W]/,                  1 => [:type,   :nonword,         '\W',         1, 3]
+
+  include_examples 'scan', /[a-b]/,                 1 => [:literal, :literal,        'a',          1, 2]
+  include_examples 'scan', /[a-c]/,                 2 => [:set,     :range,          '-',          2, 3]
+  include_examples 'scan', /[a-d]/,                 3 => [:literal, :literal,        'd',          3, 4]
+  include_examples 'scan', /[a-b-]/,                4 => [:literal, :literal,        '-',          4, 5]
+  include_examples 'scan', /[-a]/,                  1 => [:literal, :literal,        '-',          1, 2]
+  include_examples 'scan', /[a-c^]/,                4 => [:literal, :literal,        '^',          4, 5]
+  include_examples 'scan', /[a-bd-f]/,              2 => [:set,    :range,           '-',          2, 3]
+  include_examples 'scan', /[a-cd-f]/,              5 => [:set,    :range,           '-',          5, 6]
+
+  include_examples 'scan', /[a[:digit:]c]/,         2 => [:posixclass,    :digit,    '[:digit:]',  2, 11]
+  include_examples 'scan', /[[:digit:][:space:]]/,  2 => [:posixclass,    :space,    '[:space:]', 10, 19]
+  include_examples 'scan', /[[:^digit:]]/,          1 => [:nonposixclass, :digit,    '[:^digit:]', 1, 11]
+
+  include_examples 'scan', /[a[.a-b.]c]/,           2 => [:set,    :collation,       '[.a-b.]',    2,  9]
+  include_examples 'scan', /[a[=e=]c]/,             2 => [:set,    :equivalent,      '[=e=]',      2,  7]
+
+  include_examples 'scan', /[a-d&&g-h]/,            4 => [:set,    :intersection,    '&&',         4, 6]
+  include_examples 'scan', /[a&&]/,                 2 => [:set,    :intersection,    '&&',         2, 4]
+  include_examples 'scan', /[&&z]/,                 1 => [:set,    :intersection,    '&&',         1, 3]
+
+  include_examples 'scan', /[a\p{digit}c]/,         2 => [:property,    :digit,      '\p{digit}',  2, 11]
+  include_examples 'scan', /[a\P{digit}c]/,         2 => [:nonproperty, :digit,      '\P{digit}',  2, 11]
+  include_examples 'scan', /[a\p{^digit}c]/,        2 => [:nonproperty, :digit,      '\p{^digit}', 2, 12]
+  include_examples 'scan', /[a\P{^digit}c]/,        2 => [:property,    :digit,      '\P{^digit}', 2, 12]
+
+  include_examples 'scan', /[a\p{ALPHA}c]/,         2 => [:property,    :alpha,      '\p{ALPHA}',  2, 11]
+  include_examples 'scan', /[a\p{P}c]/,             2 => [:property,    :punctuation,'\p{P}',      2, 7]
+  include_examples 'scan', /[a\p{P}\P{P}c]/,        3 => [:nonproperty, :punctuation,'\P{P}',      7, 12]
+
+  include_examples 'scan', /[\x20-\x27]/,
+    1 => [:escape, :hex,             '\x20',       1, 5],
+    2 => [:set,    :range,           '-',          5, 6],
+    3 => [:escape, :hex,             '\x27',       6, 10]
+
+  include_examples 'scan', /[a-w&&[^c-g]z]/,
+    5 => [:set,    :open,            '[',          6, 7],
+    6 => [:set,    :negate,          '^',          7, 8],
+    8 => [:set,    :range,           '-',          9, 10],
+    10=> [:set,    :close,           ']',          11, 12]
+
+  specify('set literal encoding') do
+    text = RS.scan('[a]')[1][2].to_s
+    expect(text).to eq 'a'
+    expect(text.encoding.to_s).to eq 'UTF-8'
+
+    text = RS.scan("[\u{1F632}]")[1][2].to_s
+    expect(text).to eq "\u{1F632}"
+    expect(text.encoding.to_s).to eq 'UTF-8'
+  end
+end

data/spec/scanner/types_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+RSpec.describe('Type scanning') do
+  include_examples 'scan', 'a\\dc', 1 => [:type, :digit,     '\\d', 1, 3]
+  include_examples 'scan', 'a\\Dc', 1 => [:type, :nondigit,  '\\D', 1, 3]
+  include_examples 'scan', 'a\\hc', 1 => [:type, :hex,       '\\h', 1, 3]
+  include_examples 'scan', 'a\\Hc', 1 => [:type, :nonhex,    '\\H', 1, 3]
+  include_examples 'scan', 'a\\sc', 1 => [:type, :space,     '\\s', 1, 3]
+  include_examples 'scan', 'a\\Sc', 1 => [:type, :nonspace,  '\\S', 1, 3]
+  include_examples 'scan', 'a\\wc', 1 => [:type, :word,      '\\w', 1, 3]
+  include_examples 'scan', 'a\\Wc', 1 => [:type, :nonword,   '\\W', 1, 3]
+  include_examples 'scan', 'a\\Rc', 1 => [:type, :linebreak, '\\R', 1, 3]
+  include_examples 'scan', 'a\\Xc', 1 => [:type, :xgrapheme, '\\X', 1, 3]
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'regexp_parser'
+require 'regexp_property_values'
+require_relative 'support/shared_examples'
+
+RS = Regexp::Scanner
+RL = Regexp::Lexer
+RP = Regexp::Parser
+RE = Regexp::Expression
+T = Regexp::Syntax::Token
+
+include Regexp::Expression
+
+def ruby_version_at_least(version)
+  Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new(version)
+end

data/{test → spec}/support/runner.rb

@@ -1,6 +1,7 @@
 require 'pathname'
+require 'rspec'
 
-module RegexpParserTest
+module RegexpParserSpec
   class Runner
     def initialize(arguments, warning_whitelist)
       @arguments = arguments
@@ -8,14 +9,14 @@ module RegexpParserTest
     end
 
     def run
-      test_status = nil
+      spec_status = nil
 
       Warning::Filter.new(warning_whitelist).assert_expected_warnings_only do
         setup
-        test_status = run_test_unit
+        spec_status = run_rspec
       end
 
-      test_status
+      spec_status
     end
 
     private
@@ -23,14 +24,14 @@ module RegexpParserTest
     def setup
       $VERBOSE = true
 
-      test_files.each(&method(:require))
+      spec_files.each(&method(:require))
     end
 
-    def run_test_unit
-      Test::Unit::AutoRunner.run
+    def run_rspec
+      RSpec::Core::Runner.run([])
     end
 
-    def test_files
+    def spec_files
       arguments
         .map { |path| Pathname.new(path).expand_path.freeze }
         .select(&:file?)

data/spec/support/shared_examples.rb

@@ -0,0 +1,77 @@
+RSpec.shared_examples 'syntax' do |klass, opts|
+  opts[:implements].each do |type, tokens|
+    tokens.each do |token|
+      it("implements #{token} #{type}") do
+        expect(klass.implements?(type, token)).to be true
+      end
+    end
+  end
+
+  opts[:excludes] && opts[:excludes].each do |type, tokens|
+    tokens.each do |token|
+      it("does not implement #{token} #{type}") do
+        expect(klass.implements?(type, token)).to be false
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'scan' do |pattern, checks|
+  context "given the pattern #{pattern}" do
+    before(:all) { @tokens = Regexp::Scanner.scan(pattern) }
+
+    checks.each do |index, (type, token, text, ts, te)|
+      it "scans token #{index} as #{token} #{type} at #{ts}..#{te}" do
+        result = @tokens.at(index)
+
+        expect(result[0]).to eq type
+        expect(result[1]).to eq token
+        expect(result[2]).to eq text
+        expect(result[3]).to eq ts
+        expect(result[4]).to eq te
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'lex' do |pattern, checks|
+  context "given the pattern #{pattern}" do
+    before(:all) { @tokens = Regexp::Lexer.lex(pattern) }
+
+    checks.each do |index, (type, token, text, ts, te, lvl, set_lvl, cond_lvl)|
+      it "lexes token #{index} as #{token} #{type} at #{lvl}, #{set_lvl}, #{cond_lvl}" do
+        struct = @tokens.at(index)
+
+        expect(struct.type).to eq type
+        expect(struct.token).to eq token
+        expect(struct.text).to eq text
+        expect(struct.ts).to eq ts
+        expect(struct.te).to eq te
+        expect(struct.level).to eq lvl
+        expect(struct.set_level).to eq set_lvl
+        expect(struct.conditional_level).to eq cond_lvl
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'parse' do |pattern, checks|
+  context "given the pattern #{pattern}" do
+    before(:all) { @root = Regexp::Parser.parse(pattern, '*') }
+
+    checks.each do |path, (type, token, klass, attributes)|
+      it "parses expression at #{path} as #{klass}" do
+        exp = @root.dig(*path)
+
+        expect(exp).to be_instance_of(klass)
+        expect(exp.type).to eq type
+        expect(exp.token).to eq token
+
+        attributes && attributes.each do |method, value|
+          expect(exp.send(method)).to eq(value),
+            "expected expression at #{path} to have #{method} #{value}"
+        end
+      end
+    end
+  end
+end

data/{test → spec}/support/warning_extractor.rb

@@ -1,7 +1,7 @@
 require 'set'
 require 'delegate'
 
-module RegexpParserTest
+module RegexpParserSpec
   class Warning
     class UnexpectedWarnings < StandardError
       MSG = 'Unexpected warnings: %s'.freeze
@@ -22,22 +22,20 @@ module RegexpParserTest
 
         yield
 
-        if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.0.0')
-          assert_no_warnings($stderr.warnings)
-        end
+        assert_no_warnings($stderr.warnings)
       ensure
         $stderr = original
       end
 
-    private
+      private
 
       def assert_no_warnings(warnings)
-        fail UnexpectedWarnings, warnings.to_a if warnings.any?
+        raise UnexpectedWarnings, warnings.to_a if warnings.any?
       end
     end
 
     class Extractor < DelegateClass(IO)
-      PATTERN = /\A(?:.+):(?:\d+): warning: (?:.+)\n\z/.freeze
+      PATTERN = /\A(?:.+):(?:\d+): warning: (?:.+)\n\z/
 
       def initialize(io, whitelist)
         @whitelist = whitelist

data/spec/syntax/syntax_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Syntax) do
+  specify('unknown name') do
+    expect { Regexp::Syntax.new('ruby/1.0') }.to raise_error(Regexp::Syntax::UnknownSyntaxNameError)
+  end
+
+  specify('new') do
+    expect(Regexp::Syntax.new('ruby/1.9.3')).to be_instance_of(Regexp::Syntax::V1_9_3)
+  end
+
+  specify('new any') do
+    expect(Regexp::Syntax.new('any')).to be_instance_of(Regexp::Syntax::Any)
+    expect(Regexp::Syntax.new('*')).to be_instance_of(Regexp::Syntax::Any)
+  end
+
+  specify('not implemented') do
+    expect { RP.parse('\\p{alpha}', 'ruby/1.8') }.to raise_error(Regexp::Syntax::NotImplementedError)
+  end
+
+  specify('supported?') do
+    expect(Regexp::Syntax.supported?('ruby/1.1.1')).to be false
+    expect(Regexp::Syntax.supported?('ruby/2.4.3')).to be true
+    expect(Regexp::Syntax.supported?('ruby/2.5')).to be true
+  end
+
+  specify('invalid version') do
+    expect { Regexp::Syntax.version_class('2.0.0') }.to raise_error(Regexp::Syntax::InvalidVersionNameError)
+
+    expect { Regexp::Syntax.version_class('ruby/20') }.to raise_error(Regexp::Syntax::InvalidVersionNameError)
+  end
+
+  specify('version class tiny version') do
+    expect(Regexp::Syntax.version_class('ruby/1.9.3')).to eq Regexp::Syntax::V1_9_3
+
+    expect(Regexp::Syntax.version_class('ruby/2.3.1')).to eq Regexp::Syntax::V2_3_1
+  end
+
+  specify('version class minor version') do
+    expect(Regexp::Syntax.version_class('ruby/1.9')).to eq Regexp::Syntax::V1_9
+
+    expect(Regexp::Syntax.version_class('ruby/2.3')).to eq Regexp::Syntax::V2_3
+  end
+
+  specify('raises for unknown constant lookups') do
+    expect { Regexp::Syntax::V1 }.to raise_error(/V1/)
+  end
+end

data/spec/syntax/syntax_token_map_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Syntax::Token::Map) do
+  let(:map) { Regexp::Syntax::Token::Map }
+
+  specify('is complete') do
+    latest_syntax = Regexp::Syntax.new('ruby/2.9')
+
+    latest_syntax.features.each do |type, tokens|
+      tokens.each { |token| expect(map[type]).to include(token) }
+    end
+  end
+
+  specify('contains no duplicate type/token combinations') do
+    combinations = map.flat_map do |type, tokens|
+      tokens.map { |token| "#{type} #{token}" }
+    end
+
+    non_uniq = combinations.group_by { |str| str }.select { |_, v| v.count > 1 }
+
+    expect(non_uniq.keys).to be_empty
+  end
+end

data/spec/syntax/versions/1.8.6_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Syntax::V1_8_6) do
+  include_examples 'syntax', Regexp::Syntax.new('ruby/1.8.6'),
+  implements: {
+    assertion: T::Assertion::Lookahead,
+    backref: [:number],
+    escape: T::Escape::Basic + T::Escape::ASCII + T::Escape::Meta + T::Escape::Control,
+    group: T::Group::V1_8_6,
+    quantifier: T::Quantifier::Greedy + T::Quantifier::Reluctant + T::Quantifier::Interval + T::Quantifier::IntervalReluctant
+  },
+  excludes: {
+    assertion: T::Assertion::Lookbehind,
+    backref: T::Backreference::All - [:number] + T::SubexpressionCall::All,
+    quantifier: T::Quantifier::Possessive
+  }
+end

data/spec/syntax/versions/1.9.1_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+RSpec.describe(Regexp::Syntax::V1_9_1) do
+  include_examples 'syntax', Regexp::Syntax.new('ruby/1.9.1'),
+  implements: {
+    escape: T::Escape::Hex + T::Escape::Octal + T::Escape::Unicode,
+    type: T::CharacterType::Hex,
+    quantifier: T::Quantifier::Greedy + T::Quantifier::Reluctant + T::Quantifier::Possessive
+  }
+end