regexp_parser 1.3.0 → 1.6.0

data/test/helpers.rb

@@ -1,10 +0,0 @@
-require "test/unit"
-require File.expand_path("../../lib/regexp_parser", __FILE__)
-require 'regexp_property_values'
-
-RS = Regexp::Scanner
-RL = Regexp::Lexer
-RP = Regexp::Parser
-RE = Regexp::Expression
-
-include Regexp::Expression

data/test/lexer/test_all.rb

@@ -1,41 +0,0 @@
-require File.expand_path("../../helpers", __FILE__)
-
-%w{
-  literals nesting refcalls
-}.each do|tc|
-  require File.expand_path("../test_#{tc}", __FILE__)
-end
-
-if RUBY_VERSION >= '2.0.0'
-  %w{conditionals keep}.each do|tc|
-    require File.expand_path("../test_#{tc}", __FILE__)
-  end
-end
-
-class TestRegexpLexer < Test::Unit::TestCase
-
-  def test_lexer_returns_an_array
-    assert_instance_of Array, RL.lex('abc')
-  end
-
-  def test_lexer_returns_tokens
-    tokens = RL.lex('^abc+[^one]{2,3}\b\d\\\C-C$')
-
-    assert tokens.all?{ |token| token.kind_of?(Regexp::Token) },
-          "Not all array members are tokens"
-
-    assert tokens.all?{ |token| token.to_a.length == 8 },
-          "Not all tokens have a length of 8"
-  end
-
-  def test_lexer_token_count
-    tokens = RL.lex(/^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i)
-
-    assert_equal 28, tokens.length
-  end
-
-  def test_lexer_scan_alias
-    assert_equal RL.lex(/a|b|c/), RL.scan(/a|b|c/)
-  end
-
-end

data/test/lexer/test_conditionals.rb

@@ -1,127 +0,0 @@
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerConditionals < Test::Unit::TestCase
-  if RUBY_VERSION >= '2.0'
-
-  # Basic lexer output and nesting tests
-  tests = {
-    '(?<A>a)(?(<A>)b|c)'  => [3, :conditional, :open,       '(?',     7,  9, 0, 0, 0],
-    '(?<B>a)(?(<B>)b|c)'  => [4, :conditional, :condition,  '(<B>)',  9, 14, 0, 0, 1],
-    '(?<C>a)(?(<C>)b|c)'  => [6, :conditional, :separator,  '|',     15, 16, 0, 0, 1],
-    '(?<D>a)(?(<D>)b|c)'  => [8, :conditional, :close,      ')',     17, 18, 0, 0, 0],
-  }
-
-  tests.each_with_index do |(pattern, (index, type, token, text, ts, te, level, set_level, conditional_level)), count|
-    define_method "test_lexer_#{type}_#{token}_#{count}" do
-      tokens = RL.lex(pattern)
-      struct = tokens.at(index)
-
-      assert_equal type,              struct.type
-      assert_equal token,             struct.token
-      assert_equal text,              struct.text
-      assert_equal ts,                struct.ts
-      assert_equal te,                struct.te
-      assert_equal level,             struct.level
-      assert_equal set_level,         struct.set_level
-      assert_equal conditional_level, struct.conditional_level
-    end
-  end
-
-  def test_lexer_conditional_mixed_nesting
-    regexp = /((?<A>a)(?<B>(?(<A>)b|((?(<B>)[e-g]|[h-j])))))/
-    tokens = RL.lex(regexp)
-
-    [
-      [ 0, :group,       :capture,          '(',       0,  1, 0, 0, 0],
-      [ 1, :group,       :named,            '(?<A>',   1,  6, 1, 0, 0],
-
-      [ 5, :conditional, :open,             '(?',     13, 15, 2, 0, 0],
-      [ 6, :conditional, :condition,        '(<A>)',  15, 20, 2, 0, 1],
-      [ 8, :conditional, :separator,        '|',      21, 22, 2, 0, 1],
-
-      [10, :conditional, :open,             '(?',     23, 25, 3, 0, 1],
-      [11, :conditional, :condition,        '(<B>)',  25, 30, 3, 0, 2],
-
-      [12, :set,         :open,             '[',      30, 31, 3, 0, 2],
-      [13, :literal,     :literal,          'e',      31, 32, 3, 1, 2],
-      [14, :set,         :range,            '-',      32, 33, 3, 1, 2],
-      [15, :literal,     :literal,          'g',      33, 34, 3, 1, 2],
-      [16, :set,         :close,            ']',      34, 35, 3, 0, 2],
-
-      [17, :conditional, :separator,        '|',      35, 36, 3, 0, 2],
-      [23, :conditional, :close,            ')',      41, 42, 3, 0, 1],
-      [25, :conditional, :close,            ')',      43, 44, 2, 0, 0],
-
-      [26, :group,       :close,            ')',      44, 45, 1, 0, 0],
-      [27, :group,       :close,            ')',      45, 46, 0, 0, 0]
-    ].each do |index, type, token, text, ts, te, level, set_level, conditional_level|
-      struct = tokens.at(index)
-
-      assert_equal type,              struct.type
-      assert_equal token,             struct.token
-      assert_equal text,              struct.text
-      assert_equal ts,                struct.ts
-      assert_equal te,                struct.te
-      assert_equal level,             struct.level
-      assert_equal set_level,         struct.set_level
-      assert_equal conditional_level, struct.conditional_level
-    end
-  end
-
-  def test_lexer_conditional_deep_nesting
-    regexp = /(a(b(c)))(?(1)(?(2)(?(3)d|e))|(?(3)(?(2)f|g)|(?(1)f|g)))/
-    tokens = RL.lex(regexp)
-
-    [
-      [ 9, :conditional, :open,       '(?',    9, 11, 0, 0, 0],
-      [10, :conditional, :condition,  '(1)',  11, 14, 0, 0, 1],
-
-      [11, :conditional, :open,       '(?',   14, 16, 0, 0, 1],
-      [12, :conditional, :condition,  '(2)',  16, 19, 0, 0, 2],
-
-      [13, :conditional, :open,       '(?',   19, 21, 0, 0, 2],
-      [14, :conditional, :condition,  '(3)',  21, 24, 0, 0, 3],
-
-      [16, :conditional, :separator,  '|',    25, 26, 0, 0, 3],
-
-      [18, :conditional, :close,      ')',    27, 28, 0, 0, 2],
-      [19, :conditional, :close,      ')',    28, 29, 0, 0, 1],
-
-      [20, :conditional, :separator,  '|',    29, 30, 0, 0, 1],
-
-      [21, :conditional, :open,       '(?',   30, 32, 0, 0, 1],
-      [22, :conditional, :condition,  '(3)',  32, 35, 0, 0, 2],
-
-      [23, :conditional, :open,       '(?',   35, 37, 0, 0, 2],
-      [24, :conditional, :condition,  '(2)',  37, 40, 0, 0, 3],
-
-      [26, :conditional, :separator,  '|',    41, 42, 0, 0, 3],
-
-      [28, :conditional, :close,      ')',    43, 44, 0, 0, 2],
-
-      [29, :conditional, :separator,  '|',    44, 45, 0, 0, 2],
-
-      [30, :conditional, :open,       '(?',   45, 47, 0, 0, 2],
-      [31, :conditional, :condition,  '(1)',  47, 50, 0, 0, 3],
-
-      [33, :conditional, :separator,  '|',    51, 52, 0, 0, 3],
-
-      [35, :conditional, :close,      ')',    53, 54, 0, 0, 2],
-      [36, :conditional, :close,      ')',    54, 55, 0, 0, 1],
-      [37, :conditional, :close,      ')',    55, 56, 0, 0, 0]
-    ].each do |index, type, token, text, ts, te, level, set_level, conditional_level|
-      struct = tokens.at(index)
-
-      assert_equal type,              struct.type
-      assert_equal token,             struct.token
-      assert_equal text,              struct.text
-      assert_equal ts,                struct.ts
-      assert_equal te,                struct.te
-      assert_equal level,             struct.level
-      assert_equal set_level,         struct.set_level
-      assert_equal conditional_level, struct.conditional_level
-    end
-  end
-
-  end # if RUBY_VERSION >= '2.0'
-end

data/test/lexer/test_keep.rb

@@ -1,24 +0,0 @@
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerKeep < Test::Unit::TestCase
-
-  def test_lex_keep_token
-    regexp = /ab\Kcd/
-    tokens = RL.lex(regexp)
-
-    assert_equal :keep, tokens[1].type
-    assert_equal :mark, tokens[1].token
-  end
-
-  def test_lex_keep_nested
-    regexp = /(a\Kb)|(c\\\Kd)ef/
-    tokens = RL.lex(regexp)
-
-    assert_equal :keep, tokens[2].type
-    assert_equal :mark, tokens[2].token
-
-    assert_equal :keep, tokens[9].type
-    assert_equal :mark, tokens[9].token
-  end
-
-end

data/test/lexer/test_literals.rb

@@ -1,130 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerLiterals < Test::Unit::TestCase
-
-  tests = {
-    # ascii, single byte characters
-    'a' => {
-      0     => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-    },
-
-    'ab+' => {
-      0     => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-      1     => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
-      2     => [:quantifier,  :one_or_more,   '+',        2, 3, 0, 0, 0],
-    },
-
-
-    # 2 byte wide characters, Arabic
-    'ا' => {
-      0     => [:literal,     :literal,       'ا',        0, 2, 0, 0, 0],
-    },
-
-    'aاbبcت' => {
-      0     => [:literal,     :literal,       'aاbبcت',   0, 9, 0, 0, 0],
-    },
-
-    'aاbبت?' => {
-      0     => [:literal,     :literal,       'aاbب',     0, 6, 0, 0, 0],
-      1     => [:literal,     :literal,       'ت',        6, 8, 0, 0, 0],
-      2     => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0, 0],
-    },
-
-    'aا?bبcت+' => {
-      0     => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-      1     => [:literal,     :literal,       'ا',        1, 3, 0, 0, 0],
-      2     => [:quantifier,  :zero_or_one,   '?',        3, 4, 0, 0, 0],
-      3     => [:literal,     :literal,       'bبc',      4, 8, 0, 0, 0],
-      4     => [:literal,     :literal,       'ت',        8, 10, 0, 0, 0],
-      5     => [:quantifier,  :one_or_more,   '+',        10, 11, 0, 0, 0],
-    },
-
-    'a(اbب+)cت?' => {
-      0     => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-      1     => [:group,       :capture,       '(',        1, 2, 0, 0, 0],
-      2     => [:literal,     :literal,       'اb',       2, 5, 1, 0, 0],
-      3     => [:literal,     :literal,       'ب',        5, 7, 1, 0, 0],
-      4     => [:quantifier,  :one_or_more,   '+',        7, 8, 1, 0, 0],
-      5     => [:group,       :close,         ')',        8, 9, 0, 0, 0],
-      6     => [:literal,     :literal,       'c',        9, 10, 0, 0, 0],
-      7     => [:literal,     :literal,       'ت',        10, 12, 0, 0, 0],
-      8     => [:quantifier,  :zero_or_one,   '?',        12, 13, 0, 0, 0],
-    },
-
-
-    # 3 byte wide characters, Japanese
-    'ab?れます+cd' => {
-      0     => [:literal,     :literal,       'a',        0, 1, 0, 0, 0],
-      1     => [:literal,     :literal,       'b',        1, 2, 0, 0, 0],
-      2     => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0, 0],
-      3     => [:literal,     :literal,       'れま',     3, 9, 0, 0, 0],
-      4     => [:literal,     :literal,       'す',       9, 12, 0, 0, 0],
-      5     => [:quantifier,  :one_or_more,   '+',        12, 13, 0, 0, 0],
-      6     => [:literal,     :literal,       'cd',       13, 15, 0, 0, 0],
-    },
-
-
-    # 4 byte wide characters, Osmanya
-    '𐒀𐒁?𐒂ab+𐒃' => {
-      0     => [:literal,     :literal,       '𐒀',        0, 4, 0, 0, 0],
-      1     => [:literal,     :literal,       '𐒁',        4, 8, 0, 0, 0],
-      2     => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0, 0],
-      3     => [:literal,     :literal,       '𐒂a',       9, 14, 0, 0, 0],
-      4     => [:literal,     :literal,       'b',        14, 15, 0, 0, 0],
-      5     => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0, 0],
-      6     => [:literal,     :literal,       '𐒃',        16, 20, 0, 0, 0],
-    },
-
-    'mu𝄞?si*𝄫c+' => {
-      0     => [:literal,     :literal,       'mu',       0, 2, 0, 0, 0],
-      1     => [:literal,     :literal,       '𝄞',        2, 6, 0, 0, 0],
-      2     => [:quantifier,  :zero_or_one,   '?',        6, 7, 0, 0, 0],
-      3     => [:literal,     :literal,       's',        7, 8, 0, 0, 0],
-      4     => [:literal,     :literal,       'i',        8, 9, 0, 0, 0],
-      5     => [:quantifier,  :zero_or_more,  '*',        9, 10, 0, 0, 0],
-      6     => [:literal,     :literal,       '𝄫',        10, 14, 0, 0, 0],
-      7     => [:literal,     :literal,       'c',        14, 15, 0, 0, 0],
-      8     => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0, 0],
-    },
-  }
-
-  tests.each_with_index do |(pattern, checks), count|
-    define_method "test_lex_literal_runs_#{count}" do
-      tokens = RL.lex(pattern)
-
-      checks.each do |index, (type, token, text, ts, te, level, set_level, conditional_level)|
-        struct = tokens.at(index)
-
-        assert_equal type,              struct.type
-        assert_equal token,             struct.token
-        assert_equal text,              struct.text
-        assert_equal ts,                struct.ts
-        assert_equal te,                struct.te
-        assert_equal level,             struct.level
-        assert_equal set_level,         struct.set_level
-        assert_equal conditional_level, struct.conditional_level
-      end
-    end
-  end
-
-  def test_lex_single_2_byte_char
-    tokens = RL.lex('ا+')
-
-    assert_equal 2, tokens.length
-  end
-
-  def test_lex_single_3_byte_char
-    tokens = RL.lex('れ+')
-
-    assert_equal 2, tokens.length
-  end
-
-  def test_lex_single_4_byte_char
-    tokens = RL.lex('𝄞+')
-
-    assert_equal 2, tokens.length
-  end
-
-end

data/test/lexer/test_nesting.rb

@@ -1,132 +0,0 @@
-require File.expand_path("../../helpers", __FILE__)
-
-class LexerNesting < Test::Unit::TestCase
-
-  tests = {
-    '(((b)))' => {
-      0     => [:group,       :capture,       '(',      0,  1, 0, 0, 0],
-      1     => [:group,       :capture,       '(',      1,  2, 1, 0, 0],
-      2     => [:group,       :capture,       '(',      2,  3, 2, 0, 0],
-      3     => [:literal,     :literal,       'b',      3,  4, 3, 0, 0],
-      4     => [:group,       :close,         ')',      4,  5, 2, 0, 0],
-      5     => [:group,       :close,         ')',      5,  6, 1, 0, 0],
-      6     => [:group,       :close,         ')',      6,  7, 0, 0, 0],
-    },
-
-    '(\((b)\))' => {
-      0     => [:group,       :capture,       '(',      0,  1, 0, 0, 0],
-      1     => [:escape,      :group_open,    '\(',     1,  3, 1, 0, 0],
-      2     => [:group,       :capture,       '(',      3,  4, 1, 0, 0],
-      3     => [:literal,     :literal,       'b',      4,  5, 2, 0, 0],
-      4     => [:group,       :close,         ')',      5,  6, 1, 0, 0],
-      5     => [:escape,      :group_close,   '\)',     6,  8, 1, 0, 0],
-      6     => [:group,       :close,         ')',      8,  9, 0, 0, 0],
-    },
-
-    '(?>a(?>b(?>c)))' => {
-      0     => [:group,       :atomic,        '(?>',    0,  3, 0, 0, 0],
-      2     => [:group,       :atomic,        '(?>',    4,  7, 1, 0, 0],
-      4     => [:group,       :atomic,        '(?>',    8, 11, 2, 0, 0],
-      6     => [:group,       :close,         ')',     12, 13, 2, 0, 0],
-      7     => [:group,       :close,         ')',     13, 14, 1, 0, 0],
-      8     => [:group,       :close,         ')',     14, 15, 0, 0, 0],
-    },
-
-    '(?:a(?:b(?:c)))' => {
-      0     => [:group,       :passive,       '(?:',    0,  3, 0, 0, 0],
-      2     => [:group,       :passive,       '(?:',    4,  7, 1, 0, 0],
-      4     => [:group,       :passive,       '(?:',    8, 11, 2, 0, 0],
-      6     => [:group,       :close,         ')',     12, 13, 2, 0, 0],
-      7     => [:group,       :close,         ')',     13, 14, 1, 0, 0],
-      8     => [:group,       :close,         ')',     14, 15, 0, 0, 0],
-    },
-
-    '(?=a(?!b(?<=c(?<!d))))' => {
-      0     => [:assertion,   :lookahead,     '(?=',    0,  3, 0, 0, 0],
-      2     => [:assertion,   :nlookahead,    '(?!',    4,  7, 1, 0, 0],
-      4     => [:assertion,   :lookbehind,    '(?<=',   8, 12, 2, 0, 0],
-      6     => [:assertion,   :nlookbehind,   '(?<!',  13, 17, 3, 0, 0],
-      8     => [:group,       :close,         ')',     18, 19, 3, 0, 0],
-      9     => [:group,       :close,         ')',     19, 20, 2, 0, 0],
-      10    => [:group,       :close,         ')',     20, 21, 1, 0, 0],
-      11    => [:group,       :close,         ')',     21, 22, 0, 0, 0],
-    },
-
-    '((?#a)b(?#c)d(?#e))' => {
-      0     => [:group,       :capture,       '(',      0,  1, 0, 0, 0],
-      1     => [:group,       :comment,       '(?#a)',  1,  6, 1, 0, 0],
-      3     => [:group,       :comment,       '(?#c)',  7, 12, 1, 0, 0],
-      5     => [:group,       :comment,       '(?#e)', 13, 18, 1, 0, 0],
-      6     => [:group,       :close,         ')',     18, 19, 0, 0, 0],
-    },
-
-    'a[b-e]f' => {
-      1     => [:set,         :open,          '[',      1,  2, 0, 0, 0],
-      2     => [:literal,     :literal,       'b',      2,  3, 0, 1, 0],
-      3     => [:set,         :range,         '-',      3,  4, 0, 1, 0],
-      4     => [:literal,     :literal,       'e',      4,  5, 0, 1, 0],
-      5     => [:set,         :close,         ']',      5,  6, 0, 0, 0],
-    },
-
-    '[[:word:]&&[^c]z]' => {
-      0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-      1     => [:posixclass,  :word, '[:word:]',        1,  9, 0, 1, 0],
-      2     => [:set,         :intersection,  '&&',     9, 11, 0, 1, 0],
-      3     => [:set,         :open,          '[',     11, 12, 0, 1, 0],
-      4     => [:set,         :negate,        '^',     12, 13, 0, 2, 0],
-      5     => [:literal,     :literal,       'c',     13, 14, 0, 2, 0],
-      6     => [:set,         :close,         ']',     14, 15, 0, 1, 0],
-      7     => [:literal,     :literal,       'z',     15, 16, 0, 1, 0],
-      8     => [:set,         :close,         ']',     16, 17, 0, 0, 0],
-    },
-
-    '[\p{word}&&[^c]z]' => {
-      0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-      1     => [:property,    :word, '\p{word}',        1,  9, 0, 1, 0],
-      2     => [:set,         :intersection,  '&&',     9, 11, 0, 1, 0],
-      3     => [:set,         :open,          '[',     11, 12, 0, 1, 0],
-      4     => [:set,         :negate,        '^',     12, 13, 0, 2, 0],
-      5     => [:literal,     :literal,       'c',     13, 14, 0, 2, 0],
-      6     => [:set,         :close,         ']',     14, 15, 0, 1, 0],
-      7     => [:literal,     :literal,       'z',     15, 16, 0, 1, 0],
-      8     => [:set,         :close,         ']',     16, 17, 0, 0, 0],
-    },
-
-    '[a[b[c[d-g]]]]' => {
-      0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-      1     => [:literal,     :literal,       'a',      1,  2, 0, 1, 0],
-      2     => [:set,         :open,          '[',      2,  3, 0, 1, 0],
-      3     => [:literal,     :literal,       'b',      3,  4, 0, 2, 0],
-      4     => [:set,         :open,          '[',      4,  5, 0, 2, 0],
-      5     => [:literal,     :literal,       'c',      5,  6, 0, 3, 0],
-      6     => [:set,         :open,          '[',      6,  7, 0, 3, 0],
-      7     => [:literal,     :literal,       'd',      7,  8, 0, 4, 0],
-      8     => [:set,         :range,         '-',      8,  9, 0, 4, 0],
-      9     => [:literal,     :literal,       'g',      9, 10, 0, 4, 0],
-      10    => [:set,         :close,         ']',     10, 11, 0, 3, 0],
-      11    => [:set,         :close,         ']',     11, 12, 0, 2, 0],
-      12    => [:set,         :close,         ']',     12, 13, 0, 1, 0],
-      13    => [:set,         :close,         ']',     13, 14, 0, 0, 0],
-    },
-  }
-
-  tests.each_with_index do |(pattern, checks), count|
-    define_method "test_lex_nesting_in_'#{pattern}'_#{count}" do
-      tokens = RL.lex(pattern, 'ruby/1.9')
-
-      checks.each do |offset, (type, token, text, ts, te, level, set_level, conditional_level)|
-        struct = tokens.at(offset)
-
-        assert_equal type,              struct.type
-        assert_equal token,             struct.token
-        assert_equal text,              struct.text
-        assert_equal ts,                struct.ts
-        assert_equal te,                struct.te
-        assert_equal level,             struct.level
-        assert_equal set_level,         struct.set_level
-        assert_equal conditional_level, struct.conditional_level
-      end
-    end
-  end
-
-end