regexp_parser 0.1.6 → 0.2.0

data/test/scanner/test_escapes.rb

@@ -27,12 +27,15 @@ class ScannerEscapes < Test::Unit::TestCase
     'a\u0640c'        => [1, :escape,  :codepoint,        '\u0640',         1,  7],
     'a\u{0640 0641}c' => [1, :escape,  :codepoint_list,   '\u{0640 0641}',  1,  14],
 
-    'a\cBc'           => [1, :escape,  :control,          '\cB',            1,  4],
-    'a\C-bc'          => [1, :escape,  :control,          '\C-b',           1,  5],
+    /a\cBc/           => [1, :escape,  :control,          '\cB',            1,  4],
+    /a\C-bc/          => [1, :escape,  :control,          '\C-b',           1,  5],
 
-    # TODO: verify these escapes
-    'a\M-Bc'          => [1, :escape,  :meta_sequence,    '\M-B',           1,  5],
-    'a\M-\C-Bc'       => [1, :escape,  :meta_sequence,    '\M-\C-B',        1,  8],
+    /a\M-Bc/n         => [1, :escape,  :meta_sequence,    '\M-B',           1,  5],
+    /a\M-\C-Bc/n      => [1, :escape,  :meta_sequence,    '\M-\C-B',        1,  8],
+
+    'ab\\\xcd'        => [1, :escape,  :backslash,        '\\\\',           2,  4],
+    'ab\\\0cd'        => [1, :escape,  :backslash,        '\\\\',           2,  4],
+    'ab\\\Kcd'        => [1, :escape,  :backslash,        '\\\\',           2,  4],
   }
 
   count = 0

data/test/scanner/test_free_space.rb

@@ -0,0 +1,133 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerFreeSpace < Test::Unit::TestCase
+
+  def test_scan_free_space_tokens
+    regexp = %r{
+      a
+      b  ?  c  *
+      d  {2,3}
+      e + | f +
+    }x
+
+    tokens = RS.scan(regexp)
+
+    0.upto(24) do |i|
+      if i % 2 == 0
+        assert_equal( :free_space, tokens[i][0] )
+        assert_equal( :whitespace, tokens[i][1] )
+      else
+        assert_not_equal( :free_space, tokens[i][0] )
+        assert_not_equal( :whitespace, tokens[i][1] )
+      end
+    end
+
+    [0, 2, 10, 14].each do |i|
+      assert_equal( "\n      ",   tokens[i][2] )
+    end
+
+    [4, 6, 8, 12].each do |i|
+      assert_equal( '  ',         tokens[i][2] )
+    end
+  end
+
+  def test_scan_free_space_comments
+    regexp = %r{
+      a + # A + comment
+      b  ?  # B ? comment
+      c  {2,3} # C {2,3} comment
+      d + | e + # D|E comment
+    }x
+
+    tokens = RS.scan(regexp)
+
+    [
+      [ 5, [:free_space, :comment,  "# A + comment\n",      11,  25]],
+      [11, [:free_space, :comment,  "# B ? comment\n",      37,  51]],
+      [17, [:free_space, :comment,  "# C {2,3} comment\n",  66,  84]],
+      [29, [:free_space, :comment,  "# D|E comment\n",     100, 114]],
+    ].each do |index, token|
+      assert_equal( token, tokens[index] )
+    end
+  end
+
+  def test_scan_free_space_inlined
+    # Matches 'a bcdef g'
+    regexp = /a b(?x:c d e)f g/
+    tokens = RS.scan(regexp)
+
+    [
+      [0, [:literal,    :literal,     'a b',   0,  3]],
+      [1, [:group,      :options,     '(?x:',  3,  7]],
+      [2, [:literal,    :literal,     'c',     7,  8]],
+      [3, [:free_space, :whitespace,  ' ',     8,  9]],
+      [4, [:literal,    :literal,     'd',     9, 10]],
+      [5, [:free_space, :whitespace,  ' ',    10, 11]],
+      [6, [:literal,    :literal,     'e',    11, 12]],
+      [7, [:group,      :close,       ')',    12, 13]],
+      [8, [:literal,    :literal,     'f g',  13, 16]]
+    ].each do |index, token|
+      assert_equal( token, tokens[index] )
+    end
+  end
+
+  def test_scan_free_space_nested
+    # Matches 'a bcde fghi j'
+    regexp = /a b(?x:c d(?-x:e f)g h)i j/
+    tokens = RS.scan(regexp)
+
+    [
+      [ 0, [:literal,     :literal,     'a b',     0,  3]],
+      [ 1, [:group,       :options,     '(?x:',    3,  7]],
+      [ 2, [:literal,     :literal,     'c',       7,  8]],
+      [ 3, [:free_space,  :whitespace,  ' ',       8,  9]],
+      [ 4, [:literal,     :literal,     'd',       9, 10]],
+      [ 5, [:group,       :options,     '(?-x:',  10, 15]],
+      [ 6, [:literal,     :literal,     'e f',    15, 18]],
+      [ 7, [:group,       :close,       ')',      18, 19]],
+      [ 8, [:literal,     :literal,     'g',      19, 20]],
+      [ 9, [:free_space,  :whitespace,  ' ',      20, 21]],
+      [10, [:literal,     :literal,     'h',      21, 22]],
+      [11, [:group,       :close,       ')',      22, 23]],
+      [12, [:literal,     :literal,     'i j',    23, 26]]
+    ].each do |index, token|
+      assert_equal( token, tokens[index] )
+    end
+  end
+
+  def test_scan_free_space_nested_groups
+    # Matches 'a bcde f g hi j'
+    regexp = /(a (b(?x: (c d) (?-x:(e f) )g) h)i j)/
+    tokens = RS.scan(regexp)
+
+    [
+      [ 0, [:group,       :capture,     '(',       0,  1]],
+      [ 1, [:literal,     :literal,     'a ',      1,  3]],
+      [ 2, [:group,       :capture,     '(',       3,  4]],
+      [ 3, [:literal,     :literal,     'b',       4,  5]],
+      [ 4, [:group,       :options,     '(?x:',    5,  9]],
+      [ 5, [:free_space,  :whitespace,  ' ',       9, 10]],
+      [ 6, [:group,       :capture,     '(',      10, 11]],
+      [ 7, [:literal,     :literal,     'c',      11, 12]],
+      [ 8, [:free_space,  :whitespace,  ' ',      12, 13]],
+      [ 9, [:literal,     :literal,     'd',      13, 14]],
+      [10, [:group,       :close,       ')',      14, 15]],
+      [11, [:free_space,  :whitespace,  ' ',      15, 16]],
+      [12, [:group,       :options,     '(?-x:',  16, 21]],
+      [13, [:group,       :capture,     '(',      21, 22]],
+      [14, [:literal,     :literal,     'e f',    22, 25]],
+      [15, [:group,       :close,       ')',      25, 26]],
+      [16, [:literal,     :literal,     ' ',      26, 27]],
+      [17, [:group,       :close,       ')',      27, 28]],
+      [18, [:literal,     :literal,     'g',      28, 29]],
+      [19, [:group,       :close,       ')',      29, 30]],
+      [20, [:literal,     :literal,     ' h',     30, 32]],
+      [21, [:group,       :close,       ')',      32, 33]],
+      [22, [:literal,     :literal,     'i j',    33, 36]],
+      [23, [:group,       :close,       ')',      36, 37]]
+    ].each do |index, token|
+      assert_equal( token, tokens[index] )
+    end
+  end
+
+end

data/test/scanner/test_groups.rb

@@ -20,6 +20,9 @@ class ScannerGroups < Test::Unit::TestCase
     '(?<name>abc)'    => [0, :group,     :named_ab,     '(?<name>',   0, 8],
     "(?'name'abc)"    => [0, :group,     :named_sq,     "(?'name'",   0, 8],
 
+    '(?<name_1>abc)'  => [0, :group,     :named_ab,     '(?<name_1>', 0,10],
+    "(?'name_1'abc)"  => [0, :group,     :named_sq,     "(?'name_1'", 0,10],
+
     '(?:abc)'         => [0, :group,     :passive,      '(?:',        0, 3],
     '(?:)'            => [0, :group,     :passive,      '(?:',        0, 3],
     '(?::)'           => [0, :group,     :passive,      '(?:',        0, 3],
@@ -45,4 +48,29 @@ class ScannerGroups < Test::Unit::TestCase
     end
   end
 
+  if RUBY_VERSION >= '2.0'
+    option_tests = {
+      '(?m-dau:abc)'    => [0, :group,     :options,      '(?m-dau:',   0, 8],
+      '(?x-dmu:abc)'    => [0, :group,     :options,      '(?x-dmu:',   0, 8],
+      '(?-dau:abc)'     => [0, :group,     :options,      '(?-dau:',    0, 7],
+      '(?d-au:abc)'     => [0, :group,     :options,      '(?d-au:',    0, 7],
+      '(?da-u:abc)'     => [0, :group,     :options,      '(?da-u:',    0, 7],
+      '(?dau:abc)'      => [0, :group,     :options,      '(?dau:',     0, 6],
+      '(?dau)'          => [0, :group,     :options,      '(?dau',      0, 5],
+      '(?d:)'           => [0, :group,     :options,      '(?d:',       0, 4],
+      '(?a:)'           => [0, :group,     :options,      '(?a:',       0, 4],
+      '(?u:)'           => [0, :group,     :options,      '(?u:',       0, 4],
+    }
+
+    tests.each do |pattern, test|
+      define_method "test_scan_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+        tokens = RS.scan(pattern)
+        assert_equal( test[1,5], tokens[test[0]])
+        assert_equal( test[3],   pattern[tokens[test[0]][3], tokens[test[0]][4]])
+
+      end
+    end
+  end
+
 end

data/test/scanner/test_keep.rb

@@ -0,0 +1,33 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerKeep < Test::Unit::TestCase
+
+  def test_scan_keep_token
+    regexp = /ab\Kcd/
+    tokens = RS.scan(regexp)
+
+    assert_equal( :keep, tokens[1][0] )
+    assert_equal( :mark, tokens[1][1] )
+    assert_equal( '\\K', tokens[1][2] )
+    assert_equal( 2,     tokens[1][3] )
+    assert_equal( 4,     tokens[1][4] )
+  end
+
+  def test_scan_keep_nested
+    regexp = /(a\Kb)|(c\\\Kd)ef/
+    tokens = RS.scan(regexp)
+
+    assert_equal( :keep, tokens[2][0] )
+    assert_equal( :mark, tokens[2][1] )
+    assert_equal( '\\K', tokens[2][2] )
+    assert_equal( 2,     tokens[2][3] )
+    assert_equal( 4,     tokens[2][4] )
+
+    assert_equal( :keep, tokens[9][0] )
+    assert_equal( :mark, tokens[9][1] )
+    assert_equal( '\\K', tokens[9][2] )
+    assert_equal( 11,    tokens[9][3] )
+    assert_equal( 13,    tokens[9][4] )
+  end
+
+end

data/test/scanner/test_properties.rb

@@ -143,6 +143,10 @@ class ScannerProperties < Test::Unit::TestCase
     'Age=5.1'                             => :age_5_1,
     'Age=5.2'                             => :age_5_2,
     'Age=6.0'                             => :age_6_0,
+    'Age=6.1'                             => :age_6_1,
+    'Age=6.2'                             => :age_6_2,
+    'Age=6.3'                             => :age_6_3,
+    'Age=7.0'                             => :age_7_0,
 
     'ahex'                                => :ascii_hex,
     'ASCII_Hex_Digit'                     => :ascii_hex,

data/test/scanner/test_scripts.rb

@@ -3,6 +3,9 @@ require File.expand_path("../../helpers", __FILE__)
 class ScannerUnicodeScripts < Test::Unit::TestCase
 
   tests = {
+    'Aghb'		                => :script_caucasian_albanian,
+    'Caucasian Albanian'		  => :script_caucasian_albanian,
+
     'Arab'		                => :script_arabic,
     'Arabic'		              => :script_arabic,
 
@@ -21,6 +24,9 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Bamu'                    => :script_bamum,
 	  'Bamum'                   => :script_bamum,
 
+    'Bass'                    => :script_bassa_vah,
+	  'Bassa Vah'               => :script_bassa_vah,
+
     'Batk'                    => :script_batak,
 	  'Batak'                   => :script_batak,
 
@@ -69,9 +75,17 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Dsrt'                    => :script_deseret,
 	  'Deseret'                 => :script_deseret,
 
+
+    'Dupl'                    => :script_duployan,
+	  'Duployan'                => :script_duployan,
+
+
     'Egyp'                    => :script_egyptian_hieroglyphs,
 	  'Egyptian Hieroglyphs'    => :script_egyptian_hieroglyphs,
 
+    'Elba'                    => :script_elbasan,
+	  'Elbasan'                 => :script_elbasan,
+
     'Ethi'                    => :script_ethiopic,
 	  'Ethiopic'                => :script_ethiopic,
 
@@ -84,6 +98,9 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Goth'                    => :script_gothic,
 	  'Gothic'                  => :script_gothic,
 
+    'Gran'                    => :script_grantha,
+	  'Grantha'                 => :script_grantha,
+
     'Grek'                    => :script_greek,
 	  'Greek'                   => :script_greek,
 
@@ -108,6 +125,9 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Hira'                    => :script_hiragana,
 	  'Hiragana'                => :script_hiragana,
 
+    'Hmng'                    => :script_pahawh_hmong,
+	  'Pahawh Hmong'            => :script_pahawh_hmong,
+
     'Hrkt'                    => :script_katakana_or_hiragana,
 	  'Katakana or Hiragana'    => :script_katakana_or_hiragana,
 
@@ -129,6 +149,9 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Khmr'                    => :script_khmer,
 	  'Khmer'                   => :script_khmer,
 
+    'Khoj'                    => :script_khojki,
+	  'Khojki'                  => :script_khojki,
+
     'Knda'                    => :script_kannada,
 	  'Kannada'                 => :script_kannada,
 
@@ -150,6 +173,9 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Limb'                    => :script_limbu,
 	  'Limbu'                   => :script_limbu,
 
+    'Lina'                    => :script_linear_a,
+	  'Linear A'                => :script_linear_a,
+
     'Linb'                    => :script_linear_b,
 	  'Linear B'                => :script_linear_b,
 
@@ -167,15 +193,35 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Mlym'                    => :script_malayalam,
 	  'Malayalam'               => :script_malayalam,
 
+    'Mahj'                    => :script_mahajani,
+	  'Mahajani'                => :script_mahajani,
+
+    'Mani'                    => :script_manichaean,
+	  'Manichaean'              => :script_manichaean,
+
+    'Mend'                    => :script_mende_kikakui,
+	  'Mende Kikakui'           => :script_mende_kikakui,
+
+    'Modi'                    => :script_modi,
+
     'Mong'                    => :script_mongolian,
 	  'Mongolian'               => :script_mongolian,
 
+    'Mroo'                    => :script_mro,
+	  'Mro'                     => :script_mro,
+
     'Mtei'                    => :script_meetei_mayek,
 	  'Meetei Mayek'            => :script_meetei_mayek,
 
     'Mymr'                    => :script_myanmar,
 	  'Myanmar'                 => :script_myanmar,
 
+    'Narb'                    => :script_old_north_arabian,
+	  'Old North Arabian'       => :script_old_north_arabian,
+
+    'Nbat'                    => :script_nabataean,
+	  'Nabataean'               => :script_nabataean,
+
     'Nkoo'                    => :script_nko,
 	  'Nko'                     => :script_nko,
 
@@ -194,12 +240,24 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Osma'                    => :script_osmanya,
 	  'Osmanya'                 => :script_osmanya,
 
+    'Palm'                    => :script_palmyrene,
+	  'Palmyrene'               => :script_palmyrene,
+
+    'Pauc'                    => :script_pau_cin_hau,
+	  'Pau Cin Hau'             => :script_pau_cin_hau,
+
+    'Perm'                    => :script_old_permic,
+	  'Old Permic'              => :script_old_permic,
+
     'Phag'                    => :script_phags_pa,
 	  'Phags Pa'                => :script_phags_pa,
 
     'Phli'                    => :script_inscriptional_pahlavi,
 	  'Inscriptional Pahlavi'   => :script_inscriptional_pahlavi,
 
+    'Phlp'                    => :script_psalter_pahlavi,
+	  'Psalter Pahlavi'         => :script_psalter_pahlavi,
+
     'Phnx'		                => :script_phoenician,
 	  'Phoenician'		          => :script_phoenician,
 
@@ -224,6 +282,12 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Shaw'                    => :script_shavian,
 	  'Shavian'                 => :script_shavian,
 
+    'Sidd'                    => :script_siddham,
+	  'Siddham'                 => :script_siddham,
+
+    'Sind'                    => :script_khudawadi,
+	  'Khudawadi'               => :script_khudawadi,
+
     'Sinh'                    => :script_sinhala,
 	  'Sinhala'                 => :script_sinhala,
 
@@ -263,17 +327,23 @@ class ScannerUnicodeScripts < Test::Unit::TestCase
     'Thaa'                    => :script_thaana,
 	  'Thaana'                  => :script_thaana,
 
-    'Thai'                    => :script_thai, 
+    'Thai'                    => :script_thai,
 
     'Tibt'                    => :script_tibetan,
 	  'Tibetan'                 => :script_tibetan,
 
+    'Tirh'                    => :script_tirhuta,
+	  'Tirhuta'                 => :script_tirhuta,
+
     'Ugar'                    => :script_ugaritic,
 	  'Ugaritic'                => :script_ugaritic,
 
     'Vaii'                    => :script_vai,
 	  'Vai'                     => :script_vai,
 
+    'Wara'                    => :script_warang_citi,
+	  'Warang Citi'             => :script_warang_citi,
+
     'Xpeo'                    => :script_old_persian,
 	  'Old Persian'             => :script_old_persian,
 

data/test/syntax/ruby/test_1.9.3.rb

@@ -10,11 +10,11 @@ class TestSyntaxRuby_V193 < Test::Unit::TestCase
   tests = {
     :implements => {
       :property => [
-        UnicodeProperty::Script_6_0 + UnicodeProperty::Age
+        UnicodeProperty::Script_6_0 + UnicodeProperty::Age_V193
       ].flatten,
 
       :nonproperty => [
-        UnicodeProperty::Script_6_0 + UnicodeProperty::Age
+        UnicodeProperty::Script_6_0 + UnicodeProperty::Age_V193
       ].flatten,
     },
 

data/test/syntax/ruby/test_2.0.0.rb

@@ -0,0 +1,38 @@
+require File.expand_path("../../../helpers", __FILE__)
+
+class TestSyntaxRuby_V200 < Test::Unit::TestCase
+  include Regexp::Syntax::Token
+
+  def setup
+    @syntax = Regexp::Syntax.new 'ruby/2.0.0'
+  end
+
+  tests = {
+    :implements => {
+      :property => [
+        UnicodeProperty::Age_V200
+      ].flatten,
+
+      :nonproperty => [
+        UnicodeProperty::Age_V200
+      ].flatten,
+    },
+
+    :excludes => {
+    },
+  }
+
+  tests.each do |method, types|
+    types.each do |type, tokens|
+      tokens.each do |token|
+        define_method "test_syntax_ruby_v200_#{method}_#{type}_#{token}" do
+          assert_equal(
+            method == :excludes ? false : true,
+            @syntax.implements?(type, token)
+          )
+        end
+      end
+    end
+  end
+
+end