regexp_parser 0.1.0

data/test/scanner/test_quantifiers.rb

@@ -0,0 +1,38 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerQuantifiers < Test::Unit::TestCase
+
+  tests = {
+   'a?'     => [:quantifier,  :zero_or_one,             '?'],
+   'a??'    => [:quantifier,  :zero_or_one_reluctant,   '??'],
+   'a?+'    => [:quantifier,  :zero_or_one_possessive,  '?+'],
+
+   'a*'     => [:quantifier,  :zero_or_more,            '*'],
+   'a*?'    => [:quantifier,  :zero_or_more_reluctant,  '*?'],
+   'a*+'    => [:quantifier,  :zero_or_more_possessive, '*+'],
+
+   'a+'     => [:quantifier,  :one_or_more,             '+'],
+   'a+?'    => [:quantifier,  :one_or_more_reluctant,   '+?'],
+   'a++'    => [:quantifier,  :one_or_more_possessive,  '++'],
+
+   'a{2}'   => [:quantifier,  :interval,                '{2}'],
+   'a{2,}'  => [:quantifier,  :interval,                '{2,}'],
+   'a{,2}'  => [:quantifier,  :interval,                '{,2}'],
+   'a{2,4}' => [:quantifier,  :interval,                '{2,4}'],
+  }
+
+  counter = 0
+  tests.each do |pattern, test|
+    name = (test[1] == :interval ? "interval_#{counter += 1}" : test[1])
+
+    [:type, :token, :text].each_with_index do |member, i|
+      define_method "test_scan_#{test[0]}_#{name}_#{member}" do
+
+        token = RS.scan(pattern).last
+        assert_equal( test[i], token[i] )
+
+      end
+    end
+  end
+
+end

data/test/scanner/test_refcalls.rb

@@ -0,0 +1,45 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerRefCalls < Test::Unit::TestCase
+
+  tests = {
+    # Group back-references, named, numbered, and relative
+    '(?<X>abc)\k<X>'  => [3, :backref, :name_ref_ab,        '\k<X>',      9, 14],
+    "(?<X>abc)\\k'X'" => [3, :backref, :name_ref_sq,        "\\k'X'",     9, 14],
+
+    '(abc)\k<1>'      => [3, :backref, :number_ref_ab,      '\k<1>',      5, 10],
+    "(abc)\\k'1'"     => [3, :backref, :number_ref_sq,      "\\k'1'",     5, 10],
+
+    '(abc)\k<-1>'     => [3, :backref, :number_rel_ref_ab,  '\k<-1>',     5, 11],
+    "(abc)\\k'-1'"    => [3, :backref, :number_rel_ref_sq,  "\\k'-1'",    5, 11],
+
+    # Sub-expression invocation, named, numbered, and relative
+    '(?<X>abc)\g<X>'  => [3, :backref, :name_call_ab,        '\g<X>',      9, 14],
+    "(?<X>abc)\\g'X'" => [3, :backref, :name_call_sq,        "\\g'X'",     9, 14],
+
+    '(abc)\g<1>'      => [3, :backref, :number_call_ab,      '\g<1>',      5, 10],
+    "(abc)\\g'1'"     => [3, :backref, :number_call_sq,      "\\g'1'",     5, 10],
+
+    '(abc)\g<-1>'     => [3, :backref, :number_rel_call_ab,  '\g<-1>',     5, 11],
+    "(abc)\\g'-1'"    => [3, :backref, :number_rel_call_sq,  "\\g'-1'",    5, 11],
+
+    # Group back-references, with nesting level
+    '(?<X>abc)\k<X-0>'  => [3, :backref, :name_nest_ref_ab,        '\k<X-0>',      9, 16],
+    "(?<X>abc)\\k'X-0'" => [3, :backref, :name_nest_ref_sq,        "\\k'X-0'",     9, 16],
+
+    '(abc)\k<1-0>'      => [3, :backref, :number_nest_ref_ab,      '\k<1-0>',      5, 12],
+    "(abc)\\k'1-0'"     => [3, :backref, :number_nest_ref_sq,      "\\k'1-0'",     5, 12],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_scan_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      assert_equal( test[1,5], tokens[test[0]])
+      assert_equal( test[3],   pattern[tokens[test[0]][3], tokens[test[0]][4]])
+
+    end
+  end
+
+end

data/test/scanner/test_scripts.rb

@@ -0,0 +1,314 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerUnicodeScripts < Test::Unit::TestCase
+
+  tests = {
+    'Arab'		                => :script_arabic,
+    'Arabic'		              => :script_arabic,
+
+    'Armi'		                => :script_imperial_aramaic,
+    'Imperial Aramaic'        => :script_imperial_aramaic,
+
+    'Armn'                    => :script_armenian,
+    'Armenian'                => :script_armenian,
+
+    'Avst'                    => :script_avestan,
+    'Avestan'                 => :script_avestan,
+
+    'Bali'                    => :script_balinese,
+    'Balinese'                => :script_balinese,
+
+    'Bamu'                    => :script_bamum,
+	  'Bamum'                   => :script_bamum,
+
+    'Batk'                    => :script_batak,
+	  'Batak'                   => :script_batak,
+
+    'Beng'                    => :script_bengali,
+	  'Bengali'                 => :script_bengali,
+
+    'Bopo'                    => :script_bopomofo,
+	  'Bopomofo'                => :script_bopomofo,
+
+    'Brah'                    => :script_brahmi,
+	  'Brahmi'                  => :script_brahmi,
+
+    'Brai'                    => :script_braille,
+	  'Braille'                 => :script_braille,
+
+    'Bugi'                    => :script_buginese,
+	  'Buginese'                => :script_buginese,
+
+    'Buhd'                    => :script_buhid,
+	  'Buhid'                   => :script_buhid,
+
+    'Cans'                    => :script_canadian_aboriginal,
+	  'Canadian Aboriginal'     => :script_canadian_aboriginal,
+
+    'Cari'                    => :script_carian,
+	  'Carian'                  => :script_carian,
+
+    'Cham'                    => :script_cham,
+
+    'Cher'                    => :script_cherokee,
+	  'Cherokee'                => :script_cherokee,
+
+    'Copt'                    => :script_coptic,
+	  'Coptic'                  => :script_coptic,
+	  'Qaac'                    => :script_coptic,
+
+    'Cprt'                    => :script_cypriot,
+	  'Cypriot'                 => :script_cypriot,
+
+    'Cyrl'                    => :script_cyrillic,
+	  'Cyrillic'                => :script_cyrillic,
+
+    'Deva'                    => :script_devanagari,
+	  'Devanagari'              => :script_devanagari,
+
+    'Dsrt'                    => :script_deseret,
+	  'Deseret'                 => :script_deseret,
+
+    'Egyp'                    => :script_egyptian_hieroglyphs,
+	  'Egyptian Hieroglyphs'    => :script_egyptian_hieroglyphs,
+
+    'Ethi'                    => :script_ethiopic,
+	  'Ethiopic'                => :script_ethiopic,
+
+    'Geor'                    => :script_georgian,
+	  'Georgian'                => :script_georgian,
+
+    'Glag'                    => :script_glagolitic,
+	  'Glagolitic'              => :script_glagolitic,
+
+    'Goth'                    => :script_gothic,
+	  'Gothic'                  => :script_gothic,
+
+    'Grek'                    => :script_greek,
+	  'Greek'                   => :script_greek,
+
+    'Gujr'                    => :script_gujarati,
+	  'Gujarati'                => :script_gujarati,
+
+    'Guru'                    => :script_gurmukhi,
+	  'Gurmukhi'                => :script_gurmukhi,
+
+    'Hang'                    => :script_hangul,
+	  'Hangul'                  => :script_hangul,
+
+    'Hani'                    => :script_han,
+	  'Han'                     => :script_han,
+
+    'Hano'                    => :script_hanunoo,
+	  'Hanunoo'                 => :script_hanunoo,
+
+    'Hebr'                    => :script_hebrew,
+	  'Hebrew'                  => :script_hebrew,
+
+    'Hira'                    => :script_hiragana,
+	  'Hiragana'                => :script_hiragana,
+
+    'Hrkt'                    => :script_katakana_or_hiragana,
+	  'Katakana or Hiragana'    => :script_katakana_or_hiragana,
+
+    'Ital'                    => :script_old_italic,
+	  'Old Italic'              => :script_old_italic,
+
+    'Java'                    => :script_javanese,
+	  'Javanese'                => :script_javanese,
+
+    'Kali'                    => :script_kayah_li,
+	  'Kayah Li'                => :script_kayah_li,
+
+    'Kana'                    => :script_katakana,
+	  'Katakana'                => :script_katakana,
+
+    'Khar'                    => :script_kharoshthi,
+	  'Kharoshthi'              => :script_kharoshthi,
+
+    'Khmr'                    => :script_khmer,
+	  'Khmer'                   => :script_khmer,
+
+    'Knda'                    => :script_kannada,
+	  'Kannada'                 => :script_kannada,
+
+    'Kthi'                    => :script_kaithi,
+	  'Kaithi'                  => :script_kaithi,
+
+    'Lana'                    => :script_tai_tham,
+	  'Tai Tham'                => :script_tai_tham,
+
+    'Laoo'                    => :script_lao,
+	  'Lao'                     => :script_lao,
+
+    'Latn'                    => :script_latin,
+	  'Latin'                   => :script_latin,
+
+    'Lepc'                    => :script_lepcha,
+	  'Lepcha'                  => :script_lepcha,
+
+    'Limb'                    => :script_limbu,
+	  'Limbu'                   => :script_limbu,
+
+    'Linb'                    => :script_linear_b,
+	  'Linear B'                => :script_linear_b,
+
+    'Lisu'                    => :script_lisu,
+
+    'Lyci'                    => :script_lycian,
+	  'Lycian'                  => :script_lycian,
+
+    'Lydi'                    => :script_lydian,
+	  'Lydian'                  => :script_lydian,
+
+    'Mand'                    => :script_mandaic,
+	  'Mandaic'                 => :script_mandaic,
+
+    'Mlym'                    => :script_malayalam,
+	  'Malayalam'               => :script_malayalam,
+
+    'Mong'                    => :script_mongolian,
+	  'Mongolian'               => :script_mongolian,
+
+    'Mtei'                    => :script_meetei_mayek,
+	  'Meetei Mayek'            => :script_meetei_mayek,
+
+    'Mymr'                    => :script_myanmar,
+	  'Myanmar'                 => :script_myanmar,
+
+    'Nkoo'                    => :script_nko,
+	  'Nko'                     => :script_nko,
+
+    'Ogam'                    => :script_ogham,
+	  'Ogham'                   => :script_ogham,
+
+    'Olck'                    => :script_ol_chiki,
+	  'Ol Chiki'                => :script_ol_chiki,
+
+    'Orkh'                    => :script_old_turkic,
+	  'Old Turkic'              => :script_old_turkic,
+
+    'Orya'                    => :script_oriya,
+	  'Oriya'                   => :script_oriya,
+
+    'Osma'                    => :script_osmanya,
+	  'Osmanya'                 => :script_osmanya,
+
+    'Phag'                    => :script_phags_pa,
+	  'Phags Pa'                => :script_phags_pa,
+
+    'Phli'                    => :script_inscriptional_pahlavi,
+	  'Inscriptional Pahlavi'   => :script_inscriptional_pahlavi,
+
+    'Phnx'		                => :script_phoenician,
+	  'Phoenician'		          => :script_phoenician,
+
+    'Prti'		                => :script_inscriptional_parthian,
+	  'Inscriptional Parthian'  => :script_inscriptional_parthian,
+
+    'Rjng'                    => :script_rejang,
+	  'Rejang'                  => :script_rejang,
+
+    'Runr'                    => :script_runic,
+	  'Runic'                   => :script_runic,
+
+    'Samr'                    => :script_samaritan,
+	  'Samaritan'               => :script_samaritan,
+
+    'Sarb'                    => :script_old_south_arabian,
+	  'Old South Arabian'       => :script_old_south_arabian,
+
+    'Saur'                    => :script_saurashtra,
+	  'Saurashtra'              => :script_saurashtra,
+
+    'Shaw'                    => :script_shavian,
+	  'Shavian'                 => :script_shavian,
+
+    'Sinh'                    => :script_sinhala,
+	  'Sinhala'                 => :script_sinhala,
+
+    'Sund'                    => :script_sundanese,
+	  'Sundanese'               => :script_sundanese,
+
+    'Sylo'                    => :script_syloti_nagri,
+	  'Syloti Nagri'            => :script_syloti_nagri,
+
+    'Syrc'                    => :script_syriac,
+	  'Syriac'                  => :script_syriac,
+
+    'Tagb'                    => :script_tagbanwa,
+	  'Tagbanwa'                => :script_tagbanwa,
+
+    'Tale'                    => :script_tai_le,
+	  'Tai Le'                  => :script_tai_le,
+
+    'Talu'                    => :script_new_tai_lue,
+	  'New Tai Lue'             => :script_new_tai_lue,
+
+    'Taml'                    => :script_tamil,
+	  'Tamil'                   => :script_tamil,
+
+    'Tavt'                    => :script_tai_viet,
+	  'Tai Viet'                => :script_tai_viet,
+
+    'Telu'                    => :script_telugu,
+	  'Telugu'                  => :script_telugu,
+
+    'Tfng'                    => :script_tifinagh,
+	  'Tifinagh'                => :script_tifinagh,
+
+    'Tglg'                    => :script_tagalog,
+	  'Tagalog'                 => :script_tagalog,
+
+    'Thaa'                    => :script_thaana,
+	  'Thaana'                  => :script_thaana,
+
+    'Thai'                    => :script_thai, 
+
+    'Tibt'                    => :script_tibetan,
+	  'Tibetan'                 => :script_tibetan,
+
+    'Ugar'                    => :script_ugaritic,
+	  'Ugaritic'                => :script_ugaritic,
+
+    'Vaii'                    => :script_vai,
+	  'Vai'                     => :script_vai,
+
+    'Xpeo'                    => :script_old_persian,
+	  'Old Persian'             => :script_old_persian,
+
+    'Xsux'                    => :script_cuneiform,
+	  'Cuneiform'               => :script_cuneiform,
+
+    'Yiii'                    => :script_yi,
+	  'Yi'                      => :script_yi,
+
+    'Zinh'                    => :script_inherited,
+	  'Inherited'               => :script_inherited,
+	  'Qaai'                    => :script_inherited,
+
+    'Zyyy'                    => :script_common,
+	  'Common'                  => :script_common,
+
+    'Zzzz'                    => :script_unknown,
+	  'Unknown'                 => :script_unknown,
+  }
+
+  count = 0
+  tests.each do |property, test|
+    define_method "test_scan_property_#{test}_#{count+=1}" do
+      token = RS.scan("a\\p{#{property}}c")[1]
+
+      assert_equal( :property,  token[0] )
+      assert_equal( test,       token[1] )
+    end
+
+    define_method "test_scan_nonproperty_#{test}_#{count+=1}" do
+      token = RS.scan("a\\P{#{property}}c")[1]
+
+      assert_equal( :nonproperty,   token[0] )
+      assert_equal( test,           token[1] )
+    end
+  end
+
+end

data/test/scanner/test_sets.rb

@@ -0,0 +1,80 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerSets < Test::Unit::TestCase
+
+  tests = {
+    '[a]'                   => [0, :set,    :open,            '[',          0, 1],
+    '[b]'                   => [2, :set,    :close,           ']',          2, 3],
+    '[^n]'                  => [1, :set,    :negate,          '^',          1, 2],
+
+    '[c]'                   => [1, :set,    :member,          'c',          1, 2],
+    '[\b]'                  => [1, :set,    :backspace,       '\b',         1, 3],
+
+    '[.]'                   => [1, :set,    :member,          '.',          1, 2],
+    '[?]'                   => [1, :set,    :member,          '?',          1, 2],
+    '[*]'                   => [1, :set,    :member,          '*',          1, 2],
+    '[+]'                   => [1, :set,    :member,          '+',          1, 2],
+    '[{]'                   => [1, :set,    :member,          '{',          1, 2],
+    '[}]'                   => [1, :set,    :member,          '}',          1, 2],
+    '[<]'                   => [1, :set,    :member,          '<',          1, 2],
+    '[>]'                   => [1, :set,    :member,          '>',          1, 2],
+
+    '[\.]'                  => [1, :set,    :escape,          '\.',         1, 3],
+    '[\!]'                  => [1, :set,    :escape,          '\!',         1, 3],
+    '[\#]'                  => [1, :set,    :escape,          '\#',         1, 3],
+    '[\]]'                  => [1, :set,    :escape,          '\]',         1, 3],
+    '[\\\]'                 => [1, :set,    :escape,          '\\\\',       1, 3],
+    '[a\-c]'                => [2, :set,    :escape,          '\-',         2, 4],
+
+    '[\d]'                  => [1, :set,    :type_digit,      '\d',         1, 3],
+    '[\D]'                  => [1, :set,    :type_nondigit,   '\D',         1, 3],
+
+    '[\h]'                  => [1, :set,    :type_hex,        '\h',         1, 3],
+    '[\H]'                  => [1, :set,    :type_nonhex,     '\H',         1, 3],
+
+    '[\s]'                  => [1, :set,    :type_space,      '\s',         1, 3],
+    '[\S]'                  => [1, :set,    :type_nonspace,   '\S',         1, 3],
+
+    '[\w]'                  => [1, :set,    :type_word,       '\w',         1, 3],
+    '[\W]'                  => [1, :set,    :type_nonword,    '\W',         1, 3],
+
+    '[a-c]'                 => [1, :set,    :range,           'a-c',        1, 4],
+    '[a-c-]'                => [2, :set,    :member,          '-',          4, 6],
+    '[a-c^]'                => [2, :set,    :member,          '^',          4, 5],
+    '[a-cd-f]'              => [2, :set,    :range,           'd-f',        4, 7],
+
+    '[a[:digit:]c]'         => [2, :set,    :class_digit,     '[:digit:]',  2, 11],
+    '[[:digit:][:space:]]'  => [2, :set,    :class_space,     '[:space:]', 10, 19],
+    '[[:^digit:]]'          => [1, :set,    :class_nondigit,  '[:^digit:]', 1, 11],
+
+    '[a[.a-b.]c]'           => [2, :set,    :collation,       '[.a-b.]',    2,  9],
+    '[a[=e=]c]'             => [2, :set,    :equivalent,      '[=e=]',      2,  7],
+
+    '[a-d&&g-h]'            => [2, :set,    :intersection,    '&&',         4, 6],
+
+    '[\\x20-\\x28]'         => [1, :set,    :range_hex,       '\x20-\x28',  1, 10],
+
+    '[a\p{digit}c]'         => [2, :set,    :digit,           '\p{digit}',  2, 11],
+    '[a\P{digit}c]'         => [2, :set,    :digit,           '\P{digit}',  2, 11],
+
+    '[a\p{ALPHA}c]'         => [2, :set,    :alpha,           '\p{ALPHA}',  2, 11],
+    '[a\p{P}c]'             => [2, :set,    :punct_any,       '\p{P}',      2, 7],
+    '[a\p{P}\P{Z}c]'        => [3, :set,    :separator_any,   '\P{Z}',      7, 12],
+
+    '[a-w&&[^c-g]z]'        => [3, :subset, :open,            '[',          6, 7],
+    '[a-w&&[^c-h]z]'        => [4, :subset, :negate,          '^',          7, 8],
+    '[a-w&&[^c-i]z]'        => [5, :subset, :range,           'c-i',        8, 11],
+    '[a-w&&[^c-j]z]'        => [6, :subset, :close,           ']',          11, 12],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_scan_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      assert_equal( test[1,5], tokens[test[0]] )
+
+    end
+  end
+
+end