regexp_parser 0.5.0 → 1.0.0

data/test/scanner/test_scripts.rb

@@ -3,365 +3,33 @@ require File.expand_path("../../helpers", __FILE__)
 class ScannerUnicodeScripts < Test::Unit::TestCase
 
   tests = {
-    'Aghb'		                => :script_caucasian_albanian,
-    'Caucasian Albanian'		  => :script_caucasian_albanian,
+    'Aghb'	                  => :caucasian_albanian,
+    'Caucasian Albanian'	    => :caucasian_albanian,
 
-    'Arab'		                => :script_arabic,
-    'Arabic'		              => :script_arabic,
+    'Arab'	                  => :arabic,
+    'Arabic'	                => :arabic,
 
-    'Armi'		                => :script_imperial_aramaic,
-    'Imperial Aramaic'        => :script_imperial_aramaic,
+    'Armi'	                  => :imperial_aramaic,
+    'Imperial Aramaic'        => :imperial_aramaic,
 
-    'Armn'                    => :script_armenian,
-    'Armenian'                => :script_armenian,
+    'Egyp'                    => :egyptian_hieroglyphs,
+    'Egyptian Hieroglyphs'    => :egyptian_hieroglyphs, # test whitespace
 
-    'Avst'                    => :script_avestan,
-    'Avestan'                 => :script_avestan,
+    'Linb'                    => :linear_b,
+    'Linear-B'                => :linear_b, # test dash
 
-    'Bali'                    => :script_balinese,
-    'Balinese'                => :script_balinese,
+    'Yiii'                    => :yi,
+    'Yi'                      => :yi,
 
-    'Bamu'                    => :script_bamum,
-	  'Bamum'                   => :script_bamum,
+    'Zinh'                    => :inherited,
+    'Inherited'               => :inherited,
+    'Qaai'                    => :inherited,
 
-    'Bass'                    => :script_bassa_vah,
-	  'Bassa Vah'               => :script_bassa_vah,
+    'Zyyy'                    => :common,
+    'Common'                  => :common,
 
-    'Batk'                    => :script_batak,
-	  'Batak'                   => :script_batak,
-
-    'Beng'                    => :script_bengali,
-	  'Bengali'                 => :script_bengali,
-
-    'Bopo'                    => :script_bopomofo,
-	  'Bopomofo'                => :script_bopomofo,
-
-    'Brah'                    => :script_brahmi,
-	  'Brahmi'                  => :script_brahmi,
-
-    'Brai'                    => :script_braille,
-	  'Braille'                 => :script_braille,
-
-    'Bugi'                    => :script_buginese,
-	  'Buginese'                => :script_buginese,
-
-    'Buhd'                    => :script_buhid,
-	  'Buhid'                   => :script_buhid,
-
-    'Cans'                    => :script_canadian_aboriginal,
-	  'Canadian Aboriginal'     => :script_canadian_aboriginal,
-
-    'Cari'                    => :script_carian,
-	  'Carian'                  => :script_carian,
-
-    'Cham'                    => :script_cham,
-
-    'Cher'                    => :script_cherokee,
-	  'Cherokee'                => :script_cherokee,
-
-    'Copt'                    => :script_coptic,
-	  'Coptic'                  => :script_coptic,
-	  'Qaac'                    => :script_coptic,
-
-    'Cprt'                    => :script_cypriot,
-	  'Cypriot'                 => :script_cypriot,
-
-    'Cyrl'                    => :script_cyrillic,
-	  'Cyrillic'                => :script_cyrillic,
-
-    'Deva'                    => :script_devanagari,
-	  'Devanagari'              => :script_devanagari,
-
-    'Dsrt'                    => :script_deseret,
-	  'Deseret'                 => :script_deseret,
-
-
-    'Dupl'                    => :script_duployan,
-	  'Duployan'                => :script_duployan,
-
-
-    'Egyp'                    => :script_egyptian_hieroglyphs,
-	  'Egyptian Hieroglyphs'    => :script_egyptian_hieroglyphs,
-
-    'Elba'                    => :script_elbasan,
-	  'Elbasan'                 => :script_elbasan,
-
-    'Ethi'                    => :script_ethiopic,
-	  'Ethiopic'                => :script_ethiopic,
-
-    'Geor'                    => :script_georgian,
-	  'Georgian'                => :script_georgian,
-
-    'Glag'                    => :script_glagolitic,
-	  'Glagolitic'              => :script_glagolitic,
-
-    'Goth'                    => :script_gothic,
-	  'Gothic'                  => :script_gothic,
-
-    'Gran'                    => :script_grantha,
-	  'Grantha'                 => :script_grantha,
-
-    'Grek'                    => :script_greek,
-	  'Greek'                   => :script_greek,
-
-    'Gujr'                    => :script_gujarati,
-	  'Gujarati'                => :script_gujarati,
-
-    'Guru'                    => :script_gurmukhi,
-	  'Gurmukhi'                => :script_gurmukhi,
-
-    'Hang'                    => :script_hangul,
-	  'Hangul'                  => :script_hangul,
-
-    'Hani'                    => :script_han,
-	  'Han'                     => :script_han,
-
-    'Hano'                    => :script_hanunoo,
-	  'Hanunoo'                 => :script_hanunoo,
-
-    'Hebr'                    => :script_hebrew,
-	  'Hebrew'                  => :script_hebrew,
-
-    'Hira'                    => :script_hiragana,
-	  'Hiragana'                => :script_hiragana,
-
-    'Hmng'                    => :script_pahawh_hmong,
-	  'Pahawh Hmong'            => :script_pahawh_hmong,
-
-    'Hrkt'                    => :script_katakana_or_hiragana,
-	  'Katakana or Hiragana'    => :script_katakana_or_hiragana,
-
-    'Ital'                    => :script_old_italic,
-	  'Old Italic'              => :script_old_italic,
-
-    'Java'                    => :script_javanese,
-	  'Javanese'                => :script_javanese,
-
-    'Kali'                    => :script_kayah_li,
-	  'Kayah Li'                => :script_kayah_li,
-
-    'Kana'                    => :script_katakana,
-	  'Katakana'                => :script_katakana,
-
-    'Khar'                    => :script_kharoshthi,
-	  'Kharoshthi'              => :script_kharoshthi,
-
-    'Khmr'                    => :script_khmer,
-	  'Khmer'                   => :script_khmer,
-
-    'Khoj'                    => :script_khojki,
-	  'Khojki'                  => :script_khojki,
-
-    'Knda'                    => :script_kannada,
-	  'Kannada'                 => :script_kannada,
-
-    'Kthi'                    => :script_kaithi,
-	  'Kaithi'                  => :script_kaithi,
-
-    'Lana'                    => :script_tai_tham,
-	  'Tai Tham'                => :script_tai_tham,
-
-    'Laoo'                    => :script_lao,
-	  'Lao'                     => :script_lao,
-
-    'Latn'                    => :script_latin,
-	  'Latin'                   => :script_latin,
-
-    'Lepc'                    => :script_lepcha,
-	  'Lepcha'                  => :script_lepcha,
-
-    'Limb'                    => :script_limbu,
-	  'Limbu'                   => :script_limbu,
-
-    'Lina'                    => :script_linear_a,
-	  'Linear A'                => :script_linear_a,
-
-    'Linb'                    => :script_linear_b,
-	  'Linear B'                => :script_linear_b,
-
-    'Lisu'                    => :script_lisu,
-
-    'Lyci'                    => :script_lycian,
-	  'Lycian'                  => :script_lycian,
-
-    'Lydi'                    => :script_lydian,
-	  'Lydian'                  => :script_lydian,
-
-    'Mand'                    => :script_mandaic,
-	  'Mandaic'                 => :script_mandaic,
-
-    'Mlym'                    => :script_malayalam,
-	  'Malayalam'               => :script_malayalam,
-
-    'Mahj'                    => :script_mahajani,
-	  'Mahajani'                => :script_mahajani,
-
-    'Mani'                    => :script_manichaean,
-	  'Manichaean'              => :script_manichaean,
-
-    'Mend'                    => :script_mende_kikakui,
-	  'Mende Kikakui'           => :script_mende_kikakui,
-
-    'Modi'                    => :script_modi,
-
-    'Mong'                    => :script_mongolian,
-	  'Mongolian'               => :script_mongolian,
-
-    'Mroo'                    => :script_mro,
-	  'Mro'                     => :script_mro,
-
-    'Mtei'                    => :script_meetei_mayek,
-	  'Meetei Mayek'            => :script_meetei_mayek,
-
-    'Mymr'                    => :script_myanmar,
-	  'Myanmar'                 => :script_myanmar,
-
-    'Narb'                    => :script_old_north_arabian,
-	  'Old North Arabian'       => :script_old_north_arabian,
-
-    'Nbat'                    => :script_nabataean,
-	  'Nabataean'               => :script_nabataean,
-
-    'Nkoo'                    => :script_nko,
-	  'Nko'                     => :script_nko,
-
-    'Ogam'                    => :script_ogham,
-	  'Ogham'                   => :script_ogham,
-
-    'Olck'                    => :script_ol_chiki,
-	  'Ol Chiki'                => :script_ol_chiki,
-
-    'Orkh'                    => :script_old_turkic,
-	  'Old Turkic'              => :script_old_turkic,
-
-    'Orya'                    => :script_oriya,
-	  'Oriya'                   => :script_oriya,
-
-    'Osma'                    => :script_osmanya,
-	  'Osmanya'                 => :script_osmanya,
-
-    'Palm'                    => :script_palmyrene,
-	  'Palmyrene'               => :script_palmyrene,
-
-    'Pauc'                    => :script_pau_cin_hau,
-	  'Pau Cin Hau'             => :script_pau_cin_hau,
-
-    'Perm'                    => :script_old_permic,
-	  'Old Permic'              => :script_old_permic,
-
-    'Phag'                    => :script_phags_pa,
-	  'Phags Pa'                => :script_phags_pa,
-
-    'Phli'                    => :script_inscriptional_pahlavi,
-	  'Inscriptional Pahlavi'   => :script_inscriptional_pahlavi,
-
-    'Phlp'                    => :script_psalter_pahlavi,
-	  'Psalter Pahlavi'         => :script_psalter_pahlavi,
-
-    'Phnx'		                => :script_phoenician,
-	  'Phoenician'		          => :script_phoenician,
-
-    'Prti'		                => :script_inscriptional_parthian,
-	  'Inscriptional Parthian'  => :script_inscriptional_parthian,
-
-    'Rjng'                    => :script_rejang,
-	  'Rejang'                  => :script_rejang,
-
-    'Runr'                    => :script_runic,
-	  'Runic'                   => :script_runic,
-
-    'Samr'                    => :script_samaritan,
-	  'Samaritan'               => :script_samaritan,
-
-    'Sarb'                    => :script_old_south_arabian,
-	  'Old South Arabian'       => :script_old_south_arabian,
-
-    'Saur'                    => :script_saurashtra,
-	  'Saurashtra'              => :script_saurashtra,
-
-    'Shaw'                    => :script_shavian,
-	  'Shavian'                 => :script_shavian,
-
-    'Sidd'                    => :script_siddham,
-	  'Siddham'                 => :script_siddham,
-
-    'Sind'                    => :script_khudawadi,
-	  'Khudawadi'               => :script_khudawadi,
-
-    'Sinh'                    => :script_sinhala,
-	  'Sinhala'                 => :script_sinhala,
-
-    'Sund'                    => :script_sundanese,
-	  'Sundanese'               => :script_sundanese,
-
-    'Sylo'                    => :script_syloti_nagri,
-	  'Syloti Nagri'            => :script_syloti_nagri,
-
-    'Syrc'                    => :script_syriac,
-	  'Syriac'                  => :script_syriac,
-
-    'Tagb'                    => :script_tagbanwa,
-	  'Tagbanwa'                => :script_tagbanwa,
-
-    'Tale'                    => :script_tai_le,
-	  'Tai Le'                  => :script_tai_le,
-
-    'Talu'                    => :script_new_tai_lue,
-	  'New Tai Lue'             => :script_new_tai_lue,
-
-    'Taml'                    => :script_tamil,
-	  'Tamil'                   => :script_tamil,
-
-    'Tavt'                    => :script_tai_viet,
-	  'Tai Viet'                => :script_tai_viet,
-
-    'Telu'                    => :script_telugu,
-	  'Telugu'                  => :script_telugu,
-
-    'Tfng'                    => :script_tifinagh,
-	  'Tifinagh'                => :script_tifinagh,
-
-    'Tglg'                    => :script_tagalog,
-	  'Tagalog'                 => :script_tagalog,
-
-    'Thaa'                    => :script_thaana,
-	  'Thaana'                  => :script_thaana,
-
-    'Thai'                    => :script_thai,
-
-    'Tibt'                    => :script_tibetan,
-	  'Tibetan'                 => :script_tibetan,
-
-    'Tirh'                    => :script_tirhuta,
-	  'Tirhuta'                 => :script_tirhuta,
-
-    'Ugar'                    => :script_ugaritic,
-	  'Ugaritic'                => :script_ugaritic,
-
-    'Vaii'                    => :script_vai,
-	  'Vai'                     => :script_vai,
-
-    'Wara'                    => :script_warang_citi,
-	  'Warang Citi'             => :script_warang_citi,
-
-    'Xpeo'                    => :script_old_persian,
-	  'Old Persian'             => :script_old_persian,
-
-    'Xsux'                    => :script_cuneiform,
-	  'Cuneiform'               => :script_cuneiform,
-
-    'Yiii'                    => :script_yi,
-	  'Yi'                      => :script_yi,
-
-    'Zinh'                    => :script_inherited,
-	  'Inherited'               => :script_inherited,
-	  'Qaai'                    => :script_inherited,
-
-    'Zyyy'                    => :script_common,
-	  'Common'                  => :script_common,
-
-    'Zzzz'                    => :script_unknown,
-	  'Unknown'                 => :script_unknown,
+    'Zzzz'                    => :unknown,
+    'Unknown'                 => :unknown,
   }
 
   tests.each_with_index do |(property, token), count|

data/test/scanner/test_sets.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path("../../helpers", __FILE__)
 
 class ScannerSets < Test::Unit::TestCase
@@ -7,77 +9,93 @@ class ScannerSets < Test::Unit::TestCase
     '[b]'                   => [2, :set,    :close,           ']',          2, 3],
     '[^n]'                  => [1, :set,    :negate,          '^',          1, 2],
 
-    '[c]'                   => [1, :set,    :member,          'c',          1, 2],
-    '[\b]'                  => [1, :set,    :backspace,       '\b',         1, 3],
-    '[A\bX]'                => [2, :set,    :backspace,       '\b',         2, 4],
-
-    '[.]'                   => [1, :set,    :member,          '.',          1, 2],
-    '[?]'                   => [1, :set,    :member,          '?',          1, 2],
-    '[*]'                   => [1, :set,    :member,          '*',          1, 2],
-    '[+]'                   => [1, :set,    :member,          '+',          1, 2],
-    '[{]'                   => [1, :set,    :member,          '{',          1, 2],
-    '[}]'                   => [1, :set,    :member,          '}',          1, 2],
-    '[<]'                   => [1, :set,    :member,          '<',          1, 2],
-    '[>]'                   => [1, :set,    :member,          '>',          1, 2],
-
-    '[\x20]'                => [1, :set,    :member_hex,      '\x20',       1, 5],
-
-    '[\.]'                  => [1, :set,    :escape,          '\.',         1, 3],
-    '[\!]'                  => [1, :set,    :escape,          '\!',         1, 3],
-    '[\#]'                  => [1, :set,    :escape,          '\#',         1, 3],
-    '[\]]'                  => [1, :set,    :escape,          '\]',         1, 3],
-    '[\\\]'                 => [1, :set,    :escape,          '\\\\',       1, 3],
-    '[a\-c]'                => [2, :set,    :escape,          '\-',         2, 4],
-
-    '[\d]'                  => [1, :set,    :type_digit,      '\d',         1, 3],
-    '[\da-z]'               => [1, :set,    :type_digit,      '\d',         1, 3],
-    '[\D]'                  => [1, :set,    :type_nondigit,   '\D',         1, 3],
-
-    '[\h]'                  => [1, :set,    :type_hex,        '\h',         1, 3],
-    '[\H]'                  => [1, :set,    :type_nonhex,     '\H',         1, 3],
-
-    '[\s]'                  => [1, :set,    :type_space,      '\s',         1, 3],
-    '[\S]'                  => [1, :set,    :type_nonspace,   '\S',         1, 3],
-
-    '[\w]'                  => [1, :set,    :type_word,       '\w',         1, 3],
-    '[\W]'                  => [1, :set,    :type_nonword,    '\W',         1, 3],
-
-    '[\R]'                  => [1, :set,    :type_linebreak,  '\R',         1, 3],
-    '[\X]'                  => [1, :set,    :type_xgrapheme,  '\X',         1, 3],
-
-    '[a-c]'                 => [1, :set,    :range,           'a-c',        1, 4],
-    '[a-c-]'                => [2, :set,    :member,          '-',          4, 6],
-    '[a-c^]'                => [2, :set,    :member,          '^',          4, 5],
-    '[a-cd-f]'              => [2, :set,    :range,           'd-f',        4, 7],
-
-    '[a[:digit:]c]'         => [2, :set,    :class_digit,     '[:digit:]',  2, 11],
-    '[[:digit:][:space:]]'  => [2, :set,    :class_space,     '[:space:]', 10, 19],
-    '[[:^digit:]]'          => [1, :set,    :class_nondigit,  '[:^digit:]', 1, 11],
+    '[c]'                   => [1, :literal, :literal,        'c',          1, 2],
+    '[\b]'                  => [1, :escape,  :backspace,      '\b',         1, 3],
+    '[A\bX]'                => [2, :escape,  :backspace,      '\b',         2, 4],
+
+    '[.]'                   => [1, :literal, :literal,        '.',          1, 2],
+    '[?]'                   => [1, :literal, :literal,        '?',          1, 2],
+    '[*]'                   => [1, :literal, :literal,        '*',          1, 2],
+    '[+]'                   => [1, :literal, :literal,        '+',          1, 2],
+    '[{]'                   => [1, :literal, :literal,        '{',          1, 2],
+    '[}]'                   => [1, :literal, :literal,        '}',          1, 2],
+    '[<]'                   => [1, :literal, :literal,        '<',          1, 2],
+    '[>]'                   => [1, :literal, :literal,        '>',          1, 2],
+
+    '[äöü]'                 => [2, :literal, :literal,        'ö',          3, 5],
+
+    '[\x20]'                => [1, :escape, :hex,             '\x20',       1, 5],
+
+    '[\.]'                  => [1, :escape, :dot,             '\.',         1, 3],
+    '[\!]'                  => [1, :escape, :literal,         '\!',         1, 3],
+    '[\#]'                  => [1, :escape, :literal,         '\#',         1, 3],
+    '[\]]'                  => [1, :escape, :set_close,       '\]',         1, 3],
+    '[\\\]'                 => [1, :escape, :backslash,       '\\\\',       1, 3],
+    '[\A]'                  => [1, :escape, :literal,         '\A',         1, 3],
+    '[\z]'                  => [1, :escape, :literal,         '\z',         1, 3],
+    '[\g]'                  => [1, :escape, :literal,         '\g',         1, 3],
+    '[\K]'                  => [1, :escape, :literal,         '\K',         1, 3],
+    '[\c2]'                 => [1, :escape, :literal,         '\c',         1, 3],
+    '[\B]'                  => [1, :escape, :literal,         '\B',         1, 3],
+    '[a\-c]'                => [2, :escape, :literal,         '\-',         2, 4],
+
+    '[\d]'                  => [1, :type,   :digit,           '\d',         1, 3],
+    '[\da-z]'               => [1, :type,   :digit,           '\d',         1, 3],
+    '[\D]'                  => [1, :type,   :nondigit,        '\D',         1, 3],
+
+    '[\h]'                  => [1, :type,   :hex,             '\h',         1, 3],
+    '[\H]'                  => [1, :type,   :nonhex,          '\H',         1, 3],
+
+    '[\s]'                  => [1, :type,   :space,           '\s',         1, 3],
+    '[\S]'                  => [1, :type,   :nonspace,        '\S',         1, 3],
+
+    '[\w]'                  => [1, :type,   :word,            '\w',         1, 3],
+    '[\W]'                  => [1, :type,   :nonword,         '\W',         1, 3],
+
+    '[\R]'                  => [1, :escape, :literal,         '\R',         1, 3],
+    '[\X]'                  => [1, :escape, :literal,         '\X',         1, 3],
+
+    '[a-b]'                 => [1, :literal, :literal,        'a',          1, 2],
+    '[a-c]'                 => [2, :set,     :range,          '-',          2, 3],
+    '[a-d]'                 => [3, :literal, :literal,        'd',          3, 4],
+    '[a-b-]'                => [4, :literal, :literal,        '-',          4, 6],
+    '[-a]'                  => [1, :literal, :literal,        '-',          1, 2],
+    '[a-c^]'                => [4, :literal, :literal,        '^',          4, 5],
+    '[a-bd-f]'              => [2, :set,    :range,           '-',          2, 3],
+    '[a-cd-f]'              => [5, :set,    :range,           '-',          5, 6],
+
+    '[a[:digit:]c]'         => [2, :posixclass,    :digit,    '[:digit:]',  2, 11],
+    '[[:digit:][:space:]]'  => [2, :posixclass,    :space,    '[:space:]', 10, 19],
+    '[[:^digit:]]'          => [1, :nonposixclass, :digit,    '[:^digit:]', 1, 11],
 
     '[a[.a-b.]c]'           => [2, :set,    :collation,       '[.a-b.]',    2,  9],
     '[a[=e=]c]'             => [2, :set,    :equivalent,      '[=e=]',      2,  7],
 
-    '[a-d&&g-h]'            => [2, :set,    :intersection,    '&&',         4, 6],
+    '[a-d&&g-h]'            => [4, :set,    :intersection,    '&&',         4, 6],
+    '[a&&]'                 => [2, :set,    :intersection,    '&&',         2, 4],
+    '[&&z]'                 => [1, :set,    :intersection,    '&&',         1, 3],
 
-    '[\\x20-\\x28]'         => [1, :set,    :range_hex,       '\x20-\x28',  1, 10],
+    '[\\x20-\\x27]'         => [1, :escape, :hex,             '\x20',       1, 5],
+    '[\\x20-\\x28]'         => [2, :set,    :range,           '-',          5, 6],
+    '[\\x20-\\x29]'         => [3, :escape, :hex,             '\x29',       6, 10],
 
-    '[a\p{digit}c]'         => [2, :set,    :digit,           '\p{digit}',  2, 11],
-    '[a\P{digit}c]'         => [2, :set,    :digit,           '\P{digit}',  2, 11],
-    '[a\p{^digit}c]'        => [2, :set,    :digit,           '\p{^digit}', 2, 12],
-    '[a\P{^digit}c]'        => [2, :set,    :digit,           '\P{^digit}', 2, 12],
+    '[a\p{digit}c]'         => [2, :property,    :digit,      '\p{digit}',  2, 11],
+    '[a\P{digit}c]'         => [2, :nonproperty, :digit,      '\P{digit}',  2, 11],
+    '[a\p{^digit}c]'        => [2, :nonproperty, :digit,      '\p{^digit}', 2, 12],
+    '[a\P{^digit}c]'        => [2, :property,    :digit,      '\P{^digit}', 2, 12],
 
-    '[a\p{ALPHA}c]'         => [2, :set,    :alpha,           '\p{ALPHA}',  2, 11],
-    '[a\p{P}c]'             => [2, :set,    :punct_any,       '\p{P}',      2, 7],
-    '[a\p{P}\P{Z}c]'        => [3, :set,    :separator_any,   '\P{Z}',      7, 12],
+    '[a\p{ALPHA}c]'         => [2, :property,    :alpha,      '\p{ALPHA}',  2, 11],
+    '[a\p{P}c]'             => [2, :property,    :punctuation,'\p{P}',      2, 7],
+    '[a\p{P}\P{P}c]'        => [3, :nonproperty, :punctuation,'\P{P}',      7, 12],
 
-    '[a-w&&[^c-g]z]'        => [3, :subset, :open,            '[',          6, 7],
-    '[a-w&&[^c-h]z]'        => [4, :subset, :negate,          '^',          7, 8],
-    '[a-w&&[^c-i]z]'        => [5, :subset, :range,           'c-i',        8, 11],
-    '[a-w&&[^c-j]z]'        => [6, :subset, :close,           ']',          11, 12],
+    '[a-w&&[^c-g]z]'        => [5, :set,    :open,            '[',          6, 7],
+    '[a-w&&[^c-h]z]'        => [6, :set,    :negate,          '^',          7, 8],
+    '[a-w&&[^c-i]z]'        => [8, :set,    :range,           '-',          9, 10],
+    '[a-w&&[^c-j]z]'        => [10,:set,    :close,           ']',          11, 12],
   }
 
   tests.each_with_index do |(pattern, (index, type, token, text, ts, te)), count|
-    define_method "test_scanner_#{type}_#{token}_#{count}" do
+    define_method "test_scanner_#{type}_#{token}_in_'#{pattern}'_#{count}" do
       tokens = RS.scan(pattern)
       result = tokens.at(index)
 
@@ -89,4 +107,13 @@ class ScannerSets < Test::Unit::TestCase
     end
   end
 
+  def test_set_literal_encoding
+    text = RS.scan('[a]')[1][2].to_s
+    assert_equal 'a',     text
+    assert_equal 'UTF-8', text.encoding.to_s
+
+    text = RS.scan('[😲]')[1][2].to_s
+    assert_equal '😲',     text
+    assert_equal 'UTF-8', text.encoding.to_s
+  end
 end