regexp_parser 0.1.0

data/test/scanner/test_anchors.rb

@@ -0,0 +1,35 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerAnchors < Test::Unit::TestCase
+
+  tests = {
+   '^abc'       => [0, :anchor,     :beginning_of_line,     '^',    0, 1],
+   'abc$'       => [1, :anchor,     :end_of_line,           '$',    3, 4],
+
+   '\Aabc'      => [0, :anchor,     :bos,                   '\A',   0, 2],
+   'abc\z'      => [1, :anchor,     :eos,                   '\z',   3, 5],
+   'abc\Z'      => [1, :anchor,     :eos_ob_eol,            '\Z',   3, 5],
+
+   'a\bc'       => [1, :anchor,     :word_boundary,         '\b',   1, 3],
+   'a\Bc'       => [1, :anchor,     :nonword_boundary,      '\B',   1, 3],
+
+   'a\Gc'       => [1, :anchor,     :match_start,           '\G',   1, 3],
+
+   "\\\\Ac"     => [0, :escape,    :backslash,              '\\\\', 0, 2],
+   "a\\\\z"     => [1, :escape,    :backslash,              '\\\\', 1, 3],
+   "a\\\\Z"     => [1, :escape,    :backslash,              '\\\\', 1, 3],
+   "a\\\\bc"    => [1, :escape,    :backslash,              '\\\\', 1, 3],
+   "a\\\\Bc"    => [1, :escape,    :backslash,              '\\\\', 1, 3],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_scanner_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      assert_equal( test[1,5], tokens[test[0]] )
+
+    end
+  end
+
+end

data/test/scanner/test_errors.rb

@@ -0,0 +1,36 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerErrors < Test::Unit::TestCase
+
+  def test_scanner_unbalanced_set
+    assert_raise( Regexp::Scanner::PrematureEndError ) { RS.scan('[[:alpha:]') }
+  end
+
+  def test_scanner_unbalanced_group
+    assert_raise( Regexp::Scanner::PrematureEndError ) { RS.scan('(abc') }
+  end
+
+  def test_scanner_unbalanced_interval
+    assert_raise( Regexp::Scanner::PrematureEndError ) { RS.scan('a{1,2') }
+  end
+
+  def test_scanner_incomplete_property
+    assert_raise( Regexp::Scanner::PrematureEndError ) { RS.scan('\p{ascii abc') }
+  end
+
+  def test_scanner_unknown_property
+    assert_raise( Regexp::Scanner::UnknownUnicodePropertyError ) { RS.scan('\p{foobar}') }
+  end
+
+  def test_scanner_incomplete_options
+    assert_raise( Regexp::Scanner::ScannerError ) { RS.scan('(?mix abc)') }
+  end
+
+  def test_scanner_eof_options
+    assert_raise( Regexp::Scanner::PrematureEndError ) { RS.scan('(?mix') }
+  end
+
+  def test_scanner_incorrect_options
+    assert_raise( Regexp::Scanner::ScannerError ) { RS.scan('(?mix^bc') }
+  end
+end

data/test/scanner/test_escapes.rb

@@ -0,0 +1,49 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerEscapes < Test::Unit::TestCase
+
+  tests = {
+    /c\at/            => [1, :escape,  :bell,             '\a',             1,  3],
+
+    # not an escape outside a character set
+    /c\bt/            => [1, :anchor,  :word_boundary,    '\b',             1, 3],
+
+    /c\ft/            => [1, :escape,  :form_feed,        '\f',             1,  3],
+    /c\nt/            => [1, :escape,  :newline,          '\n',             1,  3],
+    /c\tt/            => [1, :escape,  :tab,              '\t',             1,  3],
+    /c\vt/            => [1, :escape,  :vertical_tab,     '\v',             1,  3],
+
+    /c\qt/            => [1, :escape,  :literal,          '\q',             1,  3],
+
+    'a\012c'          => [1, :escape,  :octal,            '\012',           1,  5],
+    'a\0124'          => [1, :escape,  :octal,            '\012',           1,  5],
+    '\712+7'          => [0, :escape,  :octal,            '\712',           0,  4],
+
+    'a\x24c'          => [1, :escape,  :hex,              '\x24',           1,  5],
+    'a\x0640c'        => [1, :escape,  :hex,              '\x06',           1,  5],
+
+    'a\x{0640}c'      => [1, :escape,  :hex_wide,         '\x{0640}',       1,  9],
+
+    'a\u0640c'        => [1, :escape,  :codepoint,        '\u0640',         1,  7],
+    'a\u{0640 0641}c' => [1, :escape,  :codepoint_list,   '\u{0640 0641}',  1,  14],
+
+    'a\cCc'           => [1, :escape,  :control,          '\cC',            1,  4],
+    'a\C-cc'          => [1, :escape,  :control,          '\C-c',           1,  5],
+
+    # TODO: verify these escapes
+    'a\M-Cc'          => [1, :escape,  :meta_sequence,    '\M-C',           1,  5],
+    'a\M-\C-cc'       => [1, :escape,  :meta_sequence,    '\M-\C-c',        1,  8],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_scan_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      token = tokens[test[0]]
+      assert_equal( test[1,5], token )
+
+    end
+  end
+
+end

data/test/scanner/test_groups.rb

@@ -0,0 +1,41 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerGroups < Test::Unit::TestCase
+
+  tests = {
+   ## Options
+    '(?-mix:abc)'     => [0, :group,     :options,      '(?-mix:',    0, 7],
+    '(?m-ix:abc)'     => [0, :group,     :options,      '(?m-ix:',    0, 7],
+    '(?mi-x:abc)'     => [0, :group,     :options,      '(?mi-x:',    0, 7],
+    '(?mix:abc)'      => [0, :group,     :options,      '(?mix:',     0, 6],
+    '(?mix)'          => [0, :group,     :options,      '(?mix',      0, 5],
+
+    # Group types
+    '(?>abc)'         => [0, :group,     :atomic,       '(?>',        0, 3],
+    '(abc)'           => [0, :group,     :capture,      '(',          0, 1],
+    '(?<name>abc)'    => [0, :group,     :named_ab,     '(?<name>',   0, 8],
+    "(?'name'abc)"    => [0, :group,     :named_sq,     "(?'name'",   0, 8],
+    '(?:abc)'         => [0, :group,     :passive,      '(?:',        0, 3],
+
+    # Comments
+    '(?#abc)'         => [0, :group,     :comment,      '(?#abc)',    0, 7],
+
+    # Assertions
+    '(?=abc)'         => [0, :assertion, :lookahead,    '(?=',        0, 3],
+    '(?!abc)'         => [0, :assertion, :nlookahead,   '(?!',        0, 3],
+    '(?<=abc)'        => [0, :assertion, :lookbehind,   '(?<=',       0, 4],
+    '(?<!abc)'        => [0, :assertion, :nlookbehind,  '(?<!',       0, 4],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_scan_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      assert_equal( test[1,5], tokens[test[0]])
+      assert_equal( test[3],   pattern[tokens[test[0]][3], tokens[test[0]][4]])
+
+    end
+  end
+
+end

data/test/scanner/test_literals.rb

@@ -0,0 +1,85 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerUTF8 < Test::Unit::TestCase
+
+  tests = {
+    # ascii, single byte characters
+    'a' => {
+      0     => [:literal,     :literal,       'a',        0, 1],
+    },
+
+    'ab+' => {
+      0     => [:literal,     :literal,       'ab',       0, 2],
+      1     => [:quantifier,  :one_or_more,   '+',        2, 3],
+    },
+
+    # 2 byte wide characters, Arabic
+    'aاbبcت' => {
+      0     => [:literal,     :literal,       'aاbبcت',   0, 9],
+    },
+
+    'aاbبت?' => {
+      0     => [:literal,     :literal,       'aاbبت',    0, 8],
+      1     => [:quantifier,  :zero_or_one,   '?',        8, 9],
+    },
+
+    'aا?bبcت+' => {
+      0     => [:literal,     :literal,       'aا',       0, 3],
+      1     => [:quantifier,  :zero_or_one,   '?',        3, 4],
+      2     => [:literal,     :literal,       'bبcت',     4, 10],
+      3     => [:quantifier,  :one_or_more,   '+',        10, 11],
+    },
+
+    'a(اbب+)cت?' => {
+      0     => [:literal,     :literal,       'a',        0, 1],
+      1     => [:group,       :capture,       '(',        1, 2],
+      2     => [:literal,     :literal,       'اbب',      2, 7],
+      3     => [:quantifier,  :one_or_more,   '+',        7, 8],
+      4     => [:group,       :close,         ')',        8, 9],
+      5     => [:literal,     :literal,       'cت',       9, 12],
+      6     => [:quantifier,  :zero_or_one,   '?',        12, 13],
+    },
+
+    # 3 byte wide characters, Japanese
+    'ab?れます+cd' => {
+      0     => [:literal,     :literal,       'ab',       0, 2],
+      1     => [:quantifier,  :zero_or_one,   '?',        2, 3],
+      2     => [:literal,     :literal,       'れます',   3, 12],
+      3     => [:quantifier,  :one_or_more,   '+',        12, 13],
+      4     => [:literal,     :literal,       'cd',       13, 15],
+    },
+
+    # 4 byte wide characters, Osmanya
+    '𐒀𐒁?𐒂ab+𐒃' => {
+      0     => [:literal,     :literal,       '𐒀𐒁',       0, 8],
+      1     => [:quantifier,  :zero_or_one,   '?',        8, 9],
+      2     => [:literal,     :literal,       '𐒂ab',      9, 15],
+      3     => [:quantifier,  :one_or_more,   '+',        15, 16],
+      4     => [:literal,     :literal,       '𐒃',        16, 20],
+    },
+
+    'mu𝄞?si*𝄫c+' => {
+      0     => [:literal,     :literal,       'mu𝄞',      0, 6],
+      1     => [:quantifier,  :zero_or_one,   '?',        6, 7],
+      2     => [:literal,     :literal,       'si',       7, 9],
+      3     => [:quantifier,  :zero_or_more,  '*',        9, 10],
+      4     => [:literal,     :literal,       '𝄫c',       10, 15],
+      5     => [:quantifier,  :one_or_more,   '+',        15, 16],
+    },
+  }
+
+  count = 0
+  tests.each do |pattern, checks|
+    define_method "test_scan_utf8_runs_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      checks.each do |offset, token|
+        assert_equal( token, tokens[offset] )
+      end
+
+    end
+  end
+
+end

data/test/scanner/test_meta.rb

@@ -0,0 +1,36 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerMeta < Test::Unit::TestCase
+
+  tests = {
+    'abc??|def*+|ghi+' => {
+      0   => [:literal,     :literal,                 'abc',  0, 3],
+      1   => [:quantifier,  :zero_or_one_reluctant,   '??',   3, 5],
+      2   => [:meta,        :alternation,             '|',    5, 6],
+      3   => [:literal,     :literal,                 'def',  6, 9],
+      4   => [:quantifier,  :zero_or_more_possessive, '*+',   9, 11],
+      5   => [:meta,        :alternation,             '|',    11, 12],
+    },
+
+    '(a\|b)|(c|d)\|(e[|]f)' => {
+      2   => [:escape,      :alternation,             '\|',   2, 4],
+      5   => [:meta,        :alternation,             '|',    6, 7],
+      8   => [:meta,        :alternation,             '|',    9, 10],
+      11  => [:escape,      :alternation,             '\|',   12, 14],
+      15  => [:set,         :member,                  '|',    17, 18],
+    },
+  }
+
+  count = 0
+  tests.each do |pattern, checks|
+    define_method "test_scan_meta_alternation_#{count+=1}" do
+
+      tokens = RS.scan(pattern)
+      checks.each do |offset, token|
+        assert_equal( token, tokens[offset] )
+      end
+
+    end
+  end
+
+end

data/test/scanner/test_properties.rb

@@ -0,0 +1,315 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ScannerProperties < Test::Unit::TestCase
+
+  tests = {
+    'Alnum'                               => :alnum,
+    'Alpha'                               => :alpha,
+    'Any'                                 => :any,
+    'Ascii'                               => :ascii,
+    'Blank'                               => :blank,
+    'Cntrl'                               => :cntrl,
+    'Digit'                               => :digit,
+    'Graph'                               => :graph,
+    'Lower'                               => :lower,
+    'Newline'                             => :newline,
+    'Print'                               => :print,
+    'Punct'                               => :punct,
+    'Space'                               => :space,
+    'Upper'                               => :upper,
+    'Word'                                => :word,
+    'Xdigit'                              => :xdigit,
+
+    'L'                                   => :letter_any,
+    'Letter'                              => :letter_any,
+
+    'Lu'                                  => :letter_uppercase,
+    'Uppercase_Letter'                    => :letter_uppercase,
+
+    'Ll'                                  => :letter_lowercase,
+    'Lowercase_Letter'                    => :letter_lowercase,
+
+    'Lt'                                  => :letter_titlecase,
+    'Titlecase_Letter'                    => :letter_titlecase,
+
+    'Lm'                                  => :letter_modifier,
+    'Modifier_Letter'                     => :letter_modifier,
+
+    'Lo'                                  => :letter_other,
+    'Other_Letter'                        => :letter_other,
+
+    'M'                                   => :mark_any,
+    'Mark'                                => :mark_any,
+
+    'Mn'                                  => :mark_nonspacing,
+    'Nonspacing_Mark'                     => :mark_nonspacing,
+
+    'Mc'                                  => :mark_spacing,
+    'Spacing_Mark'                        => :mark_spacing,
+
+    'Me'                                  => :mark_enclosing,
+    'Enclosing_Mark'                      => :mark_enclosing,
+
+    'N'                                   => :number_any,
+    'Number'                              => :number_any,
+
+    'Nd'                                  => :number_decimal,
+    'Decimal_Number'                      => :number_decimal,
+
+    'Nl'                                  => :number_letter,
+    'Letter_Number'                       => :number_letter,
+
+    'No'                                  => :number_other,
+    'Other_Number'                        => :number_other,
+
+    'P'                                   => :punct_any,
+    'Punctuation'                         => :punct_any,
+
+    'Pc'                                  => :punct_connector,
+    'Connector_Punctuation'               => :punct_connector,
+
+    'Pd'                                  => :punct_dash,
+    'Dash_Punctuation'                    => :punct_dash,
+
+    'Ps'                                  => :punct_open,
+    'Open_Punctuation'                    => :punct_open,
+
+    'Pe'                                  => :punct_close,
+    'Close_Punctuation'                   => :punct_close,
+
+    'Pi'                                  => :punct_initial,
+    'Initial_Punctuation'                 => :punct_initial,
+
+    'Pf'                                  => :punct_final,
+    'Final_Punctuation'                   => :punct_final,
+
+    'Po'                                  => :punct_other,
+    'Other_Punctuation'                   => :punct_other,
+
+    'S'                                   => :symbol_any,
+    'Symbol'                              => :symbol_any,
+
+    'Sm'                                  => :symbol_math,
+    'Math_Symbol'                         => :symbol_math,
+
+    'Sc'                                  => :symbol_currency,
+    'Currency_Symbol'                     => :symbol_currency,
+
+    'Sk'                                  => :symbol_modifier,
+    'Modifier_Symbol'                     => :symbol_modifier,
+
+    'So'                                  => :symbol_other,
+    'Other_Symbol'                        => :symbol_other,
+
+    'Z'                                   => :separator_any,
+    'Separator'                           => :separator_any,
+
+    'Zs'                                  => :separator_space,
+    'Space_Separator'                     => :separator_space,
+
+    'Zl'                                  => :separator_line,
+    'Line_Separator'                      => :separator_line,
+
+    'Zp'                                  => :separator_para,
+    'Paragraph_Separator'                 => :separator_para,
+
+    'C'                                   => :other,
+    'Other'                               => :other,
+
+    'Cc'                                  => :control,
+    'Control'                             => :control,
+
+    'Cf'                                  => :format,
+    'Format'                              => :format,
+
+    'Cs'                                  => :surrogate,
+    'Surrogate'                           => :surrogate,
+
+    'Co'                                  => :private_use,
+    'Private_Use'                         => :private_use,
+
+    'Cn'                                  => :unassigned,
+    'Unassigned'                          => :unassigned,
+
+    'Age=1.1'                             => :age_1_1,
+    'Age=2.0'                             => :age_2_0,
+    'Age=2.1'                             => :age_2_1,
+    'Age=3.0'                             => :age_3_0,
+    'Age=3.1'                             => :age_3_1,
+    'Age=3.2'                             => :age_3_2,
+    'Age=4.0'                             => :age_4_0,
+    'Age=4.1'                             => :age_4_1,
+    'Age=5.0'                             => :age_5_0,
+    'Age=5.1'                             => :age_5_1,
+    'Age=5.2'                             => :age_5_2,
+    'Age=6.0'                             => :age_6_0,
+
+    'ahex'                                => :ascii_hex,
+    'ASCII_Hex_Digit'                     => :ascii_hex,
+
+    'Alphabetic'                          => :alphabetic,
+
+    'Cased'                               => :cased,
+
+    'cwcf'                                => :changes_when_casefolded,
+    'Changes_When_Casefolded'             => :changes_when_casefolded,
+
+    'cwcm'                                => :changes_when_casemapped,
+    'Changes_When_Casemapped'             => :changes_when_casemapped,
+
+    'cwl'                                 => :changes_when_lowercased,
+    'Changes_When_Lowercased'             => :changes_when_lowercased,
+
+    'cwt'                                 => :changes_when_titlecased,
+    'Changes_When_Titlecased'             => :changes_when_titlecased,
+
+    'cwu'                                 => :changes_when_uppercased,
+    'Changes_When_Uppercased'             => :changes_when_uppercased,
+
+    'ci'                                  => :case_ignorable,
+    'Case_Ignorable'                      => :case_ignorable,
+
+    'bidic'                               => :bidi_control,
+    'Bidi_Control'                        => :bidi_control,
+
+    'Dash'                                => :dash,
+
+    'dep'                                 => :deprecated,
+    'Deprecated'                          => :deprecated,
+
+    'di'                                  => :default_ignorable_cp,
+    'Default_Ignorable_Code_Point'        => :default_ignorable_cp,
+
+    'dia'                                 => :diacritic,
+    'Diacritic'                           => :diacritic,
+
+    'ext'                                 => :extender,
+    'Extender'                            => :extender,
+
+    'grbase'                              => :grapheme_base,
+    'Grapheme_Base'                       => :grapheme_base,
+
+    'grext'                               => :grapheme_extend,
+    'Grapheme_Extend'                     => :grapheme_extend,
+
+    'grlink'                              => :grapheme_link,
+    'Grapheme_Link'                       => :grapheme_link,
+
+    'hex'                                 => :hex_digit,
+    'Hex_Digit'                           => :hex_digit,
+
+    'Hyphen'                              => :hyphen,
+
+    'idc'                                 => :id_continue,
+    'ID_Continue'                         => :id_continue,
+
+    'ideo'                                => :ideographic,
+    'Ideographic'                         => :ideographic,
+
+    'ids'                                 => :id_start,
+    'ID_Start'                            => :id_start,
+
+    'idsb'                                => :ids_binary_op,
+    'IDS_Binary_Operator'                 => :ids_binary_op,
+
+    'idst'                                => :ids_trinary_op,
+    'IDS_Trinary_Operator'                => :ids_trinary_op,
+
+    'joinc'                               => :join_control,
+    'Join_Control'                        => :join_control,
+
+    'loe'                                 => :logical_order_exception,
+    'Logical_Order_Exception'             => :logical_order_exception,
+
+    'Lowercase'                           => :lowercase,
+
+    'Math'                                => :math,
+
+    'nchar'                               => :non_character_cp,
+    'Noncharacter_Code_Point'             => :non_character_cp,
+
+    'oalpha'                              => :other_alphabetic,
+    'Other_Alphabetic'                    => :other_alphabetic,
+
+    'odi'                                 => :other_default_ignorable_cp,
+    'Other_Default_Ignorable_Code_Point'  => :other_default_ignorable_cp,
+
+    'ogrext'                              => :other_grapheme_extended,
+    'Other_Grapheme_Extend'               => :other_grapheme_extended,
+
+    'oidc'                                => :other_id_continue,
+    'Other_ID_Continue'                   => :other_id_continue,
+
+    'oids'                                => :other_id_start,
+    'Other_ID_Start'                      => :other_id_start,
+
+    'olower'                              => :other_lowercase,
+    'Other_Lowercase'                     => :other_lowercase,
+
+    'omath'                               => :other_math,
+    'Other_Math'                          => :other_math,
+
+    'oupper'                              => :other_uppercase,
+    'Other_Uppercase'                     => :other_uppercase,
+
+    'patsyn'                              => :pattern_syntax,
+    'Pattern_Syntax'                      => :pattern_syntax,
+
+    'patws'                               => :pattern_whitespace,
+    'Pattern_Whitespace'                  => :pattern_whitespace,
+
+    'qmark'                               => :quotation_mark,
+    'quotationmark'                       => :quotation_mark,
+
+    'radical'                             => :radical,
+
+    'sd'                                  => :soft_dotted,
+    'Soft_Dotted'                         => :soft_dotted,
+
+    'sterm'                               => :sentence_terminal,
+
+    'term'                                => :terminal_punctuation,
+    'Terminal_Punctuation'                => :terminal_punctuation,
+
+    'uideo'                               => :unified_ideograph,
+    'Unified_Ideograph'                   => :unified_ideograph,
+
+    'Uppercase'                           => :uppercase,
+
+    'vs'                                  => :variation_selector,
+    'Variation_Selector'                  => :variation_selector,
+
+    'wspace'                              => :whitespace,
+    'whitespace'                          => :whitespace,
+
+    'xids'                                => :xid_start,
+    'XID_Start'                           => :xid_start,
+
+    'xidc'                                => :xid_continue,
+    'XID_Continue'                        => :xid_continue,
+  }
+
+  count = 0
+  tests.each do |property, test|
+    define_method "test_scan_property_#{test}_#{count+=1}" do
+      token = RS.scan("a\\p{#{property}}c")[1]
+
+      assert_equal( :property,      token[0] )
+      assert_equal( test,           token[1] )
+    end
+
+    define_method "test_scan_nonproperty_#{test}_#{count+=1}" do
+      token = RS.scan("a\\P{#{property}}c")[1]
+
+      assert_equal( :nonproperty,   token[0] )
+      assert_equal( test,           token[1] )
+    end
+
+    define_method "test_scan_caret_nonproperty_#{test}_#{count+=1}" do
+      token = RS.scan("a\\p{^#{property}}c")[1]
+
+      assert_equal( :nonproperty,   token[0] )
+      assert_equal( test,           token[1] )
+    end
+  end
+end