regexp_parser 0.5.0 → 1.0.0

data/test/helpers.rb

@@ -1,5 +1,6 @@
 require "test/unit"
 require File.expand_path("../../lib/regexp_parser", __FILE__)
+require 'regexp_property_values'
 
 RS = Regexp::Scanner
 RL = Regexp::Lexer

data/test/lexer/test_all.rb

@@ -31,7 +31,7 @@ class TestRegexpLexer < Test::Unit::TestCase
   def test_lexer_token_count
     tokens = RL.lex(/^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i)
 
-    assert_equal 26, tokens.length
+    assert_equal 28, tokens.length
   end
 
   def test_lexer_scan_alias

data/test/lexer/test_conditionals.rb

@@ -43,15 +43,17 @@ class LexerConditionals < Test::Unit::TestCase
       [11, :conditional, :condition,        '(<B>)',  25, 30, 3, 0, 2],
 
       [12, :set,         :open,             '[',      30, 31, 3, 0, 2],
-      [13, :set,         :range,            'e-g',    31, 34, 3, 1, 2],
-      [14, :set,         :close,            ']',      34, 35, 3, 0, 2],
+      [13, :literal,     :literal,          'e',      31, 32, 3, 1, 2],
+      [14, :set,         :range,            '-',      32, 33, 3, 1, 2],
+      [15, :literal,     :literal,          'g',      33, 34, 3, 1, 2],
+      [16, :set,         :close,            ']',      34, 35, 3, 0, 2],
 
-      [15, :conditional, :separator,        '|',      35, 36, 3, 0, 2],
-      [19, :conditional, :close,            ')',      41, 42, 3, 0, 1],
-      [21, :conditional, :close,            ')',      43, 44, 2, 0, 0],
+      [17, :conditional, :separator,        '|',      35, 36, 3, 0, 2],
+      [23, :conditional, :close,            ')',      41, 42, 3, 0, 1],
+      [25, :conditional, :close,            ')',      43, 44, 2, 0, 0],
 
-      [22, :group,       :close,            ')',      44, 45, 1, 0, 0],
-      [23, :group,       :close,            ')',      45, 46, 0, 0, 0]
+      [26, :group,       :close,            ')',      44, 45, 1, 0, 0],
+      [27, :group,       :close,            ')',      45, 46, 0, 0, 0]
     ].each do |index, type, token, text, ts, te, level, set_level, conditional_level|
       struct = tokens.at(index)
 

data/test/lexer/test_nesting.rb

@@ -62,38 +62,56 @@ class LexerNesting < Test::Unit::TestCase
 
     'a[b-e]f' => {
       1     => [:set,         :open,          '[',      1,  2, 0, 0, 0],
-      2     => [:set,         :range,         'b-e',    2,  5, 0, 1, 0],
-      3     => [:set,         :close,         ']',      5,  6, 0, 0, 0],
+      2     => [:literal,     :literal,       'b',      2,  3, 0, 1, 0],
+      3     => [:set,         :range,         '-',      3,  4, 0, 1, 0],
+      4     => [:literal,     :literal,       'e',      4,  5, 0, 1, 0],
+      5     => [:set,         :close,         ']',      5,  6, 0, 0, 0],
     },
 
-    '[a-w&&[^c-g]z]' => {
+    '[[:word:]&&[^c]z]' => {
       0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-      2     => [:set,         :intersection,  '&&',     4,  6, 0, 1, 0],
-      3     => [:subset,      :open,          '[',      6,  7, 0, 1, 0],
-      4     => [:subset,      :negate,        '^',      7,  8, 0, 2, 0],
-      5     => [:subset,      :range,         'c-g',    8, 11, 0, 2, 0],
-      6     => [:subset,      :close,         ']',     11, 12, 0, 1, 0],
-      8     => [:set,         :close,         ']',     13, 14, 0, 0, 0],
+      1     => [:posixclass,  :word, '[:word:]',        1,  9, 0, 1, 0],
+      2     => [:set,         :intersection,  '&&',     9, 11, 0, 1, 0],
+      3     => [:set,         :open,          '[',     11, 12, 0, 1, 0],
+      4     => [:set,         :negate,        '^',     12, 13, 0, 2, 0],
+      5     => [:literal,     :literal,       'c',     13, 14, 0, 2, 0],
+      6     => [:set,         :close,         ']',     14, 15, 0, 1, 0],
+      7     => [:literal,     :literal,       'z',     15, 16, 0, 1, 0],
+      8     => [:set,         :close,         ']',     16, 17, 0, 0, 0],
+    },
+
+    '[\p{word}&&[^c]z]' => {
+      0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
+      1     => [:property,    :word, '\p{word}',        1,  9, 0, 1, 0],
+      2     => [:set,         :intersection,  '&&',     9, 11, 0, 1, 0],
+      3     => [:set,         :open,          '[',     11, 12, 0, 1, 0],
+      4     => [:set,         :negate,        '^',     12, 13, 0, 2, 0],
+      5     => [:literal,     :literal,       'c',     13, 14, 0, 2, 0],
+      6     => [:set,         :close,         ']',     14, 15, 0, 1, 0],
+      7     => [:literal,     :literal,       'z',     15, 16, 0, 1, 0],
+      8     => [:set,         :close,         ']',     16, 17, 0, 0, 0],
     },
 
     '[a[b[c[d-g]]]]' => {
       0     => [:set,         :open,          '[',      0,  1, 0, 0, 0],
-      1     => [:set,         :member,        'a',      1,  2, 0, 1, 0],
-      2     => [:subset,      :open,          '[',      2,  3, 0, 1, 0],
-      3     => [:subset,      :member,        'b',      3,  4, 0, 2, 0],
-      4     => [:subset,      :open,          '[',      4,  5, 0, 2, 0],
-      5     => [:subset,      :member,        'c',      5,  6, 0, 3, 0],
-      6     => [:subset,      :open,          '[',      6,  7, 0, 3, 0],
-      7     => [:subset,      :range,         'd-g',    7, 10, 0, 4, 0],
-      8     => [:subset,      :close,         ']',     10, 11, 0, 3, 0],
-      9     => [:subset,      :close,         ']',     11, 12, 0, 2, 0],
-     10     => [:subset,      :close,         ']',     12, 13, 0, 1, 0],
-     11     => [:set,         :close,         ']',     13, 14, 0, 0, 0],
+      1     => [:literal,     :literal,       'a',      1,  2, 0, 1, 0],
+      2     => [:set,         :open,          '[',      2,  3, 0, 1, 0],
+      3     => [:literal,     :literal,       'b',      3,  4, 0, 2, 0],
+      4     => [:set,         :open,          '[',      4,  5, 0, 2, 0],
+      5     => [:literal,     :literal,       'c',      5,  6, 0, 3, 0],
+      6     => [:set,         :open,          '[',      6,  7, 0, 3, 0],
+      7     => [:literal,     :literal,       'd',      7,  8, 0, 4, 0],
+      8     => [:set,         :range,         '-',      8,  9, 0, 4, 0],
+      9     => [:literal,     :literal,       'g',      9, 10, 0, 4, 0],
+      10    => [:set,         :close,         ']',     10, 11, 0, 3, 0],
+      11    => [:set,         :close,         ']',     11, 12, 0, 2, 0],
+      12    => [:set,         :close,         ']',     12, 13, 0, 1, 0],
+      13    => [:set,         :close,         ']',     13, 14, 0, 0, 0],
     },
   }
 
   tests.each_with_index do |(pattern, checks), count|
-    define_method "test_lex_nesting_#{count}" do
+    define_method "test_lex_nesting_in_'#{pattern}'_#{count}" do
       tokens = RL.lex(pattern, 'ruby/1.9')
 
       checks.each do |offset, (type, token, text, ts, te, level, set_level, conditional_level)|

data/test/lexer/test_refcalls.rb

@@ -27,11 +27,11 @@ class LexerRefCalls < Test::Unit::TestCase
     "(abc)\\g'-1'"    => [3, :backref, :number_rel_call,    "\\g'-1'",    5, 11, 0, 0, 0],
 
     # Group back-references, with nesting level
-    '(?<X>abc)\k<X-0>'  => [3, :backref, :name_nest_ref,    '\k<X-0>',    9, 16, 0, 0, 0],
-    "(?<X>abc)\\k'X-0'" => [3, :backref, :name_nest_ref,    "\\k'X-0'",   9, 16, 0, 0, 0],
+    '(?<X>abc)\k<X-0>'  => [3, :backref, :name_recursion_ref,    '\k<X-0>',    9, 16, 0, 0, 0],
+    "(?<X>abc)\\k'X-0'" => [3, :backref, :name_recursion_ref,    "\\k'X-0'",   9, 16, 0, 0, 0],
 
-    '(abc)\k<1-0>'      => [3, :backref, :number_nest_ref,  '\k<1-0>',    5, 12, 0, 0, 0],
-    "(abc)\\k'1-0'"     => [3, :backref, :number_nest_ref,  "\\k'1-0'",   5, 12, 0, 0, 0],
+    '(abc)\k<1-0>'      => [3, :backref, :number_recursion_ref,  '\k<1-0>',    5, 12, 0, 0, 0],
+    "(abc)\\k'1-0'"     => [3, :backref, :number_recursion_ref,  "\\k'1-0'",   5, 12, 0, 0, 0],
   }
 
   tests.each_with_index do |(pattern, (index, type, token, text, ts, te, level, set_level, conditional_level)), count|

data/test/parser/set/test_intersections.rb

@@ -0,0 +1,127 @@
+require File.expand_path('../../../helpers', __FILE__)
+
+# edge cases with `...-&&...` and `...&&-...` are checked in test_ranges.rb
+
+class ParserSetIntersections < Test::Unit::TestCase
+  def test_parse_set_intersection
+    root = RP.parse('[a&&z]')
+    set  = root[0]
+    ints = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Intersection, ints.class
+    assert_equal 2, ints.count
+
+    seq1, seq2 = ints.expressions
+    assert_equal CharacterSet::IntersectedSequence, seq1.class
+    assert_equal 1, seq1.count
+    assert_equal 'a', seq1.first.to_s
+    assert_equal Literal, seq1.first.class
+    assert_equal CharacterSet::IntersectedSequence, seq2.class
+    assert_equal 1, seq2.count
+    assert_equal 'z', seq2.first.to_s
+    assert_equal Literal, seq2.first.class
+
+    refute       set.matches?('a')
+    refute       set.matches?('&')
+    refute       set.matches?('z')
+  end
+
+  def test_parse_set_intersection_range_and_subset
+    root = RP.parse('[a-z&&[^a]]')
+    set  = root[0]
+    ints = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Intersection, ints.class
+    assert_equal 2, ints.count
+
+    seq1, seq2 = ints.expressions
+    assert_equal CharacterSet::IntersectedSequence, seq1.class
+    assert_equal 1, seq1.count
+    assert_equal 'a-z', seq1.first.to_s
+    assert_equal CharacterSet::Range, seq1.first.class
+    assert_equal CharacterSet::IntersectedSequence, seq2.class
+    assert_equal 1, seq2.count
+    assert_equal '[^a]', seq2.first.to_s
+    assert_equal CharacterSet, seq2.first.class
+
+    refute       set.matches?('a')
+    refute       set.matches?('&')
+    assert       set.matches?('b')
+  end
+
+  def test_parse_set_intersection_trailing_range
+    root = RP.parse('[a&&a-z]')
+    set  = root[0]
+    ints = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Intersection, ints.class
+    assert_equal 2, ints.count
+
+    seq1, seq2 = ints.expressions
+    assert_equal CharacterSet::IntersectedSequence, seq1.class
+    assert_equal 1, seq1.count
+    assert_equal 'a', seq1.first.to_s
+    assert_equal Literal, seq1.first.class
+    assert_equal CharacterSet::IntersectedSequence, seq2.class
+    assert_equal 1, seq2.count
+    assert_equal 'a-z', seq2.first.to_s
+    assert_equal CharacterSet::Range, seq2.first.class
+
+    assert       set.matches?('a')
+    refute       set.matches?('&')
+    refute       set.matches?('b')
+  end
+
+  def test_parse_set_intersection_type
+    root = RP.parse('[a&&\w]')
+    set  = root[0]
+    ints = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Intersection, ints.class
+    assert_equal 2, ints.count
+
+    seq1, seq2 = ints.expressions
+    assert_equal CharacterSet::IntersectedSequence, seq1.class
+    assert_equal 1, seq1.count
+    assert_equal 'a', seq1.first.to_s
+    assert_equal Literal, seq1.first.class
+    assert_equal CharacterSet::IntersectedSequence, seq2.class
+    assert_equal 1, seq2.count
+    assert_equal '\w', seq2.first.to_s
+    assert_equal CharacterType::Word, seq2.first.class
+
+    assert       set.matches?('a')
+    refute       set.matches?('&')
+    refute       set.matches?('b')
+  end
+
+  def test_parse_set_intersection_multipart
+    root = RP.parse('[\h&&\w&&efg]')
+    set  = root[0]
+    ints = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Intersection, ints.class
+    assert_equal 3, ints.count
+
+    seq1, seq2, seq3 = ints.expressions
+    assert_equal CharacterSet::IntersectedSequence, seq1.class
+    assert_equal 1, seq1.count
+    assert_equal '\h', seq1.first.to_s
+    assert_equal CharacterSet::IntersectedSequence, seq2.class
+    assert_equal 1, seq2.count
+    assert_equal '\w', seq2.first.to_s
+    assert_equal CharacterSet::IntersectedSequence, seq3.class
+    assert_equal 3, seq3.count
+    assert_equal 'efg', seq3.to_s
+
+    assert       set.matches?('e')
+    assert       set.matches?('f')
+    refute       set.matches?('a')
+    refute       set.matches?('g')
+  end
+end

data/test/parser/set/test_ranges.rb

@@ -0,0 +1,111 @@
+require File.expand_path('../../../helpers', __FILE__)
+
+class ParserSetRangs < Test::Unit::TestCase
+  def test_parse_set_range
+    root  = RP.parse('[a-z]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Range, range.class
+    assert_equal 2, range.count
+    assert_equal 'a', range.first.to_s
+    assert_equal Literal, range.first.class
+    assert_equal 'z', range.last.to_s
+    assert_equal Literal, range.last.class
+    assert       set.matches?('m')
+  end
+
+  def test_parse_set_range_hex
+    root  = RP.parse('[\x00-\x99]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Range, range.class
+    assert_equal 2, range.count
+    assert_equal '\x00', range.first.to_s
+    assert_equal EscapeSequence::Hex, range.first.class
+    assert_equal '\x99', range.last.to_s
+    assert_equal EscapeSequence::Hex, range.last.class
+    assert       set.matches?('\x50')
+  end
+
+  def test_parse_set_range_unicode
+    root  = RP.parse('[\u{40 42}-\u1234]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal CharacterSet::Range, range.class
+    assert_equal 2, range.count
+    assert_equal '\u{40 42}', range.first.to_s
+    assert_equal EscapeSequence::CodepointList, range.first.class
+    assert_equal '\u1234', range.last.to_s
+    assert_equal EscapeSequence::Codepoint, range.last.class
+    assert       set.matches?('\u600')
+  end
+
+  def test_parse_set_range_edge_case_leading_dash
+    root  = RP.parse('[--z]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal 2, range.count
+    assert       set.matches?('a')
+  end
+
+  def test_parse_set_range_edge_case_trailing_dash
+    root  = RP.parse('[!--]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal 2, range.count
+    assert       set.matches?('$')
+  end
+
+  def test_parse_set_range_edge_case_leading_negate
+    root = RP.parse('[^-z]')
+    set  = root[0]
+
+    assert_equal 2, set.count
+    assert       set.matches?('a')
+    refute       set.matches?('z')
+  end
+
+  def test_parse_set_range_edge_case_trailing_negate
+    root  = RP.parse('[!-^]')
+    set   = root[0]
+    range = set[0]
+
+    assert_equal 1, set.count
+    assert_equal 2, range.count
+    assert       set.matches?('$')
+  end
+
+  def test_parse_set_range_edge_case_leading_intersection
+    root  = RP.parse('[[\-ab]&&-bc]')
+    set   = root[0]
+
+    assert_equal 1, set.count
+    assert_equal '-bc', set.first.last.to_s
+    assert       set.matches?('-')
+    assert       set.matches?('b')
+    refute       set.matches?('a')
+    refute       set.matches?('c')
+  end
+
+  def test_parse_set_range_edge_case_trailing_intersection
+    root  = RP.parse('[bc-&&[\-ab]]')
+    set   = root[0]
+
+    assert_equal 1, set.count
+    assert_equal 'bc-', set.first.first.to_s
+    assert       set.matches?('-')
+    assert       set.matches?('b')
+    refute       set.matches?('a')
+    refute       set.matches?('c')
+  end
+end

data/test/parser/test_all.rb

@@ -2,11 +2,14 @@ require File.expand_path("../../helpers", __FILE__)
 
 %w{
   alternation anchors errors escapes free_space groups
-  properties quantifiers refcalls sets types
+  posix_classes properties quantifiers refcalls sets types
 }.each do|tc|
   require File.expand_path("../test_#{tc}", __FILE__)
 end
 
+require File.expand_path('../set/test_ranges.rb', __FILE__)
+require File.expand_path('../set/test_intersections.rb', __FILE__)
+
 if RUBY_VERSION >= '2.0.0'
   %w{conditionals keep}.each do|tc|
     require File.expand_path("../test_#{tc}", __FILE__)

data/test/parser/test_escapes.rb

@@ -11,10 +11,6 @@ class TestParserEscapes < Test::Unit::TestCase
     /a\tc/    => [1, :escape,   :tab,               EscapeSequence::Tab],
     /a\vc/    => [1, :escape,   :vertical_tab,      EscapeSequence::VerticalTab],
 
-    # special cases
-    /a\bc/    => [1, :anchor,   :word_boundary,     Anchor::WordBoundary],
-    /a\sc/    => [1, :type,     :space,             CharacterType::Space],
-
     # meta character escapes
     /a\.c/    => [1, :escape,   :dot,               EscapeSequence::Literal],
     /a\?c/    => [1, :escape,   :zero_or_one,       EscapeSequence::Literal],
@@ -27,14 +23,15 @@ class TestParserEscapes < Test::Unit::TestCase
     /a\}c/    => [1, :escape,   :interval_close,    EscapeSequence::Literal],
 
     # unicode escapes
-    /a\u0640/       => [1, :escape, :codepoint,      EscapeSequence::Literal],
-    /a\u{41 1F60D}/ => [1, :escape, :codepoint_list, EscapeSequence::Literal],
+    /a\u0640/       => [1, :escape, :codepoint,      EscapeSequence::Codepoint],
+    /a\u{41 1F60D}/ => [1, :escape, :codepoint_list, EscapeSequence::CodepointList],
+    /a\u{10FFFF}/   => [1, :escape, :codepoint_list, EscapeSequence::CodepointList],
 
      # hex escapes
-    /a\xFF/n =>  [1, :escape, :hex,                 EscapeSequence::Literal],
+    /a\xFF/n =>  [1, :escape, :hex,                 EscapeSequence::Hex],
 
     # octal escapes
-    /a\177/n =>  [1, :escape, :octal,               EscapeSequence::Literal],
+    /a\177/n =>  [1, :escape, :octal,               EscapeSequence::Octal],
   }
 
   tests.each_with_index do |(pattern, (index, type, token, klass)), count|
@@ -50,11 +47,35 @@ class TestParserEscapes < Test::Unit::TestCase
     end
   end
 
+  def test_parse_chars_and_codepoints
+    root = RP.parse(/\n\?\101\x42\u0043\u{44 45}/)
+
+    assert_equal "\n",       root[0].char
+    assert_equal 10,         root[0].codepoint
+
+    assert_equal "?",        root[1].char
+    assert_equal 63,         root[1].codepoint
+
+    assert_equal "A",        root[2].char
+    assert_equal 65,         root[2].codepoint
+
+    assert_equal "B",        root[3].char
+    assert_equal 66,         root[3].codepoint
+
+    assert_equal "C",        root[4].char
+    assert_equal 67,         root[4].codepoint
+
+    assert_equal ["D", "E"], root[5].chars
+    assert_equal [68, 69],   root[5].codepoints
+  end
+
   def test_parse_escape_control_sequence_lower
     root = RP.parse(/a\\\c2b/)
 
     assert_equal EscapeSequence::Control, root[2].class
     assert_equal '\\c2',                  root[2].text
+    assert_equal "\u0012",                root[2].char
+    assert_equal 18,                      root[2].codepoint
   end
 
   def test_parse_escape_control_sequence_upper
@@ -62,6 +83,8 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::Control, root[2].class
     assert_equal '\\C-C',                 root[2].text
+    assert_equal "\u0003",                root[2].char
+    assert_equal 3,                       root[2].codepoint
   end
 
   def test_parse_escape_meta_sequence
@@ -69,6 +92,8 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::Meta, root[2].class
     assert_equal '\\M-Z',              root[2].text
+    assert_equal "\u00DA",             root[2].char
+    assert_equal 218,                  root[2].codepoint
   end
 
   def test_parse_escape_meta_control_sequence
@@ -76,6 +101,8 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::MetaControl, root[2].class
     assert_equal '\\M-\\C-X',                 root[2].text
+    assert_equal "\u0098",                    root[2].char
+    assert_equal 152,                         root[2].codepoint
   end
 
   def test_parse_lower_c_meta_control_sequence
@@ -83,6 +110,8 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::MetaControl, root[2].class
     assert_equal '\\M-\\cX',                  root[2].text
+    assert_equal "\u0098",                    root[2].char
+    assert_equal 152,                         root[2].codepoint
   end
 
   def test_parse_escape_reverse_meta_control_sequence
@@ -90,6 +119,8 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::MetaControl, root[2].class
     assert_equal '\\C-\\M-X',                 root[2].text
+    assert_equal "\u0098",                    root[2].char
+    assert_equal 152,                         root[2].codepoint
   end
 
   def test_parse_escape_reverse_lower_c_meta_control_sequence
@@ -97,6 +128,7 @@ class TestParserEscapes < Test::Unit::TestCase
 
     assert_equal EscapeSequence::MetaControl, root[2].class
     assert_equal '\\c\\M-X',                  root[2].text
+    assert_equal "\u0098",                    root[2].char
+    assert_equal 152,                         root[2].codepoint
   end
-
 end