regexp_parser 0.1.0

data/test/lexer/test_refcalls.rb

@@ -0,0 +1,45 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class LexerRefCalls < Test::Unit::TestCase
+
+  tests = {
+    # Group back-references, named, numbered, and relative
+    '(?<X>abc)\k<X>'  => [3, :backref, :name_ref,           '\k<X>',      9, 14, 0, 0],
+    "(?<X>abc)\\k'X'" => [3, :backref, :name_ref,           "\\k'X'",     9, 14, 0, 0],
+
+    '(abc)\k<1>'      => [3, :backref, :number_ref,         '\k<1>',      5, 10, 0, 0],
+    "(abc)\\k'1'"     => [3, :backref, :number_ref,         "\\k'1'",     5, 10, 0, 0],
+
+    '(abc)\k<-1>'     => [3, :backref, :number_rel_ref,     '\k<-1>',     5, 11, 0, 0],
+    "(abc)\\k'-1'"    => [3, :backref, :number_rel_ref,     "\\k'-1'",    5, 11, 0, 0],
+
+    # Sub-expression invocation, named, numbered, and relative
+    '(?<X>abc)\g<X>'  => [3, :backref, :name_call,          '\g<X>',      9, 14, 0, 0],
+    "(?<X>abc)\\g'X'" => [3, :backref, :name_call,          "\\g'X'",     9, 14, 0, 0],
+
+    '(abc)\g<1>'      => [3, :backref, :number_call,        '\g<1>',      5, 10, 0, 0],
+    "(abc)\\g'1'"     => [3, :backref, :number_call,        "\\g'1'",     5, 10, 0, 0],
+
+    '(abc)\g<-1>'     => [3, :backref, :number_rel_call,    '\g<-1>',     5, 11, 0, 0],
+    "(abc)\\g'-1'"    => [3, :backref, :number_rel_call,    "\\g'-1'",    5, 11, 0, 0],
+
+    # Group back-references, with nesting level
+    '(?<X>abc)\k<X-0>'  => [3, :backref, :name_nest_ref,    '\k<X-0>',    9, 16, 0, 0],
+    "(?<X>abc)\\k'X-0'" => [3, :backref, :name_nest_ref,    "\\k'X-0'",   9, 16, 0, 0],
+
+    '(abc)\k<1-0>'      => [3, :backref, :number_nest_ref,  '\k<1-0>',    5, 12, 0, 0],
+    "(abc)\\k'1-0'"     => [3, :backref, :number_nest_ref,  "\\k'1-0'",   5, 12, 0, 0],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_lexer_#{test[1]}_#{test[2]}_#{count+=1}" do
+
+      tokens = RL.scan(pattern)
+      assert_equal( test[1,7], tokens[test[0]].to_a)
+      assert_equal( test[3],   pattern[tokens[test[0]][3], tokens[test[0]][4]])
+
+    end
+  end
+
+end

data/test/parser/test_all.rb

@@ -0,0 +1,44 @@
+require File.expand_path("../../helpers", __FILE__)
+
+%w{
+  alternation anchors errors escapes expression groups properties
+  quantifiers refcalls sets
+}.each do|tc|
+  require File.expand_path("../test_#{tc}", __FILE__)
+end
+
+class TestParser < Test::Unit::TestCase
+
+  def test_parse_returns_a_root_expression
+    assert_instance_of( Regexp::Expression::Root, RP.parse('abc'))
+  end
+
+  def test_parse_root_contains_expressions
+    root = RP.parse(/^a.c+[^one]{2,3}\b\d\\\C-C$/)
+
+    assert( root.expressions.all?{|exp|
+      exp.kind_of?(Regexp::Expression::Base)},
+      "Not all nodes are instances of Regexp::Expression")
+  end
+
+  # too much going on here, it's just for development
+  def test_parse_node_types
+    root = RP.parse('^(one){2,3}([^d\]efm-qz\,\-]*)(ghi)+$')
+
+    assert( root.expressions[1].expressions[0].is_a?(Literal),
+          "Not a literal node, but should be")
+
+    assert( root.expressions[1].quantified?, "Not quanfified, but should be")
+
+    assert( root.expressions[2].expressions[0].is_a?(CharacterSet),
+          "Not a caracter set, but it should be")
+
+    assert_equal( false, root.expressions[2].quantified? )
+
+    assert( root.expressions[3].is_a?(Group::Capture),
+          "Not a group, but should be")
+
+    assert_equal( true, root.expressions[3].quantified? )
+  end
+
+end

data/test/parser/test_alternation.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ParserAlternation < Test::Unit::TestCase
+
+  # TODO: these tests pass, but they show how hard and messy the tree is
+  # to navigate
+
+  def setup
+    @root = RP.parse('(ab??|cd*+|ef+)*|(gh|ij|kl)?')
+  end
+
+  def test_parse_alternation_root
+    e = @root.expressions[0]
+    assert_equal( true,   e.is_a?(Alternation) )
+  end
+
+  def test_parse_alternation_alts
+    alts = @root.expressions[0].alternatives
+
+    assert_equal( true,   alts[0].is_a?(Sequence) )
+    assert_equal( true,   alts[1].is_a?(Sequence) )
+
+    assert_equal( true,   alts[0][0].is_a?(Group::Capture) )
+    assert_equal( true,   alts[1][0].is_a?(Group::Capture) )
+
+    assert_equal( 2,      alts.length )
+  end
+
+  def test_parse_alternation_nested
+    e = @root[0].alternatives[0][0][0]
+
+    assert_equal( true,   e.is_a?(Alternation) )
+  end
+
+  def test_parse_alternation_nested_sequence
+    alts    = @root.expressions[0][0]
+    nested  = alts.expressions[0][0][0]
+
+    assert_equal( true,   nested.is_a?(Sequence) )
+
+    assert_equal( true,   nested.expressions[0].is_a?(Literal) )
+    assert_equal( true,   nested.expressions[1].is_a?(Literal) )
+    assert_equal( 2,      nested.expressions.length )
+  end
+
+end

data/test/parser/test_anchors.rb

@@ -0,0 +1,35 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class TestParserAnchors < Test::Unit::TestCase
+
+  tests = {
+    '^a'      => [0, :anchor,   :beginning_of_line,   Anchor::BOL],
+    'a$'      => [1, :anchor,   :end_of_line,         Anchor::EOL],
+
+    '\Aa'     => [0, :anchor,   :bos,                 Anchor::BOS],
+    'a\z'     => [1, :anchor,   :eos,                 Anchor::EOS],
+    'a\Z'     => [1, :anchor,   :eos_ob_eol,          Anchor::EOSobEOL],
+
+    'a\b'     => [1, :anchor,   :word_boundary,       Anchor::WordBoundary],
+    'a\B'     => [1, :anchor,   :nonword_boundary,    Anchor::NonWordBoundary],
+
+    'a\G'     => [1, :anchor,   :match_start,         Anchor::MatchStart],
+
+    "\\\\Aa"  => [0, :escape,   :backslash,           EscapeSequence::Literal],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_parse_anchor_#{test[2]}_#{count+=1}" do
+      root = RP.parse(pattern, 'ruby/1.9')
+      exp  = root.expressions[test[0]]
+
+      assert( exp.is_a?( test[3] ),
+             "Expected #{test[3]}, but got #{exp.class.name}")
+
+      assert_equal( test[1], exp.type )
+      assert_equal( test[2], exp.token )
+    end
+  end
+
+end

data/test/parser/test_errors.rb

@@ -0,0 +1,59 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ParserErrors < Test::Unit::TestCase
+
+  def test_parser_unknown_token_type
+    assert_raise( Regexp::Parser::UnknownTokenTypeError ) {
+      RP.parse_token(Regexp::Token.new(:foo, :bar))
+    }
+  end
+
+  def test_parser_unknown_set_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:set, :foo))
+    }
+  end
+
+  def test_parser_unknown_meta_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:meta, :foo))
+    }
+  end
+
+  def test_parser_unknown_character_type_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:type, :foo))
+    }
+  end
+
+  def test_parser_unknown_unicode_property_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:property, :foo))
+    }
+  end
+
+  def test_parser_unknown_unicode_nonproperty_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:nonproperty, :foo))
+    }
+  end
+
+  def test_parser_unknown_anchor_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:anchor, :foo))
+    }
+  end
+
+  def test_parser_unknown_quantifier_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:quantifier, :foo))
+    }
+  end
+
+  def test_parser_unknown_group_open_token
+    assert_raise( Regexp::Parser::UnknownTokenError ) {
+      RP.parse_token(Regexp::Token.new(:group, :foo))
+    }
+  end
+
+end

data/test/parser/test_escapes.rb

@@ -0,0 +1,48 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class TestParserEscapes < Test::Unit::TestCase
+
+  def test_parse_control_sequence_short
+    #root = RP.parse(/\b\d\\\c2\C-C\M-\C-2/)
+  end
+
+  tests = {
+    /a\ac/    => [1, :escape,   :bell,              EscapeSequence::Bell],
+    /a\ec/    => [1, :escape,   :escape,            EscapeSequence::AsciiEscape],
+    /a\fc/    => [1, :escape,   :form_feed,         EscapeSequence::FormFeed],
+    /a\nc/    => [1, :escape,   :newline,           EscapeSequence::Newline],
+    /a\rc/    => [1, :escape,   :carriage,          EscapeSequence::Return],
+    /a\tc/    => [1, :escape,   :tab,               EscapeSequence::Tab],
+    /a\vc/    => [1, :escape,   :vertical_tab,      EscapeSequence::VerticalTab],
+
+    # special cases
+    /a\bc/    => [1, :anchor,   :word_boundary,     Anchor::WordBoundary],
+    /a\sc/    => [1, :type,     :space,             CharacterType::Space],
+
+    # meta character escapes
+    /a\.c/    => [1, :escape,   :dot,               EscapeSequence::Literal],
+    /a\?c/    => [1, :escape,   :zero_or_one,       EscapeSequence::Literal],
+    /a\*c/    => [1, :escape,   :zero_or_more,      EscapeSequence::Literal],
+    /a\+c/    => [1, :escape,   :one_or_more,       EscapeSequence::Literal],
+    /a\|c/    => [1, :escape,   :alternation,       EscapeSequence::Literal],
+    /a\(c/    => [1, :escape,   :group_open,        EscapeSequence::Literal],
+    /a\)c/    => [1, :escape,   :group_close,       EscapeSequence::Literal],
+    /a\{c/    => [1, :escape,   :interval_open,     EscapeSequence::Literal],
+    /a\}c/    => [1, :escape,   :interval_close,    EscapeSequence::Literal],
+  }
+
+  count = 0
+  tests.each do |pattern, test|
+    define_method "test_parse_anchor_#{test[2]}_#{count+=1}" do
+      root = RP.parse(pattern, 'ruby/1.9')
+      exp  = root.expressions[test[0]]
+
+      assert( exp.is_a?( test[3] ),
+             "Expected #{test[3]}, but got #{exp.class.name}")
+
+      assert_equal( test[1], exp.type )
+      assert_equal( test[2], exp.token )
+    end
+  end
+
+end

data/test/parser/test_expression.rb

@@ -0,0 +1,51 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ParserExpression < Test::Unit::TestCase
+
+  def test_parse_expression_to_s_literal_alternation
+    pattern = 'abcd|ghij|klmn|pqur'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_quantified_alternations
+    pattern = '(?:a?[b]+(c){2}|d+[e]*(f)?)|(?:g+[h]?(i){2,3}|j*[k]{3,5}(l)?)'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_quantified_sets
+    pattern = '[abc]+|[^def]{3,6}'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_property_sets
+    pattern = '[\a\b\p{Lu}\P{Z}\c\d]+'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_groups
+    pattern = "(a(?>b(?:c(?<n>d(?'N'e)??f)+g)*+h)*i)++"
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_assertions
+    pattern = '(a+(?=b+(?!c+(?<=d+(?<!e+)?f+)?g+)?h+)?i+)?'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_comments
+    pattern = '(?#start)a(?#middle)b(?#end)'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_options
+    pattern = '(?mix:start)a(?-mix:middle)b(?i-mx:end)'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+  def test_parse_expression_to_s_url
+    pattern = '(^$)|(^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*'+
+              '\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$)'
+    assert_equal( pattern, RP.parse(pattern).to_s )
+  end
+
+end

data/test/parser/test_groups.rb

@@ -0,0 +1,69 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class TestParserGroups < Test::Unit::TestCase
+
+  def test_parse_root_options_mi
+    t = RP.parse((/[abc]/mi).to_s)
+
+    assert_equal( true,  t.m? )
+    assert_equal( true,  t.i? )
+    assert_equal( false, t.x? )
+  end
+
+  def test_parse_nested_options_m
+    t = RP.parse('(?xi-m:a(?m-ix:b))')
+
+    assert_equal( true,  t.expressions[0].expressions[1].m? )
+    assert_equal( false, t.expressions[0].expressions[1].i? )
+    assert_equal( false, t.expressions[0].expressions[1].x? )
+  end
+
+  def test_parse_nested_options_xm
+    t = RP.parse(/(?i-xm:a(?mx-i:b))/)
+
+    assert_equal( true,  t.expressions[0].expressions[1].m? )
+    assert_equal( false, t.expressions[0].expressions[1].i? )
+    assert_equal( true,  t.expressions[0].expressions[1].x? )
+  end
+
+  def test_parse_nested_options_im
+    t = RP.parse(/(?x-mi:a(?mi-x:b))/)
+
+    assert_equal( true,  t.expressions[0].expressions[1].m? )
+    assert_equal( true,  t.expressions[0].expressions[1].i? )
+    assert_equal( false, t.expressions[0].expressions[1].x? )
+  end
+
+  def test_parse_lookahead
+    t = RP.parse('(?=abc)(?!def)')
+
+    assert( t.expressions[0].is_a?(Assertion::Lookahead),
+           "Expected lookahead, but got #{t.expressions[0].class.name}")
+
+    assert( t.expressions[1].is_a?(Assertion::NegativeLookahead),
+           "Expected negative lookahead, but got #{t.expressions[0].class.name}")
+  end
+
+  def test_parse_lookbehind
+    t = RP.parse('(?<=abc)(?<!def)')
+
+    assert( t.expressions[0].is_a?(Assertion::Lookbehind),
+           "Expected lookbehind, but got #{t.expressions[0].class.name}")
+
+    assert( t.expressions[1].is_a?(Assertion::NegativeLookbehind),
+           "Expected negative lookbehind, but got #{t.expressions[0].class.name}")
+  end
+
+  def test_parse_comment
+    t = RP.parse('a(?# is for apple)b(?# for boy)c(?# cat)')
+
+    [1,3,5].each do |i|
+      assert( t.expressions[i].is_a?(Group::Comment),
+             "Expected comment, but got #{t.expressions[i].class.name}")
+
+      assert_equal( :group,   t.expressions[i].type )
+      assert_equal( :comment, t.expressions[i].token )
+    end
+  end
+
+end

data/test/parser/test_properties.rb

@@ -0,0 +1,346 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ParserProperties < Test::Unit::TestCase
+
+  modes = ['p', 'P']
+  props = [
+    'Alnum',
+    'Alpha',
+    'Any',
+    'Ascii',
+    'Blank',
+    'Cntrl',
+    'Digit',
+    'Graph',
+    'Lower',
+    'Newline',
+    'Print',
+    'Punct',
+    'Space',
+    'Upper',
+    'Word',
+    'Xdigit',
+
+    'L',
+    'Letter',
+
+    'Lu',
+    'Uppercase_Letter',
+
+    'Ll',
+    'Lowercase_Letter',
+
+    'Lt',
+    'Titlecase_Letter',
+
+    'Lm',
+    'Modifier_Letter',
+
+    'Lo',
+    'Other_Letter',
+
+    'M',
+    'Mark',
+
+    'Mn',
+    'Nonspacing_Mark',
+
+    'Mc',
+    'Spacing_Mark',
+
+    'Me',
+    'Enclosing_Mark',
+
+    'N',
+    'Number',
+
+    'Nd',
+    'Decimal_Number',
+
+    'Nl',
+    'Letter_Number',
+
+    'No',
+    'Other_Number',
+
+    'P',
+    'Punctuation',
+
+    'Pc',
+    'Connector_Punctuation',
+
+    'Pd',
+    'Dash_Punctuation',
+
+    'Ps',
+    'Open_Punctuation',
+
+    'Pe',
+    'Close_Punctuation',
+
+    'Pi',
+    'Initial_Punctuation',
+
+    'Pf',
+    'Final_Punctuation',
+
+    'Po',
+    'Other_Punctuation',
+
+    'S',
+    'Symbol',
+
+    'Sm',
+    'Math_Symbol',
+
+    'Sc',
+    'Currency_Symbol',
+
+    'Sk',
+    'Modifier_Symbol',
+
+    'So',
+    'Other_Symbol',
+
+    'Z',
+    'Separator',
+
+    'Zs',
+    'Space_Separator',
+
+    'Zl',
+    'Line_Separator',
+
+    'Zp',
+    'Paragraph_Separator',
+
+    'C',
+    'Other',
+
+    'Cc',
+    'Control',
+
+    'Cf',
+    'Format',
+
+    'Cs',
+    'Surrogate',
+
+    'Co',
+    'Private_Use',
+
+    'Cn',
+    'Unassigned',
+
+    'Age=1.1',
+    'Age=2.0',
+    'Age=2.1',
+    'Age=3.0',
+    'Age=3.1',
+    'Age=3.2',
+    'Age=4.0',
+    'Age=4.1',
+    'Age=5.0',
+    'Age=5.1',
+    'Age=5.2',
+    'Age=6.0',
+
+    'ahex',
+    'ASCII_Hex_Digit',
+
+    'Alphabetic',
+
+    'Cased',
+
+    'cwcf',
+    'Changes_When_Casefolded',
+
+    'cwcm',
+    'Changes_When_Casemapped',
+
+    'cwl',
+    'Changes_When_Lowercased',
+
+    'cwt',
+    'Changes_When_Titlecased',
+
+    'cwu',
+    'Changes_When_Uppercased',
+
+    'ci',
+    'Case_Ignorable',
+
+    'bidic',
+    'Bidi_Control',
+
+    'Dash',
+
+    'dep',
+    'Deprecated',
+
+    'di',
+    'Default_Ignorable_Code_Point',
+
+    'dia',
+    'Diacritic',
+
+    'ext',
+    'Extender',
+
+    'grbase',
+    'Grapheme_Base',
+
+    'grext',
+    'Grapheme_Extend',
+
+    'grlink',
+    'Grapheme_Link',
+
+    'hex',
+    'Hex_Digit',
+
+    'Hyphen',
+
+    'idc',
+    'ID_Continue',
+
+    'ideo',
+    'Ideographic',
+
+    'ids',
+    'ID_Start',
+
+    'idsb',
+    'IDS_Binary_Operator',
+
+    'idst',
+    'IDS_Trinary_Operator',
+
+    'joinc',
+    'Join_Control',
+
+    'loe',
+    'Logical_Order_Exception',
+
+    'Lowercase',
+
+    'Math',
+
+    'nchar',
+    'Noncharacter_Code_Point',
+
+    'oalpha',
+    'Other_Alphabetic',
+
+    'odi',
+    'Other_Default_Ignorable_Code_Point',
+
+    'ogrext',
+    'Other_Grapheme_Extend',
+
+    'oidc',
+    'Other_ID_Continue',
+
+    'oids',
+    'Other_ID_Start',
+
+    'olower',
+    'Other_Lowercase',
+
+    'omath',
+    'Other_Math',
+
+    'oupper',
+    'Other_Uppercase',
+
+    'patsyn',
+    'Pattern_Syntax',
+
+    'patws',
+    'Pattern_Whitespace',
+
+    'qmark',
+    'quotationmark',
+
+    'radical',
+
+    'sd',
+    'Soft_Dotted',
+
+    'sterm',
+
+    'term',
+    'Terminal_Punctuation',
+
+    'uideo',
+    'Unified_Ideograph',
+
+    'Uppercase',
+
+    'vs',
+    'Variation_Selector',
+
+    'wspace',
+    'whitespace',
+
+    'xids',
+    'XID_Start',
+
+    'xidc',
+    'XID_Continue',
+  ]
+
+  modes.each do |mode|
+    token_type = mode == 'p' ? :property : :nonproperty
+
+    props.each do |property|
+      define_method "test_parse_#{token_type}_#{property}" do
+        t = RP.parse "ab\\#{mode}{#{property}}"
+
+        assert( t.expressions.last.is_a?(UnicodeProperty::Base),
+               "Expected property, but got #{t.expressions.last.class.name}")
+
+        assert_equal( token_type, t.expressions.last.type )
+        assert_equal( property,   t.expressions.last.name )
+      end
+    end
+  end
+
+  def test_parse_property_negative
+    t = RP.parse 'ab\p{L}cd'
+    assert_equal( false, t.expressions[1].negative? )
+  end
+
+  def test_parse_nonproperty_negative
+    t = RP.parse 'ab\P{L}cd'
+    assert_equal( true, t.expressions[1].negative? )
+  end
+
+  def test_parse_property_age
+    t = RP.parse 'ab\p{age=5.2}cd'
+
+    assert( t.expressions[1].is_a?(UnicodeProperty::Age),
+           "Expected Age property, but got #{t.expressions[1].class.name}")
+  end
+
+  def test_parse_property_derived
+    t = RP.parse 'ab\p{Math}cd'
+
+    assert( t.expressions[1].is_a?(UnicodeProperty::Derived),
+           "Expected Derived property, but got #{t.expressions[1].class.name}")
+  end
+
+  def test_parse_property_script
+    t = RP.parse 'ab\p{Hiragana}cd'
+
+    assert( t.expressions[1].is_a?(UnicodeProperty::Script),
+           "Expected Script property, but got #{t.expressions[1].class.name}")
+  end
+
+  def test_parse_property_following_literal
+    t = RP.parse 'ab\p{Lu}cd'
+
+    assert( t.expressions[2].is_a?(Literal),
+           "Expected Literal, but got #{t.expressions[2].class.name}")
+  end
+end