antlr3 1.2.3

data/test/functional/lexer/filter-mode.rb

@@ -0,0 +1,245 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class TestFilterMode < ANTLR3::Test::Functional
+
+  inline_grammar(<<-'END')
+    lexer grammar Filter;
+    options {
+        language = Ruby;
+        filter=true;
+    }
+    
+    IMPORT
+      :  'import' WS QIDStar WS? ';'
+      ;
+      
+    RETURN
+      :  'return' .* ';'
+      ;
+    
+    CLASS
+      :  'class' WS ID WS? ('extends' WS QID WS?)?
+        ('implements' WS QID WS? (',' WS? QID WS?)*)? '{'
+      ;
+      
+    COMMENT
+        :   '/*' .* '*/'
+        ;
+    
+    STRING
+        :  '"' (options {greedy=false;}: ESC | .)* '"'
+      ;
+    
+    CHAR
+      :  '\'' (options {greedy=false;}: ESC | .)* '\''
+      ;
+    
+    WS  :   (' '|'\t'|'\n')+
+        ;
+    
+    fragment
+    QID :  ID ('.' ID)*
+      ;
+      
+    /** QID cannot see beyond end of token so using QID '.*'? somewhere won't
+     *  ever match since k=1 look in the QID loop of '.' will make it loop.
+     *  I made this rule to compensate.
+     */
+    fragment
+    QIDStar
+      :  ID ('.' ID)* '.*'?
+      ;
+    
+    fragment
+    TYPE:   QID '[]'?
+        ;
+        
+    fragment
+    ARG :   TYPE WS ID
+        ;
+    
+    fragment
+    ID  :   ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')*
+        ;
+    
+    fragment
+    ESC  :  '\\' ('"'|'\''|'\\')
+      ;
+  END
+
+  example "skipping tokens that aren't important with filter mode" do
+    input = <<-END.fixed_indent(0)
+      import org.antlr.runtime.*;
+      
+      public class Main {
+        public static void main(String[] args) throws Exception {
+            for (int i=0; i<args.length; i++) {
+          CharStream input = new ANTLRFileStream(args[i]);
+          FuzzyJava lex = new FuzzyJava(input);
+          TokenStream tokens = new CommonTokenStream(lex);
+          tokens.toString();
+          //System.out.println(tokens);
+            }
+        }
+      }
+    END
+    
+    lexer = Filter::Lexer.new( input )
+    tokens = lexer.map { |tk| tk }
+  end
+  
+
+end
+
+
+class TestFuzzy < ANTLR3::Test::Functional
+
+  inline_grammar(<<-'END')
+    lexer grammar Fuzzy;
+    options {
+        language = Ruby;
+        filter=true;
+    }
+    
+    @members {
+      include ANTLR3::Test::CaptureOutput
+    }
+    
+    IMPORT
+      :  'import' WS name=QIDStar WS? ';'
+      ;
+      
+    /** Avoids having "return foo;" match as a field */
+    RETURN
+      :  'return' (options {greedy=false;}:.)* ';'
+      ;
+    
+    CLASS
+      :  'class' WS name=ID WS? ('extends' WS QID WS?)?
+        ('implements' WS QID WS? (',' WS? QID WS?)*)? '{'
+        {  
+          say("found class " << $name.text)  
+        }
+      ;
+      
+    METHOD
+        :   TYPE WS name=ID WS? '(' ( ARG WS? (',' WS? ARG WS?)* )? ')' WS? 
+           ('throws' WS QID WS? (',' WS? QID WS?)*)? '{'
+            {
+              say("found method " << $name.text)
+            }
+        ;
+    
+    FIELD
+        :   TYPE WS name=ID '[]'? WS? (';'|'=')
+            {
+              say("found var " << $name.text)
+            }
+        ;
+    
+    STAT:  ('if'|'while'|'switch'|'for') WS? '(' ;
+      
+    CALL
+        :   name=QID WS? '('
+            {
+              say("found call " << $name.text)
+            }
+        ;
+    
+    COMMENT
+        :   '/*' (options {greedy=false;} : . )* '*/'
+            {
+              say("found comment " << self.text)
+            }
+        ;
+    
+    SL_COMMENT
+        :   '//' (options {greedy=false;} : . )* '\n'
+            {
+              say("found // comment " << self.text)
+            }
+        ;
+      
+    STRING
+      :  '"' (options {greedy=false;}: ESC | .)* '"'
+      ;
+    
+    CHAR
+      :  '\'' (options {greedy=false;}: ESC | .)* '\''
+      ;
+    
+    WS  :   (' '|'\t'|'\n')+
+        ;
+    
+    fragment
+    QID :  ID ('.' ID)*
+      ;
+      
+    /** QID cannot see beyond end of token so using QID '.*'? somewhere won't
+     *  ever match since k=1 look in the QID loop of '.' will make it loop.
+     *  I made this rule to compensate.
+     */
+    fragment
+    QIDStar
+      :  ID ('.' ID)* '.*'?
+      ;
+    
+    fragment
+    TYPE:   QID '[]'?
+        ;
+        
+    fragment
+    ARG :   TYPE WS ID
+        ;
+    
+    fragment
+    ID  :   ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')*
+        ;
+    
+    fragment
+    ESC  :  '\\' ('"'|'\''|'\\')
+      ;
+  END
+  
+  example "fuzzy lexing with the filter mode option" do
+    input = <<-END.fixed_indent(0)
+      import org.antlr.runtime.*;
+      
+      public class Main {
+        public static void main(String[] args) throws Exception {
+            for (int i=0; i<args.length; i++) {
+          CharStream input = new ANTLRFileStream(args[i]);
+          FuzzyJava lex = new FuzzyJava(input);
+          TokenStream tokens = new CommonTokenStream(lex);
+          tokens.toString();
+          //System.out.println(tokens);
+            }
+        }
+      }
+    END
+    
+    expected_output = <<-END.fixed_indent(0)
+      found class Main
+      found method main
+      found var i
+      found var input
+      found call ANTLRFileStream
+      found var lex
+      found call FuzzyJava
+      found var tokens
+      found call CommonTokenStream
+      found call tokens.toString
+      found // comment //System.out.println(tokens);
+    END
+    
+    lexer = Fuzzy::Lexer.new(input)
+    lexer.each { |tk| tk }
+    lexer.output.should == expected_output
+  end
+
+
+end
+

data/test/functional/lexer/nuances.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class TestBug80 < ANTLR3::Test::Functional
+  inline_grammar(<<-'END')
+    lexer grammar Bug80;
+    options { language = Ruby; }
+     
+    ID_LIKE
+        : 'defined' 
+        | {false}? Identifier 
+        | Identifier 
+        ; 
+     
+    fragment
+    // with just 'a', output compiles
+    Identifier: 'a'..'z'+ ;
+  END
+  
+  example "um... something" do
+    lexer = Bug80::Lexer.new( 'defined' )
+    tokens = lexer.each { |tk| tk }
+  end
+end
+
+
+class TestEOF < ANTLR3::Test::Functional
+
+  inline_grammar(<<-'END')
+    lexer grammar EndOfFile;
+    
+    options {
+      language = Ruby;
+    }
+    
+    KEND: EOF;
+    SPACE: ' ';
+  END
+  
+  example 'referencing EOF in a rule' do
+    lexer = EndOfFile::Lexer.new( " " )
+    tks = lexer.map { |tk| tk }
+  end
+end
+

data/test/functional/lexer/properties.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class TestLexerRuleReference < ANTLR3::Test::Functional
+
+  inline_grammar(<<-'END')
+    lexer grammar RuleProperty;
+    options {
+      language = Ruby;
+    }
+    
+    @lexer::init {
+      @properties = []
+    }
+    @lexer::members {
+      attr_reader :properties
+    }
+    
+    IDENTIFIER: 
+            ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
+            {
+              @properties << [$text, $type, $line, $pos, $index, $channel, $start, $stop]
+            }
+        ;
+    WS: (' ' | '\n')+;
+  END
+
+  example "referencing lexer rule properties" do
+    lexer = RuleProperty::Lexer.new( "foobar _ab98 \n A12sdf" )
+    tokens = lexer.map { |tk| tk }
+    
+    lexer.properties.should have(3).things
+    text, type, line, pos, index, channel, start, stop = lexer.properties[0]
+    text.should == 'foobar'
+    type.should == RuleProperty::TokenData::IDENTIFIER
+    line.should == 1
+    pos.should == 0
+    index.should == -1
+    channel.should == ANTLR3::DEFAULT_CHANNEL
+    start.should == 0
+    stop.should == 5
+    
+    text, type, line, pos, index, channel, start, stop = lexer.properties[1]
+    text.should == '_ab98'
+    type.should == RuleProperty::TokenData::IDENTIFIER
+    line.should == 1
+    pos.should == 7
+    index.should == -1
+    channel.should == ANTLR3::DEFAULT_CHANNEL
+    start.should == 7
+    stop.should == 11
+    
+    lexer.properties.should have(3).things
+    text, type, line, pos, index, channel, start, stop = lexer.properties[2]
+    text.should == 'A12sdf'
+    type.should == RuleProperty::TokenData::IDENTIFIER
+    line.should == 2
+    pos.should == 1
+    index.should == -1
+    channel.should == ANTLR3::DEFAULT_CHANNEL
+    start.should == 15
+    stop.should == 20
+  end
+
+
+end
+
+class TestLexerRuleLabel < ANTLR3::Test::Functional
+  inline_grammar(<<-'END')
+    lexer grammar LexerRuleLabel;
+    options {
+      language = Ruby;
+    }
+    
+    @members { attr_reader :token_text }
+    
+    A: 'a'..'z' WS '0'..'9'
+            {
+              @token_text = $WS.text
+            }
+        ;
+    
+    fragment WS  :
+            (   ' '
+            |   '\t'
+            |  ( '\n'
+                |	'\r\n'
+                |	'\r'
+                )
+            )+
+            { $channel = HIDDEN }
+        ;
+  END
+  
+  example "referencing other token rule values with labels" do
+    lexer = LexerRuleLabel::Lexer.new 'a  2'
+    lexer.next_token
+    lexer.token_text.should == '  '
+  end
+
+end
+

data/test/functional/lexer/syn-pred.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class TestSyntacticPredicate < ANTLR3::Test::Functional
+  inline_grammar(<<-'END')
+    lexer grammar SyntacticPredicateGate;
+    options {
+      language = Ruby;
+    }
+    
+    FOO
+      : ('ab')=> A
+      | ('ac')=> B
+      ;
+    
+    fragment
+    A: 'a';
+    
+    fragment
+    B: 'a';
+  END
+
+  example 'gating syntactic predicate rule' do
+    lexer = SyntacticPredicateGate::Lexer.new( 'ac' )
+    token = lexer.next_token
+  end
+
+
+end
+

data/test/functional/lexer/xml.rb

@@ -0,0 +1,206 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class XMLLexerTest < ANTLR3::Test::Functional
+  inline_grammar(<<-'END')
+    lexer grammar XML;
+    options { language = Ruby; }
+    
+    @members {
+      include ANTLR3::Test::CaptureOutput
+      include ANTLR3::Test::RaiseErrors
+      
+      def quote(text)
+        text = text.gsub(/\"/, '\\"')
+        \%("#{ text }")
+      end
+    }
+    
+    DOCUMENT
+        :  XMLDECL? WS? DOCTYPE? WS? ELEMENT WS? 
+        ;
+    
+    fragment DOCTYPE
+        :
+            '<!DOCTYPE' WS rootElementName=GENERIC_ID 
+            {say("ROOTELEMENT: " + $rootElementName.text)}
+            WS
+            ( 
+                ( 'SYSTEM' WS sys1=VALUE
+            {say("SYSTEM: " + $sys1.text)}
+                    
+                | 'PUBLIC' WS pub=VALUE WS sys2=VALUE
+                    {say("PUBLIC: " + $pub.text)}
+                    {say("SYSTEM: " + $sys2.text)}   
+                )
+                ( WS )?
+            )?
+            ( dtd=INTERNAL_DTD
+                {say("INTERNAL DTD: " + $dtd.text)}
+            )?
+        '>'
+      ;
+    
+    fragment INTERNAL_DTD : '[' (options {greedy=false;} : .)* ']' ;
+    
+    fragment PI :
+            '<?' target=GENERIC_ID WS? 
+              {say("PI: " + $target.text)}
+            ( ATTRIBUTE WS? )*  '?>'
+      ;
+    
+    fragment XMLDECL :
+            '<?' ('x'|'X') ('m'|'M') ('l'|'L') WS? 
+              {say("XML declaration")}
+            ( ATTRIBUTE WS? )*  '?>'
+      ;
+    
+    
+    fragment ELEMENT
+        : ( START_TAG
+                (ELEMENT
+                | t=PCDATA
+                    {say("PCDATA: " << quote($t.text))}
+                | t=CDATA
+                    {say("CDATA: " << quote($t.text))}
+                | t=COMMENT
+                    {say("Comment: " << quote($t.text))}
+                | pi=PI
+                )*
+                END_TAG
+            | EMPTY_ELEMENT
+            )
+        ;
+    
+    fragment START_TAG 
+        : '<' WS? name=GENERIC_ID WS?
+              {say("Start Tag: " + $name.text)}
+            ( ATTRIBUTE WS? )* '>'
+        ;
+    
+    fragment EMPTY_ELEMENT 
+        : '<' WS? name=GENERIC_ID WS?
+              {say("Empty Element: " + $name.text)}
+            ( ATTRIBUTE WS? )* '/>'
+        ;
+    
+    fragment ATTRIBUTE 
+        : name=GENERIC_ID WS? '=' WS? value=VALUE
+            {say("Attr: " + $name.text + " = "+ $value.text)}
+        ;
+    
+    fragment END_TAG 
+        : '</' WS? name=GENERIC_ID WS? '>'
+            {say("End Tag: " + $name.text)}
+        ;
+    
+    fragment COMMENT
+      :	'<!--' (options {greedy=false;} : .)* '-->'
+      ;
+    
+    fragment CDATA
+      :	'<![CDATA[' (options {greedy=false;} : .)* ']]>'
+      ;
+    
+    fragment PCDATA : (~'<')+ ; 
+    
+    fragment VALUE : 
+            ( '\"' (~'\"')* '\"'
+            | '\'' (~'\'')* '\''
+            )
+      ;
+    
+    fragment GENERIC_ID 
+        : ( LETTER | '_' | ':') 
+            ( options {greedy=true;} : LETTER | '0'..'9' | '.' | '-' | '_' | ':' )*
+      ;
+    
+    fragment LETTER
+      : 'a'..'z' 
+      | 'A'..'Z'
+      ;
+    
+    fragment WS  :
+            (   ' '
+            |   '\t'
+            |  ( '\n'
+                |	'\r\n'
+                |	'\r'
+                )
+            )+
+        ;    
+  END
+  
+  it "should be valid" do
+    lexer = XML::Lexer.new(<<-'END'.fixed_indent(0))
+      <?xml version='1.0'?>
+      <!DOCTYPE component [
+      <!ELEMENT component (PCDATA|sub)*>
+      <!ATTLIST component
+                attr CDATA #IMPLIED
+                attr2 CDATA #IMPLIED
+      >
+      <!ELMENT sub EMPTY>
+      
+      ]>
+      <component attr="val'ue" attr2='val"ue'>
+      <!-- This is a comment -->
+      Text
+      <![CDATA[huhu]]>
+      öäüß
+      &amp;
+      &lt;
+      <?xtal cursor='11'?>
+      <sub/>
+      <sub></sub>
+      </component>
+    END
+    
+    lexer.map { |tk| tk }
+    
+    lexer.output.should == <<-'END'.fixed_indent(0)
+      XML declaration
+      Attr: version = '1.0'
+      ROOTELEMENT: component
+      INTERNAL DTD: [
+      <!ELEMENT component (PCDATA|sub)*>
+      <!ATTLIST component
+                attr CDATA #IMPLIED
+                attr2 CDATA #IMPLIED
+      >
+      <!ELMENT sub EMPTY>
+      
+      ]
+      Start Tag: component
+      Attr: attr = "val'ue"
+      Attr: attr2 = 'val"ue'
+      PCDATA: "
+      "
+      Comment: "<!-- This is a comment -->"
+      PCDATA: "
+      Text
+      "
+      CDATA: "<![CDATA[huhu]]>"
+      PCDATA: "
+      öäüß
+      &amp;
+      &lt;
+      "
+      PI: xtal
+      Attr: cursor = '11'
+      PCDATA: "
+      "
+      Empty Element: sub
+      PCDATA: "
+      "
+      Start Tag: sub
+      End Tag: sub
+      PCDATA: "
+      "
+      End Tag: component
+    END
+  end
+
+end