antlr3 1.3.1 → 1.4.0

data/lib/antlr3/template/parameter.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/ruby
+
+module ANTLR3
+module Template
+Parameter = Struct.new( :name, :default )
+class Parameter
+  def to_s
+    if block then "&#{ name }"
+    elsif splat then "*#{ name }"
+    elsif default then "#{ name } = #{ default }"
+    else name.dup
+    end
+  end
+end
+
+class ParameterList < ::Array
+  attr_accessor :splat, :block
+  
+  def self.default
+    new.add( :values ) do | p |
+      p.default = '{}'
+    end
+  end
+  
+  def names
+    names = map { | param | param.name.to_s }
+    @splat and names << @splat.to_s
+    @block and names << @block.to_s
+    return( names )
+  end
+  
+  def add( name, default = nil )
+    param =
+      case name
+      when Parameter then name
+      else Parameter.new( name.to_s )
+      end
+    if options
+      default = options[ :default ] and param.default = default
+      param.splat = options.fetch( :splat, false )
+      param.block = options.fetch( :block, false )
+    end
+    block_given? and yield( param )
+    push( param )
+    return( self )
+  end
+  
+  def to_s
+    signature = join( ', ' )
+    @splat and signature << ", *" << @splat.to_s
+    @block and signature << ", &" << @block.to_s
+    return( signature )
+  end
+end
+end
+end

data/lib/antlr3/test/grammar.rb

@@ -65,7 +65,7 @@ class Grammar
   ##################################################################
   def initialize(path, options = {})
     @path = path.to_s
-    @source = prepare_source(File.read(@path))
+    @source = File.read( @path )
     @output_directory = options.fetch(:output_directory, '.')
     @verbose = options.fetch( :verbose, $VERBOSE )
     study
@@ -138,7 +138,6 @@ class Grammar
     @type == "combined"
   end
   
-  
   def target_files(include_imports = true)
     targets = []
     
@@ -204,6 +203,7 @@ class Grammar
   end
   
 private
+  
   def post_compile(options)
     # do nothing for now
   end
@@ -257,10 +257,6 @@ private
     parts.map! { |part| shell_escape(part) }.join(' ') << ' 2>&1'
   end
   
-  def prepare_source(text)
-    text.gsub(/([^\\])%/,'\1\\%').freeze
-  end
-  
   def study
     @source =~ /^\s*(lexer|parser|tree)?\s*grammar\s*(\S+)\s*;/ or
       raise Grammar::FormatError[source, path]
@@ -272,7 +268,7 @@ end # class Grammar
 class Grammar::InlineGrammar < Grammar
   attr_accessor :host_file, :host_line
   
-  def initialize(source, options = {})
+  def initialize( source, options = {} )
     host = call_stack.find { |call| call.file != __FILE__ }
     
     @host_file = File.expand_path(options[:file] || host.file)
@@ -280,14 +276,14 @@ class Grammar::InlineGrammar < Grammar
     @output_directory = options.fetch(:output_directory, File.dirname(@host_file))
     @verbose = options.fetch( :verbose, $VERBOSE )
     
-    source = source.to_s.fixed_indent(0)
-    source.strip!
-    @source = prepare_source(source)
+    @source = source.to_s.fixed_indent( 0 )
+    @source.strip!
+    
     study
     write_to_disk
     build_dependencies
     
-    yield(self) if block_given?
+    yield( self ) if block_given?
   end
   
   def output_directory
@@ -303,7 +299,9 @@ class Grammar::InlineGrammar < Grammar
   def inspect
     sprintf( 'inline grammar %s (%s:%s)', name, @host_file, @host_line )
   end
+  
 private
+  
   def write_to_disk
     @path ||= output_directory / @name + '.g'
     test(?d, output_directory) or Dir.mkdir( output_directory )

data/lib/antlr3/tree.rb

@@ -1181,6 +1181,36 @@ class CommonTreeNodeStream
     return @nodes.dup
   end
   
+  def extract_text( start = nil, stop = nil )
+    @position == -1 and fill_buffer
+    start ||= @nodes.first
+    stop  ||= @nodes.last
+    
+    if @token_stream
+      case @adaptor.type_of( stop )
+      when UP
+        stop_index = @adaptor.token_stop_index( start )
+      when EOF
+        return extract_text( start, @nodes[ - 2 ] )
+      else
+        stop_index = @adaptor.token_stop_index( stop )
+      end
+      
+      start_index = @adaptor.token_start_index( start )
+      return @token_stream.extract_text( start_index, stop_index )
+    else
+      start_index = @nodes.index( start ) || @nodes.length
+      stop_index  = @nodes.index( stop )  || @nodes.length
+      return(
+        @nodes[ start_index .. stop_index ].map do | n |
+          @adaptor.text_of( n ) or " " + @adaptor.type_of( n ).to_s
+        end.join( '' )
+      )
+    end
+  end
+  
+  alias to_s extract_text
+  
 #private
 #  
 #  def linear_node_index( node )

data/lib/antlr3/version.rb

@@ -19,8 +19,8 @@ module ANTLR3
   # The version data for the current state the library itself
   # 
   MAJOR_VERSION = 1
-  MINOR_VERSION = 3
-  PATCH_VERSION = 1
+  MINOR_VERSION = 4
+  PATCH_VERSION = 0
   VERSION = [ MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION ]
   VERSION_STRING = VERSION.join('.').freeze
   

data/templates/Ruby.stg

@@ -327,22 +327,21 @@ end
 >>
 
 catch(decl, action) ::= <<
-# - - - - - - @catch e.decl - - - - - -
+# - - - - - - @catch <e.decl> - - - - - -
 rescue <e.decl>
   <e.action><\n>
 >>
 
-
 closureBlockLoop() ::= <<
-loop do  #loop <decisionNumber>
+loop do # decision <decisionNumber>
   alt_<decisionNumber> = <maxAlt>
   <@decisionBody><decision><@end>
   case alt_<decisionNumber>
   <alts:altSwitchCase(); separator="\n">
   else
-    break #loop <decisionNumber>
+    break # out of loop for decision <decisionNumber>
   end
-end
+end # loop for decision <decisionNumber>
 >>
 
 delegateName() ::= <<
@@ -555,7 +554,7 @@ memoize(__method__, <ruleDescriptor.name>_start_index, success) if @state.backtr
 
 /** helper template to format a ruby method call */
 methodCall(n, del, args) ::= <<
-<if(del)>@<del:delegateName()>.<endif><n><if(args)>(<args; seperator=", ">)<endif>
+<if(del)>@<del:delegateName()>.<endif><n><if(args)>(<args; separator=", ">)<endif>
 >>
 
 
@@ -1148,7 +1147,7 @@ ruleLabelPropertyRef_text(scope, attr) ::= <<
 >>
 
 
-ruleLabelPropertyRef_st(scope, attr) ::= "(<scope; format=\"label\">.nil? ? nil : <scope; format=\"label\">.st)"
+ruleLabelPropertyRef_st(scope, attr) ::= "(<scope; format=\"label\">.nil? ? nil : <scope; format=\"label\">.template)"
 
 /******************************************************************************
  *****************  L E X E R - O N L Y   T E M P L A T E S  ******************
@@ -1430,11 +1429,11 @@ rulePropertyRef_tree(scope, attr) ::= "return_value.tree"
 rulePropertyRef_text(scope, attr) ::= "@input.to_s(return_value.start, @input.look(-1))"
 
 /** $template in parser rule */
-rulePropertyRef_st(scope, attr) ::= "return_value.st"
+rulePropertyRef_st(scope, attr) ::= "return_value.template"
 
 ruleSetPropertyRef_tree(scope, attr, expr) ::= "return_value.tree = <expr>"
 
-ruleSetPropertyRef_st(scope, attr, expr) ::= "return_value.st = <expr>"
+ruleSetPropertyRef_st(scope, attr, expr) ::= "return_value.template = <expr>"
 
 /** How to execute an action */
 execAction(action) ::= <<
@@ -1498,4 +1497,4 @@ placeAction(scope, name) ::= <<
 <endif>
 >>
 
-runtimeLibraryVersion() ::= "1.3.1"
+runtimeLibraryVersion() ::= "1.4.0"

data/templates/ST.stg

@@ -0,0 +1,128 @@
+group ST;
+
+@parserBody.mixins() ::= <<
+<@super.mixins()>
+<if(rewriteMode)>
+include ANTLR3::Template::RewriteBuilder
+<else>
+include ANTLR3::Template::Builder
+<endif>
+>>
+
+gatedAction( block ) ::= <<
+<if(backtracking)>
+if <actions.(actionScope).synpredgate>
+  <block>
+end
+<else>
+<block>
+<endif>
+>>
+
+/** x+=rule when output=template */
+ruleRefAndListLabel(rule,label,elementIndex,args,scope) ::= <<
+<ruleRef(...)>
+<listLabel(elem={<label; format="label">.template}, ...)>
+>>
+
+rewriteTemplate(alts) ::= <<
+<gatedAction({
+<first(alts):rewriteTemplateAltFirst(); anchor>
+<rest(alts):{a| els<rewriteTemplateAltRest(a)>}; anchor, separator="\n\n">
+<if(rewriteMode)><replaceTextInLine()><endif>
+})>
+>>
+
+rewriteTemplateAltFirst(a) ::= <<
+<if(a.pred)>
+if <a.pred>
+  # <a.description>
+  return_value.template = <a.alt>
+<else>
+# <a.description>
+return_value.template = <a.alt>
+<endif>
+>>
+
+rewriteTemplateAltRest(a) ::= <<
+<if(a.pred)>
+if <a.pred>
+  # <a.description>
+  return_value.template = <a.alt>
+<else>
+e <! little hack to get if .. elsif .. else block right !>
+  # <a.description>
+  return_value.template = <a.alt>
+end
+<endif>
+>>
+
+replaceTextInLine() ::= <<
+<if(TREE_PARSER)>
+@input.token_stream.replace(
+  @input.adaptor.token_start_index( return_value.start ),
+  @input.adaptor.token_stop_index( return_value.start ),
+  return_value.template
+)
+<else>
+@input.replace(
+  return_value.start.index,
+  @input.look( -1 ).index,
+  return_value.template
+)
+<endif>
+>>
+
+rewriteEmptyTemplate(alts) ::= <<
+nil
+>>
+
+/** Invoke a template with a set of attribute name/value pairs.
+ *  Set the value of the rule's template *after* having set
+ *  the attributes because the rule's template might be used as
+ *  an attribute to build a bigger template; you get a self-embedded
+ *  template.
+ */
+rewriteExternalTemplate(name,args) ::= <<
+fetch_template( "<name>" <if(args)>,
+  <argumentMap(args)>
+<endif>)
+>>
+
+/** expr is a string expression that says what template to load */
+rewriteIndirectTemplate(expr,args) ::= <<
+fetch_template( <expr><if(args)>, <argumentMap(args)><endif> )
+>>
+
+/** Invoke an inline template with a set of attribute name/value pairs */
+rewriteInlineTemplate(args, template) ::= <<
+create_template( "<template>"<if(args)>, <argumentMap(args)><endif> )
+>>
+
+/** plain -> {foo} action */
+rewriteAction(action) ::= <<
+<action>
+>>
+
+/** An action has %st.attrName=expr; or %{st}.attrName=expr; */
+actionSetAttribute(st,attrName,expr) ::= <<
+( <st> )[ :<attrName> ] = <expr>
+>>
+
+/** Translate %{stringExpr} */
+actionStringConstructor(stringExpr) ::= <<
+create_template( <stringExpr> )
+>>
+
+/*
+rulePropertyRef_text(scope, attr) ::= <<
+@input.to_s(return_value.start, @input.look(-1))
+>>
+*/
+
+argumentMap(args) ::= <<
+<args:{a|:<a.name> => <a.value>}; separator=",\n">
+>>
+
+
+

data/test/functional/ast-output/auto-ast.rb

@@ -726,7 +726,7 @@ class TestAutoAST < ANTLR3::Test::Functional
       grammar RuleListLabelRuleRoot;
       options {language=Ruby;output=AST;}
       a returns [result] : ( x+=b^ )+ {
-      $result = "x=%s," \% $x[1].inspect
+      $result = "x=\%s," \% $x[1].inspect
       } ;
       b : ID;
       ID : 'a'..'z'+ ;

data/test/functional/ast-output/hetero-nodes.rb

@@ -202,8 +202,9 @@ class TestHeterogeneousNodeTypes < ANTLR3::Test::Functional
           end
           super(token)
         end
+        
         def to_s
-          (@token.text.to_s rescue '') << "<V>;\%d\%d\%d" % [@x, @y, @z]
+          (@token.text.to_s rescue '') << "<V>;\%d\%d\%d" \% [@x, @y, @z]
         end
       end
     

data/test/functional/template-output/template-output.rb

@@ -0,0 +1,404 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3/test/functional'
+
+class TestTemplateOutput < ANTLR3::Test::Functional
+  
+  def parse( grammar, input, options = nil )
+    @grammar = inline_grammar( grammar )
+    compile_and_load( @grammar )
+    grammar_module = self.class.const_get( @grammar.name )
+    
+    parser_options = {}
+    if options
+      rule = options.fetch( :rule ) { grammar_module::Parser.default_rule }
+      group = options[ :templates ] and parser_options[ :templates ] = group
+    else
+      rule = grammar_module::Parser.default_rule
+    end
+    
+    @lexer  = grammar_module::Lexer.new( input )
+    @parser = grammar_module::Parser.new( @lexer, parser_options )
+    
+    out = @parser.send( rule ).template
+    return( out ? out.to_s : out )
+  end
+  
+  def parse_templates( source )
+    ANTLR3::Template::Group.parse( source.fixed_indent( 0 ) )
+  end
+  
+  
+  example 'inline templates' do
+    text = parse( <<-'END', "abc 34" )
+      grammar InlineTemplates;
+      options {
+        language = Ruby;
+        output = template;
+      }
+      
+      a : ID INT
+        -> template(id={$ID.text}, int={$INT.text})
+           "id=<%= @id %>, int=<%= @int %>"
+      ;
+      
+      ID : 'a'..'z'+;
+      INT : '0'..'9'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == "id=abc, int=34"
+  end
+  
+  example 'external template' do
+    templates = ANTLR3::Template::Group.new do
+      define_template( :expr, <<-'END'.strip )
+        [<%= @args.join( @op.to_s ) %>]
+      END
+    end
+    
+    text = parse( <<-'END', 'a + b', :templates => templates )
+      grammar ExternalTemplate;
+      options {
+        language = Ruby;
+        output = template;
+      }
+      
+      a : r+=arg OP r+=arg
+        -> expr( op={$OP.text}, args={$r} )
+      ;
+      arg: ID -> template(t={$ID.text}) "<%= @t %>";
+      
+      ID : 'a'..'z'+;
+      OP: '+';
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == '[a+b]'
+  end
+
+  example "empty template" do
+    text = parse( <<-'END', 'abc 34' )
+      grammar EmptyTemplate;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      a : ID INT
+        -> 
+      ;
+      
+      ID : 'a'..'z'+;
+      INT : '0'..'9'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+      
+    END
+    text.should be_nil
+  end
+  
+  example "list" do
+    text = parse( <<-'END', "abc def ghi" )
+      grammar List;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      a: (r+=b)* EOF
+        -> template(r={$r}) "<%= @r.join(',') %>"
+      ;
+      
+      b: ID
+        -> template(t={$ID.text}) "<%= @t %>"
+      ;
+      
+      ID : 'a'..'z'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    text.should == 'abc,def,ghi'
+  end
+  
+  example 'action' do
+    text = parse( <<-'END', "abc" )
+      grammar Action;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      a: ID
+        -> { create_template( "hello" ) }
+      ;
+      
+      ID : 'a'..'z'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == 'hello'
+  end
+  
+  example "template expression in action" do
+    text = parse( <<-'END', 'abc' )
+      grammar TemplateExpressionInAction;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      a: ID
+        { $st = %{"hello"} }
+      ;
+      
+      ID : 'a'..'z'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    text.should == 'hello'
+  end
+  
+  #example "template expression in action2" do
+  #  text = parse( <<-'END', 'abc' )
+  #    grammar TemplateExpressionInAction2;
+  #    options {
+  #      language=Ruby;
+  #      output=template;
+  #    }
+  #    a: ID
+  #      {
+  #        res = %{"hello <%= @foo %>"}
+  #        %res.foo = "world";
+  #      }
+  #      -> { res }
+  #    ;
+  #    
+  #    ID : 'a'..'z'+;
+  #    WS : (' '|'\n') {$channel=HIDDEN;} ;
+  #  END
+  #  
+  #  text.should == 'hello world'
+  #end
+  
+  example "indirect template constructor" do
+    templates = ANTLR3::Template::Group.new do
+      define_template( :expr, <<-'END'.strip )
+        [<%= @args.join( @op.to_s ) %>]
+      END
+    end
+    
+    text = parse( <<-'END', 'abc', :templates => templates )
+      grammar IndirectTemplateConstructor;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      
+      a: ID
+        {
+          $st = %({"expr"})(args={[1, 2, 3]}, op={"+"})
+        }
+      ;
+      
+      ID : 'a'..'z'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == '[1+2+3]'
+  end
+  
+  example "predicates" do
+    text = parse( <<-'END', 'b 34' )
+      grammar Predicates;
+      options {
+        language=Ruby;
+        output=template;
+      }
+      a : ID INT
+        -> {$ID.text=='a'}? template(int={$INT.text})
+                            "A: <%= @int %>"
+        -> {$ID.text=='b'}? template(int={$INT.text})
+                            "B: <%= @int %>"
+        ->                  template(int={$INT.text})
+                            "C: <%= @int %>"
+      ;
+      
+      ID : 'a'..'z'+;
+      INT : '0'..'9'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == 'B: 34'
+  end
+  
+  example "backtracking mode" do
+    text = parse( <<-'END', 'abc 34' )
+      grammar BacktrackingMode;
+      options {
+        language=Ruby;
+        output=template;
+        backtrack=true;
+      }
+      a : (ID INT)=> ID INT
+        -> template(id={$ID.text}, int={$INT.text})
+           "id=<%= @id %>, int=<%= @int %>"
+      ;
+      
+      ID : 'a'..'z'+;
+      INT : '0'..'9'+;
+      WS : (' '|'\n') {$channel=HIDDEN;} ;
+    END
+    
+    text.should == "id=abc, int=34"
+  end
+  
+  example "rewrite" do
+    input = <<-'END'.here_indent!
+    | if ( foo ) {
+    |   b = /* bla */ 2;
+    |   return 1 /* foo */;
+    | }
+    | 
+    | /* gnurz */
+    | return 12;
+    END
+    expected = <<-'END'.here_indent!
+    | if ( foo ) {
+    |   b = /* bla */ 2;
+    |   return boom(1) /* foo */;
+    | }
+    | 
+    | /* gnurz */
+    | return boom(12);
+    END
+    
+    parse( <<-'END', input )
+      grammar Rewrite;
+      options {
+        language=Ruby;
+        output=template;
+        rewrite=true;
+      }
+      
+      prog: stat+;
+      
+      stat
+          : 'if' '(' expr ')' stat
+          | 'return' return_expr ';'
+          | '{' stat* '}'
+          | ID '=' expr ';'
+          ;
+      
+      return_expr
+          : expr
+            -> template(t={$text}) <<boom(<%= @t %>)>>
+          ;
+          
+      expr
+          : ID
+          | INT
+          ;
+          
+      ID:  'a'..'z'+;
+      INT: '0'..'9'+;
+      WS: (' '|'\n')+ {$channel=HIDDEN;} ;
+      COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
+    END
+    
+    @parser.input.render.should == expected
+  end
+  
+  example "tree rewrite" do
+    input = <<-'END'.here_indent!
+    | if ( foo ) {
+    |   b = /* bla */ 2;
+    |   return 1 /* foo */;
+    | }
+    | 
+    | /* gnurz */
+    | return 12;
+    END
+    expected = <<-'END'.here_indent!
+    | if ( foo ) {
+    |   b = /* bla */ 2;
+    |   return boom(1) /* foo */;
+    | }
+    | 
+    | /* gnurz */
+    | return boom(12);
+    END
+    
+    compile_and_load( inline_grammar( <<-'END' ) )
+      grammar TreeRewrite;
+      options {
+        language=Ruby;
+        output=AST;
+      }
+      
+      tokens {
+        BLOCK;
+        ASSIGN;
+      }
+      
+      prog: stat+;
+      
+      stat
+          : IF '(' e=expr ')' s=stat
+            -> ^(IF $e $s)
+          | RETURN expr ';'
+            -> ^(RETURN expr)
+          | '{' stat* '}'
+            -> ^(BLOCK stat*)
+          | ID '=' expr ';'
+            -> ^(ASSIGN ID expr)
+          ;
+          
+      expr
+          : ID
+          | INT
+          ;
+      
+      IF: 'if';
+      RETURN: 'return';
+      ID:  'a'..'z'+;
+      INT: '0'..'9'+;
+      WS: (' '|'\n')+ {$channel=HIDDEN;} ;
+      COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
+    END
+    
+    compile_and_load( inline_grammar( <<-'END' ) )
+      tree grammar TreeRewriteTG;
+      options {
+        language=Ruby;
+        tokenVocab=TreeRewrite;
+        ASTLabelType=CommonTree;
+        output=template;
+        rewrite=true;
+      }
+      
+      prog: stat+;
+      
+      stat
+          : ^(IF expr stat)
+          | ^(RETURN return_expr)                
+          | ^(BLOCK stat*)                
+          | ^(ASSIGN ID expr)
+          ;
+      
+      return_expr
+          : expr
+            -> template(t={$text}) <<boom(<%= @t %>)>>
+          ;
+      
+      expr
+          : ID
+          | INT
+          ;
+    END
+    
+    lexer = TreeRewrite::Lexer.new( input )
+    tokens = ANTLR3::TokenRewriteStream.new( lexer )
+    parser = TreeRewrite::Parser.new( tokens )
+    tree = parser.prog.tree
+    nodes = ANTLR3::AST::CommonTreeNodeStream.new( tree )
+    nodes.token_stream = tokens
+    tree_parser = TreeRewriteTG::TreeParser.new( nodes )
+    tree_parser.prog
+    tokens.render.should == expected
+  end
+end