antlr3 1.3.0 → 1.3.1

data/lib/antlr3/task.rb

@@ -1,13 +1,456 @@
 #!/usr/bin/ruby
 # encoding: utf-8
 
+require 'antlr3'
 require 'rake/tasklib'
+require 'shellwords'
 
 module ANTLR3
+
+=begin rdoc ANTLR3::CompileTask
+
+A rake task-generating utility concerning ANTLR grammar file
+compilation. This is a general utility -- the grammars do
+not have to be targetted for Ruby output; it handles all
+known ANTLR language targets.
+
+  require 'antlr3/task'
+  
+  ANTLR3::CompileTask.define(
+    :name => 'grammars', :output_directory => 'lib/parsers'
+  ) do | t |
+    t.grammar_set( 'antlr/MainParser.g', 'antlr/MainTree.g' )
+    
+    t.grammar_set( 'antlr/Template.g' ) do | gram |
+      gram.output_directory = 'lib/parsers/template'
+      gram.debug = true
+    end
+  end
+  
+
+TODO: finish documentation
+
+=end
+
 class CompileTask < Rake::TaskLib
+  attr_reader :grammar_sets, :options
+  attr_accessor :name
+  
+  def self.define( *grammar_files )
+    lib = new( *grammar_files )
+    block_given? and yield( lib )
+    lib.define
+    return( lib )
+  end
+  
+  def initialize( *grammar_files )
+    grammar_files = [ grammar_files ].flatten!
+    options = Hash === grammar_files.last ? grammar_files.pop : {}
+    @grammar_sets = []
+    @name = options.fetch( :name, 'antlr-grammars' )
+    @options = options
+    grammar_files.empty? or grammar_set( grammar_files )
+  end
+  
+  def target_files
+    @grammar_sets.inject( [] ) do | list, set |
+      list.concat( set.target_files )
+    end
+  end
+  
+  def grammar_set( *grammar_files )
+    grammar_files = [ grammar_files ].flatten!
+    options = @options.merge(
+      Hash === grammar_files.last ? grammar_files.pop : {}
+    )
+    set = GrammarSet.new( grammar_files, options )
+    block_given? and yield( set )
+    @grammar_sets << set
+    return( set )
+  end
   
+  def define
+    namespace( @name ) do
+      desc( "trash all ANTLR-generated source code")
+      task( 'clobber' ) do
+        for set in @grammar_sets
+          set.clean
+        end
+      end
+      
+      for set in @grammar_sets
+        set.define_tasks
+      end
+      
+      desc( "compile ANTLR grammars" )
+      task( 'compile' => target_files )
+    end
+  end
   
+
+class CompileTask::GrammarSet
+  attr_accessor :antlr_jar, :debug,
+                :trace, :profile, :compile_options,
+                :java_options
+  attr_reader :load_path, :grammars
+  attr_writer :output_directory
+  
+  def initialize( grammar_files, options = {} )
+    @load_path = grammar_files.map { | f | File.dirname( f ) }
+    @load_path.push( '.', @output_directory )
+    
+    if extra_load = options[ :load_path ]
+      extra_load = [ extra_load ].flatten
+      @load_path.unshift( extra_load )
+    end
+    @load_path.uniq!
+    
+    @grammars = grammar_files.map do | file |
+      GrammarFile.new( self, file )
+    end
+    @output_directory = nil
+    dir = options[ :output_directory ] and @output_directory = dir.to_s
+    
+    @antlr_jar = options.fetch( :antlr_jar, ANTLR3.antlr_jar )
+    @debug = options.fetch( :debug, false )
+    @trace = options.fetch( :trace, false )
+    @profile = options.fetch( :profile, false )
+    @compile_options =
+      case opts = options[ :compile_options ]
+      when Array then opts
+      else Shellwords.shellsplit( opts.to_s )
+      end
+    @java_options =
+      case opts = options[ :java_options ]
+      when Array then opts
+      else Shellwords.shellsplit( opts.to_s )
+      end
+  end
+  
+  def target_files
+    @grammars.map { | gram | gram.target_files }.flatten
+  end
+  
+  def output_directory
+    @output_directory || '.'
+  end
+  
+  def define_tasks
+    directory( @output_directory )
+    file( @antlr_jar )
+    
+    for grammar in @grammars
+      deps = [ @output_directory, @antlr_jar ]
+      if  vocab = grammar.token_vocab and
+          tfile = find_tokens_file( vocab, grammar )
+        file( tfile )
+        deps << tfile
+      end
+      grammar.define_tasks( deps )
+    end
+  end
+  
+  def clean
+    for grammar in @grammars
+      grammar.clean
+    end
+    if test( ?d, output_directory ) and ( Dir.entries( output_directory ) - %w( . .. ) ).empty?
+      rmdir( output_directory )
+    end
+  end
   
+  def find_tokens_file( vocab, grammar )
+    gram = @grammars.find { | gram | gram.name == vocab } and
+      return( gram.tokens_file )
+    file = locate( "#{ vocab }.tokens" ) and return( file )
+    warn( Util.tidy( <<-END, true ) )
+    | unable to locate .tokens file `#{ vocab }' referenced in #{ grammar.path }
+    | -- ignoring dependency
+    END
+    return( nil )
+  end
   
+  def locate( file_name )
+    dir = @load_path.find do | dir |
+      File.file?( File.join( dir, file_name ) )
+    end
+    dir and return( File.join( dir, file_name ) )
+  end
+  
+  def compile( grammar )
+    sh( build_command( grammar ) )
+  end
+  
+  def build_command( grammar )
+    parts = [ 'java', '-cp', @antlr_jar ]
+    parts.concat( @java_options )
+    parts << 'org.antlr.Tool' << '-fo' << output_directory
+    parts << '-debug' if @debug
+    parts << '-profile' if @profile
+    parts << '-trace' if @trace
+    parts.concat( @compile_options )
+    parts << grammar.path
+    return Shellwords.shelljoin( parts )
+  end
 end
+
+class GrammarFile
+  LANGUAGES = {
+    "ActionScript" => [".as"],
+    "CSharp2" => [".cs"],
+    "C" => [".c", ".h"],
+    "ObjC" => [".m", ".h"],
+    "CSharp3" => [".cs"],
+    "Cpp" => [".cpp", ".h"],
+    "Ruby" => [".rb"],
+    "Java" => [".java"],
+    "JavaScript" => [".js"],
+    "Python" => [".py"],
+    "Delphi" => [".pas"],
+    "Perl5" => [".pm"]
+  }.freeze
+  GRAMMAR_TYPES = %w(lexer parser tree combined)
+  
+  ##################################################################
+  ######## CONSTRUCTOR #############################################
+  ##################################################################
+  
+  def initialize( group, path, options = {} )
+    @group = group
+    @path = path.to_s
+    @imports = []
+    @language = 'Java'
+    @token_vocab = nil
+    @tasks_defined = false
+    @extra_dependencies = []
+    if extra = options[ :extra_dependencies ]
+      extra = [ extra ].flatten
+      @extra_dependencies.concat( extra )
+    end
+    
+    study
+    yield( self ) if block_given?
+    fetch_imports
+  end
+  
+  ##################################################################
+  ######## ATTRIBUTES AND ATTRIBUTE-ISH METHODS ####################
+  ##################################################################
+  attr_reader :type, :name, :language, :source,
+              :token_vocab, :imports, :imported_grammars,
+              :path, :group
+  
+  for attr in [ :output_directory, :load_path, :antlr_jar ]
+    class_eval( <<-END )
+      def #{ attr }
+        @group.#{ attr }
+      end
+    END
+  end
+  
+  def lexer_files
+    if lexer? then base = @name
+    elsif combined? then base = @name + 'Lexer'
+    else return( [] )
+    end
+    return( file_names( base ) )
+  end
+  
+  def parser_files
+    if parser? then base = @name
+    elsif combined? then base = @name + 'Parser'
+    else return( [] )
+    end
+    return( file_names( base ) )
+  end
+  
+  def tree_parser_files
+    return( tree? ? file_names( @name ) : [] )
+  end
+  
+  def file_names( base )
+    LANGUAGES.fetch( @language ).map do | ext |
+      File.join( output_directory, base + ext )
+    end
+  end
+  
+  for type in GRAMMAR_TYPES
+    class_eval( <<-END )
+      def #{ type }?
+        @type == #{ type.inspect }
+      end
+    END
+  end
+  
+  def delegate_files( delegate_suffix )
+    file_names( "#{ name }_#{ delegate_suffix }" )
+  end
+  
+  def tokens_file
+    File.join( output_directory, name + '.tokens' )
+  end
+  
+  def target_files( all = true )
+    targets = [ tokens_file ]
+    
+    for target_type in %w( lexer parser tree_parser )
+      for file in self.send( :"#{ target_type }_files")
+        targets << file
+      end
+    end
+    
+    if all
+      for grammar in @imported_grammars
+        targets.concat( grammar.target_files )
+      end
+    end
+    
+    return targets
+  end
+  
+  def update
+    touch( @path )
+  end
+  
+  def all_imported_files
+    imported_files = []
+    for grammar in @imported_grammars
+      imported_files.push( grammar.path, *grammar.all_imported_files )
+    end
+    return imported_files
+  end
+  
+  def clean
+    deleted = []
+    for target in target_files
+      if test( ?f, target )
+        rm( target )
+        deleted << target
+      end
+    end
+    
+    for grammar in @imported_grammars
+      deleted.concat( grammar.clean )
+    end
+    
+    return deleted
+  end
+  
+  def define_tasks( shared_depends )
+    unless @tasks_defined
+      depends = [ @path, *all_imported_files ]
+      for f in depends
+        file( f )
+      end
+      depends = shared_depends + depends
+      
+      target_files.each do | target |
+        file( target => depends ) do
+          @group.compile( self )
+        end
+      end
+      
+      @tasks_defined = true
+    end
+  end
+  
+private
+  
+  def fetch_imports
+    @imported_grammars = @imports.map do | imp |
+      file = group.locate( "#{ imp }.g" ) or raise( Util.tidy( <<-END ) )
+      | #{ @path }: unable to locate imported grammar file #{ imp }.g
+      | search directories ( @load_path ):
+      |   - #{ load_path.join( "\n  - " ) }
+      END
+      Imported.new( self, file )
+    end
+  end
+  
+  def study
+    @source = File.read( @path )
+    @source =~ /^\s*(lexer|parser|tree)?\s*grammar\s*(\S+)\s*;/ or
+      raise Grammar::FormatError[ @source, @path ]
+    @name = $2
+    @type = $1 || 'combined'
+    if @source =~ /^\s*options\s*\{(.*?)\}/m
+      option_block = $1
+      if option_block =~ /\s*language\s*=\s*(\S+)\s*;/
+        @language = $1
+        LANGUAGES.has_key?( @language ) or
+          raise( Grammar::FormatError, "Unknown ANTLR target language: %p" % @language )
+      end
+      option_block =~ /\s*tokenVocab\s*=\s*(\S+)\s*;/ and
+        @token_vocab = $1
+    end
+    
+    @source.scan(/^\s*import\s+(\w+\s*(?:,\s*\w+\s*)*);/) do
+      list = $1.strip
+      @imports.concat( list.split( /\s*,\s*/ ) )
+    end
+  end
+end # class Grammar
+
+class GrammarFile::Imported < GrammarFile
+  def initialize( owner, path )
+    @owner = owner
+    @path = path.to_s
+    @imports = []
+    @language = 'Java'
+    @token_vocab = nil
+    study
+    fetch_imports
+  end
+  
+  for attr in [ :load_path, :output_directory, :antlr_jar, :verbose, :group ]
+    class_eval( <<-END )
+      def #{ attr }
+        @owner.#{ attr }
+      end
+    END
+  end
+  
+  def delegate_files( suffix )
+    @owner.delegate_files( "#{ @name }_#{ suffix }" )
+  end
+  
+  def target_files
+    targets = [ tokens_file ]
+    targets.concat( @owner.delegate_files( @name ) )
+    return( targets )
+  end
 end
+
+class GrammarFile::FormatError < StandardError
+  attr_reader :file, :source
+  
+  def self.[](*args)
+    new(*args)
+  end
+  
+  def initialize(source, file = nil)
+    @file = file
+    @source = source
+    message = ''
+    if file.nil? # inline
+      message << "bad inline grammar source:\n"
+      message << ("-" * 80) << "\n"
+      message << @source
+      message[-1] == ?\n or message << "\n"
+      message << ("-" * 80) << "\n"
+      message << "could not locate a grammar name and type declaration matching\n"
+      message << "/^\s*(lexer|parser|tree)?\s*grammar\s*(\S+)\s*;/"
+    else
+      message << 'bad grammar source in file %p' % @file
+      message << ("-" * 80) << "\n"
+      message << @source
+      message[-1] == ?\n or message << "\n"
+      message << ("-" * 80) << "\n"
+      message << "could not locate a grammar name and type declaration matching\n"
+      message << "/^\s*(lexer|parser|tree)?\s*grammar\s*(\S+)\s*;/"
+    end
+    super(message)
+  end
+end # error Grammar::FormatError
+end # class CompileTask
+end # module ANTLR3

data/lib/antlr3/test/grammar.rb

@@ -5,7 +5,6 @@ require 'antlr3'
 require 'antlr3/test/core-extensions'
 require 'antlr3/test/call-stack'
 
-
 if RUBY_VERSION =~ /^1\.9/
   require 'digest/md5'
   MD5 = Digest::MD5
@@ -13,7 +12,6 @@ else
   require 'md5'
 end
 
-
 module ANTLR3
 module Test
 module DependantFile

data/lib/antlr3/tree.rb

@@ -696,6 +696,7 @@ Tree adaptors are tasked with:
 module TreeAdaptor
   include TokenFactory
   include Constants
+  include Error
   
   def add_child( tree, child )
     tree.add_child( child ) if tree and child

data/lib/antlr3/version.rb

@@ -20,7 +20,7 @@ module ANTLR3
   # 
   MAJOR_VERSION = 1
   MINOR_VERSION = 3
-  PATCH_VERSION = 0
+  PATCH_VERSION = 1
   VERSION = [ MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION ]
   VERSION_STRING = VERSION.join('.').freeze
   

data/templates/Ruby.stg

@@ -28,7 +28,7 @@ $:.unshift( this_directory ) unless $:.include?( this_directory )
 
 antlr_load_failed = proc do
   load_path = $LOAD_PATH.map { |dir| '  - ' \<\< dir }.join( $/ )
-  raise LoadError, \<\<-'END'.strip!
+  raise LoadError, \<\<-END.strip!
   
 Failed to load the ANTLR3 runtime library (version <runtimeLibraryVersion()>):
 
@@ -1498,4 +1498,4 @@ placeAction(scope, name) ::= <<
 <endif>
 >>
 
-runtimeLibraryVersion() ::= "1.3.0"
+runtimeLibraryVersion() ::= "1.3.1"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: antlr3
 version: !ruby/object:Gem::Version 
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors: 
 - Kyle Yetter
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-02 00:00:00 -05:00
+date: 2010-01-07 00:00:00 -05:00
 default_executable: 
 dependencies: []