antlr3 1.8.0 → 1.8.2

data/History.txt

@@ -1,3 +1,38 @@
+=== 1.8.2 / 10-02-10
+
+* Bug Fix: removed references to Parser#token_stream
+  - changed references to Parser#token_stream to Parser#input in `antlr3/profile.rb' to
+    avoid "private method token_stream called" errors in profile mode
+
+* Minor Enhancement: new samples included with the distribution
+  - added a newly developed `samples/JavaScript.g' file for parsing JavaScript.
+    While not entirely complete and flawless, it does a pretty good job
+    of parsing JavaScript up through Firefox's 1.6 version of the language.
+  - included the `samples/standard' directory with the package distribution,
+    which has been in the development repository for a while, but I never
+    updated the package spec to include these files.
+
+=== 1.8.0 / 07-03-10
+
+* Bug Fix / Major Enhancement: made antlr3 compatible with ruby 1.9
+  - altered ANTLR3::StringStream to always have #peek return an integer in both
+    Ruby 1.8 and 1.9. In 1.9, @data is now an array of code point values and
+    the new instance variable @string contains the actual string. In Ruby 1.8,
+    both @data and @string contain the string value.
+  - lexer code is no longer generated using '?c' character syntax; characters
+    are now represented as hexadecimal integers to make the code compatible
+    with Ruby 1.9
+
+=== 1.7.5 / 06-18-10
+
+* Minor Enhancements
+  - added #compile! and #clobber! methods to directly invoke the 'compile'
+    and 'clobber' tasks respectively
+  
+* Bug Fixes
+  - removed special ruby-stylistic formatting on syntactic predicate method names to
+    fix a bad bug in code-generation that cropped up in more complex grammars
+
 === 1.7.2 / 05-16-10
 
 * Minor Enhancements

data/Manifest.txt

@@ -44,6 +44,7 @@ test/unit/test-streams.rb
 test/unit/test-recognizers.rb
 test/unit/test-trees.rb
 test/unit/test-exceptions.rb
+test/unit/test-scope.rb
 test/unit/test-scheme.rb
 test/unit/test-template.rb
 test/functional/delegation/import.rb
@@ -86,6 +87,78 @@ templates/Support.stg
 templates/ASTTreeParser.stg
 samples/CPP.g
 samples/ANTLRv3Grammar.g
+samples/JavaScript.g
+samples/standard/island-grammar/island.rb
+samples/standard/island-grammar/Simple.g
+samples/standard/island-grammar/Javadoc.g
+samples/standard/island-grammar/input
+samples/standard/island-grammar/output
+samples/standard/tweak/tweak.rb
+samples/standard/tweak/input
+samples/standard/tweak/output
+samples/standard/tweak/Tweak.g
+samples/standard/scopes/input
+samples/standard/scopes/output
+samples/standard/scopes/SymbolTable.g
+samples/standard/scopes/scopes.rb
+samples/standard/dynamic-scope/DynamicScopes.g
+samples/standard/dynamic-scope/dynamic-scopes.rb
+samples/standard/dynamic-scope/input
+samples/standard/dynamic-scope/output
+samples/standard/hoisted-predicates/hoisted-predicates.rb
+samples/standard/hoisted-predicates/HoistedPredicates.g
+samples/standard/hoisted-predicates/input
+samples/standard/hoisted-predicates/output
+samples/standard/fuzzy/fuzzy.rb
+samples/standard/fuzzy/input
+samples/standard/fuzzy/output
+samples/standard/fuzzy/FuzzyJava.g
+samples/standard/fuzzy/fuzzy.py
+samples/standard/treeparser/LangDumpDecl.g
+samples/standard/treeparser/Lang.g
+samples/standard/treeparser/treeparser.rb
+samples/standard/treeparser/input
+samples/standard/treeparser/output
+samples/standard/LL-star/LLStar.g
+samples/standard/LL-star/input
+samples/standard/LL-star/output
+samples/standard/LL-star/ll-star.rb
+samples/standard/java/Java.g
+samples/standard/java/java.rb
+samples/standard/java/input
+samples/standard/java/output
+samples/standard/rakefile
+samples/standard/xml/README
+samples/standard/xml/xml.rb
+samples/standard/xml/XML.g
+samples/standard/xml/input
+samples/standard/xml/output
+samples/standard/calc/Calculator.g
+samples/standard/calc/Calculator.py
+samples/standard/calc/Calculator.rb
+samples/standard/C/C.g
+samples/standard/C/c.rb
+samples/standard/C/C__testrig.st
+samples/standard/C/input
+samples/standard/C/output
+samples/standard/C/C.tokens
+samples/standard/cminus/bytecode.group
+samples/standard/cminus/CMinus.g
+samples/standard/cminus/cminus.rb
+samples/standard/cminus/input
+samples/standard/cminus/python.group
+samples/standard/cminus/output
+samples/standard/cminus/java.group
+samples/standard/simplecTreeParser/SimpleC.g
+samples/standard/simplecTreeParser/SimpleCWalker.g
+samples/standard/simplecTreeParser/input
+samples/standard/simplecTreeParser/output
+samples/standard/simplecTreeParser/simplec.rb
+samples/standard/python/PythonTokenSource.rb
+samples/standard/python/python.rb
+samples/standard/python/Python.g
+samples/standard/python/input
+samples/standard/python/output
 README.txt
 ANTLR-LICENSE.txt
 History.txt

data/README.txt

@@ -1,6 +1,7 @@
 ANTLR 3 for Ruby
     by Kyle Yetter (kcy5b@yahoo.com)
-    http://antlr3.rubyforge.org
+    http://antlr.ohboyohboyohboy.org
+    http://github.com/ohboyohboyohboy/antlr3
 
 == DESCRIPTION:
 
@@ -9,7 +10,7 @@ Fully-featured ANTLR 3 parser generation for Ruby.
 ANTLR (ANother Tool for Language Recognition) is a tool that is used to generate
 code for performing a variety of language recognition tasks: lexing, parsing,
 abstract syntax tree construction and manipulation, tree structure recognition,
-and input translation. The tool operates simillarly to other parser generators,
+and input translation. The tool operates similarly to other parser generators,
 taking in a grammar specification written in the special ANTLR metalanguage and
 producing source code that implements the recognition functionality.
 
@@ -32,11 +33,11 @@ ANTLR provides for other language targets, such as Java and Python. It contains:
   java code and templates for producing fully featured language recognition
   in ruby code
 
-* a ruby runtime library that collects classes used throughout the code that
+* a ruby run-time library that collects classes used throughout the code that
   ANTLR generates
   
 * a wrapper script, `antlr4ruby', which executes the ANTLR command line tool
-  after ensuring the ANTLR jar is java's class path
+  after ensuring the ANTLR jar is Java's class path
 
 == FEATURES
 
@@ -96,21 +97,13 @@ ANTLR provides for other language targets, such as Java and Python. It contains:
   from the conventions and examples covered in the ANTLR standard documentation.
   I am still working on documenting these details.
 
-* While the target is intended to be complete, I do not provide any
-  implementation of the template-construction mode available for Java and Python
-  targets. While I'm interested in having this capability, I have not implemented
-  it yet because ANTLR forces you to use its StringTemplate templating language.
-  Thus, I would have to port the StringTemplate library to Ruby and write a target
-  for ST parser generation. I would prefer to permit template generation that uses
-  ruby's standard ERB templating library.
-
 * So far, this has only been tested on Linux with ruby 1.8.7 and ruby 1.9.1.
   I'm currently working on verifying behavior on other systems and with
   slightly older versions of ruby. 
 
 == LICENSE
 
-[The "BSD licence"]
+[The "BSD license"]
 Copyright (c) 2009-2010 Kyle Yetter
 All rights reserved.
 

data/java/RubyTarget.java

@@ -1,6 +1,6 @@
 /*
  [The "BSD licence"]
- Copyright (c) 2010 Martin Traverso
+ Copyright (c) 2010 Kyle Yetter
  All rights reserved.
 
  Redistribution and use in source and binary forms, with or without
@@ -35,8 +35,7 @@ import org.antlr.Tool;
 import org.antlr.stringtemplate.*;
 import org.antlr.tool.Grammar;
 
-public class RubyTarget
-            extends Target
+public class RubyTarget extends Target
 {
     /** A set of ruby keywords which are used to escape labels and method names
      *  which will cause parse errors in the ruby source
@@ -62,9 +61,33 @@ public class RubyTarget
 
     public static HashMap sharedActionBlocks = new HashMap();
 
-    public class RubyRenderer
-                implements AttributeRenderer
+    public class RubyRenderer implements AttributeRenderer
     {
+    	protected String[] rubyCharValueEscape = new String[256];
+    	
+    	public RubyRenderer() {
+    		for ( int i = 0; i < 16; i++ ) {
+    			rubyCharValueEscape[ i ] = "\\x0" + Integer.toHexString( i );
+    		}
+    		for ( int i = 16; i < 32; i++ ) {
+    			rubyCharValueEscape[ i ] = "\\x" + Integer.toHexString( i );
+    		}
+    		for ( char i = 32; i < 127; i++ ) {
+    			rubyCharValueEscape[ i ] = Character.toString( i );
+    		}
+    		for ( int i = 127; i < 256; i++ ) {
+    			rubyCharValueEscape[ i ] = "\\x" + Integer.toHexString( i );
+    		}
+    		
+    		rubyCharValueEscape['\n'] = "\\n";
+    		rubyCharValueEscape['\r'] = "\\r";
+    		rubyCharValueEscape['\t'] = "\\t";
+    		rubyCharValueEscape['\b'] = "\\b";
+    		rubyCharValueEscape['\f'] = "\\f";
+    		rubyCharValueEscape['\\'] = "\\\\";
+    		rubyCharValueEscape['"'] = "\\\"";
+    	}
+    	
         public String toString( Object o ) {
             return o.toString();
         }
@@ -87,7 +110,9 @@ public class RubyTarget
             } else if ( formatName.equals( "lexerRule" ) ) {
                 return lexerRule( idString );
             } else if ( formatName.equals( "constantPath" ) ) {
-                return constantPath( idString );
+            	return constantPath( idString );
+            } else if ( formatName.equals( "rubyString" ) ) {
+                return rubyString( idString );
             } else if ( formatName.equals( "label" ) ) {
                 return label( idString );
             } else if ( formatName.equals( "symbol" ) ) {
@@ -198,6 +223,18 @@ public class RubyTarget
         private String constantPath( String value ) {
             return value.replaceAll( "\\.", "::" );
         }
+        
+        private String rubyString( String value ) {
+        	StringBuilder output_buffer = new StringBuilder();
+        	int len = value.length(); 
+        	
+        	output_buffer.append( '"' );
+        	for ( int i = 0; i < len; i++ ) {
+        		output_buffer.append( rubyCharValueEscape[ value.charAt( i ) ] );
+        	}
+        	output_buffer.append( '"' );
+        	return output_buffer.toString();
+        }
 
         private String camelcase( String value ) {
             StringBuilder output_buffer = new StringBuilder();
@@ -369,19 +406,6 @@ public class RubyTarget
         }
         
         return ( "0x" + Integer.toHexString( code_point ) );
-        
-        //if ( literal.equals( "\\" ) ) {
-        //    result += "\\\\";
-        //}
-        //else if ( literal.equals( " " ) ) {
-        //    result += "\\s";
-        //}
-        //else if ( literal.startsWith( "\\u" ) ) {
-        //    result = "0x" + literal.substring( 2 );
-        //}
-        //else {
-        //    result += literal;
-        //}
     }
 
     public int getMaxCharValue( CodeGenerator generator )

data/lib/antlr3/debug.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/ruby
 # encoding: utf-8
 
+require 'antlr3'
+
 =begin LICENSE
 
 [The "BSD licence"]

data/lib/antlr3/debug/event-hub.rb

@@ -15,36 +15,36 @@ class EventHub
   include ANTLR3::Debug::EventListener
   attr_reader :listeners
   
-  def initialize(*listeners)
-    @listeners = [listeners].flatten!
+  def initialize( *listeners )
+    @listeners = [ listeners ].flatten!
     @listeners.compact!
   end
   
-  def add(*listeners)
-    @listeners.push(*listeners)
+  def add( *listeners )
+    @listeners.push( *listeners )
   end
 
-  def add_child(root, child)
+  def add_child( root, child )
     for listener in @listeners
-      listener.add_child(root, child)
+      listener.add_child( root, child )
     end
   end
   
-  def backtrack(level)
+  def backtrack( level )
     for listener in @listeners
-      listener.backtrack(level)
+      listener.backtrack( level )
     end
   end
   
-  def become_root(new_root, old_root)
+  def become_root( new_root, old_root )
     for listener in @listeners
-      listener.become_root(new_root, old_root)
+      listener.become_root( new_root, old_root )
     end
   end
   
-  def begin_backtrack(level)
+  def begin_backtrack( level )
     for listener in @listeners
-      listener.begin_backtrack(level)
+      listener.begin_backtrack( level )
     end
   end
   
@@ -60,33 +60,33 @@ class EventHub
     end
   end
   
-  def consume_hidden_token(tree)
+  def consume_hidden_token( tree )
     for listener in @listeners
-      listener.consume_hidden_token(tree)
+      listener.consume_hidden_token( tree )
     end
   end
   
-  def consume_node(tree)
+  def consume_node( tree )
     for listener in @listeners
-      listener.consume_node(tree)
+      listener.consume_node( tree )
     end
   end
   
-  def consume_token(tree)
+  def consume_token( tree )
     for listener in @listeners
-      listener.consume_token(tree)
+      listener.consume_token( tree )
     end
   end
   
-  def create_node(node, token)
+  def create_node( node, token )
     for listener in @listeners
-      listener.create_node(node, token)
+      listener.create_node( node, token )
     end
   end
   
-  def end_backtrack(level, successful)
+  def end_backtrack( level, successful )
     for listener in @listeners
-      listener.end_backtrack(level, successful)
+      listener.end_backtrack( level, successful )
     end
   end
   
@@ -96,81 +96,81 @@ class EventHub
     end
   end
   
-  def enter_alternative(alt)
+  def enter_alternative( alt )
     for listener in @listeners
-      listener.enter_alternative(alt)
+      listener.enter_alternative( alt )
     end
   end
   
-  def enter_decision(decision_number)
+  def enter_decision( decision_number )
     for listener in @listeners
-      listener.enter_decision(decision_number)
+      listener.enter_decision( decision_number )
     end
   end
   
-  def enter_rule(grammar_file_name, rule_name)
+  def enter_rule( grammar_file_name, rule_name )
     for listener in @listeners
-      listener.enter_rule(grammar_file_name, rule_name)
+      listener.enter_rule( grammar_file_name, rule_name )
     end
   end
   
-  def enter_sub_rule(decision_number)
+  def enter_sub_rule( decision_number )
     for listener in @listeners
-      listener.enter_sub_rule(decision_number)
+      listener.enter_sub_rule( decision_number )
     end
   end
   
-  def error_node(tree)
+  def error_node( tree )
     for listener in @listeners
-      listener.error_node(tree)
+      listener.error_node( tree )
     end
   end
   
-  def exit_decision(decision_number)
+  def exit_decision( decision_number )
     for listener in @listeners
-      listener.exit_decision(decision_number)
+      listener.exit_decision( decision_number )
     end
   end
   
-  def exit_rule(grammar_file_name, rule_name)
+  def exit_rule( grammar_file_name, rule_name )
     for listener in @listeners
-      listener.exit_rule(grammar_file_name, rule_name)
+      listener.exit_rule( grammar_file_name, rule_name )
     end
   end
   
-  def exit_sub_rule(decision_number)
+  def exit_sub_rule( decision_number )
     for listener in @listeners
-      listener.exit_sub_rule(decision_number)
+      listener.exit_sub_rule( decision_number )
     end
   end
   
-  def flat_node(tree)
+  def flat_node( tree )
     for listener in @listeners
-      listener.flat_node(tree)
+      listener.flat_node( tree )
     end
   end
   
-  def location(line, position)
+  def location( line, position )
     for listener in @listeners
-      listener.location(line, position)
+      listener.location( line, position )
     end
   end
   
-  def look(i, tree)
+  def look( i, tree )
     for listener in @listeners
-      listener.look(i, tree)
+      listener.look( i, tree )
     end
   end
   
-  def mark(marker)
+  def mark( marker )
     for listener in @listeners
-      listener.mark(marker)
+      listener.mark( marker )
     end
   end
   
-  def recognition_exception(exception)
+  def recognition_exception( exception )
     for listener in @listeners
-      listener.recognition_exception(exception)
+      listener.recognition_exception( exception )
     end
   end
   
@@ -180,21 +180,21 @@ class EventHub
     end
   end
   
-  def rewind(marker)
+  def rewind( marker )
     for listener in @listeners
-      listener.rewind(marker)
+      listener.rewind( marker )
     end
   end
   
-  def semantic_predicate(result, predicate)
+  def semantic_predicate( result, predicate )
     for listener in @listeners
-      listener.semantic_predicate(result, predicate)
+      listener.semantic_predicate( result, predicate )
     end
   end
   
-  def set_token_boundaries(tree, token_start_index, token_stop_index)
+  def set_token_boundaries( tree, token_start_index, token_stop_index )
     for listener in @listeners
-      listener.set_token_boundaries(tree, token_start_index, token_stop_index)
+      listener.set_token_boundaries( tree, token_start_index, token_stop_index )
     end
   end
   
@@ -207,4 +207,4 @@ class EventHub
 end
 
 end
-end
+end