antlr3 1.8.0 → 1.8.2

data/samples/standard/python/PythonTokenSource.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+module Python
+  unless defined?(TokenData)
+    require 'PythonLexer'
+    require 'PythonParser'
+  end
+  
+  class TokenSource
+    include ANTLR3::TokenSource
+    include ANTLR3::Constants
+    include TokenData
+    
+    def initialize(lexer)
+      @lexer = lexer
+      @indent_stack = []
+      @eof = nil
+      @token_queue = []
+    end
+    
+    def next_token
+      queue_tokens if @token_queue.empty? and @eof.nil?
+      return @token_queue.shift || @eof
+    end
+    
+    def queue_tokens
+      begin
+        t = fetch or return false
+        @token_queue << t
+      end until t.type == NEWLINE
+      
+      while t = fetch and t.hidden?
+        # skip past things like comments
+        @token_queue << t
+      end
+      
+      if t.nil? # EOF
+        if prior = (@token_queue.last || @eof)
+          line = prior.line || 0
+          column = prior.column + prior.text.to_s.length rescue -1
+        else
+          line, column = 0, -1
+        end
+        insert_indentation(0, line, column)
+      elsif t.type == LEADING_WS
+        t.channel = ANTLR3::HIDDEN_CHANNEL
+        @token_queue << t
+        width = measure(t.text)
+        insert_indentation(width, t.line, t.column)
+      else
+        insert_indentation(0, t.line, t.column)
+        @token_queue << t
+      end
+      return true
+    end
+    
+    def insert_indentation(width, line, col)
+      case width <=> current_indent
+      when 1  # an indent
+        @indent_stack.push(width)
+        tk = Token.new(INDENT, DEFAULT, '')
+        tk.line    = line
+        tk.column  = col
+        @token_queue << tk
+      when -1 # a dedent
+        begin
+          @indent_stack.pop || break
+          tk = Token.new(DEDENT, DEFAULT, '')
+          tk.line = line
+          tk.column = col
+          @token_queue << tk
+        end until current_indent <= width
+      when 0  # same level
+        # do nothing
+      end
+    end
+    
+    def current_indent
+      @indent_stack.last || 0
+    end
+    
+    def measure(space)
+      size = 0
+      space.each_byte do |char|
+        case char
+        when ?\s then size += 1
+        when ?\t
+          size += 8
+          size -= size % 8
+        else break
+        end
+      end
+      return size
+    end
+    
+    def fetch
+      @eof and return nil
+      t = @lexer.next_token
+      if t.nil? or t.type == EOF
+        @eof = t || EOFToken
+        return nil
+      end
+      return t
+    end
+  end
+end

data/samples/standard/python/input

@@ -0,0 +1,210 @@
+"""This is a substantially improved version of the older Interpreter.py demo
+It creates a simple GUI JPython console window with simple history
+as well as the ability to interupt running code (with the ESC key).
+
+Like Interpreter.py, this is still just a demo, and needs substantial
+work before serious use.
+
+Thanks to Geza Groma (groma@everx.szbk.u-szeged.hu) for several valuable
+ideas for this tool -- his JPConsole is a more refined implementation
+of similar ideas.
+"""
+
+from Styles import Styles
+from Keymap import Keymap
+
+from pawt import swing, colors
+from java.awt.event.KeyEvent import VK_UP, VK_DOWN
+from java.awt.event import ActionEvent
+from java.lang import Thread, System
+from code import compile_command
+import string, sys, re
+
+class OutputBuffer:
+	def __init__(self, console, stylename):
+		self.console = console
+		self.stylename = stylename
+		
+	def flush(self):
+		pass
+		
+	def write(self, text):
+		self.console.write(text, self.stylename)
+
+class Console:
+	def __init__(self, styles=None, keymap=None):
+		if styles is None:
+			styles = Styles()
+			basic = styles.add('normal', tabsize=3, fontSize=12, fontFamily="Courier")
+			styles.add('error', parent=basic, foreground=colors.red)
+			styles.add('output', parent=basic, foreground=colors.blue)
+			styles.add('input', parent=basic, foreground=colors.black)
+			styles.add('prompt', parent=basic, foreground=colors.purple)
+		self.styles = styles
+		
+		# This is a hack to get at an inner class
+		# This will not be required in JPython-1.1
+		ForegroundAction = getattr(swing.text, 'StyledEditorKit$ForegroundAction')
+		self.inputAction = ForegroundAction("start input", colors.black)
+
+		if keymap is None:
+			keymap = Keymap()
+		keymap.bind('enter', self.enter)
+		keymap.bind('tab', self.tab)
+		keymap.bind('escape', self.escape)
+		keymap.bind('up', self.uphistory)
+		keymap.bind('down', self.downhistory)
+		
+		self.keymap = keymap
+		
+		self.document = swing.text.DefaultStyledDocument(self.styles)
+		self.document.setLogicalStyle(0, self.styles.get('normal'))
+
+		self.textpane = swing.JTextPane(self.document)
+		self.textpane.keymap = self.keymap
+		
+		self.history = []
+		self.oldHistoryLength = 0
+		self.historyPosition = 0
+		
+		self.command = []
+		self.locals = {}
+
+	def write(self, text, stylename='normal'):
+		style = self.styles.get(stylename)
+		self.document.insertString(self.document.length, text, style)
+		
+	def beep(self):
+		self.textpane.toolkit.beep()
+
+	def startUserInput(self, prompt=None):
+		if prompt is not None:
+			self.write(prompt, 'prompt')
+		self.startInput = self.document.createPosition(self.document.length-1)
+		#self.document.setCharacterAttributes(self.document.length-1, 1, self.styles.get('input'), 1)
+		self.textpane.caretPosition = self.document.length
+		ae = ActionEvent(self.textpane, ActionEvent.ACTION_PERFORMED, 'start input')
+		self.inputAction.actionPerformed(ae)
+
+	def getinput(self):
+		offset = self.startInput.offset
+		line = self.document.getText(offset+1, self.document.length-offset)
+		return string.rstrip(line)
+
+	def replaceinput(self, text):
+		offset = self.startInput.offset + 1
+		self.document.remove(offset, self.document.length-offset)
+		self.write(text, 'input')
+		
+	def enter(self):
+		line = self.getinput()
+		self.write('\n', 'input')
+		
+		self.history.append(line)
+		self.handleLine(line)
+		
+	def gethistory(self, direction):
+		historyLength = len(self.history)
+		if self.oldHistoryLength < historyLength:
+			# new line was entered after last call
+			self.oldHistoryLength = historyLength
+			if self.history[self.historyPosition] != self.history[-1]:
+				self.historyPosition = historyLength
+
+		pos = self.historyPosition + direction
+
+		if 0 <= pos < historyLength:
+			self.historyPosition = pos
+			self.replaceinput(self.history[pos])
+		else:
+			self.beep()
+
+	def uphistory(self):
+		self.gethistory(-1)
+
+	def downhistory(self):
+		self.gethistory(1)
+
+	def tab(self):
+		self.write('\t', 'input')
+		
+	def escape(self):
+		if (not hasattr(self, 'pythonThread') or self.pythonThread is None or not self.pythonThread.alive):
+			self.beep()
+			return
+			
+		self.pythonThread.stopPython()
+
+	def capturePythonOutput(self, stdoutStyle='output', stderrStyle='error'):
+		import sys
+		sys.stdout = OutputBuffer(self, stdoutStyle)
+		sys.stderr = OutputBuffer(self, stderrStyle)
+
+	def handleLine(self, text):
+		self.command.append(text)
+		
+		try:
+			code = compile_command(string.join(self.command, '\n'))
+		except SyntaxError:
+			traceback.print_exc(0)
+			self.command = []
+			self.startUserInput(str(sys.ps1)+'\t')
+			return
+
+		if code is None:
+			self.startUserInput(str(sys.ps2)+'\t')
+			return
+		
+		self.command = []
+		
+		pt = PythonThread(code, self)
+		self.pythonThread = pt
+		pt.start()
+		
+	def newInput(self):
+		self.startUserInput(str(sys.ps1)+'\t')
+		
+import traceback
+
+class PythonThread(Thread):
+	def __init__(self, code, console):
+		self.code = code
+		self.console = console
+		self.locals = console.locals
+		
+	def run(self):
+		try:
+			exec self.code in self.locals
+			
+		#Include these lines to actually exit on a sys.exit() call
+		#except SystemExit, value:
+		#	raise SystemExit, value
+		
+		except:
+			exc_type, exc_value, exc_traceback = sys.exc_info()
+			l = len(traceback.extract_tb(sys.exc_traceback))
+			try:
+				1/0
+			except:
+				m = len(traceback.extract_tb(sys.exc_traceback))
+			traceback.print_exception(exc_type, exc_value, exc_traceback, l-m)
+			
+		self.console.newInput()
+
+	def stopPython(self):
+		#Should spend 2 seconds trying to kill thread in nice Python style first...
+		self.stop()
+
+header = """\
+JPython %(version)s on %(platform)s
+%(copyright)s
+""" % {'version':sys.version, 'platform':sys.platform, 'copyright':sys.copyright}
+
+if __name__ == '__main__':
+	c = Console()
+	pane = swing.JScrollPane(c.textpane)
+	swing.test(pane, size=(500,400), name='JPython Console')
+	c.write(header, 'output')
+	c.capturePythonOutput()
+	c.textpane.requestFocus()
+	c.newInput()

data/samples/standard/python/output

@@ -0,0 +1,24 @@
+found method def __init__
+found method def flush
+found method def write
+found class def OutputBuffer
+found method def __init__
+found method def write
+found method def beep
+found method def startUserInput
+found method def getinput
+found method def replaceinput
+found method def enter
+found method def gethistory
+found method def uphistory
+found method def downhistory
+found method def tab
+found method def escape
+found method def capturePythonOutput
+found method def handleLine
+found method def newInput
+found class def Console
+found method def __init__
+found method def run
+found method def stopPython
+found class def PythonThread

data/samples/standard/python/python.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+$:.unshift( File.dirname( __FILE__ ) )
+require 'PythonLexer'
+require 'PythonParser'
+require 'PythonTokenSource'
+
+for file in ARGV
+  input = ANTLR3::FileStream.new( file )
+  lexer = Python::Lexer.new( input )
+  source = Python::TokenSource.new( lexer )
+  parser = Python::Parser.new( source )
+  parser.file_input
+end

data/samples/standard/rakefile

@@ -0,0 +1,18 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+load '../../config/antlr3.rb'
+require 'antlr3/task'
+
+ANTLR3::CompileTask.define do | t |
+  for dir in Dir[ '*/' ]
+    grammars = Dir[ dir / '*.g' ]
+    t.grammar_set( grammars, :output_directory => dir )
+  end
+end
+
+desc( "compile all example grammars" )
+task( :default => 'antlr-grammars:compile' )
+
+desc( "trash all generated output" )
+task( :clean => 'antlr-grammars:clobber' )

data/samples/standard/scopes/SymbolTable.g

@@ -0,0 +1,66 @@
+grammar SymbolTable;
+
+options { language = Ruby; }
+
+/* Scope of symbol names.  Both globals and block rules need to push a new
+ * symbol table upon entry and they must use the same stack.  So, I must
+ * define a global scope and say that globals and block use this by saying
+ * 'scope Symbols;' in those rule definitions.
+ */
+scope Symbols {
+names
+}
+
+@init {
+@level = 0
+}
+
+prog: globals (method)*
+    ;
+
+globals
+scope Symbols;
+@init {
+@level += 1
+$Symbols::names = []
+}
+    :   (decl)*
+        {
+        puts( "globals: [#{ $Symbols::names.join( ', ' ) }]" )
+        @level -= 1
+        }
+    ;
+
+method
+    :   'method' ID '(' ')' block
+    ;
+
+block
+scope Symbols;
+@init {
+@level += 1
+$Symbols::names = []
+}
+    : '{' (decl)* (stat)* '}'
+      {
+      puts( "level #@level symbols: [#{ $Symbols::names.join( ', ' ) }]" )
+      @level -= 1
+      }
+    ;
+
+stat:   ID '=' INT ';'
+    |   block
+    ;
+
+decl: 'int' ID ';'
+      { $Symbols::names << $ID } // add to current symbol table
+    ;
+
+ID  :   ('a'..'z')+
+    ;
+
+INT :   ('0'..'9')+
+    ;
+
+WS  :   (' '|'\n'|'\r')+ {$channel=HIDDEN}
+    ;

data/samples/standard/scopes/input

@@ -0,0 +1,12 @@
+int i;
+int j;
+method foo() {
+  int i;
+  int k;
+  i=3;
+  {
+    int j;
+    int k;
+    k = 4;
+  }
+}