testautoa 0.4.0.pre16

data/test-server/instrumentation-backend/src/org/antlr/runtime/BitSet.java

@@ -0,0 +1,325 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.List;
+
+/**A stripped-down version of org.antlr.misc.BitSet that is just
+ * good enough to handle runtime requirements such as FOLLOW sets
+ * for automatic error recovery.
+ */
+public class BitSet implements Cloneable {
+    protected final static int BITS = 64;    // number of bits / long
+    protected final static int LOG_BITS = 6; // 2^6 == 64
+
+    /* We will often need to do a mod operator (i mod nbits).  Its
+     * turns out that, for powers of two, this mod operation is
+     * same as (i & (nbits-1)).  Since mod is slow, we use a
+     * precomputed mod mask to do the mod instead.
+     */
+    protected final static int MOD_MASK = BITS - 1;
+
+    /** The actual data bits */
+    protected long bits[];
+
+    /** Construct a bitset of size one word (64 bits) */
+    public BitSet() {
+        this(BITS);
+    }
+
+    /** Construction from a static array of longs */
+    public BitSet(long[] bits_) {
+        bits = bits_;
+    }
+
+	/** Construction from a list of integers */
+	public BitSet(List items) {
+		this();
+		for (int i = 0; i < items.size(); i++) {
+			Integer v = (Integer) items.get(i);
+			add(v.intValue());
+		}
+	}
+
+    /** Construct a bitset given the size
+     * @param nbits The size of the bitset in bits
+     */
+    public BitSet(int nbits) {
+        bits = new long[((nbits - 1) >> LOG_BITS) + 1];
+    }
+
+	public static BitSet of(int el) {
+		BitSet s = new BitSet(el + 1);
+		s.add(el);
+		return s;
+	}
+
+	public static BitSet of(int a, int b) {
+		BitSet s = new BitSet(Math.max(a,b)+1);
+		s.add(a);
+		s.add(b);
+		return s;
+	}
+
+	public static BitSet of(int a, int b, int c) {
+		BitSet s = new BitSet();
+		s.add(a);
+		s.add(b);
+		s.add(c);
+		return s;
+	}
+
+	public static BitSet of(int a, int b, int c, int d) {
+		BitSet s = new BitSet();
+		s.add(a);
+		s.add(b);
+		s.add(c);
+		s.add(d);
+		return s;
+	}
+
+	/** return this | a in a new set */
+	public BitSet or(BitSet a) {
+		if ( a==null ) {
+			return this;
+		}
+		BitSet s = (BitSet)this.clone();
+		s.orInPlace(a);
+		return s;
+	}
+
+	/** or this element into this set (grow as necessary to accommodate) */
+	public void add(int el) {
+		int n = wordNumber(el);
+		if (n >= bits.length) {
+			growToInclude(el);
+		}
+		bits[n] |= bitMask(el);
+	}
+
+	/**
+	 * Grows the set to a larger number of bits.
+	 * @param bit element that must fit in set
+	 */
+	public void growToInclude(int bit) {
+		int newSize = Math.max(bits.length << 1, numWordsToHold(bit));
+		long newbits[] = new long[newSize];
+		System.arraycopy(bits, 0, newbits, 0, bits.length);
+		bits = newbits;
+	}
+
+	public void orInPlace(BitSet a) {
+		if ( a==null ) {
+			return;
+		}
+		// If this is smaller than a, grow this first
+		if (a.bits.length > bits.length) {
+			setSize(a.bits.length);
+		}
+		int min = Math.min(bits.length, a.bits.length);
+		for (int i = min - 1; i >= 0; i--) {
+			bits[i] |= a.bits[i];
+		}
+	}
+
+	/**
+	 * Sets the size of a set.
+	 * @param nwords how many words the new set should be
+	 */
+	private void setSize(int nwords) {
+		long newbits[] = new long[nwords];
+		int n = Math.min(nwords, bits.length);
+		System.arraycopy(bits, 0, newbits, 0, n);
+		bits = newbits;
+	}
+
+    private final static long bitMask(int bitNumber) {
+        int bitPosition = bitNumber & MOD_MASK; // bitNumber mod BITS
+        return 1L << bitPosition;
+    }
+
+    public Object clone() {
+        BitSet s;
+        try {
+            s = (BitSet)super.clone();
+            s.bits = new long[bits.length];
+            System.arraycopy(bits, 0, s.bits, 0, bits.length);
+        }
+        catch (CloneNotSupportedException e) {
+            throw new InternalError();
+        }
+        return s;
+    }
+
+    public int size() {
+        int deg = 0;
+        for (int i = bits.length - 1; i >= 0; i--) {
+            long word = bits[i];
+            if (word != 0L) {
+                for (int bit = BITS - 1; bit >= 0; bit--) {
+                    if ((word & (1L << bit)) != 0) {
+                        deg++;
+                    }
+                }
+            }
+        }
+        return deg;
+    }
+
+    public boolean equals(Object other) {
+        if ( other == null || !(other instanceof BitSet) ) {
+            return false;
+        }
+
+        BitSet otherSet = (BitSet)other;
+
+        int n = Math.min(this.bits.length, otherSet.bits.length);
+
+        // for any bits in common, compare
+        for (int i=0; i<n; i++) {
+            if (this.bits[i] != otherSet.bits[i]) {
+                return false;
+            }
+        }
+
+        // make sure any extra bits are off
+
+        if (this.bits.length > n) {
+            for (int i = n+1; i<this.bits.length; i++) {
+                if (this.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+        else if (otherSet.bits.length > n) {
+            for (int i = n+1; i<otherSet.bits.length; i++) {
+                if (otherSet.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    public boolean member(int el) {
+		if ( el<0 ) {
+			return false;
+		}
+        int n = wordNumber(el);
+        if (n >= bits.length) return false;
+        return (bits[n] & bitMask(el)) != 0;
+    }
+
+	// remove this element from this set
+	public void remove(int el) {
+		int n = wordNumber(el);
+		if (n < bits.length) {
+			bits[n] &= ~bitMask(el);
+		}
+	}
+
+    public boolean isNil() {
+        for (int i = bits.length - 1; i >= 0; i--) {
+            if (bits[i] != 0) return false;
+        }
+        return true;
+    }
+
+    private final int numWordsToHold(int el) {
+        return (el >> LOG_BITS) + 1;
+    }
+
+    public int numBits() {
+        return bits.length << LOG_BITS; // num words * bits per word
+    }
+
+    /** return how much space is being used by the bits array not
+     *  how many actually have member bits on.
+     */
+    public int lengthInLongWords() {
+        return bits.length;
+    }
+
+    /**Is this contained within a? */
+    /*
+	public boolean subset(BitSet a) {
+        if (a == null || !(a instanceof BitSet)) return false;
+        return this.and(a).equals(this);
+    }
+	*/
+
+    public int[] toArray() {
+        int[] elems = new int[size()];
+        int en = 0;
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                elems[en++] = i;
+            }
+        }
+        return elems;
+    }
+
+    public long[] toPackedArray() {
+        return bits;
+    }
+
+	private final static int wordNumber(int bit) {
+		return bit >> LOG_BITS; // bit / BITS
+	}
+
+	public String toString() {
+		return toString(null);
+	}
+
+	public String toString(String[] tokenNames) {
+		StringBuffer buf = new StringBuffer();
+		String separator = ",";
+		boolean havePrintedAnElement = false;
+		buf.append('{');
+
+		for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+			if (member(i)) {
+				if (i > 0 && havePrintedAnElement ) {
+					buf.append(separator);
+				}
+				if ( tokenNames!=null ) {
+					buf.append(tokenNames[i]);
+				}
+				else {
+					buf.append(i);
+				}
+				havePrintedAnElement = true;
+			}
+		}
+		buf.append('}');
+		return buf.toString();
+	}
+
+
+}

data/test-server/instrumentation-backend/src/org/antlr/runtime/BufferedTokenStream.java

@@ -0,0 +1,272 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+/** Buffer all input tokens but do on-demand fetching of new tokens from
+ *  lexer. Useful when the parser or lexer has to set context/mode info before
+ *  proper lexing of future tokens. The ST template parser needs this,
+ *  for example, because it has to constantly flip back and forth between
+ *  inside/output templates. E.g., <names:{hi, <it>}> has to parse names
+ *  as part of an expression but "hi, <it>" as a nested template.
+ *
+ *  You can't use this stream if you pass whitespace or other off-channel
+ *  tokens to the parser. The stream can't ignore off-channel tokens.
+ *  (UnbufferedTokenStream is the same way.)
+ *
+ *  This is not a subclass of UnbufferedTokenStream because I don't want
+ *  to confuse small moving window of tokens it uses for the full buffer.
+ */
+public class BufferedTokenStream implements TokenStream {
+    protected TokenSource tokenSource;
+
+    /** Record every single token pulled from the source so we can reproduce
+     *  chunks of it later.  The buffer in LookaheadStream overlaps sometimes
+     *  as its moving window moves through the input.  This list captures
+     *  everything so we can access complete input text.
+     */
+    protected List<Token> tokens = new ArrayList<Token>(100);
+
+    /** Track the last mark() call result value for use in rewind(). */
+    protected int lastMarker;
+
+    /** The index into the tokens list of the current token (next token
+     *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
+     *  to initialize with first token.  The ctor doesn't get a token.
+     *  First call to LT(1) or whatever gets the first token and sets p=0;
+     */
+    protected int p = -1;
+
+	protected int range = -1; // how deep have we gone?
+
+    public BufferedTokenStream() {;}
+
+    public BufferedTokenStream(TokenSource tokenSource) {
+        this.tokenSource = tokenSource;
+    }
+
+    public TokenSource getTokenSource() { return tokenSource; }
+
+	public int index() { return p; }
+
+	public int range() { return range; }
+
+    public int mark() {
+        if ( p == -1 ) setup();
+		lastMarker = index();
+		return lastMarker;
+	}
+
+	public void release(int marker) {
+		// no resources to release
+	}
+
+    public void rewind(int marker) {
+        seek(marker);
+    }
+
+    public void rewind() {
+        seek(lastMarker);
+    }
+
+    public void reset() {
+        p = 0;
+        lastMarker = 0;
+    }
+
+    public void seek(int index) { p = index; }
+
+    public int size() { return tokens.size(); }
+
+    /** Move the input pointer to the next incoming token.  The stream
+     *  must become active with LT(1) available.  consume() simply
+     *  moves the input pointer so that LT(1) points at the next
+     *  input symbol. Consume at least one token.
+     *
+     *  Walk past any token not on the channel the parser is listening to.
+     */
+    public void consume() {
+        if ( p == -1 ) setup();
+        p++;
+        sync(p);
+    }
+
+    /** Make sure index i in tokens has a token. */
+    protected void sync(int i) {
+        int n = i - tokens.size() + 1; // how many more elements we need?
+        //System.out.println("sync("+i+") needs "+n);
+        if ( n > 0 ) fetch(n);
+    }
+
+    /** add n elements to buffer */
+    protected void fetch(int n) {
+        for (int i=1; i<=n; i++) {
+            Token t = tokenSource.nextToken();
+            t.setTokenIndex(tokens.size());
+            //System.out.println("adding "+t+" at index "+tokens.size());
+            tokens.add(t);
+            if ( t.getType()==Token.EOF ) break;
+        }
+    }
+
+    public Token get(int i) {
+        if ( i < 0 || i >= tokens.size() ) {
+            throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1));
+        }
+        return tokens.get(i);
+    }
+
+	/** Get all tokens from start..stop inclusively */
+	public List get(int start, int stop) {
+		if ( start<0 || stop<0 ) return null;
+		if ( p == -1 ) setup();
+		List subset = new ArrayList();
+		if ( stop>=tokens.size() ) stop = tokens.size()-1;
+		for (int i = start; i <= stop; i++) {
+			Token t = tokens.get(i);
+			if ( t.getType()==Token.EOF ) break;
+			subset.add(t);
+		}
+		return subset;
+	}
+
+	public int LA(int i) { return LT(i).getType(); }
+
+    protected Token LB(int k) {
+        if ( (p-k)<0 ) return null;
+        return tokens.get(p-k);
+    }
+
+    public Token LT(int k) {
+        if ( p == -1 ) setup();
+        if ( k==0 ) return null;
+        if ( k < 0 ) return LB(-k);
+
+		int i = p + k - 1;
+		sync(i);
+        if ( i >= tokens.size() ) { // return EOF token
+            // EOF must be last token
+            return tokens.get(tokens.size()-1);
+        }
+		if ( i>range ) range = i; 		
+        return tokens.get(i);
+    }
+
+    protected void setup() { sync(0); p = 0; }
+
+    /** Reset this token stream by setting its token source. */
+    public void setTokenSource(TokenSource tokenSource) {
+        this.tokenSource = tokenSource;
+        tokens.clear();
+        p = -1;
+    }
+    
+    public List getTokens() { return tokens; }
+
+    public List getTokens(int start, int stop) {
+        return getTokens(start, stop, (BitSet)null);
+    }
+
+    /** Given a start and stop index, return a List of all tokens in
+     *  the token type BitSet.  Return null if no tokens were found.  This
+     *  method looks at both on and off channel tokens.
+     */
+    public List getTokens(int start, int stop, BitSet types) {
+        if ( p == -1 ) setup();
+        if ( stop>=tokens.size() ) stop=tokens.size()-1;
+        if ( start<0 ) start=0;
+        if ( start>stop ) return null;
+
+        // list = tokens[start:stop]:{Token t, t.getType() in types}
+        List<Token> filteredTokens = new ArrayList<Token>();
+        for (int i=start; i<=stop; i++) {
+            Token t = tokens.get(i);
+            if ( types==null || types.member(t.getType()) ) {
+                filteredTokens.add(t);
+            }
+        }
+        if ( filteredTokens.size()==0 ) {
+            filteredTokens = null;
+        }
+        return filteredTokens;
+    }
+
+    public List getTokens(int start, int stop, List types) {
+        return getTokens(start,stop,new BitSet(types));
+    }
+
+    public List getTokens(int start, int stop, int ttype) {
+        return getTokens(start,stop,BitSet.of(ttype));
+    }
+
+    public String getSourceName() {	return tokenSource.getSourceName();	}
+
+    /** Grab *all* tokens from stream and return string */
+    public String toString() {
+        if ( p == -1 ) setup();
+        fill();
+        return toString(0, tokens.size()-1);
+    }
+
+    public String toString(int start, int stop) {
+        if ( start<0 || stop<0 ) return null;
+        if ( p == -1 ) setup();
+        if ( stop>=tokens.size() ) stop = tokens.size()-1;
+        StringBuffer buf = new StringBuffer();
+        for (int i = start; i <= stop; i++) {
+            Token t = tokens.get(i);
+            if ( t.getType()==Token.EOF ) break;
+            buf.append(t.getText());
+        }
+        return buf.toString();
+    }
+
+    public String toString(Token start, Token stop) {
+        if ( start!=null && stop!=null ) {
+            return toString(start.getTokenIndex(), stop.getTokenIndex());
+        }
+        return null;
+    }
+
+    /** Get all tokens from lexer until EOF */
+    public void fill() {
+        if ( p == -1 ) setup();
+        if ( tokens.get(p).getType()==Token.EOF ) return;
+
+        int i = p+1;
+        sync(i);
+        while ( tokens.get(i).getType()!=Token.EOF ) {
+            i++;
+            sync(i);
+        }
+    }
+}

data/test-server/instrumentation-backend/src/org/antlr/runtime/CharStream.java

@@ -0,0 +1,57 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A source of characters for an ANTLR lexer */
+public interface CharStream extends IntStream {
+    public static final int EOF = -1;
+
+	/** For infinite streams, you don't need this; primarily I'm providing
+	 *  a useful interface for action code.  Just make sure actions don't
+	 *  use this on streams that don't support it.
+	 */
+	public String substring(int start, int stop);
+
+	/** Get the ith character of lookahead.  This is the same usually as
+	 *  LA(i).  This will be used for labels in the generated
+	 *  lexer code.  I'd prefer to return a char here type-wise, but it's
+	 *  probably better to be 32-bit clean and be consistent with LA.
+	 */
+	public int LT(int i);
+
+	/** ANTLR tracks the line information automatically */
+	int getLine();
+
+	/** Because this stream can rewind, we need to be able to reset the line */
+	void setLine(int line);
+
+	void setCharPositionInLine(int pos);
+
+	/** The index of the character relative to the beginning of the line 0..n-1 */
+	int getCharPositionInLine();
+}

data/test-server/instrumentation-backend/src/org/antlr/runtime/CharStreamState.java

@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** When walking ahead with cyclic DFA or for syntactic predicates,
+ *  we need to record the state of the input stream (char index,
+ *  line, etc...) so that we can rewind the state after scanning ahead.
+ *
+ *  This is the complete state of a stream.
+ */
+public class CharStreamState {
+	/** Index into the char stream of next lookahead char */
+	int p;
+
+	/** What line number is the scanner at before processing buffer[p]? */
+	int line;
+	
+	/** What char position 0..n-1 in line is scanner before processing buffer[p]? */
+	int charPositionInLine;
+}