@harmoniclabs/pebble 0.1.0-dev2 → 0.1.0-dev3

package/dist/tokenizer/Tokenizer.js

@@ -0,0 +1,1323 @@
+import { SourceRange } from "../ast/Source/SourceRange.js";
+import { DiagnosticEmitter } from "../diagnostics/DiagnosticEmitter.js";
+import { DiagnosticCode } from "../diagnostics/diagnosticMessages.generated.js";
+import { assert } from "../utils/assert.js";
+import { isHighSurrogate, combineSurrogates, isIdentifierStart, numCodeUnits, isIdentifierPart, isWhiteSpace, isLineBreak, isDecimal, isOctal, isHexBase } from "../utils/text.js";
+import { IdentifierHandling } from "./IdentifierHandling.js";
+import { OnNewLine } from "./OnNewLine.js";
+import { Token } from "./Token.js";
+import { tokenFromKeyword } from "./utils/tokenFromKeyword.js";
+import { tokenIsAlsoIdentifier } from "./utils/tokenIsAlsoIdentifier.js";
+import { fromHex } from "@harmoniclabs/uint8array-utils";
+/** Comment kinds. */
+export var CommentKind;
+(function (CommentKind) {
+    /** Line comment. */
+    CommentKind[CommentKind["Line"] = 0] = "Line";
+    /** Triple-slash line comment. */
+    CommentKind[CommentKind["Triple"] = 1] = "Triple";
+    /** Block comment. */
+    CommentKind[CommentKind["Block"] = 2] = "Block";
+})(CommentKind || (CommentKind = {}));
+Object.freeze(CommentKind);
+const fakeToken = -1;
+/**
+ * Takes a {@link Source} and tokenizes its contents.
+ *
+ * It acts as an iterator, so that the text is never broken down
+ * but we can still get out single tokens.
+**/
+export class Tokenizer extends DiagnosticEmitter {
+    source;
+    end = 0;
+    // tokenizer state ( pos, token, tokenPos )
+    pos = 0;
+    token = fakeToken;
+    tokenPos = 0;
+    // next token state ( nextToken, nextTokenPos, nextTokenOnNewLine )
+    nextToken = fakeToken;
+    nextTokenPos = 0;
+    // cache result of `isNextTokenOnNewLine`
+    nextTokenOnNewLine = OnNewLine.Unknown;
+    // custom comment handling
+    onComment = undefined;
+    constructor(source, diagnostics = undefined) {
+        super(diagnostics);
+        if (!Array.isArray(diagnostics))
+            diagnostics = [];
+        this.diagnostics = diagnostics;
+        this.source = source;
+        let text = source.text;
+        let end = text.length;
+        let pos = 0;
+        // skip bom
+        if (pos < end &&
+            text.charCodeAt(pos) === 65279 /* CharCode.ByteOrderMark */) {
+            ++pos;
+        }
+        // skip shebang
+        if (pos + 1 < end &&
+            text.charCodeAt(pos) === 35 /* CharCode.Hash */ &&
+            text.charCodeAt(pos + 1) === 33 /* CharCode.Exclamation */) {
+            pos += 2;
+            while (pos < end &&
+                text.charCodeAt(pos) !== 10 /* CharCode.LineFeed */) {
+                ++pos;
+            }
+            // 'next' now starts at lf or eof
+        }
+        this.pos = pos;
+        this.end = end;
+    }
+    /*
+    toArray(): Token[] {
+        let tokens = new Array<Token>();
+        let token: Token;
+        while ((token = this.next()) !== Token.EndOfFile) {
+            if( token === Token.Identifier ) this.readIdentifier();
+            if( token === Token.IntegerLiteral ) this.readInteger();
+
+            tokens.push( token );
+        }
+        return tokens;
+    }
+    */
+    /** advances the tokenizer and returns the new token */
+    next(identifierHandling = IdentifierHandling.Default) {
+        this.clearNextToken();
+        let token;
+        do
+            token = this.unsafeNext(identifierHandling);
+        while (token === Token.Invalid);
+        this.token = token;
+        return token;
+    }
+    /**
+     * Retrieves the next token from the source text,
+     * handling identifiers and token length.
+     * This method is considered unsafe because
+     * it does not perform validation on the token.
+     *
+     * @param identifierHandling - Specifies how identifiers should be handled.
+     *
+     * @param maxTokenLength - The maximum length of the token to retrieve.
+     * Defaults to Number.MAX_SAFE_INTEGER.
+     *
+     * @returns The next token from the source text.
+     */
+    unsafeNext(identifierHandling = IdentifierHandling.Default, maxTokenLength = Number.MAX_SAFE_INTEGER) {
+        let text = this.source.text;
+        let end = this.end;
+        let pos = this.pos;
+        while (pos < end) {
+            this.tokenPos = pos;
+            let c = text.charCodeAt(pos);
+            switch (c) {
+                case 13 /* CharCode.CarriageReturn */: {
+                    if (!(++pos < end &&
+                        text.charCodeAt(pos) === 10 /* CharCode.LineFeed */))
+                        break;
+                    // otherwise fall-through
+                }
+                case 10 /* CharCode.LineFeed */:
+                case 9 /* CharCode.Tab */:
+                case 11 /* CharCode.VerticalTab */:
+                case 12 /* CharCode.FormFeed */:
+                case 32 /* CharCode.Space */: {
+                    ++pos;
+                    break;
+                }
+                case 33 /* CharCode.Exclamation */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end &&
+                        text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                        ++pos;
+                        if (maxTokenLength > 2 && pos < end &&
+                            text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Exclamation_Equals_Equals;
+                        }
+                        this.pos = pos;
+                        return Token.Exclamation_Equals;
+                    }
+                    else if (maxTokenLength > 1 && pos < end &&
+                        text.charCodeAt(pos) === 46 /* CharCode.Dot */) {
+                        this.pos = pos + 1;
+                        return Token.Exclamation_Dot; // !.
+                    }
+                    this.pos = pos;
+                    return Token.Exclamation;
+                }
+                case 34 /* CharCode.DoubleQuote */:
+                case 39 /* CharCode.SingleQuote */: {
+                    this.pos = pos;
+                    return Token.StringLiteral;
+                }
+                case 96 /* CharCode.Backtick */: {
+                    this.pos = pos;
+                    return Token.StringTemplateLiteralQuote;
+                }
+                case 37 /* CharCode.Percent */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end &&
+                        text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                        this.pos = pos + 1;
+                        return Token.Percent_Equals;
+                    }
+                    this.pos = pos;
+                    return Token.Percent;
+                }
+                case 38 /* CharCode.Ampersand */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 38 /* CharCode.Ampersand */) {
+                            // this.pos = pos + 1;
+                            ++pos;
+                            if (maxTokenLength > 2 && pos < end &&
+                                text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                this.pos = pos + 1;
+                                return Token.Ampersand_Ampersand_Equals;
+                            }
+                            this.pos = pos;
+                            return Token.Ampersand_Ampersand;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Ampersand_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Ampersand;
+                }
+                case 40 /* CharCode.OpenParen */: {
+                    this.pos = pos + 1;
+                    return Token.OpenParen;
+                }
+                case 41 /* CharCode.CloseParen */: {
+                    this.pos = pos + 1;
+                    return Token.CloseParen;
+                }
+                case 42 /* CharCode.Asterisk */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Asterisk_Equals;
+                        }
+                        if (chr === 42 /* CharCode.Asterisk */) {
+                            ++pos;
+                            if (maxTokenLength > 2 && pos < end &&
+                                text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                this.pos = pos + 1;
+                                return Token.Asterisk_Asterisk_Equals;
+                            }
+                            this.pos = pos;
+                            return Token.Asterisk_Asterisk;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Asterisk;
+                }
+                case 43 /* CharCode.Plus */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 43 /* CharCode.Plus */) {
+                            this.pos = pos + 1;
+                            return Token.Plus_Plus;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Plus_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Plus;
+                }
+                case 44 /* CharCode.Comma */: {
+                    this.pos = pos + 1;
+                    return Token.Comma;
+                }
+                case 45 /* CharCode.Minus */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 45 /* CharCode.Minus */) {
+                            this.pos = pos + 1;
+                            return Token.Minus_Minus;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Minus_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Minus;
+                }
+                case 46 /* CharCode.Dot */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        // if (isDecimal(chr)) {
+                        //     this.pos = pos - 1;
+                        //     return Token.FloatLiteral; // expects a call to readFloat
+                        // }
+                        if (maxTokenLength > 2 && pos + 1 < end &&
+                            chr === 46 /* CharCode.Dot */ &&
+                            text.charCodeAt(pos + 1) === 46 /* CharCode.Dot */) {
+                            this.pos = pos + 2;
+                            return Token.Dot_Dot_Dot;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Dot;
+                }
+                case 47 /* CharCode.Slash */: {
+                    let commentStartPos = pos;
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 47 /* CharCode.Slash */) { // single-line
+                            let commentKind = CommentKind.Line;
+                            if (pos + 1 < end &&
+                                text.charCodeAt(pos + 1) === 47 /* CharCode.Slash */) {
+                                ++pos;
+                                commentKind = CommentKind.Triple;
+                            }
+                            while (++pos < end) {
+                                if (text.charCodeAt(pos) === 10 /* CharCode.LineFeed */) {
+                                    ++pos;
+                                    break;
+                                }
+                            }
+                            if (this.onComment) {
+                                this.onComment(commentKind, text.substring(commentStartPos, pos), this.range(commentStartPos, pos));
+                            }
+                            break;
+                        }
+                        if (chr === 42 /* CharCode.Asterisk */) { // multi-line
+                            let closed = false;
+                            while (++pos < end) {
+                                c = text.charCodeAt(pos);
+                                if (c === 42 /* CharCode.Asterisk */ &&
+                                    pos + 1 < end &&
+                                    text.charCodeAt(pos + 1) === 47 /* CharCode.Slash */) {
+                                    pos += 2;
+                                    closed = true;
+                                    break;
+                                }
+                            }
+                            if (!closed) {
+                                this.error(DiagnosticCode._0_expected, this.range(pos), "*/");
+                            }
+                            else if (this.onComment) {
+                                this.onComment(CommentKind.Block, text.substring(commentStartPos, pos), this.range(commentStartPos, pos));
+                            }
+                            break;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Slash_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Slash;
+                }
+                case 48 /* CharCode._0 */:
+                case 49 /* CharCode._1 */:
+                case 50 /* CharCode._2 */:
+                case 51 /* CharCode._3 */:
+                case 52 /* CharCode._4 */:
+                case 53 /* CharCode._5 */:
+                case 54 /* CharCode._6 */:
+                case 55 /* CharCode._7 */:
+                case 56 /* CharCode._8 */:
+                case 57 /* CharCode._9 */: {
+                    this.pos = pos;
+                    if (!this.testInteger()) { }
+                    return Token.IntegerLiteral;
+                    // return this.testInteger()
+                    //     ? Token.IntegerLiteral // expects a call to readInteger
+                    //     : Token.FloatLiteral;  // expects a call to readFloat
+                }
+                case 35 /* CharCode.Hash */: {
+                    this.pos = pos;
+                    return Token.HexBytesLiteral;
+                    break;
+                }
+                case 58 /* CharCode.Colon */: {
+                    this.pos = pos + 1;
+                    return Token.Colon;
+                }
+                case 59 /* CharCode.Semicolon */: {
+                    this.pos = pos + 1;
+                    return Token.Semicolon;
+                }
+                case 60 /* CharCode.LessThan */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 60 /* CharCode.LessThan */) {
+                            ++pos;
+                            if (maxTokenLength > 2 &&
+                                pos < end &&
+                                text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                this.pos = pos + 1;
+                                return Token.LessThan_LessThan_Equals;
+                            }
+                            this.pos = pos;
+                            return Token.LessThan_LessThan;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.LessThan_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.LessThan;
+                }
+                case 61 /* CharCode.Equals */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 61 /* CharCode.Equals */) {
+                            ++pos;
+                            if (maxTokenLength > 2 &&
+                                pos < end &&
+                                text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                this.pos = pos + 1;
+                                return Token.Equals_Equals_Equals;
+                            }
+                            this.pos = pos;
+                            return Token.Equals_Equals;
+                        }
+                        if (chr === 62 /* CharCode.GreaterThan */) {
+                            this.pos = pos + 1;
+                            return Token.FatArrow;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Equals;
+                }
+                case 62 /* CharCode.GreaterThan */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 62 /* CharCode.GreaterThan */) {
+                            ++pos;
+                            if (maxTokenLength > 2 && pos < end) {
+                                chr = text.charCodeAt(pos);
+                                if (chr === 62 /* CharCode.GreaterThan */) {
+                                    ++pos;
+                                    if (maxTokenLength > 3 && pos < end &&
+                                        text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                        this.pos = pos + 1;
+                                        return Token.GreaterThan_GreaterThan_GreaterThan_Equals;
+                                    }
+                                    this.pos = pos;
+                                    return Token.GreaterThan_GreaterThan_GreaterThan;
+                                }
+                                if (chr === 61 /* CharCode.Equals */) {
+                                    this.pos = pos + 1;
+                                    return Token.GreaterThan_GreaterThan_Equals;
+                                }
+                            }
+                            this.pos = pos;
+                            return Token.GreaterThan_GreaterThan;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.GreaterThan_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.GreaterThan;
+                }
+                case 63 /* CharCode.Question */: {
+                    // this.pos = pos + 1;
+                    pos++;
+                    if (maxTokenLength > 1 && pos < end &&
+                        text.charCodeAt(pos) === 63 /* CharCode.Question */) {
+                        pos++;
+                        if (maxTokenLength > 2 && pos < end &&
+                            text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Question_Question_Equals;
+                        }
+                        this.pos = pos;
+                        return Token.Question_Question;
+                    }
+                    else if (text.charCodeAt(pos) === 46 /* CharCode.Dot */) {
+                        this.pos = pos + 1;
+                        return Token.Question_Dot;
+                    }
+                    this.pos = pos;
+                    return Token.Question;
+                }
+                case 91 /* CharCode.OpenBracket */: {
+                    this.pos = pos + 1;
+                    return Token.OpenBracket;
+                }
+                case 93 /* CharCode.CloseBracket */: {
+                    this.pos = pos + 1;
+                    return Token.CloseBracket;
+                }
+                case 94 /* CharCode.Caret */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end &&
+                        text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                        this.pos = pos + 1;
+                        return Token.Caret_Equals;
+                    }
+                    this.pos = pos;
+                    return Token.Caret;
+                }
+                case 123 /* CharCode.OpenBrace */: {
+                    this.pos = pos + 1;
+                    return Token.OpenBrace;
+                }
+                case 124 /* CharCode.Bar */: {
+                    ++pos;
+                    if (maxTokenLength > 1 && pos < end) {
+                        let chr = text.charCodeAt(pos);
+                        if (chr === 124 /* CharCode.Bar */) {
+                            // this.pos = pos + 1;
+                            ++pos;
+                            if (maxTokenLength > 2 && pos < end &&
+                                text.charCodeAt(pos) === 61 /* CharCode.Equals */) {
+                                this.pos = pos + 1;
+                                return Token.Bar_Bar_Equals;
+                            }
+                            this.pos = pos;
+                            return Token.Bar_Bar;
+                        }
+                        if (chr === 61 /* CharCode.Equals */) {
+                            this.pos = pos + 1;
+                            return Token.Bar_Equals;
+                        }
+                    }
+                    this.pos = pos;
+                    return Token.Bar;
+                }
+                case 125 /* CharCode.CloseBrace */: {
+                    this.pos = pos + 1;
+                    return Token.CloseBrace;
+                }
+                case 126 /* CharCode.Tilde */: {
+                    this.pos = pos + 1;
+                    return Token.Tilde;
+                }
+                case 64 /* CharCode.At */: {
+                    this.pos = pos + 1;
+                    return Token.At;
+                }
+                default: {
+                    // Unicode-aware from here on
+                    if (isHighSurrogate(c) && pos + 1 < end) {
+                        c = combineSurrogates(c, text.charCodeAt(pos + 1));
+                    }
+                    if (isIdentifierStart(c)) {
+                        let posBefore = pos;
+                        while ((pos += numCodeUnits(c)) < end &&
+                            isIdentifierPart(c = text.codePointAt(pos))) { /* nop */ }
+                        if (identifierHandling !== IdentifierHandling.Always) {
+                            let maybeKeywordToken = tokenFromKeyword(text.substring(posBefore, pos));
+                            if (maybeKeywordToken !== Token.Invalid &&
+                                !(identifierHandling === IdentifierHandling.Prefer &&
+                                    tokenIsAlsoIdentifier(maybeKeywordToken))) {
+                                this.pos = pos;
+                                return maybeKeywordToken;
+                            }
+                        }
+                        this.pos = posBefore;
+                        return Token.Identifier;
+                    }
+                    else if (isWhiteSpace(c)) {
+                        ++pos; // assume no supplementary whitespaces
+                        break;
+                    }
+                    let start = pos;
+                    pos += numCodeUnits(c);
+                    this.error(DiagnosticCode.Invalid_character, this.range(start, pos));
+                    this.pos = pos;
+                    return Token.Invalid;
+                }
+            }
+        }
+        this.pos = pos;
+        return Token.EndOfFile;
+    }
+    peek(identifierHandling = IdentifierHandling.Default, maxCompoundLength = Number.MAX_SAFE_INTEGER) {
+        let nextToken = this.nextToken;
+        if (nextToken < 0) {
+            // save current state
+            let posBefore = this.pos;
+            let tokenBefore = this.token;
+            let tokenPosBefore = this.tokenPos;
+            // take next valid token (modifies state)
+            do
+                nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
+            while (nextToken === Token.Invalid);
+            // save next token
+            this.nextToken = nextToken;
+            this.nextTokenPos = this.tokenPos;
+            this.nextTokenOnNewLine = OnNewLine.Unknown;
+            // restore current state
+            this.pos = posBefore;
+            this.token = tokenBefore;
+            this.tokenPos = tokenPosBefore;
+        }
+        return nextToken;
+    }
+    isNextTokenOnNewLine() {
+        switch (this.nextTokenOnNewLine) {
+            case OnNewLine.No: return false;
+            case OnNewLine.Yes: return true;
+        }
+        // case OnNewLine.Unknown
+        this.peek();
+        let text = this.source.text;
+        for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) {
+            if (isLineBreak(text.charCodeAt(pos))) {
+                this.nextTokenOnNewLine = OnNewLine.Yes;
+                return true;
+            }
+        }
+        this.nextTokenOnNewLine = OnNewLine.No;
+        return false;
+    }
+    skipIdentifier(identifierHandling = IdentifierHandling.Prefer) {
+        return this.skip(Token.Identifier, identifierHandling);
+    }
+    /**
+     *
+     * @param expectedToken The token type that the tokenizer should skip if it matches the current token.
+     * @param identifierHandling
+     * @returns {boolean} Returns true if the current token matches the expectedToken and the tokenizer's position is advanced.
+     * Returns false if the current token does not match the expectedToken.
+     *
+     * ### side effects
+     *
+     * - Advances the Tokenizer Position:
+     * If the current token matches the expectedToken,
+     * the tokenizer's position is advanced to the next token.
+     *
+     * - Updates Internal State:
+     * The method updates the internal state of the tokenizer,
+     * including the current token and its position.
+     *
+     */
+    skip(expectedToken, identifierHandling = IdentifierHandling.Default) {
+        let posBefore = this.pos;
+        let tokenBefore = this.token;
+        let tokenPosBefore = this.tokenPos;
+        let maxCompoundLength = Number.MAX_SAFE_INTEGER;
+        if (expectedToken === Token.GreaterThan) { // where parsing type arguments
+            maxCompoundLength = 1;
+        }
+        // take the first valid token
+        let nextToken;
+        do
+            nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
+        while (nextToken === Token.Invalid);
+        if (nextToken === expectedToken) {
+            this.token = expectedToken;
+            this.clearNextToken();
+            return true;
+        }
+        else {
+            this.pos = posBefore;
+            this.token = tokenBefore;
+            this.tokenPos = tokenPosBefore;
+            return false;
+        }
+    }
+    eof() {
+        return this.skip(Token.EndOfFile);
+    }
+    // state management
+    /** #__PURE__ gets the current state */
+    mark() {
+        // let state = reusableState;
+        // if (state) {
+        //     reusableState = undefined;
+        //     state.pos = this.pos;
+        //     state.token = this.token;
+        //     state.tokenPos = this.tokenPos;
+        // } else {
+        //     state = new TokenizerState(this.pos, this.token, this.tokenPos);
+        // }
+        return new TokenizerState(this.pos, this.token, this.tokenPos);
+    }
+    // discard(state: TokenizerState): void {
+    //     reusableState = state;
+    // }
+    reset(state) {
+        this.pos = state.pos;
+        this.token = state.token;
+        this.tokenPos = state.tokenPos;
+        this.clearNextToken();
+    }
+    /**
+     * ### Side Effects
+     * - Resets `nextToken`: Sets the nextToken property to an invalid state, typically -1.
+     * - Resets `nextTokenPos`: Sets the nextTokenPos property to 0 or another default value.
+     * - Resets `nextTokenOnNewLine`: Sets the nextTokenOnNewLine property to OnNewLine.
+     * Unknown or another default value.
+     */
+    clearNextToken() {
+        this.nextToken = fakeToken;
+        this.nextTokenPos = 0;
+        this.nextTokenOnNewLine = OnNewLine.Unknown;
+    }
+    /**
+     * pure function for the same state
+    **/
+    range(start = -1, end = -1) {
+        if (start < 0) { // default range
+            start = this.tokenPos;
+            end = this.pos;
+        }
+        else if (end < 0) {
+            end = start;
+        }
+        return new SourceRange(this.source, start, end);
+    }
+    readIdentifier() {
+        let text = this.source.text;
+        let end = this.end;
+        let pos = this.pos;
+        let start = pos;
+        let c = text.codePointAt(pos);
+        assert(isIdentifierStart(c), "token is not an identifier");
+        while ((pos += numCodeUnits(c)) < end &&
+            isIdentifierPart(c = text.codePointAt(pos)))
+            ;
+        this.pos = pos;
+        return text.substring(start, pos);
+    }
+    readingTemplateString = false;
+    readStringStart = 0;
+    readStringEnd = 0;
+    readString(quote = 0, isTaggedTemplate = false) {
+        let text = this.source.text;
+        let end = this.end;
+        let pos = this.pos;
+        if (!quote)
+            quote = text.charCodeAt(pos++);
+        let start = pos;
+        this.readStringStart = start;
+        let result = "";
+        while (true) {
+            if (pos >= end) {
+                result += text.substring(start, pos);
+                this.error(DiagnosticCode.Unterminated_string_literal, this.range(start - 1, end));
+                this.readStringEnd = end;
+                break;
+            }
+            let c = text.charCodeAt(pos);
+            if (c === quote) {
+                this.readStringEnd = pos;
+                result += text.substring(start, pos++);
+                break;
+            }
+            if (c === 92 /* CharCode.Backslash */) {
+                result += text.substring(start, pos);
+                this.pos = pos; // save
+                result += this.readEscapeSequence(isTaggedTemplate);
+                pos = this.pos; // restore
+                start = pos;
+                continue;
+            }
+            if (quote === 96 /* CharCode.Backtick */) {
+                if (c === 36 /* CharCode.Dollar */ && pos + 1 < end && text.charCodeAt(pos + 1) === 123 /* CharCode.OpenBrace */) {
+                    result += text.substring(start, pos);
+                    this.readStringEnd = pos;
+                    this.pos = pos + 2;
+                    this.readingTemplateString = true;
+                    return result;
+                }
+            }
+            else if (isLineBreak(c)) {
+                result += text.substring(start, pos);
+                this.error(DiagnosticCode.Unterminated_string_literal, this.range(start - 1, pos));
+                this.readStringEnd = pos;
+                break;
+            }
+            ++pos;
+        }
+        this.pos = pos;
+        this.readingTemplateString = false;
+        return result;
+    }
+    readEscapeSequence(isTaggedTemplate = false) {
+        // for context on isTaggedTemplate, see: https://tc39.es/proposal-template-literal-revision/
+        let start = this.pos;
+        let end = this.end;
+        if (++this.pos >= end) {
+            this.error(DiagnosticCode.Unexpected_end_of_text, this.range(end));
+            return "";
+        }
+        let text = this.source.text;
+        let c = text.charCodeAt(this.pos++);
+        switch (c) {
+            case 48 /* CharCode._0 */: {
+                if (isTaggedTemplate && this.pos < end && isDecimal(text.charCodeAt(this.pos))) {
+                    ++this.pos;
+                    return text.substring(start, this.pos);
+                }
+                return "\0";
+            }
+            case 98 /* CharCode.b */: return "\b";
+            case 116 /* CharCode.t */: return "\t";
+            case 110 /* CharCode.n */: return "\n";
+            case 118 /* CharCode.v */: return "\v";
+            case 102 /* CharCode.f */: return "\f";
+            case 114 /* CharCode.r */: return "\r";
+            case 39 /* CharCode.SingleQuote */: return "'";
+            case 34 /* CharCode.DoubleQuote */: return "\"";
+            case 117 /* CharCode.u */: {
+                if (this.pos < end &&
+                    text.charCodeAt(this.pos) === 123 /* CharCode.OpenBrace */) {
+                    ++this.pos;
+                    return this.readExtendedUnicodeEscape(isTaggedTemplate ? start : -1); // \u{DDDDDDDD}
+                }
+                return this.readUnicodeEscape(isTaggedTemplate ? start : -1); // \uDDDD
+            }
+            case 120 /* CharCode.x */: {
+                return this.readHexadecimalEscape(2, isTaggedTemplate ? start : -1); // \xDD
+            }
+            case 13 /* CharCode.CarriageReturn */: {
+                if (this.pos < end &&
+                    text.charCodeAt(this.pos) === 10 /* CharCode.LineFeed */) {
+                    ++this.pos;
+                }
+                // fall through
+            }
+            case 10 /* CharCode.LineFeed */:
+            case 8232 /* CharCode.LineSeparator */:
+            case 8233 /* CharCode.ParagraphSeparator */: return "";
+            default: return String.fromCodePoint(c);
+        }
+    }
+    readRegexpPattern() {
+        let text = this.source.text;
+        let start = this.pos;
+        let end = this.end;
+        let escaped = false;
+        while (true) {
+            if (this.pos >= end) {
+                this.error(DiagnosticCode.Unterminated_regular_expression_literal, this.range(start, end));
+                break;
+            }
+            if (text.charCodeAt(this.pos) === 92 /* CharCode.Backslash */) {
+                ++this.pos;
+                escaped = true;
+                continue;
+            }
+            let c = text.charCodeAt(this.pos);
+            if (!escaped && c === 47 /* CharCode.Slash */)
+                break;
+            if (isLineBreak(c)) {
+                this.error(DiagnosticCode.Unterminated_regular_expression_literal, this.range(start, this.pos));
+                break;
+            }
+            ++this.pos;
+            escaped = false;
+        }
+        return text.substring(start, this.pos);
+    }
+    readRegexpFlags() {
+        let text = this.source.text;
+        let start = this.pos;
+        let end = this.end;
+        let flags = 0;
+        while (this.pos < end) {
+            let c = text.charCodeAt(this.pos);
+            if (!isIdentifierPart(c))
+                break;
+            ++this.pos;
+            // make sure each supported flag is unique
+            switch (c) {
+                case 103 /* CharCode.g */: {
+                    flags |= flags & 1 ? -1 : 1;
+                    break;
+                }
+                case 105 /* CharCode.i */: {
+                    flags |= flags & 2 ? -1 : 2;
+                    break;
+                }
+                case 109 /* CharCode.m */: {
+                    flags |= flags & 4 ? -1 : 4;
+                    break;
+                }
+                default: {
+                    flags = -1;
+                    break;
+                }
+            }
+        }
+        if (flags === -1) {
+            this.error(DiagnosticCode.Invalid_regular_expression_flags, this.range(start, this.pos));
+        }
+        return text.substring(start, this.pos);
+    }
+    testInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        // test for hex, binary or octal
+        if (pos + 1 < end && text.charCodeAt(pos) === 48 /* CharCode._0 */) {
+            switch (text.charCodeAt(pos + 2) | 32) {
+                case 120 /* CharCode.x */:
+                case 98 /* CharCode.b */:
+                case 111 /* CharCode.o */: return true;
+            }
+        }
+        // test decimal
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            if (c === 46 /* CharCode.Dot */ || (c | 32) === 101 /* CharCode.e */)
+                return false;
+            if (c !== 95 /* CharCode._ */ && (c < 48 /* CharCode._0 */ || c > 57 /* CharCode._9 */))
+                break;
+            // does not validate separator placement (this is done in readXYInteger)
+            pos++;
+        }
+        return true;
+    }
+    readInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        if (pos + 2 < this.end && text.charCodeAt(pos) === 48 /* CharCode._0 */) {
+            switch (text.charCodeAt(pos + 1) | 32) {
+                case 120 /* CharCode.x */: {
+                    this.pos = pos + 2;
+                    return this.readHexInteger();
+                }
+                case 98 /* CharCode.b */: {
+                    this.pos = pos + 2;
+                    return this.readBinaryInteger();
+                }
+                case 111 /* CharCode.o */: {
+                    this.pos = pos + 2;
+                    return this.readOctalInteger();
+                }
+            }
+            if (isOctal(text.charCodeAt(pos + 1))) {
+                let start = pos;
+                this.pos = pos + 1;
+                let value = this.readOctalInteger();
+                this.error(DiagnosticCode.Octal_literals_are_not_allowed_in_strict_mode, this.range(start, this.pos));
+                return value;
+            }
+        }
+        return this.readDecimalInteger();
+    }
+    readHexBytes() {
+        let text = this.source.text;
+        let pos = this.pos;
+        pos = text[pos] === "#" ? pos + 1 : pos;
+        this.pos = pos;
+        let end = this.end;
+        let start = pos;
+        let length = 0;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            if (isHexBase(c)) {
+                length++;
+                pos++;
+            }
+            else {
+                break;
+            }
+        }
+        if (length === 0) {
+            return new Uint8Array(0);
+        }
+        this.pos = pos;
+        const oddLen = length % 2 === 1;
+        const substr = (oddLen ? "0" : "") + text.substring(start, pos);
+        return fromHex(substr);
+    }
+    ;
+    readHexInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        let start = pos;
+        let sepEnd = start;
+        let value = BigInt(0);
+        let i64_4 = BigInt(4);
+        let nextValue = value;
+        let overflowOccurred = false;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            const bc = BigInt(c);
+            if (isDecimal(c)) {
+                // (value << 4) + c - CharCode._0
+                nextValue = (value << BigInt(4)) + bc - BigInt(48 /* CharCode._0 */);
+            }
+            else if (isHexBase(c)) {
+                // (value << 4) + (c | 32) + (10 - CharCode.a)
+                nextValue = (value << BigInt(4)) + BigInt(c | 32) + BigInt(10 - 97 /* CharCode.a */);
+            }
+            else if (c === 95 /* CharCode._ */) {
+                if (sepEnd === pos) {
+                    this.error(sepEnd === start
+                        ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+                        : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(pos));
+                }
+                sepEnd = pos + 1;
+            }
+            else {
+                break;
+            }
+            // if (i64_gt_u(value, nextValue)) {
+            //     // Unsigned overflow occurred
+            //     overflowOccurred = true;
+            // }
+            value = nextValue;
+            ++pos;
+        }
+        if (pos === start) {
+            this.error(DiagnosticCode.Hexadecimal_digit_expected, this.range(start));
+        }
+        else if (sepEnd === pos) {
+            this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1));
+        }
+        if (overflowOccurred) {
+            this.error(DiagnosticCode.Literal_0_does_not_fit_into_i64_or_u64_types, this.range(start - 2, pos), this.source.text.substring(start - 2, pos));
+        }
+        this.pos = pos;
+        return value;
+    }
+    readDecimalInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        let start = pos;
+        let sepEnd = start;
+        let value = BigInt(0);
+        let i64_10 = BigInt(10);
+        let nextValue = value;
+        let overflowOccurred = false;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            const bc = BigInt(c);
+            if (isDecimal(c)) {
+                // value = value * 10 + c - CharCode._0;
+                nextValue = value * i64_10 + bc - BigInt(48 /* CharCode._0 */);
+            }
+            else if (c === 95 /* CharCode._ */) {
+                if (sepEnd === pos) {
+                    this.error(sepEnd === start
+                        ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+                        : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(pos));
+                }
+                else if (pos - 1 === start && text.charCodeAt(pos - 1) === 48 /* CharCode._0 */) {
+                    this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(pos));
+                }
+                sepEnd = pos + 1;
+            }
+            else {
+                break;
+            }
+            // if (i64_gt_u(value, nextValue)) {
+            //     // Unsigned overflow occurred
+            //     overflowOccurred = true;
+            // }
+            value = nextValue;
+            ++pos;
+        }
+        if (pos === start) {
+            this.error(DiagnosticCode.Digit_expected, this.range(start));
+        }
+        else if (sepEnd === pos) {
+            this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1));
+        }
+        else if (overflowOccurred) {
+            this.error(DiagnosticCode.Literal_0_does_not_fit_into_i64_or_u64_types, this.range(start, pos), this.source.text.substring(start, pos));
+        }
+        this.pos = pos;
+        return value;
+    }
+    readOctalInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        let start = pos;
+        let sepEnd = start;
+        let value = BigInt(0);
+        let i64_3 = BigInt(3);
+        let nextValue = value;
+        let overflowOccurred = false;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            if (isOctal(c)) {
+                // (value << 3) + c - CharCode._0
+                nextValue = (value << BigInt(3)) + BigInt(c) - BigInt(48 /* CharCode._0 */);
+            }
+            else if (c === 95 /* CharCode._ */) {
+                if (sepEnd === pos) {
+                    this.error(sepEnd === start
+                        ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+                        : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(pos));
+                }
+                sepEnd = pos + 1;
+            }
+            else {
+                break;
+            }
+            // if (i64_gt_u(value, nextValue)) {
+            //     // Unsigned overflow occurred
+            //     overflowOccurred = true;
+            // }
+            value = nextValue;
+            ++pos;
+        }
+        if (pos === start) {
+            this.error(DiagnosticCode.Octal_digit_expected, this.range(start));
+        }
+        else if (sepEnd === pos) {
+            this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1));
+        }
+        else if (overflowOccurred) {
+            this.error(DiagnosticCode.Literal_0_does_not_fit_into_i64_or_u64_types, this.range(start - 2, pos), this.source.text.substring(start - 2, pos));
+        }
+        this.pos = pos;
+        return value;
+    }
+    readBinaryInteger() {
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        let start = pos;
+        let sepEnd = start;
+        let value = BigInt(0);
+        let nextValue = value;
+        let overflowOccurred = false;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            if (c === 48 /* CharCode._0 */) {
+                // value << 1 | 0
+                nextValue = value << BigInt(1);
+            }
+            else if (c === 49 /* CharCode._1 */) {
+                // value << 1 | 1
+                nextValue = value << BigInt(1) | BigInt(1);
+            }
+            else if (c === 95 /* CharCode._ */) {
+                if (sepEnd === pos) {
+                    this.error(sepEnd === start
+                        ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+                        : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(pos));
+                }
+                sepEnd = pos + 1;
+            }
+            else {
+                break;
+            }
+            // if (i64_gt(value, nextValue)) {
+            //     // Overflow occurred
+            //     overflowOccurred = true;
+            // }
+            value = nextValue;
+            ++pos;
+        }
+        if (pos === start) {
+            this.error(DiagnosticCode.Binary_digit_expected, this.range(start));
+        }
+        else if (sepEnd === pos) {
+            this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1));
+        }
+        else if (overflowOccurred) {
+            this.error(DiagnosticCode.Literal_0_does_not_fit_into_i64_or_u64_types, this.range(start - 2, pos), this.source.text.substring(start - 2, pos));
+        }
+        this.pos = pos;
+        return value;
+    }
+    readFloat() {
+        // let text = this.source.text;
+        // if (text.charCodeAt(this.pos) === CharCode._0 && this.pos + 2 < this.end) {
+        //   switch (text.charCodeAt(this.pos + 1)) {
+        //     case CharCode.X:
+        //     case CharCode.x: {
+        //       this.pos += 2;
+        //       return this.readHexFloat();
+        //     }
+        //   }
+        // }
+        return this.readDecimalFloat();
+    }
+    readDecimalFloat() {
+        let text = this.source.text;
+        let end = this.end;
+        let start = this.pos;
+        let sepCount = this.readDecimalFloatPartial(false);
+        if (this.pos < end && text.charCodeAt(this.pos) === 46 /* CharCode.Dot */) {
+            ++this.pos;
+            sepCount += this.readDecimalFloatPartial();
+        }
+        if (this.pos < end) {
+            let c = text.charCodeAt(this.pos);
+            if ((c | 32) === 101 /* CharCode.e */) {
+                if (++this.pos < end &&
+                    (c = text.charCodeAt(this.pos)) === 45 /* CharCode.Minus */ || c === 43 /* CharCode.Plus */ &&
+                    isDecimal(text.charCodeAt(this.pos + 1))) {
+                    ++this.pos;
+                }
+                sepCount += this.readDecimalFloatPartial();
+            }
+        }
+        let result = text.substring(start, this.pos);
+        if (sepCount)
+            result = result.replace(/_/g, "");
+        return parseFloat(result);
+    }
+    /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
+    readDecimalFloatPartial(allowLeadingZeroSep = true) {
+        let text = this.source.text;
+        let pos = this.pos;
+        let start = pos;
+        let end = this.end;
+        let sepEnd = start;
+        let sepCount = 0;
+        while (pos < end) {
+            let c = text.charCodeAt(pos);
+            if (c === 95 /* CharCode._ */) {
+                if (sepEnd === pos) {
+                    this.error(sepEnd === start
+                        ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+                        : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(pos));
+                }
+                else if (!allowLeadingZeroSep && pos - 1 === start && text.charCodeAt(pos - 1) === 48 /* CharCode._0 */) {
+                    this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(pos));
+                }
+                sepEnd = pos + 1;
+                ++sepCount;
+            }
+            else if (!isDecimal(c)) {
+                break;
+            }
+            ++pos;
+        }
+        if (pos !== start && sepEnd === pos) {
+            this.error(DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1));
+        }
+        this.pos = pos;
+        return sepCount;
+    }
+    readHexFloat() {
+        throw new Error("not implemented"); // TBD
+    }
+    readHexadecimalEscape(remain = 2, startIfTaggedTemplate = -1) {
+        let value = 0;
+        let text = this.source.text;
+        let pos = this.pos;
+        let end = this.end;
+        while (pos < end) {
+            let c = text.charCodeAt(pos++);
+            if (isDecimal(c)) {
+                value = (value << 4) + c - 48 /* CharCode._0 */;
+            }
+            else if (isHexBase(c)) {
+                value = (value << 4) + (c | 32) + (10 - 97 /* CharCode.a */);
+            }
+            else if (~startIfTaggedTemplate) {
+                this.pos = --pos;
+                return text.substring(startIfTaggedTemplate, pos);
+            }
+            else {
+                this.pos = pos;
+                this.error(DiagnosticCode.Hexadecimal_digit_expected, this.range(pos - 1, pos));
+                return "";
+            }
+            if (--remain === 0)
+                break;
+        }
+        if (remain) { // invalid
+            this.pos = pos;
+            if (~startIfTaggedTemplate) {
+                return text.substring(startIfTaggedTemplate, pos);
+            }
+            this.error(DiagnosticCode.Unexpected_end_of_text, this.range(pos));
+            return "";
+        }
+        this.pos = pos;
+        return String.fromCodePoint(value);
+    }
+    checkForIdentifierStartAfterNumericLiteral() {
+        // TODO: BigInt n
+        let pos = this.pos;
+        if (pos < this.end && isIdentifierStart(this.source.text.charCodeAt(pos))) {
+            this.error(DiagnosticCode.An_identifier_or_keyword_cannot_immediately_follow_a_numeric_literal, this.range(pos));
+        }
+    }
+    readUnicodeEscape(startIfTaggedTemplate = -1) {
+        return this.readHexadecimalEscape(4, startIfTaggedTemplate);
+    }
+    readExtendedUnicodeEscape(startIfTaggedTemplate = -1) {
+        let start = this.pos;
+        let value = this.readHexInteger();
+        let value32 = i64_low(value);
+        let invalid = false;
+        assert(!i64_high(value));
+        if (value32 > 0x10FFFF) {
+            if (startIfTaggedTemplate === -1) {
+                this.error(DiagnosticCode.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive, this.range(start, this.pos));
+            }
+            invalid = true;
+        }
+        let end = this.end;
+        let text = this.source.text;
+        if (this.pos >= end) {
+            if (startIfTaggedTemplate === -1) {
+                this.error(DiagnosticCode.Unexpected_end_of_text, this.range(start, end));
+            }
+            invalid = true;
+        }
+        else if (text.charCodeAt(this.pos) === 125 /* CharCode.CloseBrace */) {
+            ++this.pos;
+        }
+        else {
+            if (startIfTaggedTemplate === -1) {
+                this.error(DiagnosticCode.Unterminated_Unicode_escape_sequence, this.range(start, this.pos));
+            }
+            invalid = true;
+        }
+        if (invalid) {
+            return ~startIfTaggedTemplate
+                ? text.substring(startIfTaggedTemplate, this.pos)
+                : "";
+        }
+        return String.fromCodePoint(value32);
+    }
+}
+/** Tokenizer state as returned by {@link Tokenizer#mark} and consumed by {@link Tokenizer#reset}. */
+export class TokenizerState {
+    pos;
+    token;
+    tokenPos;
+    constructor(
+    /** Current position. */
+    pos, 
+    /** Current token. */
+    token, 
+    /** Current token's position. */
+    tokenPos) {
+        this.pos = pos;
+        this.token = token;
+        this.tokenPos = tokenPos;
+    }
+}
+// Reusable state object to reduce allocations
+// let reusableState: TokenizerState | undefined = undefined;
+function i64_low(value) {
+    return Number(value & BigInt(0xFFFFFFFF));
+}
+function i64_high(value) {
+    return i64_low(value >> BigInt(32));
+}