@ibyar/expressions 0.5.4 → 0.6.0

package/v8/stream.js

@@ -1,6 +1,12 @@
 import { Token, TokenExpression } from './token.js';
-import { BigIntLiteral, GlobalThisNode, NumberLiteral, OfNode, Identifier, RegExpLiteral, StringLiteral, SymbolNode, AsNode } from '../api/definition/values.js';
-import { BreakStatement, ContinueStatement } from '../api/statement/control/terminate.js';
+import { GlobalThisNode, OfNode, Identifier, SymbolNode, AsNode, Literal, ThisNode, NullNode, UndefinedNode, TrueNode, FalseNode, DefaultNode, YieldIdentifier, ConstructorIdentifier, ArgumentsIdentifier, NameIdentifier, EvalIdentifier, SuperIdentifier, LetIdentifier, SetIdentifier, GetIdentifier, AwaitIdentifier, AsyncIdentifier } from '../api/definition/values.js';
+import { PrivateIdentifier } from '../api/class/class.js';
+import { DebuggerStatement } from '../api/computing/debugger.js';
+import { isStrict, LanguageMode } from './enums.js';
+const FORBIDDEN_CODE_POINT = ['200b', '200c', '200d', 'feff'];
+export const IdentifierRegex = /^[_A-Za-z\u00A1-\uFFFF][_A-Za-z0-9\u00A1-\uFFFF]*/;
+export const IdentifierStartRegex = /[_$a-zA-Z\xA0-\uFFFF]/;
+export const IdentifierPartRegex = /[_$a-zA-Z0-9\xA0-\uFFFF]/;
 const EOFToken = Object.freeze(new TokenExpression(Token.EOS));
 ;
 export class PreTemplateLiteral {
@@ -9,20 +15,33 @@ export class PreTemplateLiteral {
         this.expressions = expressions;
     }
 }
+;
+export class TemplateStringLiteral {
+    constructor(string) {
+        this.string = string;
+    }
+}
 export class TokenStream {
-    constructor() {
+    constructor(mode = LanguageMode.Strict) {
+        this.mode = mode;
         this.pos = 0;
         this.savedPosition = 0;
     }
-    static getTokenStream(source) {
+    static getTokenStream(source, mode) {
         if (Array.isArray(source)) {
-            return new TokenStreamer(source);
+            return new TokenStreamer(source, mode);
         }
         else if (typeof source === 'string') {
-            return new TokenStreamImpl(source);
+            return new TokenStreamImpl(source, mode);
         }
         throw new Error(`Can't build token stream for ${source}`);
     }
+    setLanguageMode(mode) {
+        this.mode = mode;
+    }
+    getLanguageMode() {
+        return this.mode;
+    }
     save() {
         this.savedPosition = this.pos;
         this.savedCurrent = this.current;
@@ -103,6 +122,7 @@ export class TokenStream {
         }
     }
     readTokensPair(open, close, tokens) {
+        // check pair
         let count = 0;
         let token;
         while (true) {
@@ -185,8 +205,8 @@ export class TokenStream {
     }
 }
 export class TokenStreamer extends TokenStream {
-    constructor(tokens) {
-        super();
+    constructor(tokens, mode) {
+        super(mode);
         this.tokens = tokens;
     }
     next() {
@@ -202,10 +222,16 @@ export class TokenStreamer extends TokenStream {
     hasLineTerminatorBeforeNext() {
         return false;
     }
+    hasLineTerminatorAfterNext() {
+        return false;
+    }
+    scanTemplateContinuation() {
+        return this.next();
+    }
 }
 export class TokenStreamImpl extends TokenStream {
-    constructor(expression) {
-        super();
+    constructor(expression, mode) {
+        super(mode);
         this.expression = expression;
     }
     newToken(type, value) {
@@ -213,7 +239,7 @@ export class TokenStreamImpl extends TokenStream {
     }
     next() {
         if (this.pos >= this.expression.length) {
-            return EOFToken;
+            return this.current = EOFToken;
         }
         this.last = this.current;
         if (this.isWhitespace() || this.isComment()) {
@@ -221,6 +247,7 @@ export class TokenStreamImpl extends TokenStream {
         }
         else if (this.isRadixInteger()
             || this.isNumber()
+            // || this.isRegExp()
             || this.isString()
             || this.isTemplateLiteral()
             || this.isCurlY()
@@ -233,6 +260,9 @@ export class TokenStreamImpl extends TokenStream {
             || this.isProperty()) {
             return this.current;
         }
+        else if (this.pos >= this.expression.length) {
+            return this.current = EOFToken;
+        }
         else {
             throw new Error(this.createError('Unknown character "' + this.expression.charAt(this.pos) + '"'));
         }
@@ -242,20 +272,71 @@ export class TokenStreamImpl extends TokenStream {
         const str = this.expression.substring(this.pos, limit);
         return /(?:\r?\n)/g.test(str);
     }
+    hasLineTerminatorAfterNext() {
+        const start = this.peekPosition();
+        const end = this.peekAheadPosition();
+        const str = this.expression.substring(start, end);
+        return /(?:\r?\n)/g.test(str);
+    }
     isString() {
         const quote = this.expression.charAt(this.pos);
         if (quote === '\'' || quote === '"') {
-            const startPos = this.pos;
-            let index = this.expression.indexOf(quote, startPos + 1);
-            while (index >= 0 && this.pos < this.expression.length) {
-                this.pos = index + 1;
-                if (this.expression.charAt(index - 1) !== '\\') {
-                    const rawString = this.expression.substring(startPos + 1, index);
-                    const stringNode = new StringLiteral(this.unescape(rawString), quote);
+            const start = this.pos;
+            let literal = [];
+            let c0 = '';
+            const advance = () => {
+                c0 = this.expression.charAt(++this.pos);
+            };
+            const advanceUntil = (check) => {
+                while (true) {
+                    c0 = this.expression.charAt(++this.pos);
+                    if (this.pos >= this.expression.length) {
+                        return;
+                    }
+                    const result = check(c0);
+                    if (result) {
+                        break;
+                    }
+                }
+            };
+            const addLiteralChar = (char) => literal.push(char);
+            while (true) {
+                advanceUntil(char => {
+                    if (char.charCodeAt(0) > 127) {
+                        if (this.isStringLiteralLineTerminator(char)) {
+                            return true;
+                        }
+                        addLiteralChar(char);
+                        return false;
+                    }
+                    if (this.mayTerminateString(char))
+                        return true;
+                    addLiteralChar(c0);
+                    return false;
+                });
+                while (c0 === '\\') {
+                    advance();
+                    const char = this.scanEscape(false);
+                    if (char == '' || !char) {
+                        return false;
+                    }
+                    addLiteralChar(char);
+                    advance();
+                }
+                if (c0 === quote) {
+                    advance();
+                    const end = this.pos;
+                    const string = literal.join('');
+                    const rawString = this.expression.substring(start, end);
+                    const stringNode = new Literal(string, rawString);
                     this.current = this.newToken(Token.STRING, stringNode);
                     return true;
                 }
-                index = this.expression.indexOf(quote, index + 1);
+                if (c0 == '' || this.pos >= this.expression.length) {
+                    // Unterminated string literal
+                    break;
+                }
+                addLiteralChar(c0);
             }
         }
         return false;
@@ -263,50 +344,252 @@ export class TokenStreamImpl extends TokenStream {
     isTemplateLiteral() {
         const quote = this.expression.charAt(this.pos);
         if (quote === '`') {
-            const strings = [], exprs = [];
-            let start = this.pos + 1;
-            let i = start;
-            for (; i < this.expression.length; i++) {
-                if (this.expression[i] === '$' && this.expression[i + 1] === '{') {
-                    strings.push(this.expression.substring(start, i));
-                    i += 2;
-                    start = i;
-                    let openCount = 0;
-                    for (; i < this.expression.length; i++) {
-                        if (this.expression[i] === '$' && this.expression[i + 1] === '{') {
-                            openCount++;
-                            i++;
-                        }
-                        else if (openCount === 0 && this.expression[i] === '}') {
-                            exprs.push(this.expression.substring(start, i));
-                            break;
-                        }
-                        else if (this.expression[i] === '}') {
-                            openCount--;
-                        }
+            this.pos++;
+            this.current = this.scanTemplateSpan();
+            return true;
+        }
+        return false;
+    }
+    scanTemplateSpan() {
+        // When scanning a TemplateSpan, we are looking for the following construct:
+        // TEMPLATE_SPAN ::
+        //     ` LiteralChars* ${
+        //   | } LiteralChars* ${
+        //
+        // TEMPLATE_TAIL ::
+        //     ` LiteralChars* `
+        //   | } LiteralChar* `
+        //
+        // A TEMPLATE_SPAN should always be followed by an Expression, while a
+        // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be
+        // followed by an Expression.
+        // These scoped helpers save and restore the original error state, so that we
+        // can specially treat invalid escape sequences in templates (which are
+        // handled by the parser).
+        let result = Token.TEMPLATE_SPAN;
+        let literal = [], c0 = this.expression.charAt(this.pos);
+        const advance = () => {
+            this.pos++;
+            c0 = this.expression.charAt(this.pos);
+        };
+        const addLiteralChar = (char) => literal.push(char);
+        while (true) {
+            let c = c0;
+            if (c == '`') {
+                advance(); // Consume '`'
+                result = Token.TEMPLATE_TAIL;
+                break;
+            }
+            else if (c == '$' && this.expression.charAt(this.pos + 1) == '{') {
+                advance(); // Consume '$'
+                advance(); // Consume '{'
+                break;
+            }
+            else if (c == '\\') {
+                advance(); // Consume '\\'
+                if (this.isLineTerminator(c0)) {
+                    // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
+                    // code unit sequence.
+                    let lastChar = c0;
+                    advance();
+                    if (lastChar == '\r') {
+                        // Also skip \n.
+                        if (c0 == '\n')
+                            advance();
+                        lastChar = '\n';
                     }
-                    start = i + 1;
+                    addLiteralChar(lastChar);
                 }
-                else if (this.expression[i] === '`') {
-                    break;
+                else {
+                    const char = this.scanEscape(true);
+                    if (char === false) {
+                        throw new SyntaxError(this.createError('Unterminated Template Escape char'));
+                    }
+                    addLiteralChar(char);
+                    advance();
                 }
             }
-            strings.push(this.expression.substring(start, i));
-            this.current = this.newToken(Token.TEMPLATE_LITERALS, new PreTemplateLiteral(strings, exprs));
-            this.pos = i + 1;
-            return true;
+            else if (c == '' || this.pos >= this.expression.length) {
+                // Unterminated template literal
+                break;
+            }
+            else {
+                advance(); // Consume c.
+                // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A.
+                // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence
+                // consisting of the CV 0x000A.
+                if (c == '\r') {
+                    if (c0 == '\n')
+                        advance(); // Consume '\n'
+                    c = '\n';
+                }
+                addLiteralChar(c);
+            }
         }
-        return false;
+        return this.newToken(result, new TemplateStringLiteral(literal.join('')));
+    }
+    scanTemplateContinuation() {
+        if (this.current.isNotType(Token.RBRACE)) {
+            throw new SyntaxError(this.createError('Unterminated Template Expr'));
+        }
+        return this.scanTemplateSpan();
+    }
+    scanEscape(scanTemplateSpan) {
+        let c0 = this.expression.charAt(this.pos);
+        let c = c0;
+        // Skip escaped newlines.
+        if (!scanTemplateSpan && this.isLineTerminator(c)) {
+            // Allow escaped CR+LF newlines in multiline string literals.
+            if (c == '\r' && this.expression.charAt(this.pos + 1) == '\n')
+                this.pos++;
+            return '\n';
+        }
+        switch (c) {
+            case 'b':
+                c = '\b';
+                break;
+            case 'f':
+                c = '\f';
+                break;
+            case 'n':
+                c = '\n';
+                break;
+            case 'r':
+                c = '\r';
+                break;
+            case 't':
+                c = '\t';
+                break;
+            case 'u': {
+                this.pos++;
+                c = this.scanUnicodeEscape();
+                if (c === false)
+                    return false;
+                break;
+            }
+            case 'v':
+                c = '\v';
+                break;
+            case 'x': {
+                this.pos++;
+                c = this.scanHexNumber(2);
+                if (c === false)
+                    return false;
+                break;
+            }
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+                this.pos++;
+                c = this.scanOctalEscape(c, 2);
+                break;
+            case '8':
+            case '9':
+                // '\8' and '\9' are disallowed in strict mode.
+                // Re-use the octal error state to propagate the error.
+                throw new SyntaxError(this.createError(`'\\8' and '\\9' are disallowed in strict mode.`));
+        }
+        // Other escaped characters are interpreted as their non-escaped version.
+        return c;
+    }
+    scanUnicodeEscape() {
+        // Accept both \uxxxx and \u{xxxxxx}. In the latter case, the number of
+        // hex digits between { } is arbitrary. \ and u have already been read.
+        let c0 = this.expression.charAt(this.pos);
+        if (c0 == '{') {
+            this.pos++;
+            const cp = this.scanUnlimitedLengthHexNumber(0x10ffff);
+            c0 = this.expression.charAt(++this.pos);
+            if (c0 != '}') {
+                return false;
+            }
+            return cp;
+        }
+        return this.scanHexNumber(4);
+    }
+    scanHexNumber(length) {
+        if (length > 4) { // prevent overflow
+            throw new RangeError(this.createError('ScanHexNumber: length should be less than or equal to 4'));
+        }
+        let x = 0;
+        let scanned = this.pos;
+        let c0 = this.expression.charAt(scanned);
+        for (let i = 0; i < length; i++) {
+            let d = this.hexValue(c0);
+            if (d < 0) {
+                return false;
+            }
+            x = x * 16 + d;
+            c0 = this.expression.charAt(++scanned);
+        }
+        this.pos = scanned - 1;
+        return String.fromCodePoint(x);
+    }
+    scanUnlimitedLengthHexNumber(max_value) {
+        let x = 0;
+        let scanned = this.pos;
+        let c0 = this.expression.charAt(scanned);
+        let d = this.hexValue(c0);
+        if (d < 0) {
+            throw new RangeError(this.createError('Invalid code'));
+        }
+        while (d >= 0) {
+            x = x * 16 + d;
+            if (x > max_value) {
+                throw new RangeError(this.createError('Undefined Unicode Code Point'));
+            }
+            c0 = this.expression.charAt(++scanned);
+            d = this.hexValue(c0);
+        }
+        this.pos = scanned - 1;
+        return String.fromCodePoint(x);
+    }
+    hexValue(c) {
+        let x = c.charCodeAt(0) - '0'.charCodeAt(0);
+        if (x <= 9)
+            return x;
+        x = (x | 0x20) - ('a'.charCodeAt(0) - '0'.charCodeAt(0)); // detect 0x11..0x16 and 0x31..0x36.
+        if (x <= 5)
+            return x + 10;
+        return -1;
+    }
+    scanOctalEscape(c, length) {
+        if (!('0' <= c && c <= '7')) {
+            throw new SyntaxError(this.createError('Invalid octal escape sequence'));
+        }
+        const zero = '0'.charCodeAt(0);
+        let x = c.charCodeAt(0) - zero;
+        let scanned = 0;
+        let i = 0;
+        let char = '';
+        for (; i < length; i++) {
+            char = this.expression.charAt(this.pos + i);
+            let d = char.charCodeAt(0) - zero;
+            if (d < 0 || d > 7)
+                break;
+            const nx = x * 8 + d;
+            if (nx >= 256)
+                break;
+            x = nx;
+            scanned++;
+        }
+        this.pos += scanned - 1;
+        return String.fromCharCode(x);
     }
     isParentheses() {
         const char = this.expression.charAt(this.pos);
         if (char === '(') {
-            this.current = this.newToken(Token.L_PARENTHESES);
+            this.current = this.newToken(Token.LPAREN);
             this.pos++;
             return true;
         }
         else if (char === ')') {
-            this.current = this.newToken(Token.R_PARENTHESES);
+            this.current = this.newToken(Token.RPAREN);
             this.pos++;
             return true;
         }
@@ -315,12 +598,12 @@ export class TokenStreamImpl extends TokenStream {
     isBracket() {
         const char = this.expression.charAt(this.pos);
         if (char === '[') {
-            this.current = this.newToken(Token.L_BRACKETS);
+            this.current = this.newToken(Token.LBRACK);
             this.pos++;
             return true;
         }
         else if (char === ']') {
-            this.current = this.newToken(Token.R_BRACKETS);
+            this.current = this.newToken(Token.RBRACK);
             this.pos++;
             return true;
         }
@@ -329,12 +612,12 @@ export class TokenStreamImpl extends TokenStream {
     isCurlY() {
         const char = this.expression.charAt(this.pos);
         if (char === '{') {
-            this.current = this.newToken(Token.L_CURLY);
+            this.current = this.newToken(Token.LBRACE);
             this.pos++;
             return true;
         }
         else if (char === '}') {
-            this.current = this.newToken(Token.R_CURLY);
+            this.current = this.newToken(Token.RBRACE);
             this.pos++;
             return true;
         }
@@ -361,47 +644,82 @@ export class TokenStreamImpl extends TokenStream {
     isProperty() {
         let startPos = this.pos;
         let i = startPos;
-        let hasLetter = false;
         let isPrivate = false;
-        if (this.expression.charAt(i) === '#') {
+        let escape = false;
+        let escapeFlag = false;
+        // check is private property 
+        let char = this.expression.charAt(i);
+        if (char === '#') {
             isPrivate = true;
+            startPos++;
+            i++;
+        }
+        char = this.expression.charAt(i);
+        if (!IdentifierStartRegex.test(char)) {
+            if (char === '\\' && this.expression.charAt(i + 1) === 'u') {
+                i += 2;
+                escapeFlag = escape = true;
+                // need to valid the unicode char
+            }
+            else {
+                return false;
+            }
+        }
+        else {
             i++;
         }
         for (; i < this.expression.length; i++) {
-            const char = this.expression.charAt(i);
-            if (char.toUpperCase() === char.toLowerCase()) {
-                if (i === this.pos && (char === '$' || char === '_')) {
-                    if (char === '_') {
-                        hasLetter = true;
-                    }
+            char = this.expression.charAt(i);
+            if (!IdentifierPartRegex.test(char)) {
+                if (char === '\\' && this.expression.charAt(i + 1) === 'u') {
+                    i++;
+                    escapeFlag = escape = true;
                     continue;
                 }
-                else if (i === this.pos || !hasLetter || (char !== '_' && (char < '0' || char > '9'))) {
-                    break;
+                if ('{' === char && escapeFlag) {
+                    const end = this.expression.indexOf('}', i + 1);
+                    if (end === -1) {
+                        return false;
+                    }
+                    escapeFlag = false;
+                    i = end;
+                    continue;
                 }
+                break;
             }
-            else {
-                hasLetter = true;
+        }
+        let identifierName = this.expression.substring(startPos, i);
+        if (escape) {
+            identifierName = this.unescape(identifierName);
+            if (!IdentifierRegex.test(identifierName)) {
+                return false;
             }
+            this.pos = i;
+        }
+        else {
+            this.pos += identifierName.length + (isPrivate ? 1 : 0);
+        }
+        let node;
+        if (isPrivate) {
+            node = new PrivateIdentifier(identifierName);
+            this.current = this.newToken(Token.PRIVATE_NAME, node);
         }
-        if (hasLetter) {
-            const prop = this.expression.substring(startPos, i);
-            let node;
-            switch (prop) {
+        else {
+            switch (identifierName) {
                 case 'this':
-                    this.current = this.newToken(Token.THIS);
+                    this.current = this.newToken(Token.THIS, ThisNode);
                     break;
                 case 'null':
-                    this.current = this.newToken(Token.NULL_LITERAL);
+                    this.current = this.newToken(Token.NULL_LITERAL, NullNode);
                     break;
                 case 'undefined':
-                    this.current = this.newToken(Token.UNDEFINED_LITERAL);
+                    this.current = this.newToken(Token.UNDEFINED_LITERAL, UndefinedNode);
                     break;
                 case 'true':
-                    this.current = this.newToken(Token.TRUE_LITERAL);
+                    this.current = this.newToken(Token.TRUE_LITERAL, TrueNode);
                     break;
                 case 'false':
-                    this.current = this.newToken(Token.FALSE_LITERAL);
+                    this.current = this.newToken(Token.FALSE_LITERAL, FalseNode);
                     break;
                 case 'globalThis':
                     this.current = this.newToken(Token.IDENTIFIER, GlobalThisNode);
@@ -415,24 +733,49 @@ export class TokenStreamImpl extends TokenStream {
                 case 'as':
                     this.current = this.newToken(Token.IDENTIFIER, AsNode);
                     break;
+                case 'default':
+                    this.current = this.newToken(Token.DEFAULT, DefaultNode);
+                    break;
+                case 'yield':
+                    this.current = this.newToken(Token.YIELD, YieldIdentifier);
+                    break;
+                case 'constructor':
+                    this.current = this.newToken(Token.IDENTIFIER, ConstructorIdentifier);
+                    break;
+                case 'arguments':
+                    this.current = this.newToken(Token.IDENTIFIER, ArgumentsIdentifier);
+                    break;
+                case 'name':
+                    this.current = this.newToken(Token.IDENTIFIER, NameIdentifier);
+                    break;
+                case 'eval':
+                    this.current = this.newToken(Token.IDENTIFIER, EvalIdentifier);
+                    break;
+                case 'debugger':
+                    this.current = this.newToken(Token.DEBUGGER, DebuggerStatement.INSTANCE);
+                    break;
+                case 'class':
+                    this.current = this.newToken(Token.CLASS);
+                    break;
+                case 'await':
+                    this.current = this.newToken(Token.AWAIT, AwaitIdentifier);
+                    break;
+                case 'async':
+                    this.current = this.newToken(Token.ASYNC, AsyncIdentifier);
+                    break;
                 default:
-                    if (isPrivate) {
-                        node = new Identifier(`#${prop}`);
-                        this.current = this.newToken(Token.PRIVATE_NAME, node);
-                    }
-                    else {
-                        node = new Identifier(prop);
-                        this.current = this.newToken(Token.IDENTIFIER, node);
-                    }
+                    node = new Identifier(identifierName);
+                    this.current = this.newToken(Token.IDENTIFIER, node);
                     break;
             }
-            this.pos += prop.length;
-            return true;
         }
-        return false;
+        return true;
     }
     isWhitespace() {
         let result = false;
+        while (FORBIDDEN_CODE_POINT.includes(this.expression.charCodeAt(this.pos).toString(16))) {
+            this.pos++;
+        }
         let char = this.expression.charAt(this.pos);
         while (/\s/.test(char)) {
             result = true;
@@ -444,48 +787,130 @@ export class TokenStreamImpl extends TokenStream {
         }
         return result;
     }
+    /**
+     * [JavaScript 5 Comment Types](https://developer.adobe.com/experience-manager/reference-materials/6-5/javadoc/org/mozilla/javascript/ast/Comment.html)
+     * @returns boolean
+     */
     isComment() {
         const char = this.expression.charAt(this.pos);
         const nextChar = this.expression.charAt(this.pos + 1);
-        if (char === '/' && nextChar === '*') {
-            this.pos = this.expression.indexOf('*/', this.pos) + 2;
-            if (this.pos === 1) {
-                this.pos = this.expression.length;
-            }
-            return true;
-        }
-        if (char === '/' && nextChar === '/') {
-            this.pos = this.expression.indexOf('\n', this.pos) + 1;
-            if (this.pos === -1) {
-                this.pos = this.expression.length;
-            }
-            return true;
+        let closeTag;
+        let endPos;
+        switch (true) {
+            // // line comments
+            case (char === '/' && nextChar === '/'):
+                closeTag = '\n';
+                endPos = this.expression.indexOf(closeTag, this.pos);
+                break;
+            // /* block comments *\/ and /** jsdoc comments *\/
+            case (char === '/' && nextChar === '*'):
+                closeTag = '*/';
+                endPos = this.expression.indexOf(closeTag, this.pos);
+                if (endPos === -1) {
+                    throw new SyntaxError(this.createError('Invalid or unexpected token'));
+                }
+                break;
+            // <!-- html-open line comments
+            case (char === '<' && nextChar === '!'):
+                if ('<!--' === this.expression.substring(this.pos, this.pos + 4)) {
+                    if (isStrict(this.mode)) {
+                        throw new SyntaxError('Html Comment not allowed In Module');
+                    }
+                    const nextLinePos = this.expression.indexOf('\n', this.pos);
+                    const closePos = this.expression.indexOf('-->', this.pos);
+                    if (nextLinePos < closePos) {
+                        closeTag = '\n';
+                        endPos = nextLinePos;
+                    }
+                    else if (closePos < nextLinePos) {
+                        closeTag = '-->';
+                        endPos = closePos;
+                    }
+                    else {
+                        closeTag = '';
+                        endPos = -1;
+                    }
+                    break;
+                }
+            // ^\\s*--> html-close line comments
+            case (char === '-' && nextChar === '-'):
+                if ('-->' === this.expression.substring(this.pos, this.pos + 3)) {
+                    if (isStrict(this.mode)) {
+                        throw new SyntaxError('Html Comment not allowed In Module');
+                    }
+                    // check if '-->' is the first non-whitespace on the line
+                    let indexOfLastLine = this.expression.lastIndexOf('\n', this.pos + 1);
+                    if (indexOfLastLine == -1) {
+                        indexOfLastLine = 0;
+                    }
+                    let lastLine = this.expression.substring(indexOfLastLine, this.pos + 3);
+                    if (!/^\s*-->$/g.test(lastLine)) {
+                        // test with close of block comments
+                        let lastBlockCommentPos = lastLine.lastIndexOf('*/');
+                        if (lastBlockCommentPos == -1) {
+                            return false;
+                        }
+                        lastLine = lastLine.substring(lastBlockCommentPos + 2);
+                        if (!/^\s*-->$/g.test(lastLine)) {
+                            return false;
+                        }
+                    }
+                    endPos = this.expression.indexOf('\n', this.pos);
+                    closeTag = '\n';
+                    break;
+                }
+            default:
+                return false;
         }
-        return false;
+        this.pos = endPos == -1 ? this.expression.length : endPos + closeTag.length;
+        return true;
+    }
+    isLineTerminator(c) {
+        return /[\n\r\u2028\u2029]/.test(c);
+    }
+    isStringLiteralLineTerminator(c) {
+        return /[\n\r]/.test(c);
+    }
+    mayTerminateString(c) {
+        return (c == '\'' || c == '"' || c == '\n' || c == '\r' || c == '\\');
     }
     scanRegExpPattern() {
         const peek = this.peek();
         if (peek.isType(Token.DIV) || peek.isType(Token.DIV_ASSIGN)) {
             this.next();
             let pattern = peek.isType(Token.DIV_ASSIGN) ? '=' : '';
-            let start = this.pos;
             let currentPos = this.pos;
-            currentPos = this.expression.indexOf('/', currentPos + 1);
-            while (currentPos !== -1 && this.expression.charAt(currentPos - 1) === '\\') {
-                pattern += this.expression.substring(start, currentPos - 1);
-                start = currentPos;
-                currentPos = this.expression.indexOf('/', currentPos + 1);
-            }
-            if (currentPos === -1) {
-                return false;
+            let inCharacterClass = false;
+            let c = this.expression.charAt(currentPos);
+            while (c !== '/' || inCharacterClass) {
+                if (c == '' || this.isLineTerminator(c)) {
+                    return false;
+                }
+                if (c === '\\') { // Escape sequence.
+                    pattern += c;
+                    c = this.expression.charAt(++currentPos);
+                    if (c == '' || this.isLineTerminator(c)) {
+                        return false;
+                    }
+                    pattern += c;
+                    c = this.expression.charAt(++currentPos);
+                }
+                else {
+                    if (c == '[')
+                        inCharacterClass = true;
+                    if (c == ']')
+                        inCharacterClass = false;
+                    pattern += c;
+                    c = this.expression.charAt(++currentPos);
+                }
             }
-            pattern += this.expression.substring(start, currentPos);
-            currentPos++;
+            currentPos++; // consume '/'
             let flags = '';
             const remainFlags = TokenStreamImpl.REGEXP_FLAGS.slice();
             while (true) {
+                const code = this.expression.charCodeAt(currentPos);
                 const nextChar = this.expression.charAt(currentPos);
-                if (/[\.\s]/.test(nextChar)) {
+                if (/[\.\s]/.test(nextChar) || Number.isNaN(code) /* || FORBIDDEN_CODE_POINT.includes(code.toString(16))*/) {
                     break;
                 }
                 const nextCharIndex = remainFlags.indexOf(nextChar);
@@ -501,7 +926,7 @@ export class TokenStreamImpl extends TokenStream {
                     currentPos++;
                 }
             }
-            const regexNode = new RegExpLiteral(new RegExp(pattern, flags));
+            const regexNode = new Literal(new RegExp(pattern, flags), `/${pattern}/${flags}`, { pattern, flags });
             this.current = this.newToken(Token.REGEXP_LITERAL, regexNode);
             this.pos = currentPos;
             return true;
@@ -517,52 +942,104 @@ export class TokenStreamImpl extends TokenStream {
         while (index >= 0) {
             const c = v.charAt(++index);
             switch (c) {
-                case '\'':
-                case '"':
-                case '\\':
-                case '/':
-                    buffer += c;
+                case '\b':
+                case '\f':
+                case '\n':
+                case '\r':
+                case '\t':
+                case '\v':
+                    index++;
+                    buffer += '\\' + c;
                     break;
                 case 'b':
+                    index++;
                     buffer += '\b';
                     break;
                 case 'f':
+                    index++;
                     buffer += '\f';
                     break;
                 case 'n':
+                    index++;
                     buffer += '\n';
                     break;
                 case 'r':
+                    index++;
                     buffer += '\r';
                     break;
                 case 't':
+                    index++;
                     buffer += '\t';
                     break;
+                case 'v':
+                    index++;
+                    buffer += '\v';
+                    break;
                 case 'u': {
-                    let codePoint = v.substring(index + 1, index + 5);
-                    if (!TokenStreamImpl.UnicodePattern.test(codePoint)) {
-                        throw new Error(this.createError('Illegal escape sequence: \\u' + codePoint));
+                    index++;
+                    // interpret the following 4 characters as the hex of the unicode code point
+                    if ('{' === v.charAt(index)) {
+                        index++;
+                        const start = index;
+                        const end = v.indexOf('}', start + 1);
+                        const codePoint = v.substring(start, end);
+                        buffer += String.fromCodePoint(parseInt(codePoint, 16));
+                        index = end + 1;
+                    }
+                    else {
+                        const codePoint = v.substring(index, index + 4);
+                        if (!TokenStreamImpl.UnicodePattern.test(codePoint)) {
+                            throw new Error(this.createError('Illegal escape sequence: \\u ' + codePoint));
+                        }
+                        buffer += String.fromCodePoint(parseInt(codePoint, 16));
+                        index += 4;
                     }
-                    buffer += String.fromCharCode(parseInt(codePoint, 16));
-                    index += 4;
                     break;
                 }
                 case 'x': {
-                    let codePoint = v.substring(index + 1, index + 3);
+                    index++;
+                    // interpret the following 2 characters as the hex of the decimal code point
+                    let codePoint = v.substring(index, index + 2);
                     if (!TokenStreamImpl.DecimalPattern.test(codePoint)) {
-                        throw new Error(this.createError('Illegal escape sequence: \\x' + codePoint));
+                        throw new Error(this.createError('Illegal escape sequence: \\x ' + codePoint));
                     }
                     buffer += String.fromCharCode(parseInt(codePoint, 16));
                     index += 2;
                     break;
                 }
                 default:
-                    throw new Error(this.createError('Illegal escape sequence: "\\' + c + '"'));
+                    if (!Number.isNaN(parseInt(c, 8))) {
+                        let codePoint = c;
+                        for (const i of [1, 2]) {
+                            const tempChar = v.charAt(index + i);
+                            if (!Number.isNaN(parseInt(tempChar, 8))) {
+                                codePoint += tempChar;
+                            }
+                            else {
+                                break;
+                            }
+                        }
+                        if (!TokenStreamImpl.AsciiPattern.test(codePoint)) {
+                            throw new Error(this.createError('Illegal escape sequence: ASCII 8 base, ' + codePoint));
+                        }
+                        buffer += String.fromCharCode(parseInt(codePoint, 8));
+                        index += codePoint.length;
+                        break;
+                    }
+                    index++;
+                    buffer += c;
+                    break;
+                // throw new Error(this.createError('Illegal escape sequence: "\\' + c + '"'));
             }
-            ++index;
             let backslash = v.indexOf('\\', index);
-            buffer += v.substring(index, backslash < 0 ? v.length : backslash);
-            index = backslash;
+            if (backslash == -1) {
+                buffer += v.substring(index, v.length);
+                break;
+            }
+            else {
+                buffer += v.substring(index, backslash);
+                index = backslash;
+            }
         }
         return buffer;
     }
@@ -600,7 +1077,9 @@ export class TokenStreamImpl extends TokenStream {
             }
         }
         if (valid) {
-            const numNode = new NumberLiteral(parseInt(this.expression.substring(startPos, pos), radix));
+            const string = this.expression.substring(startPos, pos);
+            const rawString = this.expression.substring(this.pos, pos);
+            const numNode = new Literal(parseInt(string, radix), rawString);
             this.current = this.newToken(Token.NUMBER, numNode);
             this.pos = pos;
         }
@@ -613,6 +1092,7 @@ export class TokenStreamImpl extends TokenStream {
         let resetPos = pos;
         let foundDot = false;
         let foundDigits = false;
+        let isOctal = false;
         let c;
         while (pos < this.expression.length) {
             c = this.expression.charAt(pos);
@@ -626,6 +1106,15 @@ export class TokenStreamImpl extends TokenStream {
                 pos++;
                 valid = foundDigits;
             }
+            else if ('o' == c.toLowerCase()) {
+                if (pos == startPos + 1) {
+                    pos++;
+                    isOctal = true;
+                }
+                else {
+                    break;
+                }
+            }
             else {
                 break;
             }
@@ -656,13 +1145,26 @@ export class TokenStreamImpl extends TokenStream {
             }
         }
         if (valid) {
+            const rawString = this.expression.substring(startPos, pos);
             if (this.expression.charAt(pos) === 'n') {
-                const bigintNode = new BigIntLiteral(BigInt(this.expression.substring(startPos, pos)));
+                const bigintNode = new Literal(BigInt(rawString), rawString + 'n', undefined, rawString);
                 this.current = this.newToken(Token.BIGINT, bigintNode);
                 pos++;
             }
             else {
-                const numNode = new NumberLiteral(parseFloat(this.expression.substring(startPos, pos)));
+                let numNode;
+                if (isOctal) {
+                    const octal = rawString.substring(2);
+                    if (TokenStreamImpl.OctalPattern.test(octal)) {
+                        numNode = new Literal(parseInt(octal, 8), rawString);
+                    }
+                    else {
+                        return false;
+                    }
+                }
+                else {
+                    numNode = new Literal(parseFloat(rawString), rawString);
+                }
                 this.current = this.newToken(Token.NUMBER, numNode);
             }
             this.pos = pos;
@@ -954,6 +1456,7 @@ export class TokenStreamImpl extends TokenStream {
                     this.pos++;
                 }
                 return true;
+            // no break
             case ':':
                 if (this.expression.charAt(this.pos + 1) === ':') {
                     this.current = this.newToken(Token.QUESTION_BIND);
@@ -970,17 +1473,17 @@ export class TokenStreamImpl extends TokenStream {
                 return true;
             case 'a':
                 if (/async\s/.test(this.expression.substring(this.pos, this.pos + 6))) {
-                    this.current = this.newToken(Token.ASYNC);
+                    this.current = this.newToken(Token.ASYNC, AsyncIdentifier);
                     this.pos += 5;
                     return true;
                 }
                 if (/await\s/.test(this.expression.substring(this.pos, this.pos + 6))) {
-                    this.current = this.newToken(Token.AWAIT);
+                    this.current = this.newToken(Token.AWAIT, AwaitIdentifier);
                     this.pos += 5;
                     return true;
                 }
                 return false;
-            case 'i':
+            case 'i': // in, instanceof
                 if (this.expression.charAt(this.pos + 1) === 'n') {
                     if (/instanceof\s/.test(this.expression.substring(this.pos, this.pos + 11))) {
                         this.current = this.newToken(Token.INSTANCEOF);
@@ -994,53 +1497,41 @@ export class TokenStreamImpl extends TokenStream {
                     }
                 }
                 return false;
-            case 't':
+            case 't': // typeof
                 if (/typeof\s/.test(this.expression.substring(this.pos, this.pos + 7))) {
                     this.current = this.newToken(Token.TYPEOF);
                     this.pos += 7;
                     return true;
                 }
                 return false;
-            case 'v':
+            case 'v': // void
                 if (/void\s/.test(this.expression.substring(this.pos, this.pos + 5))) {
                     this.current = this.newToken(Token.VOID);
                     this.pos += 5;
                     return true;
                 }
                 return false;
-            case 'd':
+            case 'd': // delete
                 if (/delete\s/.test(this.expression.substring(this.pos, this.pos + 7))) {
                     this.current = this.newToken(Token.DELETE);
                     this.pos += 7;
                     return true;
                 }
                 return false;
-            case 'n':
+            case 'n': // new
                 if (/new\s/.test(this.expression.substring(this.pos, this.pos + 4))) {
                     this.current = this.newToken(Token.NEW);
                     this.pos += 4;
                     return true;
                 }
                 return false;
-            case 'n':
+            case 'n': // new
                 if (/new\s/.test(this.expression.substring(this.pos, this.pos + 4))) {
                     this.current = this.newToken(Token.NEW);
                     this.pos += 4;
                     return true;
                 }
                 return false;
-            case 'y':
-                if (/yield\s/.test(this.expression.substring(this.pos, this.pos + 6))) {
-                    this.current = this.newToken(Token.YIELD);
-                    this.pos += 6;
-                    return true;
-                }
-                else if (/yield\*\s/.test(this.expression.substring(this.pos, this.pos + 7))) {
-                    this.current = this.newToken(Token.YIELD);
-                    this.pos += 5;
-                    return true;
-                }
-                return false;
             default:
                 return false;
         }
@@ -1048,9 +1539,16 @@ export class TokenStreamImpl extends TokenStream {
     isStatement() {
         const char = this.expression.charAt(this.pos);
         switch (char) {
+            // case 'a':
+            // 	if (/as\s/.test(this.expression.substring(this.pos, this.pos + 3))) {
+            // 		this.current = this.newToken(Token.AS);
+            // 		this.pos += 3;
+            // 		return true;
+            // 	}
+            // return false;
             case 'b':
                 if (/break\s?;?/.test(this.expression.substring(this.pos, this.pos + 7))) {
-                    this.current = this.newToken(Token.BREAK, BreakStatement.BREAK_INSTANCE);
+                    this.current = this.newToken(Token.BREAK);
                     this.pos += 5;
                     return true;
                 }
@@ -1077,25 +1575,20 @@ export class TokenStreamImpl extends TokenStream {
                     return true;
                 }
                 else if (/continue[\s;]/.test(this.expression.substring(this.pos, this.pos + 9))) {
-                    this.current = this.newToken(Token.CONTINUE, ContinueStatement.CONTINUE_INSTANCE);
+                    this.current = this.newToken(Token.CONTINUE);
                     this.pos += 8;
                     return true;
                 }
                 return false;
             case 'd':
-                if (/^(do\s?\{)/.test(this.expression.substring(this.pos, this.pos + 4))) {
+                if (/do[\s|\{]/.test(this.expression.substring(this.pos, this.pos + 3))) {
                     this.current = this.newToken(Token.DO);
                     this.pos += 2;
                     return true;
                 }
-                else if (/^(default\s?:)/.test(this.expression.substring(this.pos, this.pos + 9))) {
-                    this.current = this.newToken(Token.DEFAULT);
-                    this.pos += 7;
-                    return true;
-                }
                 return false;
             case 'e':
-                if (/else(\s|\{])/.test(this.expression.substring(this.pos, this.pos + 5))) {
+                if (/else[\s|\{]/.test(this.expression.substring(this.pos, this.pos + 5))) {
                     this.current = this.newToken(Token.ELSE);
                     this.pos += 4;
                     return true;
@@ -1113,7 +1606,7 @@ export class TokenStreamImpl extends TokenStream {
                 return false;
             case 'g':
                 if (/get\s/.test(this.expression.substring(this.pos, this.pos + 4))) {
-                    this.current = this.newToken(Token.GET);
+                    this.current = this.newToken(Token.GET, GetIdentifier);
                     this.pos += 4;
                     return true;
                 }
@@ -1136,7 +1629,7 @@ export class TokenStreamImpl extends TokenStream {
                 }
                 return false;
             case 'i':
-                if (/if(\s|\(])/.test(this.expression.substring(this.pos, this.pos + 3))) {
+                if (/if[\s|\(]/.test(this.expression.substring(this.pos, this.pos + 3))) {
                     this.current = this.newToken(Token.IF);
                     this.pos += 2;
                     return true;
@@ -1154,7 +1647,7 @@ export class TokenStreamImpl extends TokenStream {
                 return false;
             case 'l':
                 if (/let\s/.test(this.expression.substring(this.pos, this.pos + 4))) {
-                    this.current = this.newToken(Token.LET);
+                    this.current = this.newToken(Token.LET, LetIdentifier);
                     this.pos += 4;
                     return true;
                 }
@@ -1178,6 +1671,13 @@ export class TokenStreamImpl extends TokenStream {
                     return true;
                 }
                 return false;
+            // case 'of':
+            // 	if (/of\s/.test(this.expression.substring(this.pos, this.pos + 3))) {
+            // 		this.current = this.newToken(Token.OF);
+            // 		this.pos += 3;
+            // 		return true;
+            // 	}
+            // 	return false;
             case 's':
                 if (/switch(\s|\()/.test(this.expression.substring(this.pos, this.pos + 7))) {
                     this.current = this.newToken(Token.SWITCH);
@@ -1190,12 +1690,12 @@ export class TokenStreamImpl extends TokenStream {
                     return true;
                 }
                 if (/set\s/.test(this.expression.substring(this.pos, this.pos + 4))) {
-                    this.current = this.newToken(Token.SET);
+                    this.current = this.newToken(Token.SET, SetIdentifier);
                     this.pos += 4;
                     return true;
                 }
                 if (/super[\.\(]?/.test(this.expression.substring(this.pos, this.pos + 6))) {
-                    this.current = this.newToken(Token.SUPER);
+                    this.current = this.newToken(Token.SUPER, SuperIdentifier);
                     this.pos += 5;
                     return true;
                 }
@@ -1229,14 +1729,28 @@ export class TokenStreamImpl extends TokenStream {
     }
     createError(message) {
         let coords = this.getCoordinates();
-        const subStart = Math.max(0, this.pos - 25);
-        const subEnd = Math.min(this.pos + 25, this.expression.length);
-        return `@[line: ${coords.line}, column: ${coords.column}] current token: ${JSON.stringify(this.current)}
-			${message}
-			\`${this.expression.substring(subStart, subEnd)}\``;
+        const lastLinePos = this.expression.lastIndexOf('\n', this.pos) + 1;
+        let nextLinePos = this.expression.indexOf('\n', this.pos + 1);
+        if (nextLinePos == -1) {
+            nextLinePos = this.expression.length;
+        }
+        const subStart = Math.max(lastLinePos, this.pos - 25);
+        const subEnd = Math.min(nextLinePos, this.pos + 25, this.expression.length);
+        const errorAt = this.expression.substring(subStart, subEnd);
+        const indictor = new Array(subEnd - subStart).fill(' ');
+        indictor[this.pos - subStart] = '^';
+        const errorIndictor = indictor.join('');
+        return `
+			> ${message}
+			> token name: '${this.current?.token?.getName()}' ${this.current?.value ? `parsed: ${this.current.value.toString()}` : ''}
+			> ${coords.line}:${coords.column}\t${errorAt}
+			> ${coords.line}:${coords.column}\t${errorIndictor}
+			`;
     }
 }
 TokenStreamImpl.REGEXP_FLAGS = ['g', 'i', 'm', 's', 'u', 'y'];
 TokenStreamImpl.UnicodePattern = /^[0-9a-f]{4}$/i;
 TokenStreamImpl.DecimalPattern = /^[0-9a-f]{2}$/i;
+TokenStreamImpl.AsciiPattern = /^[0-7]{1,3}$/i;
+TokenStreamImpl.OctalPattern = /^[0-7]+$/i;
 //# sourceMappingURL=stream.js.map