imapflow 2.2.5 → 2.2.7

package/dist/esm/handler/token-parser.js

@@ -1,13 +1,12 @@
 /* eslint new-cap: 0 */
 import imapFormalSyntax from './imap-formal-syntax.js';
-import { MAX_LITERAL_SIZE, normalizeLimit, createLiteralTooLargeError } from './limits.js';
+import { MAX_LITERAL_SIZE, ERROR_CONTEXT_LENGTH, boundedInput, normalizeLimit, createLiteralTooLargeError } from './limits.js';
 const STATE_ATOM = 0x001;
 const STATE_LITERAL = 0x002;
 const STATE_NORMAL = 0x003;
 const STATE_PARTIAL = 0x004;
 const STATE_SEQUENCE = 0x005;
 const STATE_STRING = 0x006;
-const RE_DIGITS = /^\d+$/;
 const RE_SINGLE_DIGIT = /^\d$/;
 // Prevents stack overflow from maliciously crafted deeply-nested IMAP input (e.g., (((((...))))))
 const MAX_NODE_DEPTH = 25;
@@ -68,7 +67,7 @@ export class TokenParser {
             if (!node.isClosed) {
                 let error = new Error(`Unexpected end of input at position ${this.pos + this.str.length - 1} [E9]`);
                 error.code = 'ParserError9';
-                error.parserContext = { input: this.str, pos: this.pos + this.str.length - 1 };
+                error.parserContext = { ...boundedInput(this.str), pos: this.pos + this.str.length - 1 };
                 throw error;
             }
             let type = (node.type || '').toString().toUpperCase();
@@ -142,7 +141,8 @@ export class TokenParser {
         if (node.depth > MAX_NODE_DEPTH) {
             let error = new Error('Too much nesting in IMAP string');
             error.code = 'MAX_IMAP_NESTING_REACHED';
-            error._imapStr = this.str;
+            // A bounded prefix only, the error travels whole into logs and rejections
+            error._imapStr = this.str.slice(0, ERROR_CONTEXT_LENGTH);
             throw error;
         }
         if (typeof startPos === 'number') {
@@ -169,6 +169,63 @@ export class TokenParser {
                 i++;
             }
         };
+        // Whether chr is the closing delimiter of node: ")" ends a LIST, "]" ends a SECTION
+        const closesNode = (c, node) => !!node && ((c === ')' && node.type === 'LIST') || (c === ']' && node.type === 'SECTION'));
+        // Facts about the ATOM, SEQUENCE or PARTIAL token being read, kept up to date as its
+        // characters are appended. Reading them back from the value instead (value.at(-1),
+        // value.includes('*'), a regex test) makes V8 flatten the string built with += on every
+        // character, which is quadratic in the token length, and the server decides that length:
+        // a 600 KB ESEARCH result took seconds of event loop time
+        let tokenLength = 0;
+        let tokenLast = '';
+        let tokenPrev = '';
+        let tokenDigitsOnly = true;
+        let tokenHasStar = false;
+        let tokenHasDot = false;
+        const appendToToken = (c) => {
+            this.currentNode.value += c;
+            tokenLength++;
+            tokenPrev = tokenLast;
+            tokenLast = c;
+            tokenDigitsOnly = tokenDigitsOnly && RE_SINGLE_DIGIT.test(c);
+            tokenHasStar = tokenHasStar || c === '*';
+            tokenHasDot = tokenHasDot || c === '.';
+        };
+        // A digit-led token is only a guess at a sequence set. ":" and "," are ATOM-CHARs, so a
+        // server sends a mailbox name, keyword or label such as "2024:Q1" or "1,a" unquoted, and
+        // rejecting it would drop the whole response line: the mailbox missing from LIST, the
+        // message missing from FETCH. When the sequence grammar breaks, such a token is read on as
+        // an atom, and the atom rules decide whether the current character is acceptable. A token
+        // holding "*" can not be an atom ("*" is not an ATOM-CHAR), so it keeps the sequence errors
+        const continueAsAtom = () => {
+            this.currentNode.type = 'ATOM';
+            this.state = STATE_ATOM;
+            // read the current character again, in STATE_ATOM
+            i--;
+        };
+        // Starts the value of the current node as a new token, with its first character if any
+        const startToken = (first) => {
+            this.currentNode.value = '';
+            tokenLength = 0;
+            tokenLast = tokenPrev = '';
+            tokenDigitsOnly = true;
+            tokenHasStar = tokenHasDot = false;
+            if (first) {
+                appendToToken(first);
+            }
+        };
+        // ImapStream supplies one buffer per literal marker it framed, {0} included. A marker
+        // without a buffer means the line and its literals do not match up, so fail with a parser
+        // error instead of reading a missing buffer or shifting every later literal by one
+        const takeLiteral = (literals) => {
+            if (!literals.length) {
+                let error = new Error(`Literal without data at position ${this.pos + i} [E35]`);
+                error.code = 'ParserError35';
+                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
+                throw error;
+            }
+            return literals.shift();
+        };
         // Any ATOM supported char starts a new Atom sequence, otherwise throw an error
         // Allow \ as the first char for atom to support system flags
         // Allow % to support LIST '' %
@@ -179,12 +236,12 @@ export class TokenParser {
             if (!imapFormalSyntax['ATOM-CHAR']().includes(chr) && chr !== '\\' && chr !== '%' && chr.charCodeAt(0) < 0x80) {
                 let error = new Error(`Unexpected char at position ${this.pos + i} [E13: ${JSON.stringify(chr)}]`);
                 error.code = 'ParserError13';
-                error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                 throw error;
             }
             this.currentNode = this.createNode(this.currentNode, this.pos + i);
             this.currentNode.type = 'ATOM';
-            this.currentNode.value = chr;
+            startToken(chr);
             this.state = STATE_ATOM;
         };
         for (i = 0, len = this.str.length; i < len; i++) {
@@ -210,7 +267,7 @@ export class TokenParser {
                             if (this.currentNode.type !== 'LIST') {
                                 let error = new Error(`Unexpected list terminator ) at position ${this.pos + i} [E10]`);
                                 error.code = 'ParserError10';
-                                error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                                 throw error;
                             }
                             this.currentNode.isClosed = true;
@@ -223,7 +280,7 @@ export class TokenParser {
                             if (this.currentNode.type !== 'SECTION') {
                                 let error = new Error(`Unexpected section terminator ] at position ${this.pos + i} [E11]`);
                                 error.code = 'ParserError11';
-                                error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                                 throw error;
                             }
                             this.currentNode.isClosed = true;
@@ -239,12 +296,13 @@ export class TokenParser {
                             if (this.str.charAt(i - 1) !== ']') {
                                 this.currentNode = this.createNode(this.currentNode, this.pos + i);
                                 this.currentNode.type = 'ATOM';
-                                this.currentNode.value = chr;
+                                startToken(chr);
                                 this.state = STATE_ATOM;
                             }
                             else {
                                 this.currentNode = this.createNode(this.currentNode, this.pos + i);
                                 this.currentNode.type = 'PARTIAL';
+                                startToken();
                                 this.state = STATE_PARTIAL;
                                 this.currentNode.isClosed = false;
                             }
@@ -254,17 +312,19 @@ export class TokenParser {
                         case '~': {
                             let nextChr = this.str.charAt(i + 1);
                             if (nextChr !== '{') {
-                                if (imapFormalSyntax['ATOM-CHAR']().includes(nextChr)) {
-                                    // '~' not followed by '{' but followed by an ATOM char: treat as ATOM
-                                    this.currentNode = this.createNode(this.currentNode, this.pos + i);
-                                    this.currentNode.type = 'ATOM';
-                                    this.currentNode.value = chr;
-                                    this.state = STATE_ATOM;
+                                // '~' is an ATOM-CHAR itself, so when it does not start a literal8
+                                // it starts an atom, including the one-character atom "~" that a
+                                // space, the end of the line or a closing delimiter ends
+                                if (imapFormalSyntax['ATOM-CHAR']().includes(nextChr) ||
+                                    nextChr === ' ' ||
+                                    nextChr === '' ||
+                                    closesNode(nextChr, this.currentNode)) {
+                                    startAtom();
                                     break;
                                 }
                                 let error = new Error(`Unexpected literal8 marker at position ${this.pos + i} [E12]`);
                                 error.code = 'ParserError12';
-                                error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                                 throw error;
                             }
                             // Mark the next literal as literal8 type; consumed when '{' is encountered
@@ -285,7 +345,7 @@ export class TokenParser {
                         case '*':
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'SEQUENCE';
-                            this.currentNode.value = chr;
+                            startToken(chr);
                             this.currentNode.isClosed = false;
                             this.state = STATE_SEQUENCE;
                             break;
@@ -366,8 +426,7 @@ export class TokenParser {
                         break;
                     }
                     // ')' or ']' terminates the atom AND closes the enclosing LIST or SECTION
-                    if (this.currentNode.parentNode &&
-                        ((chr === ')' && this.currentNode.parentNode.type === 'LIST') || (chr === ']' && this.currentNode.parentNode.type === 'SECTION'))) {
+                    if (closesNode(chr, this.currentNode.parentNode)) {
                         this.currentNode.endPos = this.pos + i - 1;
                         this.currentNode = this.currentNode.parentNode;
                         this.currentNode.isClosed = true;
@@ -403,14 +462,16 @@ export class TokenParser {
                     }
                     // If the atom so far is all digits and we see ',' or ':', it is actually
                     // a sequence set (e.g., "1:5" or "1,3,5"), so reclassify and switch state
-                    if ((chr === ',' || chr === ':') && RE_DIGITS.test(this.currentNode.value)) {
+                    if ((chr === ',' || chr === ':') && tokenLength && tokenDigitsOnly) {
                         this.currentNode.type = 'SEQUENCE';
                         this.currentNode.isClosed = true;
                         this.state = STATE_SEQUENCE;
                     }
                     // [ starts a section group for this element
                     // Allowed only for selected elements, otherwise falls through to regular ATOM processing
-                    if (chr === '[' && ['BODY', 'BODY.PEEK', 'BINARY', 'BINARY.PEEK'].includes(this.currentNode.value.toUpperCase())) {
+                    if (chr === '[' &&
+                        tokenLength <= 11 &&
+                        ['BODY', 'BODY.PEEK', 'BINARY', 'BINARY.PEEK'].includes(this.currentNode.value.toUpperCase())) {
                         this.currentNode.endPos = this.pos + i;
                         this.currentNode = this.createNode(this.currentNode.parentNode, this.pos + i);
                         this.currentNode.type = 'SECTION';
@@ -423,19 +484,19 @@ export class TokenParser {
                         chr.charCodeAt(0) < 0x80 && // allow 8bit (presumably unicode) bytes
                         chr !== ']' &&
                         !(chr === '*' && this.currentNode.value === '\\') &&
-                        (!this.parent || !this.parent.command || !['NO', 'BAD', 'OK'].includes(this.parent.command))) {
+                        (!this.parent || !this.parent.command || !['NO', 'BAD', 'OK'].includes(this.parent.command.toUpperCase()))) {
                         let error = new Error(`Unexpected char at position ${this.pos + i} [E16: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError16';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
                     else if (this.currentNode.value === '\\*') {
                         let error = new Error(`Unexpected char at position ${this.pos + i} [E17: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError17';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
-                    this.currentNode.value += chr;
+                    appendToToken(chr);
                     break;
                 case STATE_STRING:
                     // DQUOTE ends the string sequence
@@ -453,7 +514,7 @@ export class TokenParser {
                         if (i >= len) {
                             let error = new Error(`Unexpected end of input at position ${this.pos + i} [E18]`);
                             error.code = 'ParserError18';
-                            error.parserContext = { input: this.str, pos: this.pos + i };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i };
                             throw error;
                         }
                         chr = this.str.charAt(i);
@@ -462,10 +523,10 @@ export class TokenParser {
                     break;
                 case STATE_PARTIAL:
                     if (chr === '>') {
-                        if (this.currentNode.value.at(-1) === '.') {
+                        if (tokenLast === '.') {
                             let error = new Error(`Unexpected end of partial at position ${this.pos + i} [E19]`);
                             error.code = 'ParserError19';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                         this.currentNode.endPos = this.pos + i;
@@ -475,25 +536,25 @@ export class TokenParser {
                         checkSP();
                         break;
                     }
-                    if (chr === '.' && (this.currentNode.value === '' || this.currentNode.value.includes('.'))) {
+                    if (chr === '.' && (!tokenLength || tokenHasDot)) {
                         let error = new Error(`Unexpected partial separator . at position ${this.pos + i} [E20]`);
                         error.code = 'ParserError20';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
                     if (!imapFormalSyntax.DIGIT().includes(chr) && chr !== '.') {
                         let error = new Error(`Unexpected char at position ${this.pos + i} [E21: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError21';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
-                    if ((this.currentNode.value === '0' || this.currentNode.value.endsWith('.0')) && chr !== '.') {
+                    if (tokenLast === '0' && (tokenLength === 1 || tokenPrev === '.') && chr !== '.') {
                         let error = new Error(`Invalid partial at position ${this.pos + i} [E22: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError22';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
-                    this.currentNode.value += chr;
+                    appendToToken(chr);
                     break;
                 case STATE_LITERAL:
                     if (this.currentNode.started) {
@@ -525,7 +586,7 @@ export class TokenParser {
                         if (!('literalLength' in this.currentNode)) {
                             let error = new Error(`Unexpected literal prefix end char } at position ${this.pos + i} [E23]`);
                             error.code = 'ParserError23';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                         if (this.str.charAt(i + 1) === '\n') {
@@ -537,19 +598,17 @@ export class TokenParser {
                         else {
                             let error = new Error(`Unexpected char at position ${this.pos + i} [E24: ${JSON.stringify(chr)}]`);
                             error.code = 'ParserError24';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                         this.currentNode.literalLength = Number(this.currentNode.literalLength);
                         if (!this.currentNode.literalLength) {
                             // special case where literal content length is 0
                             // close the node right away, do not wait for additional input
-                            if (this.options.literals && this.options.literals.length) {
-                                // ImapStream queues a Buffer for every literal marker it
-                                // extracts, including {0} - consume the queue entry so
-                                // subsequent literals in the same response stay aligned
-                                // with their markers instead of shifting by one
-                                this.currentNode.value = this.options.literals.shift();
+                            if (this.options.literals) {
+                                // consume the queue entry of the {0} marker too, so later
+                                // literals in the same response stay aligned with their markers
+                                this.currentNode.value = takeLiteral(this.options.literals);
                             }
                             this.currentNode.endPos = this.pos + i;
                             this.currentNode.isClosed = true;
@@ -559,7 +618,7 @@ export class TokenParser {
                         }
                         else if (this.options.literals) {
                             // use the next precached literal values
-                            this.currentNode.value = this.options.literals.shift();
+                            this.currentNode.value = takeLiteral(this.options.literals);
                             // only APPEND arguments are kept as Buffers
                             /*
                             if ((this.parent.command || '').toString().toUpperCase() !== 'APPEND') {
@@ -589,7 +648,7 @@ export class TokenParser {
                             if (literalLength > this.maxLiteralSize || literalLength > available) {
                                 let overMax = literalLength > this.maxLiteralSize;
                                 let error = createLiteralTooLargeError(literalLength, overMax ? this.maxLiteralSize : available, overMax ? null : `the ${available} bytes available in the input`);
-                                error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                                error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                                 throw error;
                             }
                             this.currentNode.started = true;
@@ -603,95 +662,120 @@ export class TokenParser {
                     if (!imapFormalSyntax.DIGIT().includes(chr)) {
                         let error = new Error(`Unexpected char at position ${this.pos + i} [E25: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError25';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
                     if (this.currentNode.literalLength === '0') {
                         let error = new Error(`Invalid literal at position ${this.pos + i} [E26]`);
                         error.code = 'ParserError26';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
                     this.currentNode.literalLength = (this.currentNode.literalLength || '') + chr;
                     break;
-                case STATE_SEQUENCE:
-                    // space finishes the sequence set
-                    if (chr === ' ') {
-                        if (!RE_SINGLE_DIGIT.test(this.currentNode.value.at(-1)) && this.currentNode.value.at(-1) !== '*') {
-                            let error = new Error(`Unexpected whitespace at position ${this.pos + i} [E27]`);
+                case STATE_SEQUENCE: {
+                    // A space ends the sequence set, and so does the closing delimiter of the
+                    // enclosing list or section ("PARTIAL (1:100 5,7,9)", "[COPYUID 1 1:3 4,5]")
+                    let closesParent = closesNode(chr, this.currentNode.parentNode);
+                    if (chr === ' ' || closesParent) {
+                        if (!RE_SINGLE_DIGIT.test(tokenLast) && tokenLast !== '*') {
+                            if (!tokenHasStar) {
+                                // a dangling separator, "10:" or "1,"
+                                continueAsAtom();
+                                break;
+                            }
+                            let error = new Error(`Unexpected end of sequence at position ${this.pos + i} [E27: ${JSON.stringify(chr)}]`);
                             error.code = 'ParserError27';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
-                        if (this.currentNode.value !== '*' &&
-                            this.currentNode.value.at(-1) === '*' &&
-                            this.currentNode.value.at(-2) !== ':') {
-                            let error = new Error(`Unexpected whitespace at position ${this.pos + i} [E28]`);
+                        if (this.currentNode.value !== '*' && tokenLast === '*' && tokenPrev !== ':') {
+                            let error = new Error(`Unexpected end of sequence at position ${this.pos + i} [E28: ${JSON.stringify(chr)}]`);
                             error.code = 'ParserError28';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                         this.currentNode.isClosed = true;
                         this.currentNode.endPos = this.pos + i - 1;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = STATE_NORMAL;
-                        break;
-                    }
-                    else if (this.currentNode.parentNode && chr === ']' && this.currentNode.parentNode.type === 'SECTION') {
-                        this.currentNode.endPos = this.pos + i - 1;
-                        this.currentNode = this.currentNode.parentNode;
-                        this.currentNode.isClosed = true;
-                        this.currentNode.endPos = this.pos + i;
-                        this.currentNode = this.currentNode.parentNode;
-                        this.state = STATE_NORMAL;
-                        checkSP();
+                        if (closesParent) {
+                            this.currentNode.isClosed = true;
+                            this.currentNode.endPos = this.pos + i;
+                            this.currentNode = this.currentNode.parentNode;
+                            checkSP();
+                        }
                         break;
                     }
                     if (chr === ':') {
-                        if (!RE_SINGLE_DIGIT.test(this.currentNode.value.at(-1)) && this.currentNode.value.at(-1) !== '*') {
+                        if (!RE_SINGLE_DIGIT.test(tokenLast) && tokenLast !== '*') {
+                            if (!tokenHasStar) {
+                                continueAsAtom();
+                                break;
+                            }
                             let error = new Error(`Unexpected range separator : at position ${this.pos + i} [E29]`);
                             error.code = 'ParserError29';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                     }
                     else if (chr === '*') {
-                        if (![',', ':'].includes(this.currentNode.value.at(-1))) {
+                        if (![',', ':'].includes(tokenLast)) {
                             let error = new Error(`Unexpected range wildcard at position ${this.pos + i} [E30]`);
                             error.code = 'ParserError30';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                     }
                     else if (chr === ',') {
-                        if (!RE_SINGLE_DIGIT.test(this.currentNode.value.at(-1)) && this.currentNode.value.at(-1) !== '*') {
+                        if (!RE_SINGLE_DIGIT.test(tokenLast) && tokenLast !== '*') {
+                            if (!tokenHasStar) {
+                                continueAsAtom();
+                                break;
+                            }
                             let error = new Error(`Unexpected sequence separator , at position ${this.pos + i} [E31]`);
                             error.code = 'ParserError31';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
-                        if (this.currentNode.value.at(-1) === '*' && this.currentNode.value.at(-2) !== ':') {
+                        if (tokenLast === '*' && tokenPrev !== ':') {
                             let error = new Error(`Unexpected sequence separator , at position ${this.pos + i} [E32]`);
                             error.code = 'ParserError32';
-                            error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                            error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                             throw error;
                         }
                     }
                     else if (!RE_SINGLE_DIGIT.test(chr)) {
+                        if (!tokenHasStar) {
+                            continueAsAtom();
+                            break;
+                        }
                         let error = new Error(`Unexpected char at position ${this.pos + i} [E33: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError33';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
-                    if (RE_SINGLE_DIGIT.test(chr) && this.currentNode.value.at(-1) === '*') {
+                    if (RE_SINGLE_DIGIT.test(chr) && tokenLast === '*') {
                         let error = new Error(`Unexpected number at position ${this.pos + i} [E34: ${JSON.stringify(chr)}]`);
                         error.code = 'ParserError34';
-                        error.parserContext = { input: this.str, pos: this.pos + i, chr };
+                        error.parserContext = { ...boundedInput(this.str), pos: this.pos + i, chr };
                         throw error;
                     }
-                    this.currentNode.value += chr;
+                    appendToToken(chr);
                     break;
+                }
             }
         }
+        // the same applies to a digit-led token that ends the input on a dangling separator ("10:")
+        if (this.state === STATE_SEQUENCE && (tokenLast === ':' || tokenLast === ',') && !tokenHasStar) {
+            this.currentNode.type = 'ATOM';
+        }
+        // A star-led sequence set ("*:4") is only closed by a space or a closing delimiter, so one
+        // that ends the input is closed here, under the same rules a space applies to it. A bare "*"
+        // is left to getAttributes(), which reads it as an atom
+        if (this.state === STATE_SEQUENCE && !this.currentNode.isClosed && (RE_SINGLE_DIGIT.test(tokenLast) || (tokenLast === '*' && tokenPrev === ':'))) {
+            this.currentNode.isClosed = true;
+            this.currentNode.endPos = this.pos + this.str.length - 1;
+        }
     }
 }