@mojir/lits 2.1.28 → 2.1.29

package/dist/src/index.d.ts

@@ -4,7 +4,7 @@ export { type LitsFunction, type NativeJsFunction } from './parser/types';
 export type { Context } from './evaluator/interface';
 export type { Ast } from './parser/types';
 export type { SourceCodeInfo } from './tokenizer/token';
-export type { Token } from './tokenizer/token';
+export type { Token, TokenType } from './tokenizer/token';
 export { normalExpressionKeys, specialExpressionKeys } from './builtin';
 export { Lits } from './Lits/Lits';
 export { type LitsError, isLitsError } from './errors';

package/dist/src/tokenizer/token.d.ts

@@ -1,10 +1,11 @@
 import type { ReservedSymbol } from './reservedNames';
 import { type SymbolicBinaryOperator, type SymbolicOperator } from './operators';
-export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "String", "Symbol", "Whitespace"];
+export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "Error", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "String", "Symbol", "Whitespace"];
 export type TokenType = typeof tokenTypes[number];
 declare const modifierNames: readonly ["&rest", "&let", "&when", "&while"];
 export type ModifierName = typeof modifierNames[number];
 type GenericToken<T extends TokenType, V extends string = string> = [T, V] | [T, V, SourceCodeInfo];
+export type ErrorToken = ['Error', string, SourceCodeInfo | undefined, string];
 export type LBraceToken = GenericToken<'LBrace', '{'>;
 export type LBracketToken = GenericToken<'LBracket', '['>;
 export type LParenToken = GenericToken<'LParen', '('>;
@@ -22,7 +23,7 @@ export type StringToken = GenericToken<'String'>;
 export type DocStringToken = GenericToken<'DocString'>;
 export type SymbolToken<T extends string = string> = GenericToken<'Symbol', T>;
 export type WhitespaceToken = GenericToken<'Whitespace'>;
-export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | StringToken | SymbolToken | WhitespaceToken;
+export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | ErrorToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | StringToken | SymbolToken | WhitespaceToken;
 export type TokenDescriptor<T extends Token> = [length: number, token?: T];
 export interface SourceCodeInfo {
     position: {

package/dist/src/tokenizer/tokenizers.d.ts

@@ -1,5 +1,5 @@
-import type { BasePrefixedNumberToken, DocStringToken, LBraceToken, LBracketToken, LParenToken, MultiLineCommentToken, NumberToken, OperatorToken, RBraceToken, RBracketToken, RParenToken, RegexpShorthandToken, ReservedSymbolToken, SingleLineCommentToken, StringToken, SymbolToken, Token, TokenDescriptor, WhitespaceToken } from './token';
-export type Tokenizer<T extends Token> = (input: string, position: number) => TokenDescriptor<T>;
+import type { BasePrefixedNumberToken, DocStringToken, ErrorToken, LBraceToken, LBracketToken, LParenToken, MultiLineCommentToken, NumberToken, OperatorToken, RBraceToken, RBracketToken, RParenToken, RegexpShorthandToken, ReservedSymbolToken, SingleLineCommentToken, StringToken, SymbolToken, Token, TokenDescriptor, WhitespaceToken } from './token';
+export type Tokenizer<T extends Token> = (input: string, position: number) => TokenDescriptor<T | ErrorToken>;
 export declare const NO_MATCH: TokenDescriptor<never>;
 export declare const tokenizeDocString: Tokenizer<DocStringToken>;
 export declare const tokenizeWhitespace: Tokenizer<WhitespaceToken>;

package/dist/testFramework.esm.js

@@ -13311,7 +13311,7 @@ var tokenizeDocString = function (input, position) {
         nextThreeChars = input.slice(position + length, position + length + 3);
     }
     if (!char) {
-        throw new LitsError("Unclosed doc string at position ".concat(position, "."), undefined);
+        return [length, ['Error', value, undefined, "Unclosed doc string at position ".concat(position)]];
     }
     value += '"""'; // closing quote
     return [length + 3, ['DocString', value]];
@@ -13338,7 +13338,7 @@ var tokenizeString = function (input, position) {
         char = input[position + length];
     }
     if (!char) {
-        throw new LitsError("Unclosed string at position ".concat(position, "."), undefined);
+        return [length, ['Error', value, undefined, "Unclosed string at position ".concat(position)]];
     }
     value += '"'; // closing quote
     return [length + 1, ['String', value]];
@@ -13349,16 +13349,20 @@ var tokenizeRegexpShorthand = function (input, position) {
     var _a = __read(tokenizeString(input, position + 1), 2), stringLength = _a[0], token = _a[1];
     if (!token)
         return NO_MATCH;
+    if (token[0] === 'Error') {
+        var errorToken = ['Error', "#".concat(token[1]), undefined, "Unclosed regexp at position ".concat(position)];
+        return [stringLength + 1, errorToken];
+    }
     position += stringLength + 1;
     var length = stringLength + 1;
     var options = '';
     while (input[position] === 'g' || input[position] === 'i') {
-        if (options.includes(input[position])) {
-            throw new LitsError("Duplicated regexp option \"".concat(input[position], "\" at position ").concat(position, "."), undefined);
-        }
         options += input[position];
         length += 1;
         position += 1;
+        if (options.includes(input[position])) {
+            return [length, ['Error', "#".concat(token[1]).concat(options), undefined, "Duplicated regexp option \"".concat(input[position], "\"")]];
+        }
     }
     return [length, ['RegexpShorthand', "#".concat(token[1]).concat(options)]];
 };
@@ -13402,7 +13406,7 @@ var tokenizeNumber = function (input, position) {
                 if (i === start) {
                     return NO_MATCH;
                 }
-                throw new LitsError("Invalid number format at position ".concat(i, "."), undefined);
+                return [i - position + 1, ['Error', input.substring(position, i + 1), undefined, "Invalid number format at position ".concat(i + 1)]];
             }
         }
         else if (char === '.') {
@@ -13410,7 +13414,7 @@ var tokenizeNumber = function (input, position) {
                 return NO_MATCH;
             }
             if (hasDecimalPoint || hasExponent) {
-                throw new LitsError("Invalid number format at position ".concat(i, "."), undefined);
+                return [i - position + 1, ['Error', input.substring(position, i + 1), undefined, "Invalid number format at position ".concat(i + 1)]];
             }
             hasDecimalPoint = true;
         }
@@ -13419,10 +13423,10 @@ var tokenizeNumber = function (input, position) {
                 return NO_MATCH;
             }
             if (hasExponent) {
-                throw new LitsError("Invalid number format at position ".concat(i, "."), undefined);
+                return [i - position + 1, ['Error', input.substring(position, i + 1), undefined, "Invalid number format at position ".concat(i + 1)]];
             }
             if (input[i - 1] === '.' || input[i - 1] === '+' || input[i - 1] === '-') {
-                throw new LitsError("Invalid number format at position ".concat(i, "."), undefined);
+                return [i - position + 1, ['Error', input.substring(position, i + 1), undefined, "Invalid number format at position ".concat(i + 1)]];
             }
             if (input[i + 1] === '+' || input[i + 1] === '-') {
                 i += 1;
@@ -13442,7 +13446,7 @@ var tokenizeNumber = function (input, position) {
     }
     var nextChar = input[i];
     if (nextChar && nextChar !== ':' && !postNumberRegExp.test(nextChar)) {
-        throw new LitsError("Invalid number format at position ".concat(i, "."), undefined);
+        return [i - position + 1, ['Error', input.substring(position, i + 1), undefined, "Invalid number format at position ".concat(i + 1)]];
     }
     return [length, ['Number', input.substring(position, i)]];
 };
@@ -13492,7 +13496,7 @@ var tokenizeSymbol = function (input, position) {
         var escaping = false;
         while (char !== '\'' || escaping) {
             if (char === undefined)
-                throw new LitsError("Unclosed string at position ".concat(position, "."), undefined);
+                return [length_1, ['Error', value, undefined, "Unclosed quoted symbol at position ".concat(position)]];
             length_1 += 1;
             if (escaping) {
                 escaping = false;
@@ -13563,7 +13567,7 @@ var tokenizeMultiLineComment = function (input, position) {
             length_2 += 1;
         }
         if (position + length_2 + 1 >= input.length) {
-            throw new LitsError('Comment not closed', undefined);
+            return [length_2, ['Error', value, undefined, "Unclosed multi-line comment at position ".concat(position)]];
         }
         value += '*/';
         length_2 += 2;
@@ -13616,9 +13620,6 @@ function tokenize(input, debug, filePath) {
             ? createSourceCodeInfo(input, position, filePath)
             : undefined;
         var tokenDescriptor = getCurrentToken(input, position);
-        if (!tokenDescriptor) {
-            throw new LitsError("Unrecognized character '".concat(input[position], "'."), sourceCodeInfo);
-        }
         var _a = __read(tokenDescriptor, 2), count = _a[0], token = _a[1];
         position += count;
         if (token) {
@@ -13669,7 +13670,7 @@ function getCurrentToken(input, position) {
         }
         finally { if (e_1) throw e_1.error; }
     }
-    return null;
+    return [1, ['Error', input[initialPosition], undefined, 'Unrecognized character']];
 }
 
 function isSymbolToken(token, symbolName) {
@@ -13957,6 +13958,11 @@ var Parser = /** @class */ (function () {
         this.parseState.position += 1;
     };
     Parser.prototype.parse = function () {
+        this.tokenStream.tokens.forEach(function (token) {
+            if (token[0] === 'Error') {
+                throw new LitsError(token[3], token[2]);
+            }
+        });
         var nodes = [];
         while (!this.isAtEnd()) {
             nodes.push(this.parseExpression(0, true));
@@ -15023,20 +15029,14 @@ var AutoCompleter = /** @class */ (function () {
         this.suggestions = [];
         this.suggestionIndex = null;
         var partialProgram = this.originalProgram.slice(0, this.originalPosition);
-        var tokenStream = null;
-        try {
-            tokenStream = lits.tokenize(partialProgram);
-        }
-        catch (_a) {
-            // do nothing
-        }
-        if (!tokenStream) {
-            return;
-        }
+        var tokenStream = lits.tokenize(partialProgram);
         var lastToken = tokenStream.tokens.at(-1);
         if (!lastToken) {
             return;
         }
+        if (lastToken[0] === 'Error') {
+            return;
+        }
         this.searchString = lastToken[1];
         this.prefixProgram = this.originalProgram.slice(0, this.originalPosition - this.searchString.length);
         this.suffixProgram = this.originalProgram.slice(this.prefixProgram.length + this.searchString.length);