@mojir/lits 2.0.20 → 2.0.22

package/dist/src/Lits/Lits.d.ts

@@ -40,7 +40,9 @@ export declare class Lits {
     run(program: string, params?: LitsParams): unknown;
     context(program: string, params?: LitsParams): Context;
     analyze(program: string, params?: LitsParams): Analysis;
-    tokenize(program: string, tokenizeParams?: Pick<TokenizeParams, 'filePath'>): TokenStream;
+    tokenize(program: string, tokenizeParams?: Pick<TokenizeParams, 'filePath'> & {
+        minify?: boolean;
+    }): TokenStream;
     parse(tokenStream: TokenStream): Ast;
     evaluate(ast: Ast, params: LitsParams): Any;
     transform(tokenStream: TokenStream, transformer: (name: string) => string): TokenStream;

package/dist/src/tokenizer/minifyTokenStream.d.ts

@@ -0,0 +1,4 @@
+import type { TokenStream } from './interface';
+export declare function minifyTokenStream(tokenStream: TokenStream, { removeWhiteSpace }: {
+    removeWhiteSpace: boolean;
+}): TokenStream;

package/dist/testFramework.esm.js

@@ -6103,6 +6103,28 @@ function analyze(ast, params) {
     };
 }
 
+function minifyTokenStream(tokenStream, _a) {
+    var removeWhiteSpace = _a.removeWhiteSpace;
+    var tokens = tokenStream.tokens
+        .filter(function (token) {
+        if (isP_CommentToken(token)
+            || isA_CommentToken(token)
+            || isA_MultiLineCommentToken(token)
+            || (removeWhiteSpace && isA_WhitespaceToken(token))
+            || (removeWhiteSpace && isP_WhitespaceToken(token))) {
+            return false;
+        }
+        return true;
+    })
+        .map(function (token) {
+        if (isA_WhitespaceToken(token) || isP_WhitespaceToken(token)) {
+            return __assign(__assign({}, token), { value: ' ' });
+        }
+        return token;
+    });
+    return __assign(__assign({}, tokenStream), { tokens: tokens });
+}
+
 function parseSymbol(tokenStream, parseState) {
     var _a;
     var tkn = asToken(tokenStream.tokens[parseState.position++]);
@@ -7621,7 +7643,7 @@ function parsePolishToken(tokenStream, parseState) {
 }
 
 function parse(tokenStream) {
-    tokenStream = removeUnnecessaryTokens(tokenStream);
+    tokenStream = minifyTokenStream(tokenStream, { removeWhiteSpace: true });
     var algebraic = tokenStream.algebraic;
     var ast = {
         b: [],
@@ -7642,19 +7664,6 @@ function parse(tokenStream) {
     }
     return ast;
 }
-function removeUnnecessaryTokens(tokenStream) {
-    var tokens = tokenStream.tokens.filter(function (token) {
-        if (isP_CommentToken(token)
-            || isA_CommentToken(token)
-            || isA_MultiLineCommentToken(token)
-            || isA_WhitespaceToken(token)
-            || isP_WhitespaceToken(token)) {
-            return false;
-        }
-        return true;
-    });
-    return __assign(__assign({}, tokenStream), { tokens: tokens });
-}
 function parseToken(tokenStream, parseState) {
     return parsePolishToken(tokenStream, parseState);
 }
@@ -8481,7 +8490,8 @@ var Lits = /** @class */ (function () {
         if (tokenizeParams === void 0) { tokenizeParams = {}; }
         var debug = this.debug;
         var algebraic = this.algebraic;
-        return tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        var tokenStream = tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        return tokenizeParams.minify ? minifyTokenStream(tokenStream, { removeWhiteSpace: false }) : tokenStream;
     };
     Lits.prototype.parse = function (tokenStream) {
         return parse(tokenStream);