@mojir/lits 2.0.20 → 2.0.22

package/dist/cli/cli.js

@@ -92,7 +92,7 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 };
 
-var version = "2.0.20";
+var version = "2.0.22";
 
 var AstNodeType;
 (function (AstNodeType) {
@@ -6111,6 +6111,28 @@ function analyze(ast, params) {
     };
 }
 
+function minifyTokenStream(tokenStream, _a) {
+    var removeWhiteSpace = _a.removeWhiteSpace;
+    var tokens = tokenStream.tokens
+        .filter(function (token) {
+        if (isP_CommentToken(token)
+            || isA_CommentToken(token)
+            || isA_MultiLineCommentToken(token)
+            || (removeWhiteSpace && isA_WhitespaceToken(token))
+            || (removeWhiteSpace && isP_WhitespaceToken(token))) {
+            return false;
+        }
+        return true;
+    })
+        .map(function (token) {
+        if (isA_WhitespaceToken(token) || isP_WhitespaceToken(token)) {
+            return __assign(__assign({}, token), { value: ' ' });
+        }
+        return token;
+    });
+    return __assign(__assign({}, tokenStream), { tokens: tokens });
+}
+
 function parseSymbol(tokenStream, parseState) {
     var _a;
     var tkn = asToken(tokenStream.tokens[parseState.position++]);
@@ -7629,7 +7651,7 @@ function parsePolishToken(tokenStream, parseState) {
 }
 
 function parse(tokenStream) {
-    tokenStream = removeUnnecessaryTokens(tokenStream);
+    tokenStream = minifyTokenStream(tokenStream, { removeWhiteSpace: true });
     var algebraic = tokenStream.algebraic;
     var ast = {
         b: [],
@@ -7650,19 +7672,6 @@ function parse(tokenStream) {
     }
     return ast;
 }
-function removeUnnecessaryTokens(tokenStream) {
-    var tokens = tokenStream.tokens.filter(function (token) {
-        if (isP_CommentToken(token)
-            || isA_CommentToken(token)
-            || isA_MultiLineCommentToken(token)
-            || isA_WhitespaceToken(token)
-            || isP_WhitespaceToken(token)) {
-            return false;
-        }
-        return true;
-    });
-    return __assign(__assign({}, tokenStream), { tokens: tokens });
-}
 function parseToken(tokenStream, parseState) {
     return parsePolishToken(tokenStream, parseState);
 }
@@ -8490,7 +8499,8 @@ var Lits = /** @class */ (function () {
         if (tokenizeParams === void 0) { tokenizeParams = {}; }
         var debug = this.debug;
         var algebraic = this.algebraic;
-        return tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        var tokenStream = tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        return tokenizeParams.minify ? minifyTokenStream(tokenStream, { removeWhiteSpace: false }) : tokenStream;
     };
     Lits.prototype.parse = function (tokenStream) {
         return parse(tokenStream);

package/dist/cli/src/Lits/Lits.d.ts

@@ -40,7 +40,9 @@ export declare class Lits {
     run(program: string, params?: LitsParams): unknown;
     context(program: string, params?: LitsParams): Context;
     analyze(program: string, params?: LitsParams): Analysis;
-    tokenize(program: string, tokenizeParams?: Pick<TokenizeParams, 'filePath'>): TokenStream;
+    tokenize(program: string, tokenizeParams?: Pick<TokenizeParams, 'filePath'> & {
+        minify?: boolean;
+    }): TokenStream;
     parse(tokenStream: TokenStream): Ast;
     evaluate(ast: Ast, params: LitsParams): Any;
     transform(tokenStream: TokenStream, transformer: (name: string) => string): TokenStream;

package/dist/cli/src/tokenizer/minifyTokenStream.d.ts

@@ -0,0 +1,4 @@
+import type { TokenStream } from './interface';
+export declare function minifyTokenStream(tokenStream: TokenStream, { removeWhiteSpace }: {
+    removeWhiteSpace: boolean;
+}): TokenStream;

package/dist/index.esm.js

@@ -6141,6 +6141,28 @@ function analyze(ast, params) {
     };
 }
 
+function minifyTokenStream(tokenStream, _a) {
+    var removeWhiteSpace = _a.removeWhiteSpace;
+    var tokens = tokenStream.tokens
+        .filter(function (token) {
+        if (isP_CommentToken(token)
+            || isA_CommentToken(token)
+            || isA_MultiLineCommentToken(token)
+            || (removeWhiteSpace && isA_WhitespaceToken(token))
+            || (removeWhiteSpace && isP_WhitespaceToken(token))) {
+            return false;
+        }
+        return true;
+    })
+        .map(function (token) {
+        if (isA_WhitespaceToken(token) || isP_WhitespaceToken(token)) {
+            return __assign(__assign({}, token), { value: ' ' });
+        }
+        return token;
+    });
+    return __assign(__assign({}, tokenStream), { tokens: tokens });
+}
+
 function parseSymbol(tokenStream, parseState) {
     var _a;
     var tkn = asToken(tokenStream.tokens[parseState.position++]);
@@ -7659,7 +7681,7 @@ function parsePolishToken(tokenStream, parseState) {
 }
 
 function parse(tokenStream) {
-    tokenStream = removeUnnecessaryTokens(tokenStream);
+    tokenStream = minifyTokenStream(tokenStream, { removeWhiteSpace: true });
     var algebraic = tokenStream.algebraic;
     var ast = {
         b: [],
@@ -7680,19 +7702,6 @@ function parse(tokenStream) {
     }
     return ast;
 }
-function removeUnnecessaryTokens(tokenStream) {
-    var tokens = tokenStream.tokens.filter(function (token) {
-        if (isP_CommentToken(token)
-            || isA_CommentToken(token)
-            || isA_MultiLineCommentToken(token)
-            || isA_WhitespaceToken(token)
-            || isP_WhitespaceToken(token)) {
-            return false;
-        }
-        return true;
-    });
-    return __assign(__assign({}, tokenStream), { tokens: tokens });
-}
 function parseToken(tokenStream, parseState) {
     return parsePolishToken(tokenStream, parseState);
 }
@@ -8520,7 +8529,8 @@ var Lits = /** @class */ (function () {
         if (tokenizeParams === void 0) { tokenizeParams = {}; }
         var debug = this.debug;
         var algebraic = this.algebraic;
-        return tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        var tokenStream = tokenize(program, __assign(__assign({}, tokenizeParams), { debug: debug, algebraic: algebraic }));
+        return tokenizeParams.minify ? minifyTokenStream(tokenStream, { removeWhiteSpace: false }) : tokenStream;
     };
     Lits.prototype.parse = function (tokenStream) {
         return parse(tokenStream);