@mojir/lits 2.1.36 → 2.1.38

package/dist/cli/cli/src/modules/index.d.ts

@@ -0,0 +1,4 @@
+import type { JsFunction } from '../../../src/Lits/Lits';
+export declare const jsFunctions: {
+    [x: string]: JsFunction;
+};

package/dist/cli/cli/src/modules/sys/index.d.ts

@@ -0,0 +1,3 @@
+export declare function sys_cwd(): string;
+export declare function sys_readFile(filePath: string): string;
+export declare function sys_writeFile(filePath: string, content: string): void;

package/dist/cli/cli.js

@@ -5,6 +5,7 @@ var fs = require('node:fs');
 var path = require('node:path');
 var readline = require('node:readline');
 var os = require('node:os');
+var process$1 = require('node:process');
 
 /******************************************************************************
 Copyright (c) Microsoft Corporation.
@@ -92,7 +93,7 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 };
 
-var version = "2.1.36";
+var version = "2.1.38";
 
 function getCodeMarker(sourceCodeInfo) {
     if (!sourceCodeInfo.position || !sourceCodeInfo.code)
@@ -11805,7 +11806,6 @@ var lambdaSpecialExpression = {
             _b.arity = arity,
             _b.docString = docString,
             _b);
-        evaluatedFunction[2].self = { value: litsFunction };
         return litsFunction;
     },
     getUndefinedSymbols: function (node, contextStack, _a) {
@@ -12480,13 +12480,13 @@ var functionExecutors = {
             var args = evaluatedFunction[0];
             var nbrOfNonRestArgs = args.filter(function (arg) { return arg[0] !== bindingTargetTypes.rest; }).length;
             var newContextStack = contextStack.create(fn.evaluatedfunction[2]);
-            var newContext = {};
+            var newContext = { self: { value: fn } };
             var rest = [];
             for (var i = 0; i < params.length; i += 1) {
                 if (i < nbrOfNonRestArgs) {
                     var param = toAny(params[i]);
-                    var valueRecord = evalueateBindingNodeValues(args[i], param, function (Node) {
-                        return evaluateNode(Node, newContextStack.create(newContext));
+                    var valueRecord = evalueateBindingNodeValues(args[i], param, function (node) {
+                        return evaluateNode(node, newContextStack.create(newContext));
                     });
                     Object.entries(valueRecord).forEach(function (_a) {
                         var _b = __read(_a, 2), key = _b[0], value = _b[1];
@@ -12500,8 +12500,8 @@ var functionExecutors = {
             for (var i = params.length; i < nbrOfNonRestArgs; i++) {
                 var arg = args[i];
                 var defaultValue = evaluateNode(arg[1][1], contextStack.create(newContext));
-                var valueRecord = evalueateBindingNodeValues(arg, defaultValue, function (Node) {
-                    return evaluateNode(Node, contextStack.create(newContext));
+                var valueRecord = evalueateBindingNodeValues(arg, defaultValue, function (node) {
+                    return evaluateNode(node, contextStack.create(newContext));
                 });
                 Object.entries(valueRecord).forEach(function (_a) {
                     var _b = __read(_a, 2), key = _b[0], value = _b[1];
@@ -12510,7 +12510,7 @@ var functionExecutors = {
             }
             var restArgument = args.find(function (arg) { return arg[0] === bindingTargetTypes.rest; });
             if (restArgument !== undefined) {
-                var valueRecord = evalueateBindingNodeValues(restArgument, rest, function (Node) { return evaluateNode(Node, contextStack.create(newContext)); });
+                var valueRecord = evalueateBindingNodeValues(restArgument, rest, function (node) { return evaluateNode(node, contextStack.create(newContext)); });
                 Object.entries(valueRecord).forEach(function (_a) {
                     var _b = __read(_a, 2), key = _b[0], value = _b[1];
                     newContext[key] = { value: value };
@@ -13049,6 +13049,21 @@ var ContextStackImpl = /** @class */ (function () {
     };
     return ContextStackImpl;
 }());
+function checkNotDefined(name) {
+    if (specialExpressionKeys.includes(name)) {
+        console.warn("Cannot shadow special expression \"".concat(name, "\", ignoring."));
+        return false;
+    }
+    if (normalExpressionKeys.includes(name)) {
+        console.warn("Cannot shadow builtin function \"".concat(name, "\", ignoring."));
+        return false;
+    }
+    if (name === 'self') {
+        console.warn("Cannot shadow builtin value \"".concat(name, "\", ignoring."));
+        return false;
+    }
+    return true;
+}
 function createContextStack(params) {
     var _a;
     if (params === void 0) { params = {}; }
@@ -13060,26 +13075,53 @@ function createContextStack(params) {
         values: params.values,
         nativeJsFunctions: params.jsFunctions
             && Object.entries(params.jsFunctions).reduce(function (acc, _a) {
-                var _b;
-                var _c, _d;
-                var _e = __read(_a, 2), name = _e[0], jsFunction = _e[1];
-                if (specialExpressionKeys.includes(name)) {
-                    console.warn("Cannot shadow special expression \"".concat(name, "\", ignoring."));
+                var e_5, _b, _c;
+                var _d, _e;
+                var _f = __read(_a, 2), identifier = _f[0], entry = _f[1];
+                var identifierParts = identifier.split('.');
+                var name = identifierParts.pop();
+                if (/^[A-Z]/.test(name)) {
+                    console.warn("Invalid identifier \"".concat(identifier, "\" in jsFunctions, function name must not start with an uppercase letter"), undefined);
                     return acc;
                 }
-                if (normalExpressionKeys.includes(name)) {
-                    console.warn("Cannot shadow builtin function \"".concat(name, "\", ignoring."));
-                    return acc;
+                var scope = acc;
+                try {
+                    for (var identifierParts_1 = __values(identifierParts), identifierParts_1_1 = identifierParts_1.next(); !identifierParts_1_1.done; identifierParts_1_1 = identifierParts_1.next()) {
+                        var part = identifierParts_1_1.value;
+                        if (part.length === 0) {
+                            console.warn("Invalid empty identifier \"".concat(identifier, "\" in nativeJsFunctions"), undefined);
+                            return acc;
+                        }
+                        if (!/^[A-Z]/.test(part)) {
+                            console.warn("Invalid identifier \"".concat(identifier, "\" in jsFunctions, namespace must start with an uppercase letter"), undefined);
+                            return acc;
+                        }
+                        if (!scope[part]) {
+                            scope[part] = {};
+                        }
+                        scope = scope[part];
+                    }
+                }
+                catch (e_5_1) { e_5 = { error: e_5_1 }; }
+                finally {
+                    try {
+                        if (identifierParts_1_1 && !identifierParts_1_1.done && (_b = identifierParts_1.return)) _b.call(identifierParts_1);
+                    }
+                    finally { if (e_5) throw e_5.error; }
                 }
-                acc[name] = (_b = {
+                var natifeFn = (_c = {
                         functionType: 'NativeJsFunction',
-                        nativeFn: jsFunction,
+                        nativeFn: entry,
                         name: name
                     },
-                    _b[FUNCTION_SYMBOL] = true,
-                    _b.arity = (_c = jsFunction.arity) !== null && _c !== void 0 ? _c : {},
-                    _b.docString = (_d = jsFunction.docString) !== null && _d !== void 0 ? _d : '',
-                    _b);
+                    _c[FUNCTION_SYMBOL] = true,
+                    _c.arity = (_d = entry.arity) !== null && _d !== void 0 ? _d : {},
+                    _c.docString = (_e = entry.docString) !== null && _e !== void 0 ? _e : '',
+                    _c);
+                if (scope === acc && !checkNotDefined(name)) {
+                    return acc;
+                }
+                scope[name] = natifeFn;
                 return acc;
             }, {}),
     });
@@ -13495,10 +13537,10 @@ var tokenizeMultiLineComment = function (input, position) {
     }
     return NO_MATCH;
 };
-var tokenizeSingleLineComment = function (input, position) {
-    if (input[position] === '/' && input[position + 1] === '/') {
+var tokenizeShebang = function (input, position) {
+    if (input[position] === '#' && input[position + 1] === '!') {
         var length_3 = 2;
-        var value = '//';
+        var value = '#!';
         while (input[position + length_3] !== '\n' && position + length_3 < input.length) {
             value += input[position + length_3];
             length_3 += 1;
@@ -13507,6 +13549,18 @@ var tokenizeSingleLineComment = function (input, position) {
     }
     return NO_MATCH;
 };
+var tokenizeSingleLineComment = function (input, position) {
+    if (input[position] === '/' && input[position + 1] === '/') {
+        var length_4 = 2;
+        var value = '//';
+        while (input[position + length_4] !== '\n' && position + length_4 < input.length) {
+            value += input[position + length_4];
+            length_4 += 1;
+        }
+        return [length_4, ['SingleLineComment', value]];
+    }
+    return NO_MATCH;
+};
 // All tokenizers, order matters!
 var tokenizers = [
     tokenizeWhitespace,
@@ -13572,10 +13626,17 @@ function createSourceCodeInfo(input, position, filePath) {
 function getCurrentToken(input, position) {
     var e_1, _a;
     var initialPosition = position;
+    if (position === 0) {
+        var _b = __read(tokenizeShebang(input, position), 2), nbrOfCharacters = _b[0], token = _b[1];
+        position += nbrOfCharacters;
+        if (nbrOfCharacters > 0) {
+            return [position - initialPosition, token];
+        }
+    }
     try {
         for (var tokenizers_1 = __values(tokenizers), tokenizers_1_1 = tokenizers_1.next(); !tokenizers_1_1.done; tokenizers_1_1 = tokenizers_1.next()) {
             var tokenizer = tokenizers_1_1.value;
-            var _b = __read(tokenizer(input, position), 2), nbrOfCharacters = _b[0], token = _b[1];
+            var _c = __read(tokenizer(input, position), 2), nbrOfCharacters = _c[0], token = _c[1];
             position += nbrOfCharacters;
             if (nbrOfCharacters === 0) {
                 continue;
@@ -13629,6 +13690,9 @@ function asReservedSymbolToken(token, symbolName) {
     assertReservedSymbolToken(token, symbolName);
     return token;
 }
+function isShebangToken(token) {
+    return (token === null || token === void 0 ? void 0 : token[0]) === 'Shebang';
+}
 function isSingleLineCommentToken(token) {
     return (token === null || token === void 0 ? void 0 : token[0]) === 'SingleLineComment';
 }
@@ -13728,6 +13792,7 @@ function minifyTokenStream(tokenStream, _a) {
         .filter(function (token) {
         if (isSingleLineCommentToken(token)
             || isMultiLineCommentToken(token)
+            || isShebangToken(token)
             || (removeWhiteSpace && isWhitespaceToken(token))) {
             return false;
         }
@@ -32081,15 +32146,42 @@ function createReadlineInterface(options) {
     return rl;
 }
 
+function sys_cwd() {
+    return process$1.cwd();
+}
+function sys_readFile(filePath) {
+    return fs.readFileSync(filePath, { encoding: 'utf8' });
+}
+function sys_writeFile(filePath, content) {
+    fs.writeFileSync(content, filePath, { encoding: 'utf8' });
+}
+
+var sysFunctions = {
+    'Sys.cwd': {
+        fn: sys_cwd,
+        docString: 'Returns the current working directory of the process.',
+        arity: { min: 0, max: 0 },
+    },
+    'Sys.read-file': {
+        fn: sys_readFile,
+        docString: 'Reads the content of a file at the specified path.',
+        arity: { min: 1, max: 1 },
+    },
+    'Sys.write-file': {
+        fn: sys_writeFile,
+        docString: 'Writes content to a file at the specified path.',
+        arity: { min: 2, max: 2 },
+    },
+};
+var jsFunctions = __assign({}, sysFunctions);
+
 /* eslint-disable node/prefer-global/process */
 /* eslint-disable no-console */
-var _a;
 var useColor = !process.env.NO_COLOR;
 var fmt = createColorizer(useColor);
 var HIST_SIZE = 1000;
 var PROMPT = fmt.bright.gray('> ');
 var historyResults = [];
-var lits = new Lits({ debug: true });
 var formatValue = getInlineCodeFormatter(fmt);
 var commands = ['`help', '`quit', '`builtins', '`context'];
 var expressionRegExp = new RegExp("^(.*\\(\\s*)(".concat(polishSymbolFirstCharacterClass).concat(polishSymbolCharacterClass, "*)$"));
@@ -32097,20 +32189,44 @@ var nameRegExp = new RegExp("^(.*?)(".concat(polishSymbolFirstCharacterClass).co
 var helpRegExp = new RegExp("^`help\\s+(".concat(polishSymbolFirstCharacterClass).concat(polishSymbolCharacterClass, "+)\\s*$"));
 var expressions = __spreadArray(__spreadArray([], __read(normalExpressionKeys), false), __read(specialExpressionKeys), false);
 var config = processArguments(process.argv.slice(2));
-if (config.eval) {
-    var success = execute(config.eval);
-    process.exit(success ? 0 : 1);
-}
-else if (config.evalFilename) {
-    var content = fs.readFileSync(config.evalFilename, { encoding: 'utf-8' });
-    var success = execute(content);
-    process.exit(success ? 0 : 1);
+var lits = (function () {
+    var _lits = new Lits({ debug: true });
+    return {
+        run: function (program) {
+            var _a;
+            return _lits.run(program, {
+                globalContext: (_a = config.context) !== null && _a !== void 0 ? _a : undefined,
+                jsFunctions: jsFunctions,
+                globalModuleScope: true,
+            });
+        },
+        context: function (program) {
+            var _a;
+            return _lits.context(program, {
+                globalContext: (_a = config.context) !== null && _a !== void 0 ? _a : undefined,
+                jsFunctions: jsFunctions,
+                globalModuleScope: true,
+            });
+        },
+    };
+})();
+if (config.eval || config.evalFilename) {
+    try {
+        var program = config.eval || fs.readFileSync(config.evalFilename, { encoding: 'utf-8' });
+        var result = lits.run(program);
+        if (config.printResult) {
+            console.log(result);
+        }
+        process.exit(0);
+    }
+    catch (error) {
+        printErrorMessage("".concat(error));
+        process.exit(1);
+    }
 }
 else if (config.loadFilename) {
     var content = fs.readFileSync(config.loadFilename, { encoding: 'utf-8' });
-    config.context = lits.context(content, {
-        globalContext: (_a = config.context) !== null && _a !== void 0 ? _a : undefined,
-    });
+    config.context = lits.context(content);
     runREPL();
 }
 else if (config.testFilename) {
@@ -32134,16 +32250,14 @@ function runLitsTest(testPath, testNamePattern) {
         process.exit(1);
 }
 function execute(expression) {
-    var _a;
     try {
-        var result = lits.run(expression, {
-            globalContext: (_a = config.context) !== null && _a !== void 0 ? _a : undefined,
-        });
+        var result = lits.run(expression);
         historyResults.unshift(result);
-        if (historyResults.length > 9)
+        if (historyResults.length > 9) {
             historyResults.length = 9;
+        }
         setReplHistoryVariables();
-        console.log(formatValue(stringifyValue(result, false)));
+        console.log(stringifyValue(result, false));
         return true;
     }
     catch (error) {
@@ -32173,6 +32287,9 @@ function setReplHistoryVariables() {
 }
 function parseOption(args, i) {
     var option = args[i];
+    if (option === '-p') {
+        return { option: option, argument: null, count: 1 };
+    }
     if (/^-[a-z]$/i.test(option))
         return { option: option, argument: args[i + 1], count: 2 };
     var match = /^(--[a-z-]+)(?:=(.*))?$/i.exec(option);
@@ -32188,6 +32305,7 @@ function processArguments(args) {
         loadFilename: null,
         context: {},
         eval: null,
+        printResult: false,
     };
     var i = 0;
     while (i < args.length) {
@@ -32199,7 +32317,6 @@ function processArguments(args) {
         var option = parsedOption.option, argument = parsedOption.argument, count = parsedOption.count;
         i += count;
         switch (option) {
-            case '-t':
             case '--test':
                 if (!argument) {
                     printErrorMessage("Missing filename after ".concat(option));
@@ -32207,7 +32324,6 @@ function processArguments(args) {
                 }
                 defaultConfig.testFilename = argument;
                 break;
-            case '-p':
             case '--test-pattern':
                 if (!argument) {
                     printErrorMessage("Missing test name pattern after ".concat(option));
@@ -32266,6 +32382,10 @@ function processArguments(args) {
                     process.exit(1);
                 }
                 break;
+            case '-p':
+            case '--print-result':
+                defaultConfig.printResult = true;
+                break;
             case '-e':
             case '--eval':
                 if (!argument) {
@@ -32368,7 +32488,7 @@ function printHelp() {
     console.log("\n`builtins                 Print all builtin functions\n`context                  Print context\n`help                     Print this help message\n`help [builtin function]  Print help for [builtin function]\n`quit                     Quit\n".trim());
 }
 function printUsage() {
-    console.log("\nUsage: lits [options]\n\nOptions:\n  -c, --context=...               Context as a JSON string\n  -C, --context-file=...          Context file (.json file)\n  -e, --eval=...                  Evaluate Lits expression\n  -f, --file=...                  Evaluate .lits file\n  -p, --test-pattern=...          Test name pattern, used together with --test\n  -t, --test=...                  Test .test.lits file\n  --help                          Show this help\n  --version                       Print lits version\n".trim());
+    console.log("\nUsage: lits [options]\n\nOptions:\n  -c, --context=...               Context as a JSON string\n  -C, --context-file=...          Context file (.json file)\n  -e, --eval=...                  Evaluate Lits expression\n  -f, --file=...                  Evaluate .lits file\n      --test-pattern=...          Test name pattern, used together with --test\n      --test=...                  Test .test.lits file\n  -p  --print-result              Test .test.lits file\n  --help                          Show this help\n  --version                       Print lits version\n".trim());
 }
 function printContext() {
     var context = config.context;

package/dist/cli/src/evaluator/ContextStack.d.ts

@@ -1,6 +1,6 @@
 import type { Any } from '../interface';
 import type { ContextParams } from '../Lits/Lits';
-import type { NativeJsFunction, SymbolNode, UserDefinedSymbolNode } from '../parser/types';
+import type { NativeJsNamespace, SymbolNode, UserDefinedSymbolNode } from '../parser/types';
 import type { SourceCodeInfo } from '../tokenizer/token';
 import type { Context, LookUpResult } from './interface';
 export type ContextStack = ContextStackImpl;
@@ -12,7 +12,7 @@ export declare class ContextStackImpl {
     constructor({ contexts, values: hostValues, nativeJsFunctions, }: {
         contexts: Context[];
         values?: Record<string, unknown>;
-        nativeJsFunctions?: Record<string, NativeJsFunction>;
+        nativeJsFunctions?: NativeJsNamespace;
     });
     create(context: Context): ContextStack;
     new(context: Context): ContextStack;

package/dist/cli/src/modules/index.d.ts

@@ -0,0 +1,4 @@
+import type { JsFunction } from '../../../src/Lits/Lits';
+export declare const jsFunctions: {
+    [x: string]: JsFunction;
+};

package/dist/cli/src/modules/sys/index.d.ts

@@ -0,0 +1,3 @@
+export declare function sys_cwd(): string;
+export declare function sys_readFile(filePath: string): string;
+export declare function sys_writeFile(filePath: string, content: string): void;

package/dist/cli/src/parser/types.d.ts

@@ -30,6 +30,9 @@ export interface NativeJsFunction extends GenericLitsFunction {
     nativeFn: JsFunction;
     docString: string;
 }
+export interface NativeJsNamespace {
+    [key: string]: NativeJsFunction | NativeJsNamespace;
+}
 export interface UserDefinedFunction extends GenericLitsFunction {
     functionType: 'UserDefined';
     name: string | undefined;

package/dist/cli/src/tokenizer/token.d.ts

@@ -1,6 +1,6 @@
 import type { ReservedSymbol } from './reservedNames';
 import { type SymbolicBinaryOperator, type SymbolicOperator } from './operators';
-export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "Error", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "String", "Symbol", "Whitespace"];
+export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "Error", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "Shebang", "String", "Symbol", "Whitespace"];
 export type TokenType = typeof tokenTypes[number];
 declare const modifierNames: readonly ["&rest", "&let", "&when", "&while"];
 export type ModifierName = typeof modifierNames[number];
@@ -19,11 +19,12 @@ export type OperatorToken<T extends SymbolicOperator = SymbolicOperator> = Gener
 export type RegexpShorthandToken = GenericToken<'RegexpShorthand'>;
 export type ReservedSymbolToken<T extends ReservedSymbol = ReservedSymbol> = GenericToken<'ReservedSymbol', T>;
 export type SingleLineCommentToken = GenericToken<'SingleLineComment'>;
+export type ShebangToken = GenericToken<'Shebang'>;
 export type StringToken = GenericToken<'String'>;
 export type DocStringToken = GenericToken<'DocString'>;
 export type SymbolToken<T extends string = string> = GenericToken<'Symbol', T>;
 export type WhitespaceToken = GenericToken<'Whitespace'>;
-export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | ErrorToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | StringToken | SymbolToken | WhitespaceToken;
+export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | ErrorToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | ShebangToken | StringToken | SymbolToken | WhitespaceToken;
 export type TokenDescriptor<T extends Token> = [length: number, token?: T];
 export interface SourceCodeInfo {
     position: {
@@ -39,12 +40,9 @@ export declare function asSymbolToken<T extends string>(token: Token | undefined
 export declare function isReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): token is ReservedSymbolToken<T>;
 export declare function assertReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): asserts token is ReservedSymbolToken<T>;
 export declare function asReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): ReservedSymbolToken<T>;
+export declare function isShebangToken(token: Token | undefined): token is SingleLineCommentToken;
 export declare function isSingleLineCommentToken(token: Token | undefined): token is SingleLineCommentToken;
-export declare function assertSingleLineCommentToken(token: Token | undefined): asserts token is SingleLineCommentToken;
-export declare function asSingleLineCommentToken(token: Token | undefined): SingleLineCommentToken;
 export declare function isMultiLineCommentToken(token: Token | undefined): token is MultiLineCommentToken;
-export declare function assertMultiLineCommentToken(token: Token | undefined): asserts token is MultiLineCommentToken;
-export declare function asMultiLineCommentToken(token: Token | undefined): MultiLineCommentToken;
 export declare function isOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): token is OperatorToken<T>;
 export declare function assertOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): asserts token is OperatorToken<T>;
 export declare function asOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): OperatorToken<T>;

package/dist/cli/src/tokenizer/tokenizers.d.ts

@@ -9,5 +9,6 @@ export declare const tokenizeSymbol: Tokenizer<SymbolToken>;
 export declare const tokenizeReservedSymbolToken: Tokenizer<ReservedSymbolToken>;
 export declare const tokenizeOperator: Tokenizer<OperatorToken>;
 export declare const tokenizeMultiLineComment: Tokenizer<MultiLineCommentToken>;
+export declare const tokenizeShebang: Tokenizer<SingleLineCommentToken>;
 export declare const tokenizeSingleLineComment: Tokenizer<SingleLineCommentToken>;
 export declare const tokenizers: [Tokenizer<WhitespaceToken>, Tokenizer<MultiLineCommentToken>, Tokenizer<SingleLineCommentToken>, Tokenizer<ReservedSymbolToken>, Tokenizer<LParenToken>, Tokenizer<RParenToken>, Tokenizer<LBracketToken>, Tokenizer<RBracketToken>, Tokenizer<LBraceToken>, Tokenizer<RBraceToken>, Tokenizer<DocStringToken>, Tokenizer<StringToken>, Tokenizer<RegexpShorthandToken>, Tokenizer<BasePrefixedNumberToken>, Tokenizer<NumberToken>, Tokenizer<OperatorToken>, Tokenizer<SymbolToken>];