@mojir/lits 2.1.37 → 2.1.38

package/dist/src/tokenizer/token.d.ts

@@ -1,6 +1,6 @@
 import type { ReservedSymbol } from './reservedNames';
 import { type SymbolicBinaryOperator, type SymbolicOperator } from './operators';
-export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "Error", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "String", "Symbol", "Whitespace"];
+export declare const tokenTypes: readonly ["LBrace", "LBracket", "RBrace", "RBracket", "LParen", "RParen", "BasePrefixedNumber", "DocString", "Error", "MultiLineComment", "Number", "Operator", "RegexpShorthand", "ReservedSymbol", "SingleLineComment", "Shebang", "String", "Symbol", "Whitespace"];
 export type TokenType = typeof tokenTypes[number];
 declare const modifierNames: readonly ["&rest", "&let", "&when", "&while"];
 export type ModifierName = typeof modifierNames[number];
@@ -19,11 +19,12 @@ export type OperatorToken<T extends SymbolicOperator = SymbolicOperator> = Gener
 export type RegexpShorthandToken = GenericToken<'RegexpShorthand'>;
 export type ReservedSymbolToken<T extends ReservedSymbol = ReservedSymbol> = GenericToken<'ReservedSymbol', T>;
 export type SingleLineCommentToken = GenericToken<'SingleLineComment'>;
+export type ShebangToken = GenericToken<'Shebang'>;
 export type StringToken = GenericToken<'String'>;
 export type DocStringToken = GenericToken<'DocString'>;
 export type SymbolToken<T extends string = string> = GenericToken<'Symbol', T>;
 export type WhitespaceToken = GenericToken<'Whitespace'>;
-export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | ErrorToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | StringToken | SymbolToken | WhitespaceToken;
+export type Token = LBraceToken | LBracketToken | LParenToken | RBraceToken | RBracketToken | RParenToken | BasePrefixedNumberToken | DocStringToken | ErrorToken | MultiLineCommentToken | NumberToken | OperatorToken | RegexpShorthandToken | ReservedSymbolToken | SingleLineCommentToken | ShebangToken | StringToken | SymbolToken | WhitespaceToken;
 export type TokenDescriptor<T extends Token> = [length: number, token?: T];
 export interface SourceCodeInfo {
     position: {
@@ -39,12 +40,9 @@ export declare function asSymbolToken<T extends string>(token: Token | undefined
 export declare function isReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): token is ReservedSymbolToken<T>;
 export declare function assertReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): asserts token is ReservedSymbolToken<T>;
 export declare function asReservedSymbolToken<T extends ReservedSymbol>(token: Token | undefined | undefined, symbolName?: T): ReservedSymbolToken<T>;
+export declare function isShebangToken(token: Token | undefined): token is SingleLineCommentToken;
 export declare function isSingleLineCommentToken(token: Token | undefined): token is SingleLineCommentToken;
-export declare function assertSingleLineCommentToken(token: Token | undefined): asserts token is SingleLineCommentToken;
-export declare function asSingleLineCommentToken(token: Token | undefined): SingleLineCommentToken;
 export declare function isMultiLineCommentToken(token: Token | undefined): token is MultiLineCommentToken;
-export declare function assertMultiLineCommentToken(token: Token | undefined): asserts token is MultiLineCommentToken;
-export declare function asMultiLineCommentToken(token: Token | undefined): MultiLineCommentToken;
 export declare function isOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): token is OperatorToken<T>;
 export declare function assertOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): asserts token is OperatorToken<T>;
 export declare function asOperatorToken<T extends SymbolicOperator>(token: Token | undefined | undefined, operatorName?: T): OperatorToken<T>;

package/dist/src/tokenizer/tokenizers.d.ts

@@ -9,5 +9,6 @@ export declare const tokenizeSymbol: Tokenizer<SymbolToken>;
 export declare const tokenizeReservedSymbolToken: Tokenizer<ReservedSymbolToken>;
 export declare const tokenizeOperator: Tokenizer<OperatorToken>;
 export declare const tokenizeMultiLineComment: Tokenizer<MultiLineCommentToken>;
+export declare const tokenizeShebang: Tokenizer<SingleLineCommentToken>;
 export declare const tokenizeSingleLineComment: Tokenizer<SingleLineCommentToken>;
 export declare const tokenizers: [Tokenizer<WhitespaceToken>, Tokenizer<MultiLineCommentToken>, Tokenizer<SingleLineCommentToken>, Tokenizer<ReservedSymbolToken>, Tokenizer<LParenToken>, Tokenizer<RParenToken>, Tokenizer<LBracketToken>, Tokenizer<RBracketToken>, Tokenizer<LBraceToken>, Tokenizer<RBraceToken>, Tokenizer<DocStringToken>, Tokenizer<StringToken>, Tokenizer<RegexpShorthandToken>, Tokenizer<BasePrefixedNumberToken>, Tokenizer<NumberToken>, Tokenizer<OperatorToken>, Tokenizer<SymbolToken>];

package/dist/testFramework.esm.js

@@ -13529,10 +13529,10 @@ var tokenizeMultiLineComment = function (input, position) {
     }
     return NO_MATCH;
 };
-var tokenizeSingleLineComment = function (input, position) {
-    if (input[position] === '/' && input[position + 1] === '/') {
+var tokenizeShebang = function (input, position) {
+    if (input[position] === '#' && input[position + 1] === '!') {
         var length_3 = 2;
-        var value = '//';
+        var value = '#!';
         while (input[position + length_3] !== '\n' && position + length_3 < input.length) {
             value += input[position + length_3];
             length_3 += 1;
@@ -13541,6 +13541,18 @@ var tokenizeSingleLineComment = function (input, position) {
     }
     return NO_MATCH;
 };
+var tokenizeSingleLineComment = function (input, position) {
+    if (input[position] === '/' && input[position + 1] === '/') {
+        var length_4 = 2;
+        var value = '//';
+        while (input[position + length_4] !== '\n' && position + length_4 < input.length) {
+            value += input[position + length_4];
+            length_4 += 1;
+        }
+        return [length_4, ['SingleLineComment', value]];
+    }
+    return NO_MATCH;
+};
 // All tokenizers, order matters!
 var tokenizers = [
     tokenizeWhitespace,
@@ -13606,10 +13618,17 @@ function createSourceCodeInfo(input, position, filePath) {
 function getCurrentToken(input, position) {
     var e_1, _a;
     var initialPosition = position;
+    if (position === 0) {
+        var _b = __read(tokenizeShebang(input, position), 2), nbrOfCharacters = _b[0], token = _b[1];
+        position += nbrOfCharacters;
+        if (nbrOfCharacters > 0) {
+            return [position - initialPosition, token];
+        }
+    }
     try {
         for (var tokenizers_1 = __values(tokenizers), tokenizers_1_1 = tokenizers_1.next(); !tokenizers_1_1.done; tokenizers_1_1 = tokenizers_1.next()) {
             var tokenizer = tokenizers_1_1.value;
-            var _b = __read(tokenizer(input, position), 2), nbrOfCharacters = _b[0], token = _b[1];
+            var _c = __read(tokenizer(input, position), 2), nbrOfCharacters = _c[0], token = _c[1];
             position += nbrOfCharacters;
             if (nbrOfCharacters === 0) {
                 continue;
@@ -13663,6 +13682,9 @@ function asReservedSymbolToken(token, symbolName) {
     assertReservedSymbolToken(token, symbolName);
     return token;
 }
+function isShebangToken(token) {
+    return (token === null || token === void 0 ? void 0 : token[0]) === 'Shebang';
+}
 function isSingleLineCommentToken(token) {
     return (token === null || token === void 0 ? void 0 : token[0]) === 'SingleLineComment';
 }
@@ -13762,6 +13784,7 @@ function minifyTokenStream(tokenStream, _a) {
         .filter(function (token) {
         if (isSingleLineCommentToken(token)
             || isMultiLineCommentToken(token)
+            || isShebangToken(token)
             || (removeWhiteSpace && isWhitespaceToken(token))) {
             return false;
         }