lua-lexer 1.0.0

package/README.md

@@ -0,0 +1,52 @@
+# lua-lexer
+
+A fast and lightweight Lua lexer written in TypeScript.
+
+This lexer tokenizes Lua source code into an array of tokens, making it easy to build parsers, formatters, syntax highlighters, or static analysis tools.
+
+## Features
+- **TypeScript Support**: Fully typed API with included `.d.ts` files.
+- **Fast**: Designed for performance.
+- **Universal**: Supports both CommonJS and ES Modules.
+
+## Installation
+
+Install using npm:
+
+```bash
+npm install lua-lexer
+```
+
+## Basic Usage
+
+```typescript
+import { tokenize } from "lua-lexer"; // adjust based on your actual package name
+
+const code = `
+  local function hello()
+      print("Hello World!")
+  end
+`;
+
+const tokens = tokenize(code);
+console.log(tokens);
+```
+
+### Advanced Usage
+
+For parsing large files or advanced scenarios where you need granular control, you can use the `LuaLexer` class directly:
+
+```typescript
+import { LuaLexer, LuaTokenType } from "lua-lexer";
+
+const lexer = new LuaLexer();
+lexer.load("print(123)");
+
+// Read token by token
+const token = lexer.next();
+console.log(token);
+```
+
+## License
+
+This project is licensed under the MIT License.

package/build/index.d.mts

@@ -0,0 +1,91 @@
+declare const enum LuaTokenType {
+    END_OF_FILE = 0,
+    ERROR = 1,
+    AND = 2,
+    BREAK = 3,
+    DO = 4,
+    ELSE = 5,
+    ELSEIF = 6,
+    END = 7,
+    FALSE = 8,
+    FOR = 9,
+    FUNCTION = 10,
+    GLOBAL = 11,
+    GOTO = 12,
+    IF = 13,
+    IN = 14,
+    LOCAL = 15,
+    NIL = 16,
+    NOT = 17,
+    OR = 18,
+    REPEAT = 19,
+    RETURN = 20,
+    THEN = 21,
+    TRUE = 22,
+    UNTIL = 23,
+    WHILE = 24,
+    PLUS = 25,
+    MINUS = 26,
+    ASTERISK = 27,
+    SLASH = 28,
+    PERCENT = 29,
+    CARET = 30,
+    HASH = 31,
+    AMPERSAND = 32,
+    TILDE = 33,
+    PIPE = 34,
+    LESS_LESS = 35,
+    GREATER_GREATER = 36,
+    DOUBLE_SLASH = 37,
+    EQUAL_EQUAL = 38,
+    TILDE_EQUAL = 39,
+    LESS_EQUAL = 40,
+    GREATER_EQUAL = 41,
+    LESS = 42,
+    GREATER = 43,
+    EQUAL = 44,
+    LEFT_PAREN = 45,
+    RIGHT_PAREN = 46,
+    LEFT_BRACE = 47,
+    RIGHT_BRACE = 48,
+    LEFT_BRACKET = 49,
+    RIGHT_BRACKET = 50,
+    DOUBLE_COLON = 51,
+    SEMICOLON = 52,
+    COLON = 53,
+    COMMA = 54,
+    DOT = 55,
+    DOUBLE_DOT = 56,
+    TRIPLE_DOT = 57,
+    STRING_LITERAL = 58,
+    NUMBER_LITERAL = 59,
+    IDENTIFIER = 60,
+    COMMENT = 61
+}
+type LuaTokenValue = string | number | boolean | null | undefined;
+interface LuaToken {
+    type: LuaTokenType;
+    value: LuaTokenValue;
+    line: number;
+    column: number;
+    start: number;
+    length: number;
+}
+declare class LuaLexer {
+    private raw;
+    private index;
+    private line;
+    private lineStart;
+    load(source: string): void;
+    tokenizeAll(): LuaToken[];
+    private yieldToken;
+    next(): LuaToken;
+    private readDecimalLiteral;
+    private readHexadecimalLiteral;
+    private readShortStringLiteral;
+    private readLongStringLiteralOrComment;
+    private readIdentifierOrKeyword;
+}
+declare function tokenize(code: string): LuaToken[];
+
+export { LuaLexer, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };

package/build/index.d.ts

@@ -0,0 +1,91 @@
+declare const enum LuaTokenType {
+    END_OF_FILE = 0,
+    ERROR = 1,
+    AND = 2,
+    BREAK = 3,
+    DO = 4,
+    ELSE = 5,
+    ELSEIF = 6,
+    END = 7,
+    FALSE = 8,
+    FOR = 9,
+    FUNCTION = 10,
+    GLOBAL = 11,
+    GOTO = 12,
+    IF = 13,
+    IN = 14,
+    LOCAL = 15,
+    NIL = 16,
+    NOT = 17,
+    OR = 18,
+    REPEAT = 19,
+    RETURN = 20,
+    THEN = 21,
+    TRUE = 22,
+    UNTIL = 23,
+    WHILE = 24,
+    PLUS = 25,
+    MINUS = 26,
+    ASTERISK = 27,
+    SLASH = 28,
+    PERCENT = 29,
+    CARET = 30,
+    HASH = 31,
+    AMPERSAND = 32,
+    TILDE = 33,
+    PIPE = 34,
+    LESS_LESS = 35,
+    GREATER_GREATER = 36,
+    DOUBLE_SLASH = 37,
+    EQUAL_EQUAL = 38,
+    TILDE_EQUAL = 39,
+    LESS_EQUAL = 40,
+    GREATER_EQUAL = 41,
+    LESS = 42,
+    GREATER = 43,
+    EQUAL = 44,
+    LEFT_PAREN = 45,
+    RIGHT_PAREN = 46,
+    LEFT_BRACE = 47,
+    RIGHT_BRACE = 48,
+    LEFT_BRACKET = 49,
+    RIGHT_BRACKET = 50,
+    DOUBLE_COLON = 51,
+    SEMICOLON = 52,
+    COLON = 53,
+    COMMA = 54,
+    DOT = 55,
+    DOUBLE_DOT = 56,
+    TRIPLE_DOT = 57,
+    STRING_LITERAL = 58,
+    NUMBER_LITERAL = 59,
+    IDENTIFIER = 60,
+    COMMENT = 61
+}
+type LuaTokenValue = string | number | boolean | null | undefined;
+interface LuaToken {
+    type: LuaTokenType;
+    value: LuaTokenValue;
+    line: number;
+    column: number;
+    start: number;
+    length: number;
+}
+declare class LuaLexer {
+    private raw;
+    private index;
+    private line;
+    private lineStart;
+    load(source: string): void;
+    tokenizeAll(): LuaToken[];
+    private yieldToken;
+    next(): LuaToken;
+    private readDecimalLiteral;
+    private readHexadecimalLiteral;
+    private readShortStringLiteral;
+    private readLongStringLiteralOrComment;
+    private readIdentifierOrKeyword;
+}
+declare function tokenize(code: string): LuaToken[];
+
+export { LuaLexer, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };

package/build/index.js

@@ -0,0 +1,6 @@
+"use strict";var _=Object.defineProperty;var S=Object.getOwnPropertyDescriptor;var O=Object.getOwnPropertyNames;var D=Object.prototype.hasOwnProperty;var U=(A,r)=>{for(var t in r)_(A,t,{get:r[t],enumerable:!0})},k=(A,r,t,n)=>{if(r&&typeof r=="object"||typeof r=="function")for(let e of O(r))!D.call(A,e)&&e!==t&&_(A,e,{get:()=>r[e],enumerable:!(n=S(r,e))||n.enumerable});return A};var P=A=>k(_({},"__esModule",{value:!0}),A);var G={};U(G,{LuaLexer:()=>L,LuaTokenType:()=>T,tokenize:()=>N});module.exports=P(G);var T=(i=>(i[i.END_OF_FILE=0]="END_OF_FILE",i[i.ERROR=1]="ERROR",i[i.AND=2]="AND",i[i.BREAK=3]="BREAK",i[i.DO=4]="DO",i[i.ELSE=5]="ELSE",i[i.ELSEIF=6]="ELSEIF",i[i.END=7]="END",i[i.FALSE=8]="FALSE",i[i.FOR=9]="FOR",i[i.FUNCTION=10]="FUNCTION",i[i.GLOBAL=11]="GLOBAL",i[i.GOTO=12]="GOTO",i[i.IF=13]="IF",i[i.IN=14]="IN",i[i.LOCAL=15]="LOCAL",i[i.NIL=16]="NIL",i[i.NOT=17]="NOT",i[i.OR=18]="OR",i[i.REPEAT=19]="REPEAT",i[i.RETURN=20]="RETURN",i[i.THEN=21]="THEN",i[i.TRUE=22]="TRUE",i[i.UNTIL=23]="UNTIL",i[i.WHILE=24]="WHILE",i[i.PLUS=25]="PLUS",i[i.MINUS=26]="MINUS",i[i.ASTERISK=27]="ASTERISK",i[i.SLASH=28]="SLASH",i[i.PERCENT=29]="PERCENT",i[i.CARET=30]="CARET",i[i.HASH=31]="HASH",i[i.AMPERSAND=32]="AMPERSAND",i[i.TILDE=33]="TILDE",i[i.PIPE=34]="PIPE",i[i.LESS_LESS=35]="LESS_LESS",i[i.GREATER_GREATER=36]="GREATER_GREATER",i[i.DOUBLE_SLASH=37]="DOUBLE_SLASH",i[i.EQUAL_EQUAL=38]="EQUAL_EQUAL",i[i.TILDE_EQUAL=39]="TILDE_EQUAL",i[i.LESS_EQUAL=40]="LESS_EQUAL",i[i.GREATER_EQUAL=41]="GREATER_EQUAL",i[i.LESS=42]="LESS",i[i.GREATER=43]="GREATER",i[i.EQUAL=44]="EQUAL",i[i.LEFT_PAREN=45]="LEFT_PAREN",i[i.RIGHT_PAREN=46]="RIGHT_PAREN",i[i.LEFT_BRACE=47]="LEFT_BRACE",i[i.RIGHT_BRACE=48]="RIGHT_BRACE",i[i.LEFT_BRACKET=49]="LEFT_BRACKET",i[i.RIGHT_BRACKET=50]="RIGHT_BRACKET",i[i.DOUBLE_COLON=51]="DOUBLE_COLON",i[i.SEMICOLON=52]="SEMICOLON",i[i.COLON=53]="COLON",i[i.COMMA=54]="COMMA",i[i.DOT=55]="DOT",i[i.DOUBLE_DOT=56]="DOUBLE_DOT",i[i.TRIPLE_DOT=57]="TRIPLE_DOT",i[i.STRING_LITERAL=58]="STRING_LITERAL",i[i.NUMBER_LITERAL=59]="NUMBER_LITERAL",i[i.IDENTIFIER=60]="IDENTIFIER",i[i.COMMENT=61]="COMMENT",i))(T||{}),L=class{raw="";index=0;line=1;lineStart=0;load(r){this.raw=r+"\0\0\0\0\0\0\0\0",this.index=0,this.line=1,this.lineStart=0}tokenizeAll(){let r=[];for(;;){let t=this.next();if(r.push(t),t.type===0)break}return r}yieldToken(r,t,n){let e={type:r,value:t,line:this.line,column:this.index-this.lineStart+1,start:this.index,length:n};return this.index+=n,this.lineStart+=n,e}next(){let r=this.raw,t=this.index,n=this.line,e=this.lineStart;for(;;){let l=r.charCodeAt(t);switch(l){case 0:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(0,void 0,0);case 9:case 11:case 12:case 32:{t++;continue}case 13:{r.charCodeAt(t+1)===10?t+=2:t++,n++,e=t;continue}case 10:{r.charCodeAt(t+1)===13?t+=2:t++,n++,e=t;continue}case 95:return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();case 34:case 39:return this.index=t,this.line=n,this.lineStart=e,this.readShortStringLiteral(l);case 48:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===120||s===88?this.readHexadecimalLiteral():this.readDecimalLiteral()}case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:return this.index=t,this.line=n,this.lineStart=e,this.readDecimalLiteral();case 46:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===46?r.charCodeAt(t+2)===46?this.yieldToken(57,void 0,3):this.yieldToken(56,void 0,2):s-48>>>0<=9?this.readDecimalLiteral():this.yieldToken(55,void 0,1)}case 47:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===47?this.yieldToken(37,void 0,2):this.yieldToken(28,void 0,1);case 126:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(39,void 0,2):this.yieldToken(33,void 0,1);case 61:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(38,void 0,2):this.yieldToken(44,void 0,1);case 58:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===58?this.yieldToken(51,void 0,2):this.yieldToken(53,void 0,1);case 60:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===60?this.yieldToken(35,void 0,2):s===61?this.yieldToken(40,void 0,2):this.yieldToken(42,void 0,1)}case 62:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===62?this.yieldToken(36,void 0,2):s===61?this.yieldToken(41,void 0,2):this.yieldToken(43,void 0,1)}case 91:{this.index=t,this.line=n,this.lineStart=e;let s=t+1,a=0;for(;r.charCodeAt(s)===61;)a++,s++;return r.charCodeAt(s)===91?this.readLongStringLiteralOrComment(a,!1):this.yieldToken(49,void 0,1)}case 45:{if(this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)!==45)return this.yieldToken(26,void 0,1);if(r.charCodeAt(t+2)===91){let a=t+3,u=0;for(;r.charCodeAt(a)===61;)u++,a++;if(r.charCodeAt(a)===91)return this.readLongStringLiteralOrComment(u,!0)}let s=t+2;for(;s<r.length;){let a=r.charCodeAt(s);if(a===13||a===10)break;s++}return this.yieldToken(61,r.substring(t+2,s),s-t)}case 43:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(25,void 0,1);case 42:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(27,void 0,1);case 37:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(29,void 0,1);case 94:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(30,void 0,1);case 35:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(31,void 0,1);case 38:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(32,void 0,1);case 124:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(34,void 0,1);case 40:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(45,void 0,1);case 41:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(46,void 0,1);case 123:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(47,void 0,1);case 125:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(48,void 0,1);case 93:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(50,void 0,1);case 59:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(52,void 0,1);case 44:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(54,void 0,1);default:{if((l|32)-97>>>0<=25)return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();t++;continue}}}}readDecimalLiteral(){let r=this.raw,t=this.index,n=t+1,e=0,l=0,s=1,a=0,u=1,R=!1;if(r.charCodeAt(t)===46){let d=!1;for(;;){let h=r.charCodeAt(n)-48;if(h>>>0>9)break;d=!0,l=l*10+h,s*=10,n++}d||(R=!0)}else{for(e=r.charCodeAt(t)-48;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;e=e*10+d,n++}if(r.charCodeAt(n)===46)for(n++;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;l=l*10+d,s*=10,n++}}if(!R&&(r.charCodeAt(n)|32)===101){n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(u=-1,n++);let h=!1;for(;;){let E=r.charCodeAt(n)-48;if(E>>>0>9)break;h=!0,a=a*10+E,n++}h||(R=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)R=!0,n++;else break}return R?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=10**(u*a)),this.yieldToken(59,e,n-t))}readHexadecimalLiteral(){let r=this.raw,t=this.index,n=t+2,e=0,l=0,s=1,a=0,u=1,R=!1;if(r.charCodeAt(n)===46){n++;let d=!1;for(;;){let h=r.charCodeAt(n),E=h-48,f=(h|32)-97;if(E>>>0<=9)l=l*16+E;else if(f>>>0<=5)l=l*16+f+10;else break;s*=16,d=!0,n++}d||(R=!0)}else{let d=!1;for(;;){let h=r.charCodeAt(n),E=h-48,f=(h|32)-97;if(E>>>0<=9)e=e*16+E;else if(f>>>0<=5)e=e*16+f+10;else break;d=!0,n++}if(r.charCodeAt(n)===46)for(n++;;){let h=r.charCodeAt(n),E=h-48,f=(h|32)-97;if(E>>>0<=9)l=l*16+E;else if(f>>>0<=5)l=l*16+f+10;else break;d=!0,s*=16,n++}d||(R=!0)}if(!R&&(r.charCodeAt(n)|32)===112){n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(u=-1,n++);let h=!1;for(;;){let E=r.charCodeAt(n)-48;if(E>>>0>9)break;h=!0,a=a*10+E,n++}h||(R=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)R=!0,n++;else break}return R?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=2**(u*a)),this.yieldToken(59,e,n-t))}readShortStringLiteral(r){let t=this.raw,n=this.index,e=n+1,l=e,s="",a=this.line,u=this.lineStart,R=!1,d="unfinished string";for(;;){for(;;){let E=t.charCodeAt(e);if(E===r)return s+=t.substring(l,e),this.line=a,this.lineStart=u,this.yieldToken(58,s,e-n+1);if(E===92){s+=t.substring(l,e);break}if(E===13||E===10||E===0){R=!0,d="unfinished string";break}e++}if(R)break;e++;let h=t.charCodeAt(e);switch(h){case 97:s+="\x07",e++;break;case 98:s+="\b",e++;break;case 102:s+="\f",e++;break;case 110:s+=`
+`,e++;break;case 114:s+="\r",e++;break;case 116:s+="	",e++;break;case 118:s+="\v",e++;break;case 34:s+='"',e++;break;case 39:s+="'",e++;break;case 92:s+="\\",e++;break;case 13:{s+=`
+`,t.charCodeAt(e+1)===10?e+=2:e++,a++,u=e;break}case 10:{s+=`
+`,t.charCodeAt(e+1)===13?e+=2:e++,a++,u=e;break}case 120:{e++;let E=0,f=t.charCodeAt(e),c=f-48,o=(f|32)-97;if(c>>>0<=9)E=c;else if(o>>>0<=5)E=o+10;else{R=!0,d="hexadecimal digit expected";break}if(e++,f=t.charCodeAt(e),c=f-48,o=(f|32)-97,c>>>0<=9)E=E*16+c;else if(o>>>0<=5)E=E*16+o+10;else{R=!0,d="hexadecimal digit expected";break}e++,s+=String.fromCharCode(E);break}case 117:{if(e++,t.charCodeAt(e)!==123){R=!0,d="missing '{'";break}e++;let E=0,f=!1;for(;;){let c=t.charCodeAt(e),o=c-48,I=(c|32)-97;if(o>>>0<=9)E=E*16+o;else if(I>>>0<=5)E=E*16+I+10;else break;f=!0,e++}if(!f){R=!0,d="hexadecimal digit expected";break}if(t.charCodeAt(e)!==125){R=!0,d="missing '}'";break}if(e++,E>1114111){R=!0,d="UTF-8 value too large";break}s+=String.fromCodePoint(E);break}case 122:{for(e++;;){let E=t.charCodeAt(e);if(E===32||E===9||E===11||E===12)e++;else if(E===13)t.charCodeAt(e+1)===10?e+=2:e++,a++,u=e;else if(E===10)t.charCodeAt(e+1)===13?e+=2:e++,a++,u=e;else break}break}default:{let E=h-48;if(E>>>0<=9){let f=E;e++,E=t.charCodeAt(e)-48,E>>>0<=9&&(f=f*10+E,e++,E=t.charCodeAt(e)-48,E>>>0<=9&&(f=f*10+E,e++)),f>255?(R=!0,d="decimal escape too large"):s+=String.fromCharCode(f)}else h===0?(R=!0,d="unfinished string"):(R=!0,d="invalid escape sequence",e++);break}}if(R)break;l=e}for(;;){let h=t.charCodeAt(e);if(h===r){e++;break}else if(h===13||h===10||h===92)break;e++}return this.line=a,this.lineStart=u,this.yieldToken(1,d,e-n)}readLongStringLiteralOrComment(r,t){let n=this.raw,e=this.index,l;t?l=4+r:l=2+r;let s=e+l,a=this.line,u=this.lineStart,R=n.charCodeAt(s);R===13?(n.charCodeAt(s+1)===10?s+=2:s++,a++,u=s):R===10&&(n.charCodeAt(s+1)===13?s+=2:s++,a++,u=s);let d=s,h="",E=!1;for(;;){let c=n.charCodeAt(s);if(c===13){s>d&&(h+=n.substring(d,s)),h+=`
+`,n.charCodeAt(s+1)===10?s+=2:s++,a++,u=s,d=s;continue}else if(c===10){s>d&&(h+=n.substring(d,s)),h+=`
+`,n.charCodeAt(s+1)===13?s+=2:s++,a++,u=s,d=s;continue}else if(c===93){let o=0;for(;n.charCodeAt(s+1+o)===61;)o++;if(o===r&&n.charCodeAt(s+1+o)===93){s>d&&(h+=n.substring(d,s)),s+=2+o,E=!0;break}}else if(c===0)break;s++}let f;return E?f=this.yieldToken(t?61:58,h,s-e):(s>d&&(h+=n.substring(d,s)),f=this.yieldToken(1,t?"unfinished long comment":"unfinished long string",s-e)),this.line=a,this.lineStart=u,f}readIdentifierOrKeyword(){let r=this.raw,t=this.index,n=t+1;for(;;){let s=r.charCodeAt(n);if((s|32)-97>>>0<=25||s-48>>>0<=9||s===95)n++;else break}let e=n-t,l=r.substring(t,n);return e===2?l==="do"?this.yieldToken(4,void 0,2):l==="if"?this.yieldToken(13,void 0,2):l==="in"?this.yieldToken(14,void 0,2):l==="or"?this.yieldToken(18,void 0,2):this.yieldToken(60,l,e):e===3?l==="and"?this.yieldToken(2,void 0,3):l==="end"?this.yieldToken(7,void 0,3):l==="for"?this.yieldToken(9,void 0,3):l==="not"?this.yieldToken(17,void 0,3):l==="nil"?this.yieldToken(16,null,3):this.yieldToken(60,l,e):e===4?l==="else"?this.yieldToken(5,void 0,4):l==="goto"?this.yieldToken(12,void 0,4):l==="then"?this.yieldToken(21,void 0,4):l==="true"?this.yieldToken(22,!0,4):this.yieldToken(60,l,e):e===5?l==="break"?this.yieldToken(3,void 0,5):l==="local"?this.yieldToken(15,void 0,5):l==="false"?this.yieldToken(8,!1,5):l==="while"?this.yieldToken(24,void 0,5):this.yieldToken(60,l,e):e===6?l==="elseif"?this.yieldToken(6,void 0,6):l==="repeat"?this.yieldToken(19,void 0,6):l==="return"?this.yieldToken(20,void 0,6):l==="global"?this.yieldToken(11,void 0,6):this.yieldToken(60,l,e):e===8?l==="function"?this.yieldToken(10,void 0,8):this.yieldToken(60,l,e):this.yieldToken(60,l,e)}};function N(A){let r=new L;return r.load(A),r.tokenizeAll()}0&&(module.exports={LuaLexer,LuaTokenType,tokenize});

package/build/index.mjs

@@ -0,0 +1,6 @@
+var I=(i=>(i[i.END_OF_FILE=0]="END_OF_FILE",i[i.ERROR=1]="ERROR",i[i.AND=2]="AND",i[i.BREAK=3]="BREAK",i[i.DO=4]="DO",i[i.ELSE=5]="ELSE",i[i.ELSEIF=6]="ELSEIF",i[i.END=7]="END",i[i.FALSE=8]="FALSE",i[i.FOR=9]="FOR",i[i.FUNCTION=10]="FUNCTION",i[i.GLOBAL=11]="GLOBAL",i[i.GOTO=12]="GOTO",i[i.IF=13]="IF",i[i.IN=14]="IN",i[i.LOCAL=15]="LOCAL",i[i.NIL=16]="NIL",i[i.NOT=17]="NOT",i[i.OR=18]="OR",i[i.REPEAT=19]="REPEAT",i[i.RETURN=20]="RETURN",i[i.THEN=21]="THEN",i[i.TRUE=22]="TRUE",i[i.UNTIL=23]="UNTIL",i[i.WHILE=24]="WHILE",i[i.PLUS=25]="PLUS",i[i.MINUS=26]="MINUS",i[i.ASTERISK=27]="ASTERISK",i[i.SLASH=28]="SLASH",i[i.PERCENT=29]="PERCENT",i[i.CARET=30]="CARET",i[i.HASH=31]="HASH",i[i.AMPERSAND=32]="AMPERSAND",i[i.TILDE=33]="TILDE",i[i.PIPE=34]="PIPE",i[i.LESS_LESS=35]="LESS_LESS",i[i.GREATER_GREATER=36]="GREATER_GREATER",i[i.DOUBLE_SLASH=37]="DOUBLE_SLASH",i[i.EQUAL_EQUAL=38]="EQUAL_EQUAL",i[i.TILDE_EQUAL=39]="TILDE_EQUAL",i[i.LESS_EQUAL=40]="LESS_EQUAL",i[i.GREATER_EQUAL=41]="GREATER_EQUAL",i[i.LESS=42]="LESS",i[i.GREATER=43]="GREATER",i[i.EQUAL=44]="EQUAL",i[i.LEFT_PAREN=45]="LEFT_PAREN",i[i.RIGHT_PAREN=46]="RIGHT_PAREN",i[i.LEFT_BRACE=47]="LEFT_BRACE",i[i.RIGHT_BRACE=48]="RIGHT_BRACE",i[i.LEFT_BRACKET=49]="LEFT_BRACKET",i[i.RIGHT_BRACKET=50]="RIGHT_BRACKET",i[i.DOUBLE_COLON=51]="DOUBLE_COLON",i[i.SEMICOLON=52]="SEMICOLON",i[i.COLON=53]="COLON",i[i.COMMA=54]="COMMA",i[i.DOT=55]="DOT",i[i.DOUBLE_DOT=56]="DOUBLE_DOT",i[i.TRIPLE_DOT=57]="TRIPLE_DOT",i[i.STRING_LITERAL=58]="STRING_LITERAL",i[i.NUMBER_LITERAL=59]="NUMBER_LITERAL",i[i.IDENTIFIER=60]="IDENTIFIER",i[i.COMMENT=61]="COMMENT",i))(I||{}),A=class{raw="";index=0;line=1;lineStart=0;load(r){this.raw=r+"\0\0\0\0\0\0\0\0",this.index=0,this.line=1,this.lineStart=0}tokenizeAll(){let r=[];for(;;){let n=this.next();if(r.push(n),n.type===0)break}return r}yieldToken(r,n,t){let e={type:r,value:n,line:this.line,column:this.index-this.lineStart+1,start:this.index,length:t};return this.index+=t,this.lineStart+=t,e}next(){let r=this.raw,n=this.index,t=this.line,e=this.lineStart;for(;;){let l=r.charCodeAt(n);switch(l){case 0:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(0,void 0,0);case 9:case 11:case 12:case 32:{n++;continue}case 13:{r.charCodeAt(n+1)===10?n+=2:n++,t++,e=n;continue}case 10:{r.charCodeAt(n+1)===13?n+=2:n++,t++,e=n;continue}case 95:return this.index=n,this.line=t,this.lineStart=e,this.readIdentifierOrKeyword();case 34:case 39:return this.index=n,this.line=t,this.lineStart=e,this.readShortStringLiteral(l);case 48:{this.index=n,this.line=t,this.lineStart=e;let s=r.charCodeAt(n+1);return s===120||s===88?this.readHexadecimalLiteral():this.readDecimalLiteral()}case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:return this.index=n,this.line=t,this.lineStart=e,this.readDecimalLiteral();case 46:{this.index=n,this.line=t,this.lineStart=e;let s=r.charCodeAt(n+1);return s===46?r.charCodeAt(n+2)===46?this.yieldToken(57,void 0,3):this.yieldToken(56,void 0,2):s-48>>>0<=9?this.readDecimalLiteral():this.yieldToken(55,void 0,1)}case 47:return this.index=n,this.line=t,this.lineStart=e,r.charCodeAt(n+1)===47?this.yieldToken(37,void 0,2):this.yieldToken(28,void 0,1);case 126:return this.index=n,this.line=t,this.lineStart=e,r.charCodeAt(n+1)===61?this.yieldToken(39,void 0,2):this.yieldToken(33,void 0,1);case 61:return this.index=n,this.line=t,this.lineStart=e,r.charCodeAt(n+1)===61?this.yieldToken(38,void 0,2):this.yieldToken(44,void 0,1);case 58:return this.index=n,this.line=t,this.lineStart=e,r.charCodeAt(n+1)===58?this.yieldToken(51,void 0,2):this.yieldToken(53,void 0,1);case 60:{this.index=n,this.line=t,this.lineStart=e;let s=r.charCodeAt(n+1);return s===60?this.yieldToken(35,void 0,2):s===61?this.yieldToken(40,void 0,2):this.yieldToken(42,void 0,1)}case 62:{this.index=n,this.line=t,this.lineStart=e;let s=r.charCodeAt(n+1);return s===62?this.yieldToken(36,void 0,2):s===61?this.yieldToken(41,void 0,2):this.yieldToken(43,void 0,1)}case 91:{this.index=n,this.line=t,this.lineStart=e;let s=n+1,a=0;for(;r.charCodeAt(s)===61;)a++,s++;return r.charCodeAt(s)===91?this.readLongStringLiteralOrComment(a,!1):this.yieldToken(49,void 0,1)}case 45:{if(this.index=n,this.line=t,this.lineStart=e,r.charCodeAt(n+1)!==45)return this.yieldToken(26,void 0,1);if(r.charCodeAt(n+2)===91){let a=n+3,u=0;for(;r.charCodeAt(a)===61;)u++,a++;if(r.charCodeAt(a)===91)return this.readLongStringLiteralOrComment(u,!0)}let s=n+2;for(;s<r.length;){let a=r.charCodeAt(s);if(a===13||a===10)break;s++}return this.yieldToken(61,r.substring(n+2,s),s-n)}case 43:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(25,void 0,1);case 42:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(27,void 0,1);case 37:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(29,void 0,1);case 94:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(30,void 0,1);case 35:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(31,void 0,1);case 38:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(32,void 0,1);case 124:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(34,void 0,1);case 40:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(45,void 0,1);case 41:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(46,void 0,1);case 123:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(47,void 0,1);case 125:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(48,void 0,1);case 93:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(50,void 0,1);case 59:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(52,void 0,1);case 44:return this.index=n,this.line=t,this.lineStart=e,this.yieldToken(54,void 0,1);default:{if((l|32)-97>>>0<=25)return this.index=n,this.line=t,this.lineStart=e,this.readIdentifierOrKeyword();n++;continue}}}}readDecimalLiteral(){let r=this.raw,n=this.index,t=n+1,e=0,l=0,s=1,a=0,u=1,R=!1;if(r.charCodeAt(n)===46){let d=!1;for(;;){let h=r.charCodeAt(t)-48;if(h>>>0>9)break;d=!0,l=l*10+h,s*=10,t++}d||(R=!0)}else{for(e=r.charCodeAt(n)-48;;){let d=r.charCodeAt(t)-48;if(d>>>0>9)break;e=e*10+d,t++}if(r.charCodeAt(t)===46)for(t++;;){let d=r.charCodeAt(t)-48;if(d>>>0>9)break;l=l*10+d,s*=10,t++}}if(!R&&(r.charCodeAt(t)|32)===101){t++;let d=r.charCodeAt(t);d===43?t++:d===45&&(u=-1,t++);let h=!1;for(;;){let E=r.charCodeAt(t)-48;if(E>>>0>9)break;h=!0,a=a*10+E,t++}h||(R=!0)}for(;;){let d=r.charCodeAt(t);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)R=!0,t++;else break}return R?this.yieldToken(1,`Malformed number: ${r.substring(n,t)}`,t-n):(l>0&&(e+=l/s),a!==0&&(e*=10**(u*a)),this.yieldToken(59,e,t-n))}readHexadecimalLiteral(){let r=this.raw,n=this.index,t=n+2,e=0,l=0,s=1,a=0,u=1,R=!1;if(r.charCodeAt(t)===46){t++;let d=!1;for(;;){let h=r.charCodeAt(t),E=h-48,f=(h|32)-97;if(E>>>0<=9)l=l*16+E;else if(f>>>0<=5)l=l*16+f+10;else break;s*=16,d=!0,t++}d||(R=!0)}else{let d=!1;for(;;){let h=r.charCodeAt(t),E=h-48,f=(h|32)-97;if(E>>>0<=9)e=e*16+E;else if(f>>>0<=5)e=e*16+f+10;else break;d=!0,t++}if(r.charCodeAt(t)===46)for(t++;;){let h=r.charCodeAt(t),E=h-48,f=(h|32)-97;if(E>>>0<=9)l=l*16+E;else if(f>>>0<=5)l=l*16+f+10;else break;d=!0,s*=16,t++}d||(R=!0)}if(!R&&(r.charCodeAt(t)|32)===112){t++;let d=r.charCodeAt(t);d===43?t++:d===45&&(u=-1,t++);let h=!1;for(;;){let E=r.charCodeAt(t)-48;if(E>>>0>9)break;h=!0,a=a*10+E,t++}h||(R=!0)}for(;;){let d=r.charCodeAt(t);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)R=!0,t++;else break}return R?this.yieldToken(1,`Malformed number: ${r.substring(n,t)}`,t-n):(l>0&&(e+=l/s),a!==0&&(e*=2**(u*a)),this.yieldToken(59,e,t-n))}readShortStringLiteral(r){let n=this.raw,t=this.index,e=t+1,l=e,s="",a=this.line,u=this.lineStart,R=!1,d="unfinished string";for(;;){for(;;){let E=n.charCodeAt(e);if(E===r)return s+=n.substring(l,e),this.line=a,this.lineStart=u,this.yieldToken(58,s,e-t+1);if(E===92){s+=n.substring(l,e);break}if(E===13||E===10||E===0){R=!0,d="unfinished string";break}e++}if(R)break;e++;let h=n.charCodeAt(e);switch(h){case 97:s+="\x07",e++;break;case 98:s+="\b",e++;break;case 102:s+="\f",e++;break;case 110:s+=`
+`,e++;break;case 114:s+="\r",e++;break;case 116:s+="	",e++;break;case 118:s+="\v",e++;break;case 34:s+='"',e++;break;case 39:s+="'",e++;break;case 92:s+="\\",e++;break;case 13:{s+=`
+`,n.charCodeAt(e+1)===10?e+=2:e++,a++,u=e;break}case 10:{s+=`
+`,n.charCodeAt(e+1)===13?e+=2:e++,a++,u=e;break}case 120:{e++;let E=0,f=n.charCodeAt(e),c=f-48,o=(f|32)-97;if(c>>>0<=9)E=c;else if(o>>>0<=5)E=o+10;else{R=!0,d="hexadecimal digit expected";break}if(e++,f=n.charCodeAt(e),c=f-48,o=(f|32)-97,c>>>0<=9)E=E*16+c;else if(o>>>0<=5)E=E*16+o+10;else{R=!0,d="hexadecimal digit expected";break}e++,s+=String.fromCharCode(E);break}case 117:{if(e++,n.charCodeAt(e)!==123){R=!0,d="missing '{'";break}e++;let E=0,f=!1;for(;;){let c=n.charCodeAt(e),o=c-48,_=(c|32)-97;if(o>>>0<=9)E=E*16+o;else if(_>>>0<=5)E=E*16+_+10;else break;f=!0,e++}if(!f){R=!0,d="hexadecimal digit expected";break}if(n.charCodeAt(e)!==125){R=!0,d="missing '}'";break}if(e++,E>1114111){R=!0,d="UTF-8 value too large";break}s+=String.fromCodePoint(E);break}case 122:{for(e++;;){let E=n.charCodeAt(e);if(E===32||E===9||E===11||E===12)e++;else if(E===13)n.charCodeAt(e+1)===10?e+=2:e++,a++,u=e;else if(E===10)n.charCodeAt(e+1)===13?e+=2:e++,a++,u=e;else break}break}default:{let E=h-48;if(E>>>0<=9){let f=E;e++,E=n.charCodeAt(e)-48,E>>>0<=9&&(f=f*10+E,e++,E=n.charCodeAt(e)-48,E>>>0<=9&&(f=f*10+E,e++)),f>255?(R=!0,d="decimal escape too large"):s+=String.fromCharCode(f)}else h===0?(R=!0,d="unfinished string"):(R=!0,d="invalid escape sequence",e++);break}}if(R)break;l=e}for(;;){let h=n.charCodeAt(e);if(h===r){e++;break}else if(h===13||h===10||h===92)break;e++}return this.line=a,this.lineStart=u,this.yieldToken(1,d,e-t)}readLongStringLiteralOrComment(r,n){let t=this.raw,e=this.index,l;n?l=4+r:l=2+r;let s=e+l,a=this.line,u=this.lineStart,R=t.charCodeAt(s);R===13?(t.charCodeAt(s+1)===10?s+=2:s++,a++,u=s):R===10&&(t.charCodeAt(s+1)===13?s+=2:s++,a++,u=s);let d=s,h="",E=!1;for(;;){let c=t.charCodeAt(s);if(c===13){s>d&&(h+=t.substring(d,s)),h+=`
+`,t.charCodeAt(s+1)===10?s+=2:s++,a++,u=s,d=s;continue}else if(c===10){s>d&&(h+=t.substring(d,s)),h+=`
+`,t.charCodeAt(s+1)===13?s+=2:s++,a++,u=s,d=s;continue}else if(c===93){let o=0;for(;t.charCodeAt(s+1+o)===61;)o++;if(o===r&&t.charCodeAt(s+1+o)===93){s>d&&(h+=t.substring(d,s)),s+=2+o,E=!0;break}}else if(c===0)break;s++}let f;return E?f=this.yieldToken(n?61:58,h,s-e):(s>d&&(h+=t.substring(d,s)),f=this.yieldToken(1,n?"unfinished long comment":"unfinished long string",s-e)),this.line=a,this.lineStart=u,f}readIdentifierOrKeyword(){let r=this.raw,n=this.index,t=n+1;for(;;){let s=r.charCodeAt(t);if((s|32)-97>>>0<=25||s-48>>>0<=9||s===95)t++;else break}let e=t-n,l=r.substring(n,t);return e===2?l==="do"?this.yieldToken(4,void 0,2):l==="if"?this.yieldToken(13,void 0,2):l==="in"?this.yieldToken(14,void 0,2):l==="or"?this.yieldToken(18,void 0,2):this.yieldToken(60,l,e):e===3?l==="and"?this.yieldToken(2,void 0,3):l==="end"?this.yieldToken(7,void 0,3):l==="for"?this.yieldToken(9,void 0,3):l==="not"?this.yieldToken(17,void 0,3):l==="nil"?this.yieldToken(16,null,3):this.yieldToken(60,l,e):e===4?l==="else"?this.yieldToken(5,void 0,4):l==="goto"?this.yieldToken(12,void 0,4):l==="then"?this.yieldToken(21,void 0,4):l==="true"?this.yieldToken(22,!0,4):this.yieldToken(60,l,e):e===5?l==="break"?this.yieldToken(3,void 0,5):l==="local"?this.yieldToken(15,void 0,5):l==="false"?this.yieldToken(8,!1,5):l==="while"?this.yieldToken(24,void 0,5):this.yieldToken(60,l,e):e===6?l==="elseif"?this.yieldToken(6,void 0,6):l==="repeat"?this.yieldToken(19,void 0,6):l==="return"?this.yieldToken(20,void 0,6):l==="global"?this.yieldToken(11,void 0,6):this.yieldToken(60,l,e):e===8?l==="function"?this.yieldToken(10,void 0,8):this.yieldToken(60,l,e):this.yieldToken(60,l,e)}};function T(L){let r=new A;return r.load(L),r.tokenizeAll()}export{A as LuaLexer,I as LuaTokenType,T as tokenize};

package/package.json

@@ -0,0 +1,43 @@
+{
+    "name": "lua-lexer",
+    "version": "1.0.0",
+    "description": "A fast Lua lexer written in TypeScript",
+    "main": "./build/index.js",
+    "module": "./build/index.mjs",
+    "types": "./build/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./build/index.d.ts",
+            "import": "./build/index.mjs",
+            "require": "./build/index.js"
+        }
+    },
+    "files": [
+        "build"
+    ],
+    "scripts": {
+        "dev": "tsx source/index.ts",
+        "build": "tsup",
+        "prepare": "npm run build",
+        "bench:conditions": "tsx source/bench-conditions.ts",
+        "bench:conditions:gc": "node --expose-gc ./node_modules/tsx/dist/cli.mjs source/bench-conditions.ts",
+        "format": "prettier . --write"
+    },
+    "keywords": [
+        "lua",
+        "lexer",
+        "tokenizer",
+        "parser",
+        "typescript"
+    ],
+    "author": "",
+    "license": "MIT",
+    "type": "commonjs",
+    "devDependencies": {
+        "@types/node": "^25.5.2",
+        "prettier": "^3.8.1",
+        "tsup": "^8.5.1",
+        "tsx": "^4.21.0",
+        "typescript": "^6.0.2"
+    }
+}