lua-lexer 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -23
- package/build/index.d.mts +63 -2
- package/build/index.d.ts +63 -2
- package/build/index.js +5 -5
- package/build/index.mjs +5 -5
- package/package.json +14 -4
package/README.md
CHANGED
|
@@ -1,52 +1,157 @@
|
|
|
1
1
|
# lua-lexer
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://npmjs.org/package/lua-lexer)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://www.typescriptlang.org/)
|
|
6
|
+
[](https://npmjs.org/package/lua-lexer)
|
|
4
7
|
|
|
5
|
-
|
|
8
|
+
A fast, zero-dependency, and lightweight Lua lexer written in TypeScript.
|
|
9
|
+
|
|
10
|
+
This library is designed solely for tokenizing Lua source code into an array of tokens, making it a solid foundation for building parsers, formatters, syntax highlighters, or static analysis tools without the overhead of building an Abstract Syntax Tree (AST).
|
|
6
11
|
|
|
7
12
|
## Features
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
|
|
14
|
+
* **High Performance:** Optimized with zero-allocation strategies and tight loops to minimize execution time.
|
|
15
|
+
* **Zero Dependencies:** Keeps your dependency tree small and secure.
|
|
16
|
+
* **Dual API:** Choose between a standard `tokenize()` function or a Streaming API optimized for large files.
|
|
17
|
+
* **Lua 5.3+ Support:** Fully handles modern Lua lexical syntax, including bitwise operators and hexadecimal floats.
|
|
18
|
+
* **TypeScript Support:** Includes comprehensive type definitions.
|
|
19
|
+
|
|
11
20
|
|
|
12
21
|
## Installation
|
|
13
22
|
|
|
14
|
-
Install
|
|
23
|
+
Install via npm:
|
|
15
24
|
|
|
16
25
|
```bash
|
|
17
26
|
npm install lua-lexer
|
|
18
27
|
```
|
|
19
28
|
|
|
20
|
-
|
|
29
|
+
|
|
30
|
+
## Usage
|
|
31
|
+
|
|
32
|
+
### ES Modules (ESM) / TypeScript
|
|
21
33
|
|
|
22
34
|
```typescript
|
|
23
|
-
import { tokenize } from "lua-lexer";
|
|
35
|
+
import { tokenize, LuaTokenType } from "lua-lexer";
|
|
24
36
|
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
37
|
+
const tokens = tokenize("i = 0");
|
|
38
|
+
console.log(JSON.stringify(tokens, null, 2));
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### CommonJS
|
|
30
42
|
|
|
31
|
-
|
|
32
|
-
|
|
43
|
+
```javascript
|
|
44
|
+
const { tokenize, LuaTokenType } = require("lua-lexer");
|
|
45
|
+
|
|
46
|
+
const tokens = tokenize("i = 0");
|
|
47
|
+
console.log(JSON.stringify(tokens, null, 2));
|
|
33
48
|
```
|
|
34
49
|
|
|
35
|
-
|
|
50
|
+
## Lexer Interface
|
|
51
|
+
|
|
52
|
+
### 1. The `tokenize` Helper
|
|
36
53
|
|
|
37
|
-
|
|
54
|
+
The standard way to use the library is the `tokenize` function, which processes the entire string and returns an array of `LuaToken` objects:
|
|
55
|
+
|
|
56
|
+
```typescript
|
|
57
|
+
tokenize(code, options?);
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**Available Options:**
|
|
61
|
+
* `skipComments?: boolean` *(default: `false`)*: Skips parsing and yielding comment tokens to save memory and CPU cycles.
|
|
62
|
+
* `luaVersion?: "5.1" | "5.2" | "5.3" | "5.4" | "5.5"` *(default: `"5.4"`)*: The specific Lua language version to tokenize against. Determines the availability of version-specific operators, keywords, and escape sequences.
|
|
63
|
+
|
|
64
|
+
### 2. Streaming / Manual Iteration Interface
|
|
65
|
+
|
|
66
|
+
A streaming interface is also available. This is preferable when dealing with large files or when building a parser that consumes tokens on-demand.
|
|
38
67
|
|
|
39
68
|
```typescript
|
|
40
69
|
import { LuaLexer, LuaTokenType } from "lua-lexer";
|
|
41
70
|
|
|
42
|
-
const lexer = new LuaLexer();
|
|
43
|
-
lexer.load("
|
|
71
|
+
const lexer = new LuaLexer({ skipComments: true });
|
|
72
|
+
lexer.load("foo = 'bar'");
|
|
44
73
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
74
|
+
let token;
|
|
75
|
+
while (true) {
|
|
76
|
+
token = lexer.next();
|
|
77
|
+
if (token.type === LuaTokenType.END_OF_FILE) break;
|
|
78
|
+
console.log(token);
|
|
79
|
+
}
|
|
48
80
|
```
|
|
49
81
|
|
|
82
|
+
|
|
83
|
+
## Token Format
|
|
84
|
+
|
|
85
|
+
If the following code is executed:
|
|
86
|
+
|
|
87
|
+
```typescript
|
|
88
|
+
tokenize("i = 0");
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The returned array will look like:
|
|
92
|
+
|
|
93
|
+
```js
|
|
94
|
+
[
|
|
95
|
+
{
|
|
96
|
+
"type": 60,
|
|
97
|
+
"value": "i",
|
|
98
|
+
"line": 1,
|
|
99
|
+
"column": 1,
|
|
100
|
+
"start": 0,
|
|
101
|
+
"length": 1
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"type": 44,
|
|
105
|
+
"value": undefined,
|
|
106
|
+
"line": 1,
|
|
107
|
+
"column": 3,
|
|
108
|
+
"start": 2,
|
|
109
|
+
"length": 1
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"type": 59,
|
|
113
|
+
"value": 0,
|
|
114
|
+
"line": 1,
|
|
115
|
+
"column": 5,
|
|
116
|
+
"start": 4,
|
|
117
|
+
"length": 1
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"type": 0,
|
|
121
|
+
"line": 1,
|
|
122
|
+
"column": 6,
|
|
123
|
+
"start": 5,
|
|
124
|
+
"length": 0
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Token Properties:
|
|
130
|
+
* **`type`**: Expressed as an enum integer which can be matched with the exported `LuaTokenType` enum (e.g., `60` equals `LuaTokenType.IDENTIFIER`).
|
|
131
|
+
* **`value`**: The parsed value of the token (e.g., numeric value for numbers, string values for strings and identifiers).
|
|
132
|
+
* **`line`**: The 1-indexed line number where the token starts.
|
|
133
|
+
* **`column`**: The 1-indexed column number where the token starts.
|
|
134
|
+
* **`start`**: The 0-indexed character offset where the token begins in the raw source string.
|
|
135
|
+
* **`length`**: The length of the token in characters. Slicing `code.substring(start, start + length)` will return the raw token string.
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
## Performance
|
|
139
|
+
|
|
140
|
+
Below is a benchmark comparing the raw tokenization speed of `lua-lexer` against the tokenization phase of the `luaparse` package.
|
|
141
|
+
|
|
142
|
+
| Library | Avg Time (Raw Lexing) | Relative Performance |
|
|
143
|
+
| :--- | :--- | :--- |
|
|
144
|
+
| **lua-lexer** | **~0.83 ms** | **~1.50x Faster** |
|
|
145
|
+
| luaparse | ~1.24 ms | Baseline (1.0x) |
|
|
146
|
+
|
|
147
|
+
> **Note:** For full methodology, reproduction instructions, and details on optimizations like `skipComments`, please read the [Performance Benchmarks](./benchmarks/README.md) documentation.
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
## Acknowledgements
|
|
151
|
+
|
|
152
|
+
This project was heavily inspired by the excellent `luaparse` library created by Oskar Schöldström. Ultimately, `lua-lexer` started as a fun experiment with code—a playground to see just how fast and optimized a zero-dependency Lua lexer could be written in TypeScript.
|
|
153
|
+
|
|
154
|
+
|
|
50
155
|
## License
|
|
51
156
|
|
|
52
|
-
|
|
157
|
+
MIT
|
package/build/index.d.mts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the type of a scanned Lua token.
|
|
3
|
+
* Contains values for keywords, operators, literals, and special tokens.
|
|
4
|
+
*/
|
|
1
5
|
declare const enum LuaTokenType {
|
|
2
6
|
END_OF_FILE = 0,
|
|
3
7
|
ERROR = 1,
|
|
@@ -62,23 +66,80 @@ declare const enum LuaTokenType {
|
|
|
62
66
|
IDENTIFIER = 60,
|
|
63
67
|
COMMENT = 61
|
|
64
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Represents the value extracted from a token.
|
|
71
|
+
* - Strings for identifiers, string literals, and comments.
|
|
72
|
+
* - Numbers for numeric literals.
|
|
73
|
+
* - Booleans for `true` and `false`.
|
|
74
|
+
* - `null` for `nil`.
|
|
75
|
+
* - `undefined` for punctuation and generic keywords.
|
|
76
|
+
*/
|
|
65
77
|
type LuaTokenValue = string | number | boolean | null | undefined;
|
|
78
|
+
/**
|
|
79
|
+
* Represents a single lexical token emitted by the LuaLexer.
|
|
80
|
+
*/
|
|
66
81
|
interface LuaToken {
|
|
82
|
+
/** The categorized type of the token. */
|
|
67
83
|
type: LuaTokenType;
|
|
84
|
+
/** The parsed value of the token (e.g. the actual number, string content). */
|
|
68
85
|
value: LuaTokenValue;
|
|
86
|
+
/** The 1-indexed line number where the token starts. */
|
|
69
87
|
line: number;
|
|
88
|
+
/** The 0-indexed column number where the token starts. */
|
|
70
89
|
column: number;
|
|
90
|
+
/** The 0-indexed absolute string index where the token begins in the source. */
|
|
71
91
|
start: number;
|
|
92
|
+
/** The length of the token in characters. */
|
|
72
93
|
length: number;
|
|
73
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Options to configure the behavior of LuaLexer.
|
|
97
|
+
*/
|
|
98
|
+
interface LuaLexerOptions {
|
|
99
|
+
/**
|
|
100
|
+
* If true, the lexer will skip over comments and not emit them as tokens.
|
|
101
|
+
* @default false
|
|
102
|
+
*/
|
|
103
|
+
skipComments?: boolean;
|
|
104
|
+
/**
|
|
105
|
+
* The specific Lua language version to tokenize against.
|
|
106
|
+
* Determines the availability of certain operators, keywords, and escape sequences.
|
|
107
|
+
* @default "5.4"
|
|
108
|
+
*/
|
|
109
|
+
luaVersion?: "5.1" | "5.2" | "5.3" | "5.4" | "5.5";
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* A highly optimized lexical analyzer for Lua source code.
|
|
113
|
+
*/
|
|
74
114
|
declare class LuaLexer {
|
|
75
115
|
private raw;
|
|
76
116
|
private index;
|
|
77
117
|
private line;
|
|
78
118
|
private lineStart;
|
|
119
|
+
options: LuaLexerOptions;
|
|
120
|
+
private isLua52Plus;
|
|
121
|
+
private isLua53Plus;
|
|
122
|
+
private isLua55Plus;
|
|
123
|
+
private skipComments;
|
|
124
|
+
/**
|
|
125
|
+
* Creates a new instance of the LuaLexer.
|
|
126
|
+
* @param options Configuration options for the lexer behavior.
|
|
127
|
+
*/
|
|
128
|
+
constructor(options?: LuaLexerOptions);
|
|
129
|
+
/**
|
|
130
|
+
* Loads a new Lua source string into the lexer and resets its internal state.
|
|
131
|
+
* Appends an internal null-terminator sentinel to safely avoid out-of-bounds checks.
|
|
132
|
+
* @param source The raw Lua source code string to scan.
|
|
133
|
+
*/
|
|
79
134
|
load(source: string): void;
|
|
80
135
|
tokenizeAll(): LuaToken[];
|
|
81
136
|
private yieldToken;
|
|
137
|
+
/**
|
|
138
|
+
* Scans and returns the next lexical token from the loaded source code.
|
|
139
|
+
* This method automatically skips whitespaces.
|
|
140
|
+
* If the end of the file is reached, it yields an `END_OF_FILE` token.
|
|
141
|
+
* @returns The parsed LuaToken.
|
|
142
|
+
*/
|
|
82
143
|
next(): LuaToken;
|
|
83
144
|
private readDecimalLiteral;
|
|
84
145
|
private readHexadecimalLiteral;
|
|
@@ -86,6 +147,6 @@ declare class LuaLexer {
|
|
|
86
147
|
private readLongStringLiteralOrComment;
|
|
87
148
|
private readIdentifierOrKeyword;
|
|
88
149
|
}
|
|
89
|
-
declare function tokenize(code: string): LuaToken[];
|
|
150
|
+
declare function tokenize(code: string, options?: LuaLexerOptions): LuaToken[];
|
|
90
151
|
|
|
91
|
-
export { LuaLexer, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };
|
|
152
|
+
export { LuaLexer, type LuaLexerOptions, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };
|
package/build/index.d.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the type of a scanned Lua token.
|
|
3
|
+
* Contains values for keywords, operators, literals, and special tokens.
|
|
4
|
+
*/
|
|
1
5
|
declare const enum LuaTokenType {
|
|
2
6
|
END_OF_FILE = 0,
|
|
3
7
|
ERROR = 1,
|
|
@@ -62,23 +66,80 @@ declare const enum LuaTokenType {
|
|
|
62
66
|
IDENTIFIER = 60,
|
|
63
67
|
COMMENT = 61
|
|
64
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Represents the value extracted from a token.
|
|
71
|
+
* - Strings for identifiers, string literals, and comments.
|
|
72
|
+
* - Numbers for numeric literals.
|
|
73
|
+
* - Booleans for `true` and `false`.
|
|
74
|
+
* - `null` for `nil`.
|
|
75
|
+
* - `undefined` for punctuation and generic keywords.
|
|
76
|
+
*/
|
|
65
77
|
type LuaTokenValue = string | number | boolean | null | undefined;
|
|
78
|
+
/**
|
|
79
|
+
* Represents a single lexical token emitted by the LuaLexer.
|
|
80
|
+
*/
|
|
66
81
|
interface LuaToken {
|
|
82
|
+
/** The categorized type of the token. */
|
|
67
83
|
type: LuaTokenType;
|
|
84
|
+
/** The parsed value of the token (e.g. the actual number, string content). */
|
|
68
85
|
value: LuaTokenValue;
|
|
86
|
+
/** The 1-indexed line number where the token starts. */
|
|
69
87
|
line: number;
|
|
88
|
+
/** The 0-indexed column number where the token starts. */
|
|
70
89
|
column: number;
|
|
90
|
+
/** The 0-indexed absolute string index where the token begins in the source. */
|
|
71
91
|
start: number;
|
|
92
|
+
/** The length of the token in characters. */
|
|
72
93
|
length: number;
|
|
73
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Options to configure the behavior of LuaLexer.
|
|
97
|
+
*/
|
|
98
|
+
interface LuaLexerOptions {
|
|
99
|
+
/**
|
|
100
|
+
* If true, the lexer will skip over comments and not emit them as tokens.
|
|
101
|
+
* @default false
|
|
102
|
+
*/
|
|
103
|
+
skipComments?: boolean;
|
|
104
|
+
/**
|
|
105
|
+
* The specific Lua language version to tokenize against.
|
|
106
|
+
* Determines the availability of certain operators, keywords, and escape sequences.
|
|
107
|
+
* @default "5.4"
|
|
108
|
+
*/
|
|
109
|
+
luaVersion?: "5.1" | "5.2" | "5.3" | "5.4" | "5.5";
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* A highly optimized lexical analyzer for Lua source code.
|
|
113
|
+
*/
|
|
74
114
|
declare class LuaLexer {
|
|
75
115
|
private raw;
|
|
76
116
|
private index;
|
|
77
117
|
private line;
|
|
78
118
|
private lineStart;
|
|
119
|
+
options: LuaLexerOptions;
|
|
120
|
+
private isLua52Plus;
|
|
121
|
+
private isLua53Plus;
|
|
122
|
+
private isLua55Plus;
|
|
123
|
+
private skipComments;
|
|
124
|
+
/**
|
|
125
|
+
* Creates a new instance of the LuaLexer.
|
|
126
|
+
* @param options Configuration options for the lexer behavior.
|
|
127
|
+
*/
|
|
128
|
+
constructor(options?: LuaLexerOptions);
|
|
129
|
+
/**
|
|
130
|
+
* Loads a new Lua source string into the lexer and resets its internal state.
|
|
131
|
+
* Appends an internal null-terminator sentinel to safely avoid out-of-bounds checks.
|
|
132
|
+
* @param source The raw Lua source code string to scan.
|
|
133
|
+
*/
|
|
79
134
|
load(source: string): void;
|
|
80
135
|
tokenizeAll(): LuaToken[];
|
|
81
136
|
private yieldToken;
|
|
137
|
+
/**
|
|
138
|
+
* Scans and returns the next lexical token from the loaded source code.
|
|
139
|
+
* This method automatically skips whitespaces.
|
|
140
|
+
* If the end of the file is reached, it yields an `END_OF_FILE` token.
|
|
141
|
+
* @returns The parsed LuaToken.
|
|
142
|
+
*/
|
|
82
143
|
next(): LuaToken;
|
|
83
144
|
private readDecimalLiteral;
|
|
84
145
|
private readHexadecimalLiteral;
|
|
@@ -86,6 +147,6 @@ declare class LuaLexer {
|
|
|
86
147
|
private readLongStringLiteralOrComment;
|
|
87
148
|
private readIdentifierOrKeyword;
|
|
88
149
|
}
|
|
89
|
-
declare function tokenize(code: string): LuaToken[];
|
|
150
|
+
declare function tokenize(code: string, options?: LuaLexerOptions): LuaToken[];
|
|
90
151
|
|
|
91
|
-
export { LuaLexer, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };
|
|
152
|
+
export { LuaLexer, type LuaLexerOptions, type LuaToken, LuaTokenType, type LuaTokenValue, tokenize };
|
package/build/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
"use strict";var
|
|
1
|
+
"use strict";var A=Object.defineProperty;var S=Object.getOwnPropertyDescriptor;var O=Object.getOwnPropertyNames;var k=Object.prototype.hasOwnProperty;var P=(c,r)=>{for(var t in r)A(c,t,{get:r[t],enumerable:!0})},D=(c,r,t,n)=>{if(r&&typeof r=="object"||typeof r=="function")for(let e of O(r))!k.call(c,e)&&e!==t&&A(c,e,{get:()=>r[e],enumerable:!(n=S(r,e))||n.enumerable});return c};var U=c=>D(A({},"__esModule",{value:!0}),c);var x={};P(x,{LuaLexer:()=>T,LuaTokenType:()=>_,tokenize:()=>N});module.exports=U(x);var _=(i=>(i[i.END_OF_FILE=0]="END_OF_FILE",i[i.ERROR=1]="ERROR",i[i.AND=2]="AND",i[i.BREAK=3]="BREAK",i[i.DO=4]="DO",i[i.ELSE=5]="ELSE",i[i.ELSEIF=6]="ELSEIF",i[i.END=7]="END",i[i.FALSE=8]="FALSE",i[i.FOR=9]="FOR",i[i.FUNCTION=10]="FUNCTION",i[i.GLOBAL=11]="GLOBAL",i[i.GOTO=12]="GOTO",i[i.IF=13]="IF",i[i.IN=14]="IN",i[i.LOCAL=15]="LOCAL",i[i.NIL=16]="NIL",i[i.NOT=17]="NOT",i[i.OR=18]="OR",i[i.REPEAT=19]="REPEAT",i[i.RETURN=20]="RETURN",i[i.THEN=21]="THEN",i[i.TRUE=22]="TRUE",i[i.UNTIL=23]="UNTIL",i[i.WHILE=24]="WHILE",i[i.PLUS=25]="PLUS",i[i.MINUS=26]="MINUS",i[i.ASTERISK=27]="ASTERISK",i[i.SLASH=28]="SLASH",i[i.PERCENT=29]="PERCENT",i[i.CARET=30]="CARET",i[i.HASH=31]="HASH",i[i.AMPERSAND=32]="AMPERSAND",i[i.TILDE=33]="TILDE",i[i.PIPE=34]="PIPE",i[i.LESS_LESS=35]="LESS_LESS",i[i.GREATER_GREATER=36]="GREATER_GREATER",i[i.DOUBLE_SLASH=37]="DOUBLE_SLASH",i[i.EQUAL_EQUAL=38]="EQUAL_EQUAL",i[i.TILDE_EQUAL=39]="TILDE_EQUAL",i[i.LESS_EQUAL=40]="LESS_EQUAL",i[i.GREATER_EQUAL=41]="GREATER_EQUAL",i[i.LESS=42]="LESS",i[i.GREATER=43]="GREATER",i[i.EQUAL=44]="EQUAL",i[i.LEFT_PAREN=45]="LEFT_PAREN",i[i.RIGHT_PAREN=46]="RIGHT_PAREN",i[i.LEFT_BRACE=47]="LEFT_BRACE",i[i.RIGHT_BRACE=48]="RIGHT_BRACE",i[i.LEFT_BRACKET=49]="LEFT_BRACKET",i[i.RIGHT_BRACKET=50]="RIGHT_BRACKET",i[i.DOUBLE_COLON=51]="DOUBLE_COLON",i[i.SEMICOLON=52]="SEMICOLON",i[i.COLON=53]="COLON",i[i.COMMA=54]="COMMA",i[i.DOT=55]="DOT",i[i.DOUBLE_DOT=56]="DOUBLE_DOT",i[i.TRIPLE_DOT=57]="TRIPLE_DOT",i[i.STRING_LITERAL=58]="STRING_LITERAL",i[i.NUMBER_LITERAL=59]="NUMBER_LITERAL",i[i.IDENTIFIER=60]="IDENTIFIER",i[i.COMMENT=61]="COMMENT",i))(_||{}),T=class{raw="";index=0;line=1;lineStart=0;options;isLua52Plus;isLua53Plus;isLua55Plus;skipComments;constructor(r={}){this.options=r,this.skipComments=r.skipComments===!0;let t=r.luaVersion||"5.4";this.isLua52Plus=t!=="5.1",this.isLua53Plus=t!=="5.1"&&t!=="5.2",this.isLua55Plus=t==="5.5"}load(r){this.raw=r+"\0\0\0\0\0\0\0\0",this.index=0,this.line=1,this.lineStart=0}tokenizeAll(){let r=[];for(;;){let t=this.next();if(r.push(t),t.type===0)break}return r}yieldToken(r,t,n){let e={type:r,value:t,line:this.line,column:this.index-this.lineStart+1,start:this.index,length:n};return this.index+=n,this.lineStart+=n,e}next(){let r=this.raw,t=this.index,n=this.line,e=this.lineStart;for(;;){let l=r.charCodeAt(t);switch(l){case 0:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(0,void 0,0);case 9:case 11:case 12:case 32:{t++;continue}case 13:{r.charCodeAt(t+1)===10?t+=2:t++,n++,e=t;continue}case 10:{r.charCodeAt(t+1)===13?t+=2:t++,n++,e=t;continue}case 95:return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();case 34:case 39:return this.index=t,this.line=n,this.lineStart=e,this.readShortStringLiteral(l);case 48:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===120||s===88?this.readHexadecimalLiteral():this.readDecimalLiteral()}case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:return this.index=t,this.line=n,this.lineStart=e,this.readDecimalLiteral();case 46:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===46?r.charCodeAt(t+2)===46?this.yieldToken(57,void 0,3):this.yieldToken(56,void 0,2):s-48>>>0<=9?this.readDecimalLiteral():this.yieldToken(55,void 0,1)}case 47:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===47?this.isLua53Plus?this.yieldToken(37,void 0,2):this.yieldToken(28,void 0,1):this.yieldToken(28,void 0,1);case 126:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(39,void 0,2):this.isLua53Plus?this.yieldToken(33,void 0,1):this.yieldToken(1,"unexpected symbol '~'",1);case 61:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(38,void 0,2):this.yieldToken(44,void 0,1);case 58:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===58?this.isLua52Plus?this.yieldToken(51,void 0,2):this.yieldToken(53,void 0,1):this.yieldToken(53,void 0,1);case 60:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===60?this.isLua53Plus?this.yieldToken(35,void 0,2):this.yieldToken(42,void 0,1):s===61?this.yieldToken(40,void 0,2):this.yieldToken(42,void 0,1)}case 62:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===62?this.isLua53Plus?this.yieldToken(36,void 0,2):this.yieldToken(43,void 0,1):s===61?this.yieldToken(41,void 0,2):this.yieldToken(43,void 0,1)}case 91:{this.index=t,this.line=n,this.lineStart=e;let s=t+1,a=0;for(;r.charCodeAt(s)===61;)a++,s++;return r.charCodeAt(s)===91?this.readLongStringLiteralOrComment(a,!1):this.yieldToken(49,void 0,1)}case 45:{if(this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)!==45)return this.yieldToken(26,void 0,1);if(r.charCodeAt(t+2)===91){let a=t+3,f=0;for(;r.charCodeAt(a)===61;)f++,a++;if(r.charCodeAt(a)===91){let E=this.readLongStringLiteralOrComment(f,!0);if(E)return E;t=this.index,n=this.line,e=this.lineStart;continue}}let s=t+2;for(;s<r.length;){let a=r.charCodeAt(s);if(a===13||a===10)break;s++}if(this.skipComments){this.index=s,t=s;continue}return this.yieldToken(61,r.substring(t+2,s),s-t)}case 43:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(25,void 0,1);case 42:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(27,void 0,1);case 37:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(29,void 0,1);case 94:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(30,void 0,1);case 35:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(31,void 0,1);case 38:return this.index=t,this.line=n,this.lineStart=e,this.isLua53Plus?this.yieldToken(32,void 0,1):this.yieldToken(1,"unexpected symbol '&'",1);case 124:return this.index=t,this.line=n,this.lineStart=e,this.isLua53Plus?this.yieldToken(34,void 0,1):this.yieldToken(1,"unexpected symbol '|'",1);case 40:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(45,void 0,1);case 41:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(46,void 0,1);case 123:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(47,void 0,1);case 125:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(48,void 0,1);case 93:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(50,void 0,1);case 59:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(52,void 0,1);case 44:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(54,void 0,1);default:{if((l|32)-97>>>0<=25)return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();t++;continue}}}}readDecimalLiteral(){let r=this.raw,t=this.index,n=t+1,e=0,l=0,s=1,a=0,f=1,E=!1;if(r.charCodeAt(t)===46){let d=!1;for(;;){let u=r.charCodeAt(n)-48;if(u>>>0>9)break;d=!0,l=l*10+u,s*=10,n++}d||(E=!0)}else{for(e=r.charCodeAt(t)-48;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;e=e*10+d,n++}if(r.charCodeAt(n)===46)for(n++;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;l=l*10+d,s*=10,n++}}if(!E&&(r.charCodeAt(n)|32)===101){n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(f=-1,n++);let u=!1;for(;;){let h=r.charCodeAt(n)-48;if(h>>>0>9)break;u=!0,a=a*10+h,n++}u||(E=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)E=!0,n++;else break}return E?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=10**(f*a)),this.yieldToken(59,e,n-t))}readHexadecimalLiteral(){let r=this.raw,t=this.index,n=t+2,e=0,l=0,s=1,a=0,f=1,E=!1;if(r.charCodeAt(n)===46){this.isLua52Plus||(E=!0),n++;let d=!1;for(;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)l=l*16+h;else if(o>>>0<=5)l=l*16+o+10;else break;s*=16,d=!0,n++}d||(E=!0)}else{let d=!1;for(;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)e=e*16+h;else if(o>>>0<=5)e=e*16+o+10;else break;d=!0,n++}if(r.charCodeAt(n)===46)for(this.isLua52Plus||(E=!0),n++;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)l=l*16+h;else if(o>>>0<=5)l=l*16+o+10;else break;d=!0,s*=16,n++}d||(E=!0)}if(!E&&(r.charCodeAt(n)|32)===112){this.isLua52Plus||(E=!0),n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(f=-1,n++);let u=!1;for(;;){let h=r.charCodeAt(n)-48;if(h>>>0>9)break;u=!0,a=a*10+h,n++}u||(E=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)E=!0,n++;else break}return E?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=2**(f*a)),this.yieldToken(59,e,n-t))}readShortStringLiteral(r){let t=this.raw,n=this.index,e=n+1,l=e,s="",a=this.line,f=this.lineStart,E=!1,d="unfinished string";for(;;){for(;;){let h=t.charCodeAt(e);if(h===r)return s+=t.substring(l,e),this.line=a,this.lineStart=f,this.yieldToken(58,s,e-n+1);if(h===92){s+=t.substring(l,e);break}if(h===13||h===10||h===0){E=!0,d="unfinished string";break}e++}if(E)break;e++;let u=t.charCodeAt(e);switch(u){case 97:s+="\x07",e++;break;case 98:s+="\b",e++;break;case 102:s+="\f",e++;break;case 110:s+=`
|
|
2
2
|
`,e++;break;case 114:s+="\r",e++;break;case 116:s+=" ",e++;break;case 118:s+="\v",e++;break;case 34:s+='"',e++;break;case 39:s+="'",e++;break;case 92:s+="\\",e++;break;case 13:{s+=`
|
|
3
|
-
`,t.charCodeAt(e+1)===10?e+=2:e++,a++,
|
|
4
|
-
`,t.charCodeAt(e+1)===13?e+=2:e++,a++,
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
`,t.charCodeAt(e+1)===10?e+=2:e++,a++,f=e;break}case 10:{s+=`
|
|
4
|
+
`,t.charCodeAt(e+1)===13?e+=2:e++,a++,f=e;break}case 120:{if(!this.isLua52Plus){E=!0,d="invalid escape sequence",e++;break}e++;let h=0,o=t.charCodeAt(e),L=o-48,R=(o|32)-97;if(L>>>0<=9)h=L;else if(R>>>0<=5)h=R+10;else{E=!0,d="hexadecimal digit expected";break}if(e++,o=t.charCodeAt(e),L=o-48,R=(o|32)-97,L>>>0<=9)h=h*16+L;else if(R>>>0<=5)h=h*16+R+10;else{E=!0,d="hexadecimal digit expected";break}e++,s+=String.fromCharCode(h);break}case 117:{if(!this.isLua53Plus){E=!0,d="invalid escape sequence",e++;break}if(e++,t.charCodeAt(e)!==123){E=!0,d="missing '{'";break}e++;let h=0,o=!1;for(;;){let L=t.charCodeAt(e),R=L-48,I=(L|32)-97;if(R>>>0<=9)h=h*16+R;else if(I>>>0<=5)h=h*16+I+10;else break;o=!0,e++}if(!o){E=!0,d="hexadecimal digit expected";break}if(t.charCodeAt(e)!==125){E=!0,d="missing '}'";break}if(e++,h>1114111){E=!0,d="UTF-8 value too large";break}s+=String.fromCodePoint(h);break}case 122:{if(!this.isLua52Plus){E=!0,d="invalid escape sequence",e++;break}for(e++;;){let h=t.charCodeAt(e);if(h===32||h===9||h===11||h===12)e++;else if(h===13)t.charCodeAt(e+1)===10?e+=2:e++,a++,f=e;else if(h===10)t.charCodeAt(e+1)===13?e+=2:e++,a++,f=e;else break}break}default:{let h=u-48;if(h>>>0<=9){let o=h;e++,h=t.charCodeAt(e)-48,h>>>0<=9&&(o=o*10+h,e++,h=t.charCodeAt(e)-48,h>>>0<=9&&(o=o*10+h,e++)),o>255?(E=!0,d="decimal escape too large"):s+=String.fromCharCode(o)}else u===0?(E=!0,d="unfinished string"):(E=!0,d="invalid escape sequence",e++);break}}if(E)break;l=e}for(;;){let u=t.charCodeAt(e);if(u===r){e++;break}else if(u===13||u===10||u===92)break;e++}return this.line=a,this.lineStart=f,this.yieldToken(1,d,e-n)}readLongStringLiteralOrComment(r,t){let n=this.raw,e=this.index,l;t?l=4+r:l=2+r;let s=e+l,a=this.line,f=this.lineStart,E=n.charCodeAt(s);E===13?(n.charCodeAt(s+1)===10?s+=2:s++,a++,f=s):E===10&&(n.charCodeAt(s+1)===13?s+=2:s++,a++,f=s);let d=s,u="",h=!1;for(;;){let L=n.charCodeAt(s);if(L===13){(!t||!this.skipComments)&&(s>d&&(u+=n.substring(d,s)),u+=`
|
|
5
|
+
`),n.charCodeAt(s+1)===10?s+=2:s++,a++,f=s,d=s;continue}else if(L===10){(!t||!this.skipComments)&&(s>d&&(u+=n.substring(d,s)),u+=`
|
|
6
|
+
`),n.charCodeAt(s+1)===13?s+=2:s++,a++,f=s,d=s;continue}else if(L===93){let R=0;for(;n.charCodeAt(s+1+R)===61;)R++;if(R===r&&n.charCodeAt(s+1+R)===93){(!t||!this.skipComments)&&s>d&&(u+=n.substring(d,s)),s+=2+R,h=!0;break}}else if(L===0)break;s++}let o=null;return h?t&&this.skipComments?this.index=s:o=this.yieldToken(t?61:58,u,s-e):((!t||!this.skipComments)&&s>d&&(u+=n.substring(d,s)),o=this.yieldToken(1,t?"unfinished long comment":"unfinished long string",s-e)),this.line=a,this.lineStart=f,o}readIdentifierOrKeyword(){let r=this.raw,t=this.index,n=t+1;for(;;){let s=r.charCodeAt(n);if((s|32)-97>>>0<=25||s-48>>>0<=9||s===95)n++;else break}let e=n-t,l=r.substring(t,n);return e===2?l==="do"?this.yieldToken(4,void 0,2):l==="if"?this.yieldToken(13,void 0,2):l==="in"?this.yieldToken(14,void 0,2):l==="or"?this.yieldToken(18,void 0,2):this.yieldToken(60,l,e):e===3?l==="and"?this.yieldToken(2,void 0,3):l==="end"?this.yieldToken(7,void 0,3):l==="for"?this.yieldToken(9,void 0,3):l==="not"?this.yieldToken(17,void 0,3):l==="nil"?this.yieldToken(16,null,3):this.yieldToken(60,l,e):e===4?l==="else"?this.yieldToken(5,void 0,4):l==="goto"?this.isLua52Plus?this.yieldToken(12,void 0,4):this.yieldToken(60,l,4):l==="then"?this.yieldToken(21,void 0,4):l==="true"?this.yieldToken(22,!0,4):this.yieldToken(60,l,e):e===5?l==="break"?this.yieldToken(3,void 0,5):l==="local"?this.yieldToken(15,void 0,5):l==="false"?this.yieldToken(8,!1,5):l==="until"?this.yieldToken(23,void 0,5):l==="while"?this.yieldToken(24,void 0,5):this.yieldToken(60,l,e):e===6?l==="elseif"?this.yieldToken(6,void 0,6):l==="repeat"?this.yieldToken(19,void 0,6):l==="return"?this.yieldToken(20,void 0,6):l==="global"?this.isLua55Plus?this.yieldToken(11,void 0,6):this.yieldToken(60,l,6):this.yieldToken(60,l,e):e===8?l==="function"?this.yieldToken(10,void 0,8):this.yieldToken(60,l,e):this.yieldToken(60,l,e)}};function N(c,r){let t=new T(r);return t.load(c),t.tokenizeAll()}0&&(module.exports={LuaLexer,LuaTokenType,tokenize});
|
package/build/index.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
var I=(i=>(i[i.END_OF_FILE=0]="END_OF_FILE",i[i.ERROR=1]="ERROR",i[i.AND=2]="AND",i[i.BREAK=3]="BREAK",i[i.DO=4]="DO",i[i.ELSE=5]="ELSE",i[i.ELSEIF=6]="ELSEIF",i[i.END=7]="END",i[i.FALSE=8]="FALSE",i[i.FOR=9]="FOR",i[i.FUNCTION=10]="FUNCTION",i[i.GLOBAL=11]="GLOBAL",i[i.GOTO=12]="GOTO",i[i.IF=13]="IF",i[i.IN=14]="IN",i[i.LOCAL=15]="LOCAL",i[i.NIL=16]="NIL",i[i.NOT=17]="NOT",i[i.OR=18]="OR",i[i.REPEAT=19]="REPEAT",i[i.RETURN=20]="RETURN",i[i.THEN=21]="THEN",i[i.TRUE=22]="TRUE",i[i.UNTIL=23]="UNTIL",i[i.WHILE=24]="WHILE",i[i.PLUS=25]="PLUS",i[i.MINUS=26]="MINUS",i[i.ASTERISK=27]="ASTERISK",i[i.SLASH=28]="SLASH",i[i.PERCENT=29]="PERCENT",i[i.CARET=30]="CARET",i[i.HASH=31]="HASH",i[i.AMPERSAND=32]="AMPERSAND",i[i.TILDE=33]="TILDE",i[i.PIPE=34]="PIPE",i[i.LESS_LESS=35]="LESS_LESS",i[i.GREATER_GREATER=36]="GREATER_GREATER",i[i.DOUBLE_SLASH=37]="DOUBLE_SLASH",i[i.EQUAL_EQUAL=38]="EQUAL_EQUAL",i[i.TILDE_EQUAL=39]="TILDE_EQUAL",i[i.LESS_EQUAL=40]="LESS_EQUAL",i[i.GREATER_EQUAL=41]="GREATER_EQUAL",i[i.LESS=42]="LESS",i[i.GREATER=43]="GREATER",i[i.EQUAL=44]="EQUAL",i[i.LEFT_PAREN=45]="LEFT_PAREN",i[i.RIGHT_PAREN=46]="RIGHT_PAREN",i[i.LEFT_BRACE=47]="LEFT_BRACE",i[i.RIGHT_BRACE=48]="RIGHT_BRACE",i[i.LEFT_BRACKET=49]="LEFT_BRACKET",i[i.RIGHT_BRACKET=50]="RIGHT_BRACKET",i[i.DOUBLE_COLON=51]="DOUBLE_COLON",i[i.SEMICOLON=52]="SEMICOLON",i[i.COLON=53]="COLON",i[i.COMMA=54]="COMMA",i[i.DOT=55]="DOT",i[i.DOUBLE_DOT=56]="DOUBLE_DOT",i[i.TRIPLE_DOT=57]="TRIPLE_DOT",i[i.STRING_LITERAL=58]="STRING_LITERAL",i[i.NUMBER_LITERAL=59]="NUMBER_LITERAL",i[i.IDENTIFIER=60]="IDENTIFIER",i[i.COMMENT=61]="COMMENT",i))(I||{}),
|
|
1
|
+
var I=(i=>(i[i.END_OF_FILE=0]="END_OF_FILE",i[i.ERROR=1]="ERROR",i[i.AND=2]="AND",i[i.BREAK=3]="BREAK",i[i.DO=4]="DO",i[i.ELSE=5]="ELSE",i[i.ELSEIF=6]="ELSEIF",i[i.END=7]="END",i[i.FALSE=8]="FALSE",i[i.FOR=9]="FOR",i[i.FUNCTION=10]="FUNCTION",i[i.GLOBAL=11]="GLOBAL",i[i.GOTO=12]="GOTO",i[i.IF=13]="IF",i[i.IN=14]="IN",i[i.LOCAL=15]="LOCAL",i[i.NIL=16]="NIL",i[i.NOT=17]="NOT",i[i.OR=18]="OR",i[i.REPEAT=19]="REPEAT",i[i.RETURN=20]="RETURN",i[i.THEN=21]="THEN",i[i.TRUE=22]="TRUE",i[i.UNTIL=23]="UNTIL",i[i.WHILE=24]="WHILE",i[i.PLUS=25]="PLUS",i[i.MINUS=26]="MINUS",i[i.ASTERISK=27]="ASTERISK",i[i.SLASH=28]="SLASH",i[i.PERCENT=29]="PERCENT",i[i.CARET=30]="CARET",i[i.HASH=31]="HASH",i[i.AMPERSAND=32]="AMPERSAND",i[i.TILDE=33]="TILDE",i[i.PIPE=34]="PIPE",i[i.LESS_LESS=35]="LESS_LESS",i[i.GREATER_GREATER=36]="GREATER_GREATER",i[i.DOUBLE_SLASH=37]="DOUBLE_SLASH",i[i.EQUAL_EQUAL=38]="EQUAL_EQUAL",i[i.TILDE_EQUAL=39]="TILDE_EQUAL",i[i.LESS_EQUAL=40]="LESS_EQUAL",i[i.GREATER_EQUAL=41]="GREATER_EQUAL",i[i.LESS=42]="LESS",i[i.GREATER=43]="GREATER",i[i.EQUAL=44]="EQUAL",i[i.LEFT_PAREN=45]="LEFT_PAREN",i[i.RIGHT_PAREN=46]="RIGHT_PAREN",i[i.LEFT_BRACE=47]="LEFT_BRACE",i[i.RIGHT_BRACE=48]="RIGHT_BRACE",i[i.LEFT_BRACKET=49]="LEFT_BRACKET",i[i.RIGHT_BRACKET=50]="RIGHT_BRACKET",i[i.DOUBLE_COLON=51]="DOUBLE_COLON",i[i.SEMICOLON=52]="SEMICOLON",i[i.COLON=53]="COLON",i[i.COMMA=54]="COMMA",i[i.DOT=55]="DOT",i[i.DOUBLE_DOT=56]="DOUBLE_DOT",i[i.TRIPLE_DOT=57]="TRIPLE_DOT",i[i.STRING_LITERAL=58]="STRING_LITERAL",i[i.NUMBER_LITERAL=59]="NUMBER_LITERAL",i[i.IDENTIFIER=60]="IDENTIFIER",i[i.COMMENT=61]="COMMENT",i))(I||{}),c=class{raw="";index=0;line=1;lineStart=0;options;isLua52Plus;isLua53Plus;isLua55Plus;skipComments;constructor(r={}){this.options=r,this.skipComments=r.skipComments===!0;let t=r.luaVersion||"5.4";this.isLua52Plus=t!=="5.1",this.isLua53Plus=t!=="5.1"&&t!=="5.2",this.isLua55Plus=t==="5.5"}load(r){this.raw=r+"\0\0\0\0\0\0\0\0",this.index=0,this.line=1,this.lineStart=0}tokenizeAll(){let r=[];for(;;){let t=this.next();if(r.push(t),t.type===0)break}return r}yieldToken(r,t,n){let e={type:r,value:t,line:this.line,column:this.index-this.lineStart+1,start:this.index,length:n};return this.index+=n,this.lineStart+=n,e}next(){let r=this.raw,t=this.index,n=this.line,e=this.lineStart;for(;;){let l=r.charCodeAt(t);switch(l){case 0:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(0,void 0,0);case 9:case 11:case 12:case 32:{t++;continue}case 13:{r.charCodeAt(t+1)===10?t+=2:t++,n++,e=t;continue}case 10:{r.charCodeAt(t+1)===13?t+=2:t++,n++,e=t;continue}case 95:return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();case 34:case 39:return this.index=t,this.line=n,this.lineStart=e,this.readShortStringLiteral(l);case 48:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===120||s===88?this.readHexadecimalLiteral():this.readDecimalLiteral()}case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:return this.index=t,this.line=n,this.lineStart=e,this.readDecimalLiteral();case 46:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===46?r.charCodeAt(t+2)===46?this.yieldToken(57,void 0,3):this.yieldToken(56,void 0,2):s-48>>>0<=9?this.readDecimalLiteral():this.yieldToken(55,void 0,1)}case 47:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===47?this.isLua53Plus?this.yieldToken(37,void 0,2):this.yieldToken(28,void 0,1):this.yieldToken(28,void 0,1);case 126:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(39,void 0,2):this.isLua53Plus?this.yieldToken(33,void 0,1):this.yieldToken(1,"unexpected symbol '~'",1);case 61:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===61?this.yieldToken(38,void 0,2):this.yieldToken(44,void 0,1);case 58:return this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)===58?this.isLua52Plus?this.yieldToken(51,void 0,2):this.yieldToken(53,void 0,1):this.yieldToken(53,void 0,1);case 60:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===60?this.isLua53Plus?this.yieldToken(35,void 0,2):this.yieldToken(42,void 0,1):s===61?this.yieldToken(40,void 0,2):this.yieldToken(42,void 0,1)}case 62:{this.index=t,this.line=n,this.lineStart=e;let s=r.charCodeAt(t+1);return s===62?this.isLua53Plus?this.yieldToken(36,void 0,2):this.yieldToken(43,void 0,1):s===61?this.yieldToken(41,void 0,2):this.yieldToken(43,void 0,1)}case 91:{this.index=t,this.line=n,this.lineStart=e;let s=t+1,a=0;for(;r.charCodeAt(s)===61;)a++,s++;return r.charCodeAt(s)===91?this.readLongStringLiteralOrComment(a,!1):this.yieldToken(49,void 0,1)}case 45:{if(this.index=t,this.line=n,this.lineStart=e,r.charCodeAt(t+1)!==45)return this.yieldToken(26,void 0,1);if(r.charCodeAt(t+2)===91){let a=t+3,f=0;for(;r.charCodeAt(a)===61;)f++,a++;if(r.charCodeAt(a)===91){let E=this.readLongStringLiteralOrComment(f,!0);if(E)return E;t=this.index,n=this.line,e=this.lineStart;continue}}let s=t+2;for(;s<r.length;){let a=r.charCodeAt(s);if(a===13||a===10)break;s++}if(this.skipComments){this.index=s,t=s;continue}return this.yieldToken(61,r.substring(t+2,s),s-t)}case 43:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(25,void 0,1);case 42:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(27,void 0,1);case 37:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(29,void 0,1);case 94:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(30,void 0,1);case 35:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(31,void 0,1);case 38:return this.index=t,this.line=n,this.lineStart=e,this.isLua53Plus?this.yieldToken(32,void 0,1):this.yieldToken(1,"unexpected symbol '&'",1);case 124:return this.index=t,this.line=n,this.lineStart=e,this.isLua53Plus?this.yieldToken(34,void 0,1):this.yieldToken(1,"unexpected symbol '|'",1);case 40:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(45,void 0,1);case 41:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(46,void 0,1);case 123:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(47,void 0,1);case 125:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(48,void 0,1);case 93:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(50,void 0,1);case 59:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(52,void 0,1);case 44:return this.index=t,this.line=n,this.lineStart=e,this.yieldToken(54,void 0,1);default:{if((l|32)-97>>>0<=25)return this.index=t,this.line=n,this.lineStart=e,this.readIdentifierOrKeyword();t++;continue}}}}readDecimalLiteral(){let r=this.raw,t=this.index,n=t+1,e=0,l=0,s=1,a=0,f=1,E=!1;if(r.charCodeAt(t)===46){let d=!1;for(;;){let u=r.charCodeAt(n)-48;if(u>>>0>9)break;d=!0,l=l*10+u,s*=10,n++}d||(E=!0)}else{for(e=r.charCodeAt(t)-48;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;e=e*10+d,n++}if(r.charCodeAt(n)===46)for(n++;;){let d=r.charCodeAt(n)-48;if(d>>>0>9)break;l=l*10+d,s*=10,n++}}if(!E&&(r.charCodeAt(n)|32)===101){n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(f=-1,n++);let u=!1;for(;;){let h=r.charCodeAt(n)-48;if(h>>>0>9)break;u=!0,a=a*10+h,n++}u||(E=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)E=!0,n++;else break}return E?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=10**(f*a)),this.yieldToken(59,e,n-t))}readHexadecimalLiteral(){let r=this.raw,t=this.index,n=t+2,e=0,l=0,s=1,a=0,f=1,E=!1;if(r.charCodeAt(n)===46){this.isLua52Plus||(E=!0),n++;let d=!1;for(;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)l=l*16+h;else if(o>>>0<=5)l=l*16+o+10;else break;s*=16,d=!0,n++}d||(E=!0)}else{let d=!1;for(;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)e=e*16+h;else if(o>>>0<=5)e=e*16+o+10;else break;d=!0,n++}if(r.charCodeAt(n)===46)for(this.isLua52Plus||(E=!0),n++;;){let u=r.charCodeAt(n),h=u-48,o=(u|32)-97;if(h>>>0<=9)l=l*16+h;else if(o>>>0<=5)l=l*16+o+10;else break;d=!0,s*=16,n++}d||(E=!0)}if(!E&&(r.charCodeAt(n)|32)===112){this.isLua52Plus||(E=!0),n++;let d=r.charCodeAt(n);d===43?n++:d===45&&(f=-1,n++);let u=!1;for(;;){let h=r.charCodeAt(n)-48;if(h>>>0>9)break;u=!0,a=a*10+h,n++}u||(E=!0)}for(;;){let d=r.charCodeAt(n);if(d===46||(d|32)-97>>>0<=25||d-48>>>0<=9||d===95)E=!0,n++;else break}return E?this.yieldToken(1,`Malformed number: ${r.substring(t,n)}`,n-t):(l>0&&(e+=l/s),a!==0&&(e*=2**(f*a)),this.yieldToken(59,e,n-t))}readShortStringLiteral(r){let t=this.raw,n=this.index,e=n+1,l=e,s="",a=this.line,f=this.lineStart,E=!1,d="unfinished string";for(;;){for(;;){let h=t.charCodeAt(e);if(h===r)return s+=t.substring(l,e),this.line=a,this.lineStart=f,this.yieldToken(58,s,e-n+1);if(h===92){s+=t.substring(l,e);break}if(h===13||h===10||h===0){E=!0,d="unfinished string";break}e++}if(E)break;e++;let u=t.charCodeAt(e);switch(u){case 97:s+="\x07",e++;break;case 98:s+="\b",e++;break;case 102:s+="\f",e++;break;case 110:s+=`
|
|
2
2
|
`,e++;break;case 114:s+="\r",e++;break;case 116:s+=" ",e++;break;case 118:s+="\v",e++;break;case 34:s+='"',e++;break;case 39:s+="'",e++;break;case 92:s+="\\",e++;break;case 13:{s+=`
|
|
3
|
-
`,
|
|
4
|
-
`,
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
`,t.charCodeAt(e+1)===10?e+=2:e++,a++,f=e;break}case 10:{s+=`
|
|
4
|
+
`,t.charCodeAt(e+1)===13?e+=2:e++,a++,f=e;break}case 120:{if(!this.isLua52Plus){E=!0,d="invalid escape sequence",e++;break}e++;let h=0,o=t.charCodeAt(e),L=o-48,R=(o|32)-97;if(L>>>0<=9)h=L;else if(R>>>0<=5)h=R+10;else{E=!0,d="hexadecimal digit expected";break}if(e++,o=t.charCodeAt(e),L=o-48,R=(o|32)-97,L>>>0<=9)h=h*16+L;else if(R>>>0<=5)h=h*16+R+10;else{E=!0,d="hexadecimal digit expected";break}e++,s+=String.fromCharCode(h);break}case 117:{if(!this.isLua53Plus){E=!0,d="invalid escape sequence",e++;break}if(e++,t.charCodeAt(e)!==123){E=!0,d="missing '{'";break}e++;let h=0,o=!1;for(;;){let L=t.charCodeAt(e),R=L-48,A=(L|32)-97;if(R>>>0<=9)h=h*16+R;else if(A>>>0<=5)h=h*16+A+10;else break;o=!0,e++}if(!o){E=!0,d="hexadecimal digit expected";break}if(t.charCodeAt(e)!==125){E=!0,d="missing '}'";break}if(e++,h>1114111){E=!0,d="UTF-8 value too large";break}s+=String.fromCodePoint(h);break}case 122:{if(!this.isLua52Plus){E=!0,d="invalid escape sequence",e++;break}for(e++;;){let h=t.charCodeAt(e);if(h===32||h===9||h===11||h===12)e++;else if(h===13)t.charCodeAt(e+1)===10?e+=2:e++,a++,f=e;else if(h===10)t.charCodeAt(e+1)===13?e+=2:e++,a++,f=e;else break}break}default:{let h=u-48;if(h>>>0<=9){let o=h;e++,h=t.charCodeAt(e)-48,h>>>0<=9&&(o=o*10+h,e++,h=t.charCodeAt(e)-48,h>>>0<=9&&(o=o*10+h,e++)),o>255?(E=!0,d="decimal escape too large"):s+=String.fromCharCode(o)}else u===0?(E=!0,d="unfinished string"):(E=!0,d="invalid escape sequence",e++);break}}if(E)break;l=e}for(;;){let u=t.charCodeAt(e);if(u===r){e++;break}else if(u===13||u===10||u===92)break;e++}return this.line=a,this.lineStart=f,this.yieldToken(1,d,e-n)}readLongStringLiteralOrComment(r,t){let n=this.raw,e=this.index,l;t?l=4+r:l=2+r;let s=e+l,a=this.line,f=this.lineStart,E=n.charCodeAt(s);E===13?(n.charCodeAt(s+1)===10?s+=2:s++,a++,f=s):E===10&&(n.charCodeAt(s+1)===13?s+=2:s++,a++,f=s);let d=s,u="",h=!1;for(;;){let L=n.charCodeAt(s);if(L===13){(!t||!this.skipComments)&&(s>d&&(u+=n.substring(d,s)),u+=`
|
|
5
|
+
`),n.charCodeAt(s+1)===10?s+=2:s++,a++,f=s,d=s;continue}else if(L===10){(!t||!this.skipComments)&&(s>d&&(u+=n.substring(d,s)),u+=`
|
|
6
|
+
`),n.charCodeAt(s+1)===13?s+=2:s++,a++,f=s,d=s;continue}else if(L===93){let R=0;for(;n.charCodeAt(s+1+R)===61;)R++;if(R===r&&n.charCodeAt(s+1+R)===93){(!t||!this.skipComments)&&s>d&&(u+=n.substring(d,s)),s+=2+R,h=!0;break}}else if(L===0)break;s++}let o=null;return h?t&&this.skipComments?this.index=s:o=this.yieldToken(t?61:58,u,s-e):((!t||!this.skipComments)&&s>d&&(u+=n.substring(d,s)),o=this.yieldToken(1,t?"unfinished long comment":"unfinished long string",s-e)),this.line=a,this.lineStart=f,o}readIdentifierOrKeyword(){let r=this.raw,t=this.index,n=t+1;for(;;){let s=r.charCodeAt(n);if((s|32)-97>>>0<=25||s-48>>>0<=9||s===95)n++;else break}let e=n-t,l=r.substring(t,n);return e===2?l==="do"?this.yieldToken(4,void 0,2):l==="if"?this.yieldToken(13,void 0,2):l==="in"?this.yieldToken(14,void 0,2):l==="or"?this.yieldToken(18,void 0,2):this.yieldToken(60,l,e):e===3?l==="and"?this.yieldToken(2,void 0,3):l==="end"?this.yieldToken(7,void 0,3):l==="for"?this.yieldToken(9,void 0,3):l==="not"?this.yieldToken(17,void 0,3):l==="nil"?this.yieldToken(16,null,3):this.yieldToken(60,l,e):e===4?l==="else"?this.yieldToken(5,void 0,4):l==="goto"?this.isLua52Plus?this.yieldToken(12,void 0,4):this.yieldToken(60,l,4):l==="then"?this.yieldToken(21,void 0,4):l==="true"?this.yieldToken(22,!0,4):this.yieldToken(60,l,e):e===5?l==="break"?this.yieldToken(3,void 0,5):l==="local"?this.yieldToken(15,void 0,5):l==="false"?this.yieldToken(8,!1,5):l==="until"?this.yieldToken(23,void 0,5):l==="while"?this.yieldToken(24,void 0,5):this.yieldToken(60,l,e):e===6?l==="elseif"?this.yieldToken(6,void 0,6):l==="repeat"?this.yieldToken(19,void 0,6):l==="return"?this.yieldToken(20,void 0,6):l==="global"?this.isLua55Plus?this.yieldToken(11,void 0,6):this.yieldToken(60,l,6):this.yieldToken(60,l,e):e===8?l==="function"?this.yieldToken(10,void 0,8):this.yieldToken(60,l,e):this.yieldToken(60,l,e)}};function _(T,r){let t=new c(r);return t.load(T),t.tokenizeAll()}export{c as LuaLexer,I as LuaTokenType,_ as tokenize};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lua-lexer",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "A fast Lua lexer written in TypeScript",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"module": "./build/index.mjs",
|
|
@@ -15,12 +15,19 @@
|
|
|
15
15
|
"files": [
|
|
16
16
|
"build"
|
|
17
17
|
],
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/Andmarski/lua-lexer.git"
|
|
21
|
+
},
|
|
22
|
+
"bugs": {
|
|
23
|
+
"url": "https://github.com/Andmarski/lua-lexer/issues"
|
|
24
|
+
},
|
|
25
|
+
"homepage": "https://github.com/Andmarski/lua-lexer#readme",
|
|
18
26
|
"scripts": {
|
|
19
27
|
"dev": "tsx source/index.ts",
|
|
20
28
|
"build": "tsup",
|
|
21
29
|
"prepare": "npm run build",
|
|
22
|
-
"bench
|
|
23
|
-
"bench:conditions:gc": "node --expose-gc ./node_modules/tsx/dist/cli.mjs source/bench-conditions.ts",
|
|
30
|
+
"bench": "tsx benchmarks/index.ts",
|
|
24
31
|
"format": "prettier . --write"
|
|
25
32
|
},
|
|
26
33
|
"keywords": [
|
|
@@ -34,7 +41,10 @@
|
|
|
34
41
|
"license": "MIT",
|
|
35
42
|
"type": "commonjs",
|
|
36
43
|
"devDependencies": {
|
|
37
|
-
"@types/
|
|
44
|
+
"@types/luaparse": "^0.2.13",
|
|
45
|
+
"@types/node": "^25.9.5",
|
|
46
|
+
"luaparse": "^0.3.1",
|
|
47
|
+
"mitata": "^1.0.34",
|
|
38
48
|
"prettier": "^3.8.1",
|
|
39
49
|
"tsup": "^8.5.1",
|
|
40
50
|
"tsx": "^4.21.0",
|