functionalscript 0.6.8 → 0.6.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +2 -2
  2. package/bnf/data/module.f.d.ts +2 -0
  3. package/bnf/data/test.f.d.ts +1 -0
  4. package/bnf/data/test.f.js +11 -0
  5. package/bnf/module.f.d.ts +8 -0
  6. package/bnf/module.f.js +1 -1
  7. package/bnf/testlib.f.js +1 -0
  8. package/dev/module.f.d.ts +1 -1
  9. package/dev/module.f.js +7 -4
  10. package/djs/module.f.js +2 -1
  11. package/djs/parser/module.f.d.ts +7 -2
  12. package/djs/parser/module.f.js +80 -52
  13. package/djs/parser/test.f.d.ts +1 -0
  14. package/djs/parser/test.f.js +104 -76
  15. package/djs/tokenizer/module.f.d.ts +6 -2
  16. package/djs/tokenizer/module.f.js +12 -7
  17. package/djs/tokenizer/test.f.d.ts +1 -0
  18. package/djs/tokenizer/test.f.js +103 -87
  19. package/djs/transpiler/module.f.d.ts +3 -2
  20. package/djs/transpiler/module.f.js +3 -3
  21. package/djs/transpiler/test.f.js +2 -2
  22. package/io/module.js +5 -1
  23. package/issues/31-json.f.d.ts +1 -1
  24. package/issues/31-json.f.js +1 -1
  25. package/issues/demo/data/data.f.js +12 -0
  26. package/issues/demo/data/shared.f.js +3 -0
  27. package/issues/demo/fs/app.js +4 -0
  28. package/issues/demo/fs/math.f.js +4 -0
  29. package/issues/demo/test/test.f.js +13 -0
  30. package/js/tokenizer/module.f.d.ts +14 -2
  31. package/js/tokenizer/module.f.js +30 -17
  32. package/js/tokenizer/test.f.d.ts +1 -0
  33. package/js/tokenizer/test.f.js +159 -149
  34. package/json/parser/module.f.js +18 -9
  35. package/json/parser/test.f.js +16 -16
  36. package/json/tokenizer/module.f.d.ts +1 -1
  37. package/json/tokenizer/module.f.js +7 -6
  38. package/json/tokenizer/test.f.js +68 -68
  39. package/package.json +9 -11
package/js/tokenizer/test.f.js CHANGED
@@ -5,415 +5,417 @@ import * as serializer from "../../djs/serializer/module.f.js";
5
5
  import * as o from "../../types/object/module.f.js";
6
6
  const { sort } = o;
7
7
  import * as encoding from "../../text/utf16/module.f.js";
8
- const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s)));
8
+ const tokenizeString = s => toArray(list.map(withoutMetada)(tokenizer.tokenize(encoding.stringToList(s))('')));
9
+ const tokenizeStringWithMetadata = s => toArray(tokenizer.tokenize(encoding.stringToList(s))(''));
9
10
  const stringify = serializer.stringifyAsTree(sort);
11
+ const withoutMetada = tokenWithMetada => { return tokenWithMetada.token; };
10
12
  export default {
11
13
  djs: [
12
14
  () => {
13
- const result = tokenizeString('');
14
- if (result.length !== 0) {
15
+ const result = stringify(tokenizeString(''));
16
+ if (result !== '[{"kind":"eof"}]') {
15
17
  throw result;
16
18
  }
17
19
  },
18
20
  () => {
19
21
  const result = stringify(tokenizeString('{'));
20
- if (result !== '[{"kind":"{"}]') {
22
+ if (result !== '[{"kind":"{"},{"kind":"eof"}]') {
21
23
  throw result;
22
24
  }
23
25
  },
24
26
  () => {
25
27
  const result = stringify(tokenizeString('}'));
26
- if (result !== '[{"kind":"}"}]') {
28
+ if (result !== '[{"kind":"}"},{"kind":"eof"}]') {
27
29
  throw result;
28
30
  }
29
31
  },
30
32
  () => {
31
33
  const result = stringify(tokenizeString(':'));
32
- if (result !== '[{"kind":":"}]') {
34
+ if (result !== '[{"kind":":"},{"kind":"eof"}]') {
33
35
  throw result;
34
36
  }
35
37
  },
36
38
  () => {
37
39
  const result = stringify(tokenizeString(','));
38
- if (result !== '[{"kind":","}]') {
40
+ if (result !== '[{"kind":","},{"kind":"eof"}]') {
39
41
  throw result;
40
42
  }
41
43
  },
42
44
  () => {
43
45
  const result = stringify(tokenizeString('['));
44
- if (result !== '[{"kind":"["}]') {
46
+ if (result !== '[{"kind":"["},{"kind":"eof"}]') {
45
47
  throw result;
46
48
  }
47
49
  },
48
50
  () => {
49
51
  const result = stringify(tokenizeString(']'));
50
- if (result !== '[{"kind":"]"}]') {
52
+ if (result !== '[{"kind":"]"},{"kind":"eof"}]') {
51
53
  throw result;
52
54
  }
53
55
  },
54
56
  () => {
55
57
  const result = stringify(tokenizeString('ᄑ'));
56
- if (result !== '[{"kind":"error","message":"unexpected character"}]') {
58
+ if (result !== '[{"kind":"error","message":"unexpected character"},{"kind":"eof"}]') {
57
59
  throw result;
58
60
  }
59
61
  },
60
62
  () => {
61
63
  const result = stringify(tokenizeString('{ \t\n\r}'));
62
- if (result !== '[{"kind":"{"},{"kind":"nl"},{"kind":"}"}]') {
64
+ if (result !== '[{"kind":"{"},{"kind":"nl"},{"kind":"}"},{"kind":"eof"}]') {
63
65
  throw result;
64
66
  }
65
67
  },
66
68
  () => {
67
69
  const result = stringify(tokenizeString('""'));
68
- if (result !== '[{"kind":"string","value":""}]') {
70
+ if (result !== '[{"kind":"string","value":""},{"kind":"eof"}]') {
69
71
  throw result;
70
72
  }
71
73
  },
72
74
  () => {
73
75
  const result = stringify(tokenizeString('"value"'));
74
- if (result !== '[{"kind":"string","value":"value"}]') {
76
+ if (result !== '[{"kind":"string","value":"value"},{"kind":"eof"}]') {
75
77
  throw result;
76
78
  }
77
79
  },
78
80
  () => {
79
81
  const result = stringify(tokenizeString('"value'));
80
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
82
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
81
83
  throw result;
82
84
  }
83
85
  },
84
86
  () => {
85
87
  const result = stringify(tokenizeString('"value1" "value2"'));
86
- if (result !== '[{"kind":"string","value":"value1"},{"kind":"ws"},{"kind":"string","value":"value2"}]') {
88
+ if (result !== '[{"kind":"string","value":"value1"},{"kind":"ws"},{"kind":"string","value":"value2"},{"kind":"eof"}]') {
87
89
  throw result;
88
90
  }
89
91
  },
90
92
  () => {
91
93
  const result = stringify(tokenizeString('"'));
92
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
94
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
93
95
  throw result;
94
96
  }
95
97
  },
96
98
  () => {
97
99
  const result = stringify(tokenizeString('"\\\\"'));
98
- if (result !== '[{"kind":"string","value":"\\\\"}]') {
100
+ if (result !== '[{"kind":"string","value":"\\\\"},{"kind":"eof"}]') {
99
101
  throw result;
100
102
  }
101
103
  },
102
104
  () => {
103
105
  const result = stringify(tokenizeString('"\\""'));
104
- if (result !== '[{"kind":"string","value":"\\""}]') {
106
+ if (result !== '[{"kind":"string","value":"\\""},{"kind":"eof"}]') {
105
107
  throw result;
106
108
  }
107
109
  },
108
110
  () => {
109
111
  const result = stringify(tokenizeString('"\\/"'));
110
- if (result !== '[{"kind":"string","value":"/"}]') {
112
+ if (result !== '[{"kind":"string","value":"/"},{"kind":"eof"}]') {
111
113
  throw result;
112
114
  }
113
115
  },
114
116
  () => {
115
117
  const result = stringify(tokenizeString('"\\x"'));
116
- if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"}]') {
118
+ if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"},{"kind":"eof"}]') {
117
119
  throw result;
118
120
  }
119
121
  },
120
122
  () => {
121
123
  const result = stringify(tokenizeString('"\\'));
122
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
124
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
123
125
  throw result;
124
126
  }
125
127
  },
126
128
  () => {
127
129
  const result = stringify(tokenizeString('"\r"'));
128
- if (result !== '[{"kind":"error","message":"unterminated string literal"},{"kind":"nl"},{"kind":"error","message":"\\" are missing"}]') {
130
+ if (result !== '[{"kind":"error","message":"unterminated string literal"},{"kind":"nl"},{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
129
131
  throw result;
130
132
  }
131
133
  },
132
134
  () => {
133
135
  const result = stringify(tokenizeString('"\n null'));
134
- if (result !== '[{"kind":"error","message":"unterminated string literal"},{"kind":"nl"},{"kind":"null"}]') {
136
+ if (result !== '[{"kind":"error","message":"unterminated string literal"},{"kind":"nl"},{"kind":"null"},{"kind":"eof"}]') {
135
137
  throw result;
136
138
  }
137
139
  },
138
140
  () => {
139
141
  const result = stringify(tokenizeString('"\\b\\f\\n\\r\\t"'));
140
- if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"}]') {
142
+ if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"},{"kind":"eof"}]') {
141
143
  throw result;
142
144
  }
143
145
  },
144
146
  () => {
145
147
  const result = stringify(tokenizeString('"\\u1234"'));
146
- if (result !== '[{"kind":"string","value":"ሴ"}]') {
148
+ if (result !== '[{"kind":"string","value":"ሴ"},{"kind":"eof"}]') {
147
149
  throw result;
148
150
  }
149
151
  },
150
152
  () => {
151
153
  const result = stringify(tokenizeString('"\\uaBcDEeFf"'));
152
- if (result !== '[{"kind":"string","value":"ꯍEeFf"}]') {
154
+ if (result !== '[{"kind":"string","value":"ꯍEeFf"},{"kind":"eof"}]') {
153
155
  throw result;
154
156
  }
155
157
  },
156
158
  () => {
157
159
  const result = stringify(tokenizeString('"\\uEeFg"'));
158
- if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"}]') {
160
+ if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"},{"kind":"eof"}]') {
159
161
  throw result;
160
162
  }
161
163
  },
162
164
  () => {
163
165
  const result = stringify(tokenizeString('0'));
164
- if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"}]') {
166
+ if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
165
167
  throw result;
166
168
  }
167
169
  },
168
170
  () => {
169
171
  const result = stringify(tokenizeString('[0]'));
170
- if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"}]') {
172
+ if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') {
171
173
  throw result;
172
174
  }
173
175
  },
174
176
  () => {
175
177
  const result = stringify(tokenizeString('00'));
176
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
178
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
177
179
  throw result;
178
180
  }
179
181
  },
180
182
  () => {
181
183
  const result = stringify(tokenizeString('0abc,'));
182
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"abc"},{"kind":","}]') {
184
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"abc"},{"kind":","},{"kind":"eof"}]') {
183
185
  throw result;
184
186
  }
185
187
  },
186
188
  () => {
187
189
  const result = stringify(tokenizeString('123456789012345678901234567890'));
188
- if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"}]') {
190
+ if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') {
189
191
  throw result;
190
192
  }
191
193
  },
192
194
  () => {
193
195
  const result = stringify(tokenizeString('{90}'));
194
- if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"}]') {
196
+ if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') {
195
197
  throw result;
196
198
  }
197
199
  },
198
200
  () => {
199
201
  const result = stringify(tokenizeString('1 2'));
200
- if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
202
+ if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
201
203
  throw result;
202
204
  }
203
205
  },
204
206
  () => {
205
207
  const result = stringify(tokenizeString('0. 2'));
206
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
208
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
207
209
  throw result;
208
210
  }
209
211
  },
210
212
  () => {
211
213
  const result = stringify(tokenizeString('10-0'));
212
- if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"}]') {
214
+ if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
213
215
  throw result;
214
216
  }
215
217
  },
216
218
  () => {
217
219
  const result = stringify(tokenizeString('9a:'));
218
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":":"}]') {
220
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":":"},{"kind":"eof"}]') {
219
221
  throw result;
220
222
  }
221
223
  },
222
224
  () => {
223
225
  const result = stringify(tokenizeString('-10'));
224
- if (result !== '[{"kind":"-"},{"bf":[10n,0],"kind":"number","value":"10"}]') {
226
+ if (result !== '[{"kind":"-"},{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"eof"}]') {
225
227
  throw result;
226
228
  }
227
229
  },
228
230
  () => {
229
231
  const result = stringify(tokenizeString('-0'));
230
- if (result !== '[{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"}]') {
232
+ if (result !== '[{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
231
233
  throw result;
232
234
  }
233
235
  },
234
236
  () => {
235
237
  const result = stringify(tokenizeString('-00'));
236
- if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') {
238
+ if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
237
239
  throw result;
238
240
  }
239
241
  },
240
242
  () => {
241
243
  const result = stringify(tokenizeString('-.123'));
242
- if (result !== '[{"kind":"-"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"}]') {
244
+ if (result !== '[{"kind":"-"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') {
243
245
  throw result;
244
246
  }
245
247
  },
246
248
  () => {
247
249
  const result = stringify(tokenizeString('0.01'));
248
- if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"}]') {
250
+ if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') {
249
251
  throw result;
250
252
  }
251
253
  },
252
254
  () => {
253
255
  const result = stringify(tokenizeString('-0.9'));
254
- if (result !== '[{"kind":"-"},{"bf":[9n,-1],"kind":"number","value":"0.9"}]') {
256
+ if (result !== '[{"kind":"-"},{"bf":[9n,-1],"kind":"number","value":"0.9"},{"kind":"eof"}]') {
255
257
  throw result;
256
258
  }
257
259
  },
258
260
  () => {
259
261
  const result = stringify(tokenizeString('-0.'));
260
- if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') {
262
+ if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
261
263
  throw result;
262
264
  }
263
265
  },
264
266
  () => {
265
267
  const result = stringify(tokenizeString('-0.]'));
266
- if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
268
+ if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
267
269
  throw result;
268
270
  }
269
271
  },
270
272
  () => {
271
273
  const result = stringify(tokenizeString('12.34'));
272
- if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"}]') {
274
+ if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') {
273
275
  throw result;
274
276
  }
275
277
  },
276
278
  () => {
277
279
  const result = stringify(tokenizeString('-12.00'));
278
- if (result !== '[{"kind":"-"},{"bf":[1200n,-2],"kind":"number","value":"12.00"}]') {
280
+ if (result !== '[{"kind":"-"},{"bf":[1200n,-2],"kind":"number","value":"12.00"},{"kind":"eof"}]') {
279
281
  throw result;
280
282
  }
281
283
  },
282
284
  () => {
283
285
  const result = stringify(tokenizeString('-12.'));
284
- if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') {
286
+ if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
285
287
  throw result;
286
288
  }
287
289
  },
288
290
  () => {
289
291
  const result = stringify(tokenizeString('12.]'));
290
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
292
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
291
293
  throw result;
292
294
  }
293
295
  },
294
296
  () => {
295
297
  const result = stringify(tokenizeString('0e1'));
296
- if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"}]') {
298
+ if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') {
297
299
  throw result;
298
300
  }
299
301
  },
300
302
  () => {
301
303
  const result = stringify(tokenizeString('0e+2'));
302
- if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"}]') {
304
+ if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') {
303
305
  throw result;
304
306
  }
305
307
  },
306
308
  () => {
307
309
  const result = stringify(tokenizeString('0e-0'));
308
- if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"}]') {
310
+ if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') {
309
311
  throw result;
310
312
  }
311
313
  },
312
314
  () => {
313
315
  const result = stringify(tokenizeString('12e0000'));
314
- if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"}]') {
316
+ if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') {
315
317
  throw result;
316
318
  }
317
319
  },
318
320
  () => {
319
321
  const result = stringify(tokenizeString('-12e-0001'));
320
- if (result !== '[{"kind":"-"},{"bf":[12n,-1],"kind":"number","value":"12e-0001"}]') {
322
+ if (result !== '[{"kind":"-"},{"bf":[12n,-1],"kind":"number","value":"12e-0001"},{"kind":"eof"}]') {
321
323
  throw result;
322
324
  }
323
325
  },
324
326
  () => {
325
327
  const result = stringify(tokenizeString('-12.34e1234'));
326
- if (result !== '[{"kind":"-"},{"bf":[1234n,1232],"kind":"number","value":"12.34e1234"}]') {
328
+ if (result !== '[{"kind":"-"},{"bf":[1234n,1232],"kind":"number","value":"12.34e1234"},{"kind":"eof"}]') {
327
329
  throw result;
328
330
  }
329
331
  },
330
332
  () => {
331
333
  const result = stringify(tokenizeString('0e'));
332
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
334
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
333
335
  throw result;
334
336
  }
335
337
  },
336
338
  () => {
337
339
  const result = stringify(tokenizeString('0e-'));
338
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
340
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
339
341
  throw result;
340
342
  }
341
343
  },
342
344
  () => {
343
345
  const result = stringify(tokenizeString('ABCdef1234567890$_'));
344
- if (result !== '[{"kind":"id","value":"ABCdef1234567890$_"}]') {
346
+ if (result !== '[{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"eof"}]') {
345
347
  throw result;
346
348
  }
347
349
  },
348
350
  () => {
349
351
  const result = stringify(tokenizeString('{ABCdef1234567890$_}'));
350
- if (result !== '[{"kind":"{"},{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"}"}]') {
352
+ if (result !== '[{"kind":"{"},{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"}"},{"kind":"eof"}]') {
351
353
  throw result;
352
354
  }
353
355
  },
354
356
  () => {
355
357
  const result = stringify(tokenizeString('123 _123'));
356
- if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"}]') {
358
+ if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"},{"kind":"eof"}]') {
357
359
  throw result;
358
360
  }
359
361
  },
360
362
  () => {
361
363
  const result = stringify(tokenizeString('123 $123'));
362
- if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"}]') {
364
+ if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"},{"kind":"eof"}]') {
363
365
  throw result;
364
366
  }
365
367
  },
366
368
  () => {
367
369
  const result = stringify(tokenizeString('123_123'));
368
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"_123"}]') {
370
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"_123"},{"kind":"eof"}]') {
369
371
  throw result;
370
372
  }
371
373
  },
372
374
  () => {
373
375
  const result = stringify(tokenizeString('123$123'));
374
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"$123"}]') {
376
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"$123"},{"kind":"eof"}]') {
375
377
  throw result;
376
378
  }
377
379
  },
378
380
  () => {
379
381
  const result = stringify(tokenizeString('1234567890n'));
380
- if (result !== '[{"kind":"bigint","value":1234567890n}]') {
382
+ if (result !== '[{"kind":"bigint","value":1234567890n},{"kind":"eof"}]') {
381
383
  throw result;
382
384
  }
383
385
  },
384
386
  () => {
385
387
  const result = stringify(tokenizeString('0n'));
386
- if (result !== '[{"kind":"bigint","value":0n}]') {
388
+ if (result !== '[{"kind":"bigint","value":0n},{"kind":"eof"}]') {
387
389
  throw result;
388
390
  }
389
391
  },
390
392
  () => {
391
393
  const result = stringify(tokenizeString('[-1234567890n]'));
392
- if (result !== '[{"kind":"["},{"kind":"-"},{"kind":"bigint","value":1234567890n},{"kind":"]"}]') {
394
+ if (result !== '[{"kind":"["},{"kind":"-"},{"kind":"bigint","value":1234567890n},{"kind":"]"},{"kind":"eof"}]') {
393
395
  throw result;
394
396
  }
395
397
  },
396
398
  () => {
397
399
  const result = stringify(tokenizeString('123.456n'));
398
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
400
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
399
401
  throw result;
400
402
  }
401
403
  },
402
404
  () => {
403
405
  const result = stringify(tokenizeString('123e456n'));
404
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
406
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
405
407
  throw result;
406
408
  }
407
409
  },
408
410
  () => {
409
411
  const result = stringify(tokenizeString('1234567890na'));
410
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"}]') {
412
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":"eof"}]') {
411
413
  throw result;
412
414
  }
413
415
  },
414
416
  () => {
415
417
  const result = stringify(tokenizeString('1234567890nn'));
416
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
418
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
417
419
  throw result;
418
420
  }
419
421
  },
@@ -421,79 +423,79 @@ export default {
421
423
  operators: [
422
424
  () => {
423
425
  const result = stringify(tokenizeString('='));
424
- if (result !== '[{"kind":"="}]') {
426
+ if (result !== '[{"kind":"="},{"kind":"eof"}]') {
425
427
  throw result;
426
428
  }
427
429
  },
428
430
  () => {
429
431
  const result = stringify(tokenizeString('=a'));
430
- if (result !== '[{"kind":"="},{"kind":"id","value":"a"}]') {
432
+ if (result !== '[{"kind":"="},{"kind":"id","value":"a"},{"kind":"eof"}]') {
431
433
  throw result;
432
434
  }
433
435
  },
434
436
  () => {
435
437
  const result = stringify(tokenizeString('-'));
436
- if (result !== '[{"kind":"-"}]') {
438
+ if (result !== '[{"kind":"-"},{"kind":"eof"}]') {
437
439
  throw result;
438
440
  }
439
441
  },
440
442
  () => {
441
443
  const result = stringify(tokenizeString('1*2'));
442
- if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"*"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
444
+ if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"*"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
443
445
  throw result;
444
446
  }
445
447
  },
446
448
  () => {
447
449
  const result = stringify(tokenizeString('( )'));
448
- if (result !== '[{"kind":"("},{"kind":"ws"},{"kind":")"}]') {
450
+ if (result !== '[{"kind":"("},{"kind":"ws"},{"kind":")"},{"kind":"eof"}]') {
449
451
  throw result;
450
452
  }
451
453
  },
452
454
  () => {
453
455
  const result = stringify(tokenizeString('== != === !== > >= < <='));
454
- if (result !== '[{"kind":"=="},{"kind":"ws"},{"kind":"!="},{"kind":"ws"},{"kind":"==="},{"kind":"ws"},{"kind":"!=="},{"kind":"ws"},{"kind":">"},{"kind":"ws"},{"kind":">="},{"kind":"ws"},{"kind":"<"},{"kind":"ws"},{"kind":"<="}]') {
456
+ if (result !== '[{"kind":"=="},{"kind":"ws"},{"kind":"!="},{"kind":"ws"},{"kind":"==="},{"kind":"ws"},{"kind":"!=="},{"kind":"ws"},{"kind":">"},{"kind":"ws"},{"kind":">="},{"kind":"ws"},{"kind":"<"},{"kind":"ws"},{"kind":"<="},{"kind":"eof"}]') {
455
457
  throw result;
456
458
  }
457
459
  },
458
460
  () => {
459
461
  const result = stringify(tokenizeString('+ - * / % ++ -- **'));
460
- if (result !== '[{"kind":"+"},{"kind":"ws"},{"kind":"-"},{"kind":"ws"},{"kind":"*"},{"kind":"ws"},{"kind":"/"},{"kind":"ws"},{"kind":"%"},{"kind":"ws"},{"kind":"++"},{"kind":"ws"},{"kind":"--"},{"kind":"ws"},{"kind":"**"}]') {
462
+ if (result !== '[{"kind":"+"},{"kind":"ws"},{"kind":"-"},{"kind":"ws"},{"kind":"*"},{"kind":"ws"},{"kind":"/"},{"kind":"ws"},{"kind":"%"},{"kind":"ws"},{"kind":"++"},{"kind":"ws"},{"kind":"--"},{"kind":"ws"},{"kind":"**"},{"kind":"eof"}]') {
461
463
  throw result;
462
464
  }
463
465
  },
464
466
  () => {
465
467
  const result = stringify(tokenizeString('= += -= *= /= %= **='));
466
- if (result !== '[{"kind":"="},{"kind":"ws"},{"kind":"+="},{"kind":"ws"},{"kind":"-="},{"kind":"ws"},{"kind":"*="},{"kind":"ws"},{"kind":"/="},{"kind":"ws"},{"kind":"%="},{"kind":"ws"},{"kind":"**="}]') {
468
+ if (result !== '[{"kind":"="},{"kind":"ws"},{"kind":"+="},{"kind":"ws"},{"kind":"-="},{"kind":"ws"},{"kind":"*="},{"kind":"ws"},{"kind":"/="},{"kind":"ws"},{"kind":"%="},{"kind":"ws"},{"kind":"**="},{"kind":"eof"}]') {
467
469
  throw result;
468
470
  }
469
471
  },
470
472
  () => {
471
473
  const result = stringify(tokenizeString('& | ^ ~ << >> >>>'));
472
- if (result !== '[{"kind":"&"},{"kind":"ws"},{"kind":"|"},{"kind":"ws"},{"kind":"^"},{"kind":"ws"},{"kind":"~"},{"kind":"ws"},{"kind":"<<"},{"kind":"ws"},{"kind":">>"},{"kind":"ws"},{"kind":">>>"}]') {
474
+ if (result !== '[{"kind":"&"},{"kind":"ws"},{"kind":"|"},{"kind":"ws"},{"kind":"^"},{"kind":"ws"},{"kind":"~"},{"kind":"ws"},{"kind":"<<"},{"kind":"ws"},{"kind":">>"},{"kind":"ws"},{"kind":">>>"},{"kind":"eof"}]') {
473
475
  throw result;
474
476
  }
475
477
  },
476
478
  () => {
477
479
  const result = stringify(tokenizeString('&= |= ^= <<= >>= >>>='));
478
- if (result !== '[{"kind":"&="},{"kind":"ws"},{"kind":"|="},{"kind":"ws"},{"kind":"^="},{"kind":"ws"},{"kind":"<<="},{"kind":"ws"},{"kind":">>="},{"kind":"ws"},{"kind":">>>="}]') {
480
+ if (result !== '[{"kind":"&="},{"kind":"ws"},{"kind":"|="},{"kind":"ws"},{"kind":"^="},{"kind":"ws"},{"kind":"<<="},{"kind":"ws"},{"kind":">>="},{"kind":"ws"},{"kind":">>>="},{"kind":"eof"}]') {
479
481
  throw result;
480
482
  }
481
483
  },
482
484
  () => {
483
485
  const result = stringify(tokenizeString('&& || ! ??'));
484
- if (result !== '[{"kind":"&&"},{"kind":"ws"},{"kind":"||"},{"kind":"ws"},{"kind":"!"},{"kind":"ws"},{"kind":"??"}]') {
486
+ if (result !== '[{"kind":"&&"},{"kind":"ws"},{"kind":"||"},{"kind":"ws"},{"kind":"!"},{"kind":"ws"},{"kind":"??"},{"kind":"eof"}]') {
485
487
  throw result;
486
488
  }
487
489
  },
488
490
  () => {
489
491
  const result = stringify(tokenizeString('&&= ||= ??='));
490
- if (result !== '[{"kind":"&&="},{"kind":"ws"},{"kind":"||="},{"kind":"ws"},{"kind":"??="}]') {
492
+ if (result !== '[{"kind":"&&="},{"kind":"ws"},{"kind":"||="},{"kind":"ws"},{"kind":"??="},{"kind":"eof"}]') {
491
493
  throw result;
492
494
  }
493
495
  },
494
496
  () => {
495
497
  const result = stringify(tokenizeString('? ?. . =>'));
496
- if (result !== '[{"kind":"?"},{"kind":"ws"},{"kind":"?."},{"kind":"ws"},{"kind":"."},{"kind":"ws"},{"kind":"=>"}]') {
498
+ if (result !== '[{"kind":"?"},{"kind":"ws"},{"kind":"?."},{"kind":"ws"},{"kind":"."},{"kind":"ws"},{"kind":"=>"},{"kind":"eof"}]') {
497
499
  throw result;
498
500
  }
499
501
  },
@@ -501,37 +503,37 @@ export default {
501
503
  ws: [
502
504
  () => {
503
505
  const result = stringify(tokenizeString(' '));
504
- if (result !== '[{"kind":"ws"}]') {
506
+ if (result !== '[{"kind":"ws"},{"kind":"eof"}]') {
505
507
  throw result;
506
508
  }
507
509
  },
508
510
  () => {
509
511
  const result = stringify(tokenizeString('\t'));
510
- if (result !== '[{"kind":"ws"}]') {
512
+ if (result !== '[{"kind":"ws"},{"kind":"eof"}]') {
511
513
  throw result;
512
514
  }
513
515
  },
514
516
  () => {
515
517
  const result = stringify(tokenizeString(' \t'));
516
- if (result !== '[{"kind":"ws"}]') {
518
+ if (result !== '[{"kind":"ws"},{"kind":"eof"}]') {
517
519
  throw result;
518
520
  }
519
521
  },
520
522
  () => {
521
523
  const result = stringify(tokenizeString('\n'));
522
- if (result !== '[{"kind":"nl"}]') {
524
+ if (result !== '[{"kind":"nl"},{"kind":"eof"}]') {
523
525
  throw result;
524
526
  }
525
527
  },
526
528
  () => {
527
529
  const result = stringify(tokenizeString('\r'));
528
- if (result !== '[{"kind":"nl"}]') {
530
+ if (result !== '[{"kind":"nl"},{"kind":"eof"}]') {
529
531
  throw result;
530
532
  }
531
533
  },
532
534
  () => {
533
535
  const result = stringify(tokenizeString(' \t\n\r '));
534
- if (result !== '[{"kind":"nl"}]') {
536
+ if (result !== '[{"kind":"nl"},{"kind":"eof"}]') {
535
537
  throw result;
536
538
  }
537
539
  },
@@ -539,19 +541,19 @@ export default {
539
541
  id: [
540
542
  () => {
541
543
  const result = stringify(tokenizeString('err'));
542
- if (result !== '[{"kind":"id","value":"err"}]') {
544
+ if (result !== '[{"kind":"id","value":"err"},{"kind":"eof"}]') {
543
545
  throw result;
544
546
  }
545
547
  },
546
548
  () => {
547
549
  const result = stringify(tokenizeString('{e}'));
548
- if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') {
550
+ if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"},{"kind":"eof"}]') {
549
551
  throw result;
550
552
  }
551
553
  },
552
554
  () => {
553
555
  const result = stringify(tokenizeString('tru'));
554
- if (result !== '[{"kind":"id","value":"tru"}]') {
556
+ if (result !== '[{"kind":"id","value":"tru"},{"kind":"eof"}]') {
555
557
  throw result;
556
558
  }
557
559
  },
@@ -559,301 +561,301 @@ export default {
559
561
  keywords: [
560
562
  () => {
561
563
  const result = stringify(tokenizeString('true'));
562
- if (result !== '[{"kind":"true"}]') {
564
+ if (result !== '[{"kind":"true"},{"kind":"eof"}]') {
563
565
  throw result;
564
566
  }
565
567
  },
566
568
  () => {
567
569
  const result = stringify(tokenizeString('false'));
568
- if (result !== '[{"kind":"false"}]') {
570
+ if (result !== '[{"kind":"false"},{"kind":"eof"}]') {
569
571
  throw result;
570
572
  }
571
573
  },
572
574
  () => {
573
575
  const result = stringify(tokenizeString('null'));
574
- if (result !== '[{"kind":"null"}]') {
576
+ if (result !== '[{"kind":"null"},{"kind":"eof"}]') {
575
577
  throw result;
576
578
  }
577
579
  },
578
580
  () => {
579
581
  const result = stringify(tokenizeString('undefined'));
580
- if (result !== '[{"kind":"undefined"}]') {
582
+ if (result !== '[{"kind":"undefined"},{"kind":"eof"}]') {
581
583
  throw result;
582
584
  }
583
585
  },
584
586
  () => {
585
587
  const result = stringify(tokenizeString('[null]'));
586
- if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') {
588
+ if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"},{"kind":"eof"}]') {
587
589
  throw result;
588
590
  }
589
591
  },
590
592
  () => {
591
593
  const result = stringify(tokenizeString('arguments'));
592
- if (result !== '[{"kind":"arguments"}]') {
594
+ if (result !== '[{"kind":"arguments"},{"kind":"eof"}]') {
593
595
  throw result;
594
596
  }
595
597
  },
596
598
  () => {
597
599
  const result = stringify(tokenizeString('await'));
598
- if (result !== '[{"kind":"await"}]') {
600
+ if (result !== '[{"kind":"await"},{"kind":"eof"}]') {
599
601
  throw result;
600
602
  }
601
603
  },
602
604
  () => {
603
605
  const result = stringify(tokenizeString('break'));
604
- if (result !== '[{"kind":"break"}]') {
606
+ if (result !== '[{"kind":"break"},{"kind":"eof"}]') {
605
607
  throw result;
606
608
  }
607
609
  },
608
610
  () => {
609
611
  const result = stringify(tokenizeString('case'));
610
- if (result !== '[{"kind":"case"}]') {
612
+ if (result !== '[{"kind":"case"},{"kind":"eof"}]') {
611
613
  throw result;
612
614
  }
613
615
  },
614
616
  () => {
615
617
  const result = stringify(tokenizeString('catch'));
616
- if (result !== '[{"kind":"catch"}]') {
618
+ if (result !== '[{"kind":"catch"},{"kind":"eof"}]') {
617
619
  throw result;
618
620
  }
619
621
  },
620
622
  () => {
621
623
  const result = stringify(tokenizeString('class'));
622
- if (result !== '[{"kind":"class"}]') {
624
+ if (result !== '[{"kind":"class"},{"kind":"eof"}]') {
623
625
  throw result;
624
626
  }
625
627
  },
626
628
  () => {
627
629
  const result = stringify(tokenizeString('const'));
628
- if (result !== '[{"kind":"const"}]') {
630
+ if (result !== '[{"kind":"const"},{"kind":"eof"}]') {
629
631
  throw result;
630
632
  }
631
633
  },
632
634
  () => {
633
635
  const result = stringify(tokenizeString('continue'));
634
- if (result !== '[{"kind":"continue"}]') {
636
+ if (result !== '[{"kind":"continue"},{"kind":"eof"}]') {
635
637
  throw result;
636
638
  }
637
639
  },
638
640
  () => {
639
641
  const result = stringify(tokenizeString('debugger'));
640
- if (result !== '[{"kind":"debugger"}]') {
642
+ if (result !== '[{"kind":"debugger"},{"kind":"eof"}]') {
641
643
  throw result;
642
644
  }
643
645
  },
644
646
  () => {
645
647
  const result = stringify(tokenizeString('default'));
646
- if (result !== '[{"kind":"default"}]') {
648
+ if (result !== '[{"kind":"default"},{"kind":"eof"}]') {
647
649
  throw result;
648
650
  }
649
651
  },
650
652
  () => {
651
653
  const result = stringify(tokenizeString('delete'));
652
- if (result !== '[{"kind":"delete"}]') {
654
+ if (result !== '[{"kind":"delete"},{"kind":"eof"}]') {
653
655
  throw result;
654
656
  }
655
657
  },
656
658
  () => {
657
659
  const result = stringify(tokenizeString('do'));
658
- if (result !== '[{"kind":"do"}]') {
660
+ if (result !== '[{"kind":"do"},{"kind":"eof"}]') {
659
661
  throw result;
660
662
  }
661
663
  },
662
664
  () => {
663
665
  const result = stringify(tokenizeString('else'));
664
- if (result !== '[{"kind":"else"}]') {
666
+ if (result !== '[{"kind":"else"},{"kind":"eof"}]') {
665
667
  throw result;
666
668
  }
667
669
  },
668
670
  () => {
669
671
  const result = stringify(tokenizeString('enum'));
670
- if (result !== '[{"kind":"enum"}]') {
672
+ if (result !== '[{"kind":"enum"},{"kind":"eof"}]') {
671
673
  throw result;
672
674
  }
673
675
  },
674
676
  () => {
675
677
  const result = stringify(tokenizeString('eval'));
676
- if (result !== '[{"kind":"eval"}]') {
678
+ if (result !== '[{"kind":"eval"},{"kind":"eof"}]') {
677
679
  throw result;
678
680
  }
679
681
  },
680
682
  () => {
681
683
  const result = stringify(tokenizeString('export'));
682
- if (result !== '[{"kind":"export"}]') {
684
+ if (result !== '[{"kind":"export"},{"kind":"eof"}]') {
683
685
  throw result;
684
686
  }
685
687
  },
686
688
  () => {
687
689
  const result = stringify(tokenizeString('extends'));
688
- if (result !== '[{"kind":"extends"}]') {
690
+ if (result !== '[{"kind":"extends"},{"kind":"eof"}]') {
689
691
  throw result;
690
692
  }
691
693
  },
692
694
  () => {
693
695
  const result = stringify(tokenizeString('finally'));
694
- if (result !== '[{"kind":"finally"}]') {
696
+ if (result !== '[{"kind":"finally"},{"kind":"eof"}]') {
695
697
  throw result;
696
698
  }
697
699
  },
698
700
  () => {
699
701
  const result = stringify(tokenizeString('for'));
700
- if (result !== '[{"kind":"for"}]') {
702
+ if (result !== '[{"kind":"for"},{"kind":"eof"}]') {
701
703
  throw result;
702
704
  }
703
705
  },
704
706
  () => {
705
707
  const result = stringify(tokenizeString('function'));
706
- if (result !== '[{"kind":"function"}]') {
708
+ if (result !== '[{"kind":"function"},{"kind":"eof"}]') {
707
709
  throw result;
708
710
  }
709
711
  },
710
712
  () => {
711
713
  const result = stringify(tokenizeString('if'));
712
- if (result !== '[{"kind":"if"}]') {
714
+ if (result !== '[{"kind":"if"},{"kind":"eof"}]') {
713
715
  throw result;
714
716
  }
715
717
  },
716
718
  () => {
717
719
  const result = stringify(tokenizeString('implements'));
718
- if (result !== '[{"kind":"implements"}]') {
720
+ if (result !== '[{"kind":"implements"},{"kind":"eof"}]') {
719
721
  throw result;
720
722
  }
721
723
  },
722
724
  () => {
723
725
  const result = stringify(tokenizeString('import'));
724
- if (result !== '[{"kind":"import"}]') {
726
+ if (result !== '[{"kind":"import"},{"kind":"eof"}]') {
725
727
  throw result;
726
728
  }
727
729
  },
728
730
  () => {
729
731
  const result = stringify(tokenizeString('in'));
730
- if (result !== '[{"kind":"in"}]') {
732
+ if (result !== '[{"kind":"in"},{"kind":"eof"}]') {
731
733
  throw result;
732
734
  }
733
735
  },
734
736
  () => {
735
737
  const result = stringify(tokenizeString('instanceof'));
736
- if (result !== '[{"kind":"instanceof"}]') {
738
+ if (result !== '[{"kind":"instanceof"},{"kind":"eof"}]') {
737
739
  throw result;
738
740
  }
739
741
  },
740
742
  () => {
741
743
  const result = stringify(tokenizeString('interface'));
742
- if (result !== '[{"kind":"interface"}]') {
744
+ if (result !== '[{"kind":"interface"},{"kind":"eof"}]') {
743
745
  throw result;
744
746
  }
745
747
  },
746
748
  () => {
747
749
  const result = stringify(tokenizeString('let'));
748
- if (result !== '[{"kind":"let"}]') {
750
+ if (result !== '[{"kind":"let"},{"kind":"eof"}]') {
749
751
  throw result;
750
752
  }
751
753
  },
752
754
  () => {
753
755
  const result = stringify(tokenizeString('new'));
754
- if (result !== '[{"kind":"new"}]') {
756
+ if (result !== '[{"kind":"new"},{"kind":"eof"}]') {
755
757
  throw result;
756
758
  }
757
759
  },
758
760
  () => {
759
761
  const result = stringify(tokenizeString('package'));
760
- if (result !== '[{"kind":"package"}]') {
762
+ if (result !== '[{"kind":"package"},{"kind":"eof"}]') {
761
763
  throw result;
762
764
  }
763
765
  },
764
766
  () => {
765
767
  const result = stringify(tokenizeString('private'));
766
- if (result !== '[{"kind":"private"}]') {
768
+ if (result !== '[{"kind":"private"},{"kind":"eof"}]') {
767
769
  throw result;
768
770
  }
769
771
  },
770
772
  () => {
771
773
  const result = stringify(tokenizeString('protected'));
772
- if (result !== '[{"kind":"protected"}]') {
774
+ if (result !== '[{"kind":"protected"},{"kind":"eof"}]') {
773
775
  throw result;
774
776
  }
775
777
  },
776
778
  () => {
777
779
  const result = stringify(tokenizeString('public'));
778
- if (result !== '[{"kind":"public"}]') {
780
+ if (result !== '[{"kind":"public"},{"kind":"eof"}]') {
779
781
  throw result;
780
782
  }
781
783
  },
782
784
  () => {
783
785
  const result = stringify(tokenizeString('return'));
784
- if (result !== '[{"kind":"return"}]') {
786
+ if (result !== '[{"kind":"return"},{"kind":"eof"}]') {
785
787
  throw result;
786
788
  }
787
789
  },
788
790
  () => {
789
791
  const result = stringify(tokenizeString('static'));
790
- if (result !== '[{"kind":"static"}]') {
792
+ if (result !== '[{"kind":"static"},{"kind":"eof"}]') {
791
793
  throw result;
792
794
  }
793
795
  },
794
796
  () => {
795
797
  const result = stringify(tokenizeString('super'));
796
- if (result !== '[{"kind":"super"}]') {
798
+ if (result !== '[{"kind":"super"},{"kind":"eof"}]') {
797
799
  throw result;
798
800
  }
799
801
  },
800
802
  () => {
801
803
  const result = stringify(tokenizeString('switch'));
802
- if (result !== '[{"kind":"switch"}]') {
804
+ if (result !== '[{"kind":"switch"},{"kind":"eof"}]') {
803
805
  throw result;
804
806
  }
805
807
  },
806
808
  () => {
807
809
  const result = stringify(tokenizeString('this'));
808
- if (result !== '[{"kind":"this"}]') {
810
+ if (result !== '[{"kind":"this"},{"kind":"eof"}]') {
809
811
  throw result;
810
812
  }
811
813
  },
812
814
  () => {
813
815
  const result = stringify(tokenizeString('throw'));
814
- if (result !== '[{"kind":"throw"}]') {
816
+ if (result !== '[{"kind":"throw"},{"kind":"eof"}]') {
815
817
  throw result;
816
818
  }
817
819
  },
818
820
  () => {
819
821
  const result = stringify(tokenizeString('try'));
820
- if (result !== '[{"kind":"try"}]') {
822
+ if (result !== '[{"kind":"try"},{"kind":"eof"}]') {
821
823
  throw result;
822
824
  }
823
825
  },
824
826
  () => {
825
827
  const result = stringify(tokenizeString('typeof'));
826
- if (result !== '[{"kind":"typeof"}]') {
828
+ if (result !== '[{"kind":"typeof"},{"kind":"eof"}]') {
827
829
  throw result;
828
830
  }
829
831
  },
830
832
  () => {
831
833
  const result = stringify(tokenizeString('var'));
832
- if (result !== '[{"kind":"var"}]') {
834
+ if (result !== '[{"kind":"var"},{"kind":"eof"}]') {
833
835
  throw result;
834
836
  }
835
837
  },
836
838
  () => {
837
839
  const result = stringify(tokenizeString('void'));
838
- if (result !== '[{"kind":"void"}]') {
840
+ if (result !== '[{"kind":"void"},{"kind":"eof"}]') {
839
841
  throw result;
840
842
  }
841
843
  },
842
844
  () => {
843
845
  const result = stringify(tokenizeString('while'));
844
- if (result !== '[{"kind":"while"}]') {
846
+ if (result !== '[{"kind":"while"},{"kind":"eof"}]') {
845
847
  throw result;
846
848
  }
847
849
  },
848
850
  () => {
849
851
  const result = stringify(tokenizeString('with'));
850
- if (result !== '[{"kind":"with"}]') {
852
+ if (result !== '[{"kind":"with"},{"kind":"eof"}]') {
851
853
  throw result;
852
854
  }
853
855
  },
854
856
  () => {
855
857
  const result = stringify(tokenizeString('yield'));
856
- if (result !== '[{"kind":"yield"}]') {
858
+ if (result !== '[{"kind":"yield"},{"kind":"eof"}]') {
857
859
  throw result;
858
860
  }
859
861
  },
@@ -861,43 +863,51 @@ export default {
861
863
  comments: [
862
864
  () => {
863
865
  const result = stringify(tokenizeString('//singleline comment'));
864
- if (result !== '[{"kind":"//","value":"singleline comment"}]') {
866
+ if (result !== '[{"kind":"//","value":"singleline comment"},{"kind":"eof"}]') {
865
867
  throw result;
866
868
  }
867
869
  },
868
870
  () => {
869
871
  const result = stringify(tokenizeString('true//singleline comment\nfalse'));
870
- if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"}]') {
872
+ if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"},{"kind":"eof"}]') {
871
873
  throw result;
872
874
  }
873
875
  },
874
876
  () => {
875
877
  const result = stringify(tokenizeString('/* multiline comment */'));
876
- if (result !== '[{"kind":"/*","value":" multiline comment "}]') {
878
+ if (result !== '[{"kind":"/*","value":" multiline comment "},{"kind":"eof"}]') {
877
879
  throw result;
878
880
  }
879
881
  },
880
882
  () => {
881
883
  const result = stringify(tokenizeString('/* multiline comment *'));
882
- if (result !== '[{"kind":"error","message":"*/ expected"}]') {
884
+ if (result !== '[{"kind":"error","message":"*/ expected"},{"kind":"eof"}]') {
883
885
  throw result;
884
886
  }
885
887
  },
886
888
  () => {
887
889
  const result = stringify(tokenizeString('/* multiline comment '));
888
- if (result !== '[{"kind":"error","message":"*/ expected"}]') {
890
+ if (result !== '[{"kind":"error","message":"*/ expected"},{"kind":"eof"}]') {
889
891
  throw result;
890
892
  }
891
893
  },
892
894
  () => {
893
895
  const result = stringify(tokenizeString('/* multiline comment \n * **/'));
894
- if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"}]') {
896
+ if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
895
897
  throw result;
896
898
  }
897
899
  },
898
900
  () => {
899
901
  const result = stringify(tokenizeString('/* multiline comment *\n * **/'));
900
- if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"}]') {
902
+ if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
903
+ throw result;
904
+ }
905
+ },
906
+ ],
907
+ metadata: [
908
+ () => {
909
+ const result = stringify(tokenizeStringWithMetadata('[\ntrue, false\n]'));
910
+ if (result !== '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"["}},{"metadata":{"column":1,"line":2,"path":""},"token":{"kind":"nl"}},{"metadata":{"column":5,"line":2,"path":""},"token":{"kind":"true"}},{"metadata":{"column":6,"line":2,"path":""},"token":{"kind":","}},{"metadata":{"column":7,"line":2,"path":""},"token":{"kind":"ws"}},{"metadata":{"column":12,"line":2,"path":""},"token":{"kind":"false"}},{"metadata":{"column":1,"line":3,"path":""},"token":{"kind":"nl"}},{"metadata":{"column":2,"line":3,"path":""},"token":{"kind":"]"}},{"metadata":{"column":2,"line":3,"path":""},"token":{"kind":"eof"}}]') {
901
911
  throw result;
902
912
  }
903
913
  },