functionalscript 0.6.8 → 0.6.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +2 -2
  2. package/bnf/data/module.f.d.ts +2 -0
  3. package/bnf/data/test.f.d.ts +1 -0
  4. package/bnf/data/test.f.js +11 -0
  5. package/bnf/module.f.d.ts +8 -0
  6. package/bnf/module.f.js +1 -1
  7. package/bnf/testlib.f.js +1 -0
  8. package/dev/module.f.d.ts +1 -1
  9. package/dev/module.f.js +7 -4
  10. package/djs/module.f.js +2 -1
  11. package/djs/parser/module.f.d.ts +7 -2
  12. package/djs/parser/module.f.js +80 -52
  13. package/djs/parser/test.f.d.ts +1 -0
  14. package/djs/parser/test.f.js +104 -76
  15. package/djs/tokenizer/module.f.d.ts +6 -2
  16. package/djs/tokenizer/module.f.js +12 -7
  17. package/djs/tokenizer/test.f.d.ts +1 -0
  18. package/djs/tokenizer/test.f.js +103 -87
  19. package/djs/transpiler/module.f.d.ts +3 -2
  20. package/djs/transpiler/module.f.js +3 -3
  21. package/djs/transpiler/test.f.js +2 -2
  22. package/io/module.js +5 -1
  23. package/issues/31-json.f.d.ts +1 -1
  24. package/issues/31-json.f.js +1 -1
  25. package/issues/demo/data/data.f.js +12 -0
  26. package/issues/demo/data/shared.f.js +3 -0
  27. package/issues/demo/fs/app.js +4 -0
  28. package/issues/demo/fs/math.f.js +4 -0
  29. package/issues/demo/test/test.f.js +13 -0
  30. package/js/tokenizer/module.f.d.ts +14 -2
  31. package/js/tokenizer/module.f.js +30 -17
  32. package/js/tokenizer/test.f.d.ts +1 -0
  33. package/js/tokenizer/test.f.js +159 -149
  34. package/json/parser/module.f.js +18 -9
  35. package/json/parser/test.f.js +16 -16
  36. package/json/tokenizer/module.f.d.ts +1 -1
  37. package/json/tokenizer/module.f.js +7 -6
  38. package/json/tokenizer/test.f.js +68 -68
  39. package/package.json +9 -11
package/djs/tokenizer/test.f.js CHANGED
@@ -5,421 +5,423 @@ import * as serializer from "../serializer/module.f.js";
5
5
  import * as o from "../../types/object/module.f.js";
6
6
  const { sort } = o;
7
7
  import * as encoding from "../../text/utf16/module.f.js";
8
- const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s)));
8
+ const tokenizeString = s => toArray(list.map(withoutMetada)(tokenizer.tokenize(encoding.stringToList(s))('')));
9
+ const tokenizeStringWithMetadata = s => toArray(tokenizer.tokenize(encoding.stringToList(s))(''));
9
10
  const stringify = serializer.stringifyAsTree(sort);
11
+ const withoutMetada = tokenWithMetada => { return tokenWithMetada.token; };
10
12
  export default {
11
13
  djs: [
12
14
  () => {
13
- const result = tokenizeString('');
14
- if (result.length !== 0) {
15
+ const result = stringify(tokenizeString(''));
16
+ if (result !== '[{"kind":"eof"}]') {
15
17
  throw result;
16
18
  }
17
19
  },
18
20
  () => {
19
21
  const result = stringify(tokenizeString('{'));
20
- if (result !== '[{"kind":"{"}]') {
22
+ if (result !== '[{"kind":"{"},{"kind":"eof"}]') {
21
23
  throw result;
22
24
  }
23
25
  },
24
26
  () => {
25
27
  const result = stringify(tokenizeString('}'));
26
- if (result !== '[{"kind":"}"}]') {
28
+ if (result !== '[{"kind":"}"},{"kind":"eof"}]') {
27
29
  throw result;
28
30
  }
29
31
  },
30
32
  () => {
31
33
  const result = stringify(tokenizeString(':'));
32
- if (result !== '[{"kind":":"}]') {
34
+ if (result !== '[{"kind":":"},{"kind":"eof"}]') {
33
35
  throw result;
34
36
  }
35
37
  },
36
38
  () => {
37
39
  const result = stringify(tokenizeString(','));
38
- if (result !== '[{"kind":","}]') {
40
+ if (result !== '[{"kind":","},{"kind":"eof"}]') {
39
41
  throw result;
40
42
  }
41
43
  },
42
44
  () => {
43
45
  const result = stringify(tokenizeString('['));
44
- if (result !== '[{"kind":"["}]') {
46
+ if (result !== '[{"kind":"["},{"kind":"eof"}]') {
45
47
  throw result;
46
48
  }
47
49
  },
48
50
  () => {
49
51
  const result = stringify(tokenizeString(']'));
50
- if (result !== '[{"kind":"]"}]') {
52
+ if (result !== '[{"kind":"]"},{"kind":"eof"}]') {
51
53
  throw result;
52
54
  }
53
55
  },
54
56
  () => {
55
57
  const result = stringify(tokenizeString('ᄑ'));
56
- if (result !== '[{"kind":"error","message":"unexpected character"}]') {
58
+ if (result !== '[{"kind":"error","message":"unexpected character"},{"kind":"eof"}]') {
57
59
  throw result;
58
60
  }
59
61
  },
60
62
  () => {
61
63
  const result = stringify(tokenizeString('{ \t\n\r}'));
62
- if (result !== '[{"kind":"{"},{"kind":"nl"},{"kind":"}"}]') {
64
+ if (result !== '[{"kind":"{"},{"kind":"nl"},{"kind":"}"},{"kind":"eof"}]') {
63
65
  throw result;
64
66
  }
65
67
  },
66
68
  () => {
67
69
  const result = stringify(tokenizeString('""'));
68
- if (result !== '[{"kind":"string","value":""}]') {
70
+ if (result !== '[{"kind":"string","value":""},{"kind":"eof"}]') {
69
71
  throw result;
70
72
  }
71
73
  },
72
74
  () => {
73
75
  const result = stringify(tokenizeString('"value"'));
74
- if (result !== '[{"kind":"string","value":"value"}]') {
76
+ if (result !== '[{"kind":"string","value":"value"},{"kind":"eof"}]') {
75
77
  throw result;
76
78
  }
77
79
  },
78
80
  () => {
79
81
  const result = stringify(tokenizeString('"value'));
80
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
82
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
81
83
  throw result;
82
84
  }
83
85
  },
84
86
  () => {
85
87
  const result = stringify(tokenizeString('"value1" "value2"'));
86
- if (result !== '[{"kind":"string","value":"value1"},{"kind":"ws"},{"kind":"string","value":"value2"}]') {
88
+ if (result !== '[{"kind":"string","value":"value1"},{"kind":"ws"},{"kind":"string","value":"value2"},{"kind":"eof"}]') {
87
89
  throw result;
88
90
  }
89
91
  },
90
92
  () => {
91
93
  const result = stringify(tokenizeString('"'));
92
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
94
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
93
95
  throw result;
94
96
  }
95
97
  },
96
98
  () => {
97
99
  const result = stringify(tokenizeString('"\\\\"'));
98
- if (result !== '[{"kind":"string","value":"\\\\"}]') {
100
+ if (result !== '[{"kind":"string","value":"\\\\"},{"kind":"eof"}]') {
99
101
  throw result;
100
102
  }
101
103
  },
102
104
  () => {
103
105
  const result = stringify(tokenizeString('"\\""'));
104
- if (result !== '[{"kind":"string","value":"\\""}]') {
106
+ if (result !== '[{"kind":"string","value":"\\""},{"kind":"eof"}]') {
105
107
  throw result;
106
108
  }
107
109
  },
108
110
  () => {
109
111
  const result = stringify(tokenizeString('"\\/"'));
110
- if (result !== '[{"kind":"string","value":"/"}]') {
112
+ if (result !== '[{"kind":"string","value":"/"},{"kind":"eof"}]') {
111
113
  throw result;
112
114
  }
113
115
  },
114
116
  () => {
115
117
  const result = stringify(tokenizeString('"\\x"'));
116
- if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"}]') {
118
+ if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"},{"kind":"eof"}]') {
117
119
  throw result;
118
120
  }
119
121
  },
120
122
  () => {
121
123
  const result = stringify(tokenizeString('"\\'));
122
- if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
124
+ if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
123
125
  throw result;
124
126
  }
125
127
  },
126
128
  () => {
127
129
  const result = stringify(tokenizeString('"\\b\\f\\n\\r\\t"'));
128
- if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"}]') {
130
+ if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"},{"kind":"eof"}]') {
129
131
  throw result;
130
132
  }
131
133
  },
132
134
  () => {
133
135
  const result = stringify(tokenizeString('"\\u1234"'));
134
- if (result !== '[{"kind":"string","value":"ሴ"}]') {
136
+ if (result !== '[{"kind":"string","value":"ሴ"},{"kind":"eof"}]') {
135
137
  throw result;
136
138
  }
137
139
  },
138
140
  () => {
139
141
  const result = stringify(tokenizeString('"\\uaBcDEeFf"'));
140
- if (result !== '[{"kind":"string","value":"ꯍEeFf"}]') {
142
+ if (result !== '[{"kind":"string","value":"ꯍEeFf"},{"kind":"eof"}]') {
141
143
  throw result;
142
144
  }
143
145
  },
144
146
  () => {
145
147
  const result = stringify(tokenizeString('"\\uEeFg"'));
146
- if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"}]') {
148
+ if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"},{"kind":"eof"}]') {
147
149
  throw result;
148
150
  }
149
151
  },
150
152
  () => {
151
153
  const result = stringify(tokenizeString('0'));
152
- if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"}]') {
154
+ if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
153
155
  throw result;
154
156
  }
155
157
  },
156
158
  () => {
157
159
  const result = stringify(tokenizeString('[0]'));
158
- if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"}]') {
160
+ if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') {
159
161
  throw result;
160
162
  }
161
163
  },
162
164
  () => {
163
165
  const result = stringify(tokenizeString('00'));
164
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
166
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
165
167
  throw result;
166
168
  }
167
169
  },
168
170
  () => {
169
171
  const result = stringify(tokenizeString('0abc,'));
170
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"abc"},{"kind":","}]') {
172
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"abc"},{"kind":","},{"kind":"eof"}]') {
171
173
  throw result;
172
174
  }
173
175
  },
174
176
  () => {
175
177
  const result = stringify(tokenizeString('123456789012345678901234567890'));
176
- if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"}]') {
178
+ if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') {
177
179
  throw result;
178
180
  }
179
181
  },
180
182
  () => {
181
183
  const result = stringify(tokenizeString('{90}'));
182
- if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"}]') {
184
+ if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') {
183
185
  throw result;
184
186
  }
185
187
  },
186
188
  () => {
187
189
  const result = stringify(tokenizeString('1 2'));
188
- if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
190
+ if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
189
191
  throw result;
190
192
  }
191
193
  },
192
194
  () => {
193
195
  const result = stringify(tokenizeString('0. 2'));
194
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
196
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
195
197
  throw result;
196
198
  }
197
199
  },
198
200
  () => {
199
201
  const result = stringify(tokenizeString('10-0'));
200
- if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"}]') {
202
+ if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') {
201
203
  throw result;
202
204
  }
203
205
  },
204
206
  () => {
205
207
  const result = stringify(tokenizeString('9a:'));
206
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":":"}]') {
208
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":":"},{"kind":"eof"}]') {
207
209
  throw result;
208
210
  }
209
211
  },
210
212
  () => {
211
213
  const result = stringify(tokenizeString('-10'));
212
- if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"}]') {
214
+ if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"},{"kind":"eof"}]') {
213
215
  throw result;
214
216
  }
215
217
  },
216
218
  () => {
217
219
  const result = stringify(tokenizeString('-'));
218
- if (result !== '[{"kind":"error","message":"invalid token"}]') {
220
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
219
221
  throw result;
220
222
  }
221
223
  },
222
224
  () => {
223
225
  const result = stringify(tokenizeString('--'));
224
- if (result !== '[{"kind":"error","message":"invalid token"}]') {
226
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
225
227
  throw result;
226
228
  }
227
229
  },
228
230
  () => {
229
231
  const result = stringify(tokenizeString('---'));
230
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"}]') {
232
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
231
233
  throw result;
232
234
  }
233
235
  },
234
236
  () => {
235
237
  const result = stringify(tokenizeString('-0'));
236
- if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"}]') {
238
+ if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') {
237
239
  throw result;
238
240
  }
239
241
  },
240
242
  () => {
241
243
  const result = stringify(tokenizeString('-00'));
242
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
244
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
243
245
  throw result;
244
246
  }
245
247
  },
246
248
  () => {
247
249
  const result = stringify(tokenizeString('-.123'));
248
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"}]') {
250
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') {
249
251
  throw result;
250
252
  }
251
253
  },
252
254
  () => {
253
255
  const result = stringify(tokenizeString('0.01'));
254
- if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"}]') {
256
+ if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') {
255
257
  throw result;
256
258
  }
257
259
  },
258
260
  () => {
259
261
  const result = stringify(tokenizeString('-0.9'));
260
- if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"}]') {
262
+ if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"},{"kind":"eof"}]') {
261
263
  throw result;
262
264
  }
263
265
  },
264
266
  () => {
265
267
  const result = stringify(tokenizeString('-0.'));
266
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
268
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
267
269
  throw result;
268
270
  }
269
271
  },
270
272
  () => {
271
273
  const result = stringify(tokenizeString('-0.]'));
272
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
274
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
273
275
  throw result;
274
276
  }
275
277
  },
276
278
  () => {
277
279
  const result = stringify(tokenizeString('12.34'));
278
- if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"}]') {
280
+ if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') {
279
281
  throw result;
280
282
  }
281
283
  },
282
284
  () => {
283
285
  const result = stringify(tokenizeString('-12.00'));
284
- if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"}]') {
286
+ if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"},{"kind":"eof"}]') {
285
287
  throw result;
286
288
  }
287
289
  },
288
290
  () => {
289
291
  const result = stringify(tokenizeString('-12.'));
290
- if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
292
+ if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
291
293
  throw result;
292
294
  }
293
295
  },
294
296
  () => {
295
297
  const result = stringify(tokenizeString('12.]'));
296
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
298
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
297
299
  throw result;
298
300
  }
299
301
  },
300
302
  () => {
301
303
  const result = stringify(tokenizeString('0e1'));
302
- if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"}]') {
304
+ if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') {
303
305
  throw result;
304
306
  }
305
307
  },
306
308
  () => {
307
309
  const result = stringify(tokenizeString('0e+2'));
308
- if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"}]') {
310
+ if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') {
309
311
  throw result;
310
312
  }
311
313
  },
312
314
  () => {
313
315
  const result = stringify(tokenizeString('0e-0'));
314
- if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"}]') {
316
+ if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') {
315
317
  throw result;
316
318
  }
317
319
  },
318
320
  () => {
319
321
  const result = stringify(tokenizeString('12e0000'));
320
- if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"}]') {
322
+ if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') {
321
323
  throw result;
322
324
  }
323
325
  },
324
326
  () => {
325
327
  const result = stringify(tokenizeString('-12e-0001'));
326
- if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"}]') {
328
+ if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"},{"kind":"eof"}]') {
327
329
  throw result;
328
330
  }
329
331
  },
330
332
  () => {
331
333
  const result = stringify(tokenizeString('-12.34e1234'));
332
- if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"}]') {
334
+ if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"},{"kind":"eof"}]') {
333
335
  throw result;
334
336
  }
335
337
  },
336
338
  () => {
337
339
  const result = stringify(tokenizeString('0e'));
338
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
340
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
339
341
  throw result;
340
342
  }
341
343
  },
342
344
  () => {
343
345
  const result = stringify(tokenizeString('0e-'));
344
- if (result !== '[{"kind":"error","message":"invalid number"}]') {
346
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
345
347
  throw result;
346
348
  }
347
349
  },
348
350
  () => {
349
351
  const result = stringify(tokenizeString('ABCdef1234567890$_'));
350
- if (result !== '[{"kind":"id","value":"ABCdef1234567890$_"}]') {
352
+ if (result !== '[{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"eof"}]') {
351
353
  throw result;
352
354
  }
353
355
  },
354
356
  () => {
355
357
  const result = stringify(tokenizeString('{ABCdef1234567890$_}'));
356
- if (result !== '[{"kind":"{"},{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"}"}]') {
358
+ if (result !== '[{"kind":"{"},{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"}"},{"kind":"eof"}]') {
357
359
  throw result;
358
360
  }
359
361
  },
360
362
  () => {
361
363
  const result = stringify(tokenizeString('123 _123'));
362
- if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"}]') {
364
+ if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"},{"kind":"eof"}]') {
363
365
  throw result;
364
366
  }
365
367
  },
366
368
  () => {
367
369
  const result = stringify(tokenizeString('123 $123'));
368
- if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"}]') {
370
+ if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"},{"kind":"eof"}]') {
369
371
  throw result;
370
372
  }
371
373
  },
372
374
  () => {
373
375
  const result = stringify(tokenizeString('123_123'));
374
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"_123"}]') {
376
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"_123"},{"kind":"eof"}]') {
375
377
  throw result;
376
378
  }
377
379
  },
378
380
  () => {
379
381
  const result = stringify(tokenizeString('123$123'));
380
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"$123"}]') {
382
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"$123"},{"kind":"eof"}]') {
381
383
  throw result;
382
384
  }
383
385
  },
384
386
  () => {
385
387
  const result = stringify(tokenizeString('1234567890n'));
386
- if (result !== '[{"kind":"bigint","value":1234567890n}]') {
388
+ if (result !== '[{"kind":"bigint","value":1234567890n},{"kind":"eof"}]') {
387
389
  throw result;
388
390
  }
389
391
  },
390
392
  () => {
391
393
  const result = stringify(tokenizeString('0n'));
392
- if (result !== '[{"kind":"bigint","value":0n}]') {
394
+ if (result !== '[{"kind":"bigint","value":0n},{"kind":"eof"}]') {
393
395
  throw result;
394
396
  }
395
397
  },
396
398
  () => {
397
399
  const result = stringify(tokenizeString('[-1234567890n]'));
398
- if (result !== '[{"kind":"["},{"kind":"bigint","value":-1234567890n},{"kind":"]"}]') {
400
+ if (result !== '[{"kind":"["},{"kind":"bigint","value":-1234567890n},{"kind":"]"},{"kind":"eof"}]') {
399
401
  throw result;
400
402
  }
401
403
  },
402
404
  () => {
403
405
  const result = stringify(tokenizeString('123.456n'));
404
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
406
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
405
407
  throw result;
406
408
  }
407
409
  },
408
410
  () => {
409
411
  const result = stringify(tokenizeString('123e456n'));
410
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
412
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
411
413
  throw result;
412
414
  }
413
415
  },
414
416
  () => {
415
417
  const result = stringify(tokenizeString('1234567890na'));
416
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"}]') {
418
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"a"},{"kind":"eof"}]') {
417
419
  throw result;
418
420
  }
419
421
  },
420
422
  () => {
421
423
  const result = stringify(tokenizeString('1234567890nn'));
422
- if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') {
424
+ if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"},{"kind":"eof"}]') {
423
425
  throw result;
424
426
  }
425
427
  },
@@ -427,25 +429,25 @@ export default {
427
429
  id: [
428
430
  () => {
429
431
  const result = stringify(tokenizeString('err'));
430
- if (result !== '[{"kind":"id","value":"err"}]') {
432
+ if (result !== '[{"kind":"id","value":"err"},{"kind":"eof"}]') {
431
433
  throw result;
432
434
  }
433
435
  },
434
436
  () => {
435
437
  const result = stringify(tokenizeString('{e}'));
436
- if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') {
438
+ if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"},{"kind":"eof"}]') {
437
439
  throw result;
438
440
  }
439
441
  },
440
442
  () => {
441
443
  const result = stringify(tokenizeString('tru'));
442
- if (result !== '[{"kind":"id","value":"tru"}]') {
444
+ if (result !== '[{"kind":"id","value":"tru"},{"kind":"eof"}]') {
443
445
  throw result;
444
446
  }
445
447
  },
446
448
  () => {
447
449
  const result = stringify(tokenizeString('break'));
448
- if (result !== '[{"kind":"id","value":"break"}]') {
450
+ if (result !== '[{"kind":"id","value":"break"},{"kind":"eof"}]') {
449
451
  throw result;
450
452
  }
451
453
  },
@@ -453,31 +455,31 @@ export default {
453
455
  keywords: [
454
456
  () => {
455
457
  const result = stringify(tokenizeString('true'));
456
- if (result !== '[{"kind":"true"}]') {
458
+ if (result !== '[{"kind":"true"},{"kind":"eof"}]') {
457
459
  throw result;
458
460
  }
459
461
  },
460
462
  () => {
461
463
  const result = stringify(tokenizeString('false'));
462
- if (result !== '[{"kind":"false"}]') {
464
+ if (result !== '[{"kind":"false"},{"kind":"eof"}]') {
463
465
  throw result;
464
466
  }
465
467
  },
466
468
  () => {
467
469
  const result = stringify(tokenizeString('null'));
468
- if (result !== '[{"kind":"null"}]') {
470
+ if (result !== '[{"kind":"null"},{"kind":"eof"}]') {
469
471
  throw result;
470
472
  }
471
473
  },
472
474
  () => {
473
475
  const result = stringify(tokenizeString('undefined'));
474
- if (result !== '[{"kind":"undefined"}]') {
476
+ if (result !== '[{"kind":"undefined"},{"kind":"eof"}]') {
475
477
  throw result;
476
478
  }
477
479
  },
478
480
  () => {
479
481
  const result = stringify(tokenizeString('[null]'));
480
- if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') {
482
+ if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"},{"kind":"eof"}]') {
481
483
  throw result;
482
484
  }
483
485
  },
@@ -485,43 +487,57 @@ export default {
485
487
  comments: [
486
488
  () => {
487
489
  const result = stringify(tokenizeString('//singleline comment'));
488
- if (result !== '[{"kind":"//","value":"singleline comment"}]') {
490
+ if (result !== '[{"kind":"//","value":"singleline comment"},{"kind":"eof"}]') {
489
491
  throw result;
490
492
  }
491
493
  },
492
494
  () => {
493
495
  const result = stringify(tokenizeString('true//singleline comment\nfalse'));
494
- if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"}]') {
496
+ if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"},{"kind":"eof"}]') {
495
497
  throw result;
496
498
  }
497
499
  },
498
500
  () => {
499
501
  const result = stringify(tokenizeString('/* multiline comment */'));
500
- if (result !== '[{"kind":"/*","value":" multiline comment "}]') {
502
+ if (result !== '[{"kind":"/*","value":" multiline comment "},{"kind":"eof"}]') {
501
503
  throw result;
502
504
  }
503
505
  },
504
506
  () => {
505
507
  const result = stringify(tokenizeString('/* multiline comment *'));
506
- if (result !== '[{"kind":"error","message":"*/ expected"}]') {
508
+ if (result !== '[{"kind":"error","message":"*/ expected"},{"kind":"eof"}]') {
507
509
  throw result;
508
510
  }
509
511
  },
510
512
  () => {
511
513
  const result = stringify(tokenizeString('/* multiline comment '));
512
- if (result !== '[{"kind":"error","message":"*/ expected"}]') {
514
+ if (result !== '[{"kind":"error","message":"*/ expected"},{"kind":"eof"}]') {
513
515
  throw result;
514
516
  }
515
517
  },
516
518
  () => {
517
519
  const result = stringify(tokenizeString('/* multiline comment \n * **/'));
518
- if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"}]') {
520
+ if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
519
521
  throw result;
520
522
  }
521
523
  },
522
524
  () => {
523
525
  const result = stringify(tokenizeString('/* multiline comment *\n * **/'));
524
- if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"}]') {
526
+ if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
527
+ throw result;
528
+ }
529
+ },
530
+ ],
531
+ metadata: [
532
+ () => {
533
+ const result = stringify(tokenizeStringWithMetadata('[\ntrue, -1\n]'));
534
+ if (result !== '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"["}},{"metadata":{"column":1,"line":2,"path":""},"token":{"kind":"nl"}},{"metadata":{"column":5,"line":2,"path":""},"token":{"kind":"true"}},{"metadata":{"column":6,"line":2,"path":""},"token":{"kind":","}},{"metadata":{"column":7,"line":2,"path":""},"token":{"kind":"ws"}},{"metadata":{"column":9,"line":2,"path":""},"token":{"bf":[-1n,0],"kind":"number","value":"-1"}},{"metadata":{"column":1,"line":3,"path":""},"token":{"kind":"nl"}},{"metadata":{"column":2,"line":3,"path":""},"token":{"kind":"]"}},{"metadata":{"column":2,"line":3,"path":""},"token":{"kind":"eof"}}]') {
535
+ throw result;
536
+ }
537
+ },
538
+ () => {
539
+ const result = stringify(tokenizeStringWithMetadata('[-'));
540
+ if (result !== '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"["}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"eof"}}]') {
525
541
  throw result;
526
542
  }
527
543
  },