functionalscript 0.0.326 → 0.0.327
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/json/tokenizer/index.js +31 -28
- package/package.json +1 -1
package/json/tokenizer/index.js
CHANGED
|
@@ -15,18 +15,18 @@ const list = require('../../types/list')
|
|
|
15
15
|
|
|
16
16
|
/** @typedef {{readonly kind: ']'}} RightBracketToken */
|
|
17
17
|
|
|
18
|
-
/**
|
|
18
|
+
/**
|
|
19
19
|
* @typedef {{
|
|
20
20
|
* readonly kind: 'string'
|
|
21
21
|
* readonly value: string
|
|
22
|
-
* }} StringToken
|
|
22
|
+
* }} StringToken
|
|
23
23
|
* */
|
|
24
24
|
|
|
25
|
-
/**
|
|
25
|
+
/**
|
|
26
26
|
* @typedef {{
|
|
27
27
|
* readonly kind: 'number'
|
|
28
28
|
* readonly value: string
|
|
29
|
-
* }} NumberToken
|
|
29
|
+
* }} NumberToken
|
|
30
30
|
* */
|
|
31
31
|
|
|
32
32
|
/** @typedef {{readonly kind: 'true'}} TrueToken */
|
|
@@ -37,7 +37,7 @@ const list = require('../../types/list')
|
|
|
37
37
|
|
|
38
38
|
/** @typedef {{readonly kind: 'error', message: ErrorMessage}} ErrorToken */
|
|
39
39
|
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
41
|
* @typedef {|
|
|
42
42
|
* LeftBraceToken |
|
|
43
43
|
* RightBraceToken |
|
|
@@ -93,7 +93,7 @@ const letterT = 0x74
|
|
|
93
93
|
const letterU = 0x75
|
|
94
94
|
const letterZ = 0x7a
|
|
95
95
|
|
|
96
|
-
/**
|
|
96
|
+
/**
|
|
97
97
|
* @typedef {|
|
|
98
98
|
* InitialState |
|
|
99
99
|
* ParseKeywordState |
|
|
@@ -103,10 +103,10 @@ const letterZ = 0x7a
|
|
|
103
103
|
* ParseNumberState |
|
|
104
104
|
* InvalidNumberState |
|
|
105
105
|
* EofState
|
|
106
|
-
* } TokenizerState
|
|
106
|
+
* } TokenizerState
|
|
107
107
|
*/
|
|
108
108
|
|
|
109
|
-
/**
|
|
109
|
+
/**
|
|
110
110
|
* @typedef {|
|
|
111
111
|
* 'invalid keyword' |
|
|
112
112
|
* '" are missing' |
|
|
@@ -115,7 +115,7 @@ const letterZ = 0x7a
|
|
|
115
115
|
* 'unexpected character' |
|
|
116
116
|
* 'invalid number' |
|
|
117
117
|
* 'eof'
|
|
118
|
-
* } ErrorMessage
|
|
118
|
+
* } ErrorMessage
|
|
119
119
|
*/
|
|
120
120
|
|
|
121
121
|
/** @typedef {{ readonly kind: 'initial'}} InitialState */
|
|
@@ -129,7 +129,7 @@ const letterZ = 0x7a
|
|
|
129
129
|
/** @typedef {{ readonly kind: 'unicodeChar', readonly value: string, readonly unicode: number, readonly hexIndex: number}} ParseUnicodeCharState */
|
|
130
130
|
|
|
131
131
|
/**
|
|
132
|
-
* @typedef {{
|
|
132
|
+
* @typedef {{
|
|
133
133
|
* readonly kind: 'number',
|
|
134
134
|
* readonly numberKind: '0' | '-' | 'int' | '.' | 'fractional' | 'e' | 'e+' | 'e-' | 'expDigits'
|
|
135
135
|
* readonly value: string
|
|
@@ -149,11 +149,11 @@ const appendChar = old => input => input === undefined ? old : operator.concat(c
|
|
|
149
149
|
const charToString = input => input === undefined ? '' : String.fromCharCode(input)
|
|
150
150
|
|
|
151
151
|
/** @type {(state: InitialState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
152
|
-
const initialStateOp = initialState => input =>
|
|
152
|
+
const initialStateOp = initialState => input =>
|
|
153
153
|
{
|
|
154
|
-
if (input === undefined)
|
|
154
|
+
if (input === undefined)
|
|
155
155
|
{
|
|
156
|
-
return[undefined, {kind: 'eof'}]
|
|
156
|
+
return[undefined, {kind: 'eof'}]
|
|
157
157
|
}
|
|
158
158
|
if (input >= digit1 && input <= digit9)
|
|
159
159
|
{
|
|
@@ -193,7 +193,7 @@ const parseNumberStateOp = state => input =>
|
|
|
193
193
|
case '.':
|
|
194
194
|
case 'e':
|
|
195
195
|
case 'e+':
|
|
196
|
-
case 'e-': return [[{kind: 'error', message: 'invalid number'}], {kind: 'invalidNumber', }]
|
|
196
|
+
case 'e-': return [[{kind: 'error', message: 'invalid number'}], {kind: 'invalidNumber', }]
|
|
197
197
|
default: return [[{kind: 'number', value: state.value}], {kind: 'eof'}]
|
|
198
198
|
}
|
|
199
199
|
}
|
|
@@ -266,12 +266,12 @@ const parseNumberStateOp = state => input =>
|
|
|
266
266
|
case '.':
|
|
267
267
|
case 'e':
|
|
268
268
|
case 'e+':
|
|
269
|
-
case 'e-':
|
|
269
|
+
case 'e-':
|
|
270
270
|
{
|
|
271
271
|
const next = tokenizeOp({kind: 'initial'})(input)
|
|
272
272
|
return [{first: {kind: 'error', message: 'invalid number'}, tail: next[0]}, next[1]]
|
|
273
273
|
}
|
|
274
|
-
default:
|
|
274
|
+
default:
|
|
275
275
|
{
|
|
276
276
|
const next = tokenizeOp({kind: 'initial'})(input)
|
|
277
277
|
return [{first: {kind: 'number', value: state.value}, tail: next[0]}, next[1]]
|
|
@@ -304,7 +304,7 @@ const invalidNumberStateOp = state => input =>
|
|
|
304
304
|
{
|
|
305
305
|
return [[{kind: 'error', message: 'invalid number'}], {kind: 'eof'}]
|
|
306
306
|
}
|
|
307
|
-
if (isTerminalForNumber(input))
|
|
307
|
+
if (isTerminalForNumber(input))
|
|
308
308
|
{
|
|
309
309
|
const next = tokenizeOp({kind: 'initial'})(input)
|
|
310
310
|
return [{first: {kind: 'error', message: 'invalid number'}, tail: next[0]}, next[1]]
|
|
@@ -386,7 +386,7 @@ const stringToKeywordToken = s =>
|
|
|
386
386
|
}
|
|
387
387
|
|
|
388
388
|
/** @type {(state: ParseKeywordState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
389
|
-
const parseKeyWordStateOp = state => input =>
|
|
389
|
+
const parseKeyWordStateOp = state => input =>
|
|
390
390
|
{
|
|
391
391
|
if (input === undefined)
|
|
392
392
|
{
|
|
@@ -408,17 +408,20 @@ const eofStateOp = state => input => [[{kind: 'error', message: 'eof'}], state]
|
|
|
408
408
|
/** @type {operator.StateScan<JsonCharacter, TokenizerState, list.List<JsonToken>>} */
|
|
409
409
|
const tokenizeOp = state => input =>
|
|
410
410
|
{
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
411
|
+
const f = () => {
|
|
412
|
+
switch(state.kind)
|
|
413
|
+
{
|
|
414
|
+
case 'initial': return initialStateOp(state)
|
|
415
|
+
case 'keyword': return parseKeyWordStateOp(state)
|
|
416
|
+
case 'string': return parseStringStateOp(state)
|
|
417
|
+
case 'escapeChar': return parseEscapeCharStateOp(state)
|
|
418
|
+
case 'unicodeChar': return parseUnicodeCharStateOp(state)
|
|
419
|
+
case 'invalidNumber': return invalidNumberStateOp(state)
|
|
420
|
+
case 'number': return parseNumberStateOp(state)
|
|
421
|
+
case 'eof': return eofStateOp(state)
|
|
422
|
+
}
|
|
421
423
|
}
|
|
424
|
+
return f()(input)
|
|
422
425
|
}
|
|
423
426
|
|
|
424
427
|
/** @type {(input: list.List<JsonCharacter>) => list.List<JsonToken>} */
|