functionalscript 0.0.496 → 0.0.498
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/fsc/module.f.cjs +41 -37
- package/fsc/test.f.cjs +1 -1
- package/json/tokenizer/module.f.cjs +242 -167
- package/package.json +2 -2
- package/json/tokenizer_range/module.f.cjs +0 -456
- package/json/tokenizer_range/test.f.cjs +0 -257
package/fsc/module.f.cjs
CHANGED
|
@@ -12,7 +12,7 @@ const { toArray, map } = list
|
|
|
12
12
|
|
|
13
13
|
/** @typedef {readonly[readonly string[], ToResult]} Result */
|
|
14
14
|
|
|
15
|
-
/** @typedef {(
|
|
15
|
+
/** @typedef {(codePoint: number) => Result} ToResult */
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* @template T
|
|
@@ -25,10 +25,10 @@ const { toArray, map } = list
|
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
/** @type {ToResult} */
|
|
28
|
-
const
|
|
28
|
+
const unexpectedSymbol = codePoint => [[`unexpected symbol ${codePoint}`], unexpectedSymbol]
|
|
29
29
|
|
|
30
|
-
/** @type {<T>(
|
|
31
|
-
const def = () =>
|
|
30
|
+
/** @type {<T>(state: T) => ToResult} */
|
|
31
|
+
const def = () => unexpectedSymbol
|
|
32
32
|
|
|
33
33
|
/** @type {<T>(a: CreateToResult<T>) => (b: CreateToResult<T>) => CreateToResult<T>} */
|
|
34
34
|
const union = a => b => {
|
|
@@ -77,42 +77,46 @@ const create = a => {
|
|
|
77
77
|
|
|
78
78
|
const terminal = -1
|
|
79
79
|
|
|
80
|
+
/** @type {() => ToResult} */
|
|
81
|
+
const toInit = () => () => [[], init]
|
|
82
|
+
|
|
80
83
|
const init = create([
|
|
81
|
-
codePointRange(one(terminal))(
|
|
82
|
-
rangeSet(['\t', ' '
|
|
83
|
-
|
|
84
|
-
range('
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
range('
|
|
88
|
-
range('
|
|
89
|
-
range(
|
|
90
|
-
range('
|
|
91
|
-
range('
|
|
92
|
-
range('
|
|
93
|
-
range('
|
|
94
|
-
range('
|
|
95
|
-
range('
|
|
96
|
-
range('
|
|
97
|
-
range('
|
|
98
|
-
range('
|
|
99
|
-
range('
|
|
100
|
-
range('
|
|
101
|
-
range('
|
|
102
|
-
range('
|
|
103
|
-
range('
|
|
104
|
-
range('
|
|
105
|
-
range('
|
|
106
|
-
range('
|
|
107
|
-
range('
|
|
108
|
-
range('
|
|
109
|
-
range('
|
|
110
|
-
range('
|
|
111
|
-
range('
|
|
112
|
-
|
|
113
|
-
])
|
|
84
|
+
codePointRange(one(terminal))(toInit),
|
|
85
|
+
rangeSet(['\t', ' ', '\n', '\r'])(toInit),
|
|
86
|
+
range('!')(() => () => [['!'], unexpectedSymbol]),
|
|
87
|
+
range('"')(() => () => [['"'], unexpectedSymbol]),
|
|
88
|
+
rangeSet(['$', '_', 'AZ', 'az'])(() => c => [[fromCharCode(c)], unexpectedSymbol]),
|
|
89
|
+
range('%')(() => () => [['%'], unexpectedSymbol]),
|
|
90
|
+
range('&')(() => () => [['&'], unexpectedSymbol]),
|
|
91
|
+
range("'")(() => () => [["'"], unexpectedSymbol]),
|
|
92
|
+
range('(')(() => () => [['('], unexpectedSymbol]),
|
|
93
|
+
range(')')(() => () => [[')'], unexpectedSymbol]),
|
|
94
|
+
range('*')(() => () => [['*'], unexpectedSymbol]),
|
|
95
|
+
range('+')(() => () => [['+'], unexpectedSymbol]),
|
|
96
|
+
range(',')(() => () => [[','], unexpectedSymbol]),
|
|
97
|
+
range('-')(() => () => [['-'], unexpectedSymbol]),
|
|
98
|
+
range('.')(() => () => [['.'], unexpectedSymbol]),
|
|
99
|
+
range('/')(() => () => [['/'], unexpectedSymbol]),
|
|
100
|
+
range('09')(() => a => [[fromCharCode(a)], unexpectedSymbol]),
|
|
101
|
+
range(':')(() => () => [[':'], unexpectedSymbol]),
|
|
102
|
+
range(';')(() => () => [[';'], unexpectedSymbol]),
|
|
103
|
+
range('<')(() => () => [['<'], unexpectedSymbol]),
|
|
104
|
+
range('=')(() => () => [['='], unexpectedSymbol]),
|
|
105
|
+
range('>')(() => () => [['>'], unexpectedSymbol]),
|
|
106
|
+
range('?')(() => () => [['?'], unexpectedSymbol]),
|
|
107
|
+
range('[')(() => () => [['['], unexpectedSymbol]),
|
|
108
|
+
range(']')(() => () => [[']'], unexpectedSymbol]),
|
|
109
|
+
range('^')(() => () => [['^'], unexpectedSymbol]),
|
|
110
|
+
range('`')(() => () => [['`'], unexpectedSymbol]),
|
|
111
|
+
range('{')(() => () => [['{'], unexpectedSymbol]),
|
|
112
|
+
range('|')(() => () => [['|'], unexpectedSymbol]),
|
|
113
|
+
range('}')(() => () => [['}'], unexpectedSymbol]),
|
|
114
|
+
range('~')(() => () => [['~'], unexpectedSymbol]),
|
|
115
|
+
])(void 0)
|
|
114
116
|
|
|
115
117
|
module.exports = {
|
|
118
|
+
/** @readonly */
|
|
119
|
+
terminal,
|
|
116
120
|
/** @readonly */
|
|
117
121
|
init,
|
|
118
122
|
}
|
package/fsc/test.f.cjs
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
const operator = require('../../types/function/operator/module.f.cjs')
|
|
2
|
+
const range_map = require('../../types/range_map/module.f.cjs')
|
|
3
|
+
const { merge, fromRange, get } = range_map
|
|
2
4
|
const list = require('../../types/list/module.f.cjs')
|
|
3
|
-
const
|
|
4
|
-
const {
|
|
5
|
+
const _range = require('../../types/range/module.f.cjs')
|
|
6
|
+
const { one } = _range
|
|
7
|
+
const { empty, stateScan, flat, toArray, reduce: listReduce, scan } = list
|
|
5
8
|
const { fromCharCode } = String
|
|
6
9
|
const {
|
|
7
10
|
range,
|
|
@@ -72,11 +75,31 @@ const {
|
|
|
72
75
|
* } JsonToken
|
|
73
76
|
*/
|
|
74
77
|
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
const rangeOneNine = range('19')
|
|
79
|
+
|
|
80
|
+
const rangeSetWhiteSpace = [
|
|
81
|
+
one(ht),
|
|
82
|
+
one(lf),
|
|
83
|
+
one(cr),
|
|
84
|
+
one(space)
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
const rangeSetTerminalForNumber = [
|
|
88
|
+
one(ht),
|
|
89
|
+
one(lf),
|
|
90
|
+
one(cr),
|
|
91
|
+
one(space),
|
|
92
|
+
one(quotationMark),
|
|
93
|
+
one(comma),
|
|
94
|
+
one(leftCurlyBracket),
|
|
95
|
+
one(rightCurlyBracket),
|
|
96
|
+
one(leftSquareBracket),
|
|
97
|
+
one(rightSquareBracket),
|
|
98
|
+
one(colon)
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
const rangeSmallAF = range('af')
|
|
102
|
+
const rangeCapitalAF = range('AF')
|
|
80
103
|
|
|
81
104
|
/**
|
|
82
105
|
* @typedef {|
|
|
@@ -111,15 +134,22 @@ const containsSmallLetter = contains(latinSmallLetterRange)
|
|
|
111
134
|
|
|
112
135
|
/** @typedef {{ readonly kind: 'escapeChar', readonly value: string}} ParseEscapeCharState */
|
|
113
136
|
|
|
114
|
-
/**
|
|
137
|
+
/**
|
|
138
|
+
* @typedef {{
|
|
139
|
+
* readonly kind: 'unicodeChar'
|
|
140
|
+
* readonly value: string
|
|
141
|
+
* readonly unicode: number
|
|
142
|
+
* readonly hexIndex: number
|
|
143
|
+
* }} ParseUnicodeCharState
|
|
144
|
+
*/
|
|
115
145
|
|
|
116
146
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
147
|
+
* @typedef {{
|
|
148
|
+
* readonly kind: 'number',
|
|
149
|
+
* readonly numberKind: '0' | '-' | 'int' | '.' | 'fractional' | 'e' | 'e+' | 'e-' | 'expDigits'
|
|
150
|
+
* readonly value: string
|
|
121
151
|
* }} ParseNumberState
|
|
122
|
-
|
|
152
|
+
*/
|
|
123
153
|
|
|
124
154
|
/** @typedef {{ readonly kind: 'invalidNumber'}} InvalidNumberState */
|
|
125
155
|
|
|
@@ -127,192 +157,235 @@ const containsSmallLetter = contains(latinSmallLetterRange)
|
|
|
127
157
|
|
|
128
158
|
/** @typedef {number|null} CharCodeOrEof */
|
|
129
159
|
|
|
160
|
+
/** @typedef {(input: number) => readonly[list.List<JsonToken>, TokenizerState]} ToToken */
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @template T
|
|
164
|
+
* @typedef {(state: T) => ToToken} CreateToToken<T>
|
|
165
|
+
*/
|
|
166
|
+
|
|
167
|
+
/** @typedef {list.List<_range.Range>} RangeSet */
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* @template T
|
|
171
|
+
* @typedef {(def: CreateToToken<T>) => (RangeMapToToken<T>)} RangeFunc<T>
|
|
172
|
+
*/
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* @template T
|
|
176
|
+
* @typedef {range_map.RangeMapArray<CreateToToken<T>>} RangeMapToToken<T>
|
|
177
|
+
*/
|
|
178
|
+
|
|
130
179
|
/** @type {(old: string) => (input: number) => string} */
|
|
131
180
|
const appendChar = old => input => `${old}${fromCharCode(input)}`
|
|
132
181
|
|
|
182
|
+
/** @type {<T>(def: CreateToToken<T>) => (a: CreateToToken<T>) => (b: CreateToToken<T>) => CreateToToken<T>} */
|
|
183
|
+
const union = def => a => b => {
|
|
184
|
+
if (a === def || a === b) { return b }
|
|
185
|
+
if (b === def) { return a }
|
|
186
|
+
throw [a, b]
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** @type {<T>(def: CreateToToken<T>) => range_map.RangeMerge<CreateToToken<T>>} */
|
|
190
|
+
const rangeMapMerge = def => merge({
|
|
191
|
+
union: union(def),
|
|
192
|
+
equal: operator.strictEqual,
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
/** @type {<T>(r: _range.Range) => (f: CreateToToken<T>) => RangeFunc<T>} */
|
|
196
|
+
const rangeFunc = r => f => def => fromRange(def)(r)(f)
|
|
197
|
+
|
|
198
|
+
/** @type {<T>(def: CreateToToken<T>) => (operator.Scan<RangeFunc<T>, RangeMapToToken<T>>)} */
|
|
199
|
+
const scanRangeOp = def => f => [f(def), scanRangeOp(def)]
|
|
200
|
+
|
|
201
|
+
/** @type {<T>(def: CreateToToken<T>) => (a: list.List<RangeFunc<T>>) => RangeMapToToken<T>} */
|
|
202
|
+
const reduceRangeMap = def => a => {
|
|
203
|
+
const rm = scan(scanRangeOp(def))(a)
|
|
204
|
+
return toArray(listReduce(rangeMapMerge(def))(empty)(rm))
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** @type {<T>(def: CreateToToken<T>) => (f: CreateToToken<T>) => (operator.Scan<_range.Range, RangeMapToToken<T>>)} */
|
|
208
|
+
const scanRangeSetOp = def => f => r => [fromRange(def)(r)(f), scanRangeSetOp(def)(f)]
|
|
209
|
+
|
|
210
|
+
/** @type {<T>(rs: list.List<_range.Range>) => (f: CreateToToken<T>) => RangeFunc<T>} */
|
|
211
|
+
const rangeSetFunc = rs => f => def => {
|
|
212
|
+
const rm = scan(scanRangeSetOp(def)(f))(rs)
|
|
213
|
+
return toArray(listReduce(rangeMapMerge(def))(empty)(rm))
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** @type {<T>(def: CreateToToken<T>) => (a: list.List<RangeFunc<T>>) => CreateToToken<T>} */
|
|
217
|
+
const create = def => a => {
|
|
218
|
+
/** @typedef {typeof def extends CreateToToken<infer T> ? T : never} T */
|
|
219
|
+
const i = reduceRangeMap(def)(a)
|
|
220
|
+
/** @type {(v: number) => (i: RangeMapToToken<T>) => (v: T) => ToToken} */
|
|
221
|
+
const x = get(def)
|
|
222
|
+
return v => c => x(c)(i)(v)(c)
|
|
223
|
+
}
|
|
224
|
+
|
|
133
225
|
/** @type {(state: InitialState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
134
|
-
const initialStateOp =
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
226
|
+
const initialStateOp = create(state => () => [[{ kind: 'error', message: 'unexpected character' }], state])([
|
|
227
|
+
rangeFunc(rangeOneNine)(() => input => [empty, { kind: 'number', value: fromCharCode(input), numberKind: 'int' }]),
|
|
228
|
+
rangeFunc(latinSmallLetterRange)(() => input => [empty, { kind: 'keyword', value: fromCharCode(input) }]),
|
|
229
|
+
rangeSetFunc(rangeSetWhiteSpace)(state => () => [empty, state]),
|
|
230
|
+
rangeFunc(one(leftCurlyBracket))(state => () => [[{ kind: '{' }], state]),
|
|
231
|
+
rangeFunc(one(rightCurlyBracket))(state => () => [[{ kind: '}' }], state]),
|
|
232
|
+
rangeFunc(one(colon))(state => () => [[{ kind: ':' }], state]),
|
|
233
|
+
rangeFunc(one(comma))(state => () => [[{ kind: ',' }], state]),
|
|
234
|
+
rangeFunc(one(leftSquareBracket))(state => () => [[{ kind: '[' }], state]),
|
|
235
|
+
rangeFunc(one(rightSquareBracket))(state => () => [[{ kind: ']' }], state]),
|
|
236
|
+
rangeFunc(one(quotationMark))(() => () => [empty, { kind: 'string', value: '' }]),
|
|
237
|
+
rangeFunc(one(digit0))(() => input => [empty, { kind: 'number', value: fromCharCode(input), numberKind: '0' }]),
|
|
238
|
+
rangeFunc(one(hyphenMinus))(() => input => [empty, { kind: 'number', value: fromCharCode(input), numberKind: '-' }])
|
|
239
|
+
])
|
|
240
|
+
|
|
241
|
+
/** @type {(state: any) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
242
|
+
const invalidNumberToToken = () => input => tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
243
|
+
|
|
244
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
245
|
+
const fullStopToToken = state => input => {
|
|
246
|
+
switch (state.numberKind) {
|
|
247
|
+
case '0':
|
|
248
|
+
case 'int': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: '.' }]
|
|
249
|
+
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
155
250
|
}
|
|
156
251
|
}
|
|
157
252
|
|
|
158
253
|
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
254
|
+
const digit0ToToken = state => input => {
|
|
255
|
+
switch (state.numberKind) {
|
|
256
|
+
case '0': return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
257
|
+
case '-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: '0' }]
|
|
258
|
+
case '.': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'fractional' }]
|
|
259
|
+
case 'e':
|
|
260
|
+
case 'e+':
|
|
261
|
+
case 'e-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'expDigits' }]
|
|
262
|
+
default: return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: state.numberKind }]
|
|
166
263
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
switch (state.numberKind) {
|
|
180
|
-
case '0': return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
181
|
-
case '-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'int' }]
|
|
182
|
-
case '.': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'fractional' }]
|
|
183
|
-
case 'e':
|
|
184
|
-
case 'e+':
|
|
185
|
-
case 'e-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'expDigits' }]
|
|
186
|
-
default: return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: state.numberKind }]
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
if (input === latinSmallLetterE || input === latinCapitalLetterE) {
|
|
190
|
-
switch (state.numberKind) {
|
|
191
|
-
case '0':
|
|
192
|
-
case 'int':
|
|
193
|
-
case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e' }]
|
|
194
|
-
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
if (input === hyphenMinus) {
|
|
198
|
-
switch (state.numberKind) {
|
|
199
|
-
case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e-' }]
|
|
200
|
-
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
if (input === plusSign) {
|
|
204
|
-
switch (state.numberKind) {
|
|
205
|
-
case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e+' }]
|
|
206
|
-
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
207
|
-
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
267
|
+
const digit19ToToken = state => input => {
|
|
268
|
+
switch (state.numberKind) {
|
|
269
|
+
case '0': return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
270
|
+
case '-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'int' }]
|
|
271
|
+
case '.': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'fractional' }]
|
|
272
|
+
case 'e':
|
|
273
|
+
case 'e+':
|
|
274
|
+
case 'e-': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'expDigits' }]
|
|
275
|
+
default: return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: state.numberKind }]
|
|
208
276
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
|
|
219
|
-
}
|
|
220
|
-
default:
|
|
221
|
-
{
|
|
222
|
-
const next = tokenizeOp({ kind: 'initial' })(input)
|
|
223
|
-
return [{ first: { kind: 'number', value: state.value }, tail: next[0] }, next[1]]
|
|
224
|
-
}
|
|
225
|
-
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
280
|
+
const expToToken = state => input => {
|
|
281
|
+
switch (state.numberKind) {
|
|
282
|
+
case '0':
|
|
283
|
+
case 'int':
|
|
284
|
+
case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e' }]
|
|
285
|
+
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
226
286
|
}
|
|
227
|
-
return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
228
287
|
}
|
|
229
288
|
|
|
230
|
-
/** @type {(
|
|
231
|
-
const
|
|
232
|
-
|
|
233
|
-
return
|
|
289
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
290
|
+
const hyphenMinusToToken = state => input => {
|
|
291
|
+
switch (state.numberKind) {
|
|
292
|
+
case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e-' }]
|
|
293
|
+
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
234
294
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
case
|
|
241
|
-
|
|
242
|
-
case colon: return true
|
|
243
|
-
default: return false
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
298
|
+
const plusSignToToken = state => input => {
|
|
299
|
+
switch (state.numberKind) {
|
|
300
|
+
case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e+' }]
|
|
301
|
+
default: return tokenizeOp({ kind: 'invalidNumber' })(input)
|
|
244
302
|
}
|
|
245
303
|
}
|
|
246
304
|
|
|
247
|
-
/** @type {(
|
|
248
|
-
const
|
|
249
|
-
switch (
|
|
250
|
-
case
|
|
251
|
-
case
|
|
252
|
-
case
|
|
253
|
-
case
|
|
254
|
-
|
|
305
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
306
|
+
const terminalToToken = state => input => {
|
|
307
|
+
switch (state.numberKind) {
|
|
308
|
+
case '-':
|
|
309
|
+
case '.':
|
|
310
|
+
case 'e':
|
|
311
|
+
case 'e+':
|
|
312
|
+
case 'e-':
|
|
313
|
+
{
|
|
314
|
+
const next = tokenizeOp({ kind: 'initial' })(input)
|
|
315
|
+
return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
|
|
316
|
+
}
|
|
317
|
+
default:
|
|
318
|
+
{
|
|
319
|
+
const next = tokenizeOp({ kind: 'initial' })(input)
|
|
320
|
+
return [{ first: { kind: 'number', value: state.value }, tail: next[0] }, next[1]]
|
|
321
|
+
}
|
|
255
322
|
}
|
|
256
323
|
}
|
|
257
324
|
|
|
325
|
+
/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
326
|
+
const parseNumberStateOp = create(invalidNumberToToken)([
|
|
327
|
+
rangeFunc(one(fullStop))(fullStopToToken),
|
|
328
|
+
rangeFunc(one(digit0))(digit0ToToken),
|
|
329
|
+
rangeFunc(rangeOneNine)(digit19ToToken),
|
|
330
|
+
rangeSetFunc([one(latinSmallLetterE), one(latinCapitalLetterE)])(expToToken),
|
|
331
|
+
rangeFunc(one(hyphenMinus))(hyphenMinusToToken),
|
|
332
|
+
rangeFunc(one(plusSign))(plusSignToToken),
|
|
333
|
+
rangeSetFunc(rangeSetTerminalForNumber)(terminalToToken)
|
|
334
|
+
])
|
|
335
|
+
|
|
258
336
|
/** @type {(state: InvalidNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
259
|
-
const invalidNumberStateOp = () =>
|
|
260
|
-
|
|
337
|
+
const invalidNumberStateOp = create(() => () => [empty, { kind: 'invalidNumber' }])([
|
|
338
|
+
rangeSetFunc(rangeSetTerminalForNumber)(() => input => {
|
|
261
339
|
const next = tokenizeOp({ kind: 'initial' })(input)
|
|
262
340
|
return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
}
|
|
341
|
+
})
|
|
342
|
+
])
|
|
266
343
|
|
|
267
344
|
/** @type {(state: ParseStringState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
268
|
-
const parseStringStateOp = state => input => {
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
default: return [empty, { kind: 'string', value: appendChar(state.value)(input) }]
|
|
273
|
-
}
|
|
274
|
-
}
|
|
345
|
+
const parseStringStateOp = create(state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }])([
|
|
346
|
+
rangeFunc(one(quotationMark))(state => () => [[{ kind: 'string', value: state.value }], { kind: 'initial' }]),
|
|
347
|
+
rangeFunc(one(reverseSolidus))(state => () => [empty, { kind: 'escapeChar', value: state.value }])
|
|
348
|
+
])
|
|
275
349
|
|
|
276
350
|
/** @type {(state: ParseEscapeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
277
|
-
const
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
case reverseSolidus:
|
|
281
|
-
case solidus: return [empty, { kind: 'string', value: appendChar(state.value)(input) }]
|
|
282
|
-
case latinSmallLetterB: return [empty, { kind: 'string', value: appendChar(state.value)(backspace) }]
|
|
283
|
-
case latinSmallLetterF: return [empty, { kind: 'string', value: appendChar(state.value)(ff) }]
|
|
284
|
-
case latinSmallLetterN: return [empty, { kind: 'string', value: appendChar(state.value)(lf) }]
|
|
285
|
-
case latinSmallLetterR: return [empty, { kind: 'string', value: appendChar(state.value)(cr) }]
|
|
286
|
-
case latinSmallLetterT: return [empty, { kind: 'string', value: appendChar(state.value)(ht) }]
|
|
287
|
-
case latinSmallLetterU: return [empty, { kind: 'unicodeChar', value: state.value, unicode: 0, hexIndex: 0 }]
|
|
288
|
-
default: {
|
|
289
|
-
const next = tokenizeOp({ kind: 'string', value: state.value })(input)
|
|
290
|
-
return [{ first: { kind: 'error', message: 'unescaped character' }, tail: next[0] }, next[1]]
|
|
291
|
-
}
|
|
292
|
-
}
|
|
351
|
+
const parseEscapeDefault = state => input => {
|
|
352
|
+
const next = tokenizeOp({ kind: 'string', value: state.value })(input)
|
|
353
|
+
return [{ first: { kind: 'error', message: 'unescaped character' }, tail: next[0] }, next[1]]
|
|
293
354
|
}
|
|
294
355
|
|
|
295
|
-
/** @type {(
|
|
296
|
-
const
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}
|
|
356
|
+
/** @type {(state: ParseEscapeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
357
|
+
const parseEscapeCharStateOp = create(parseEscapeDefault)([
|
|
358
|
+
rangeSetFunc([one(quotationMark), one(reverseSolidus), one(solidus)])(state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }]),
|
|
359
|
+
rangeFunc(one(latinSmallLetterB))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(backspace) }]),
|
|
360
|
+
rangeFunc(one(latinSmallLetterF))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(ff) }]),
|
|
361
|
+
rangeFunc(one(latinSmallLetterN))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(lf) }]),
|
|
362
|
+
rangeFunc(one(latinSmallLetterR))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(cr) }]),
|
|
363
|
+
rangeFunc(one(latinSmallLetterT))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(ht) }]),
|
|
364
|
+
rangeFunc(one(latinSmallLetterU))(state => () => [empty, { kind: 'unicodeChar', value: state.value, unicode: 0, hexIndex: 0 }]),
|
|
365
|
+
])
|
|
302
366
|
|
|
303
367
|
/** @type {(state: ParseUnicodeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
304
|
-
const
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
368
|
+
const parseUnicodeCharDefault = state => input => {
|
|
369
|
+
const next = tokenizeOp({ kind: 'string', value: state.value })(input)
|
|
370
|
+
return [{ first: { kind: 'error', message: 'invalid hex value' }, tail: next[0] }, next[1]]
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** @type {(offser: number) => (state: ParseUnicodeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
374
|
+
const parseUnicodeCharHex = offset => state => input => {
|
|
375
|
+
const hexValue = input - offset
|
|
310
376
|
const newUnicode = state.unicode | (hexValue << (3 - state.hexIndex) * 4)
|
|
311
377
|
return [empty, state.hexIndex === 3 ?
|
|
312
378
|
{ kind: 'string', value: appendChar(state.value)(newUnicode) } :
|
|
313
379
|
{ kind: 'unicodeChar', value: state.value, unicode: newUnicode, hexIndex: state.hexIndex + 1 }]
|
|
314
380
|
}
|
|
315
381
|
|
|
382
|
+
/** @type {(state: ParseUnicodeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
383
|
+
const parseUnicodeCharStateOp = create(parseUnicodeCharDefault)([
|
|
384
|
+
rangeFunc(digitRange)(parseUnicodeCharHex(digit0)),
|
|
385
|
+
rangeFunc(rangeSmallAF)(parseUnicodeCharHex(latinSmallLetterA - 10)),
|
|
386
|
+
rangeFunc(rangeCapitalAF)(parseUnicodeCharHex(latinCapitalLetterA - 10))
|
|
387
|
+
])
|
|
388
|
+
|
|
316
389
|
/** @type {(s: string) => JsonToken} */
|
|
317
390
|
const stringToKeywordToken = s => {
|
|
318
391
|
switch (s) {
|
|
@@ -324,17 +397,19 @@ const stringToKeywordToken = s => {
|
|
|
324
397
|
}
|
|
325
398
|
|
|
326
399
|
/** @type {(state: ParseKeywordState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
327
|
-
const
|
|
328
|
-
if (containsSmallLetter(input)) {
|
|
329
|
-
return [empty, { kind: 'keyword', value: appendChar(state.value)(input) }]
|
|
330
|
-
}
|
|
400
|
+
const parseKeyWordDefault = state => input => {
|
|
331
401
|
const keyWordToken = stringToKeywordToken(state.value)
|
|
332
402
|
const next = tokenizeOp({ kind: 'initial' })(input)
|
|
333
403
|
return [{ first: keyWordToken, tail: next[0] }, next[1]]
|
|
334
404
|
}
|
|
335
405
|
|
|
406
|
+
/** @type {(state: ParseKeywordState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
407
|
+
const parseKeyWordStateOp = create(parseKeyWordDefault)([
|
|
408
|
+
rangeFunc(latinSmallLetterRange)(state => input => [empty, { kind: 'keyword', value: appendChar(state.value)(input) }])
|
|
409
|
+
])
|
|
410
|
+
|
|
336
411
|
/** @type {(state: EofState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
|
|
337
|
-
const eofStateOp = state => () => [[{ kind: 'error', message: 'eof' }], state]
|
|
412
|
+
const eofStateOp = create(state => () => [[{ kind: 'error', message: 'eof' }], state])([])
|
|
338
413
|
|
|
339
414
|
/** @type {operator.StateScan<number, TokenizerState, list.List<JsonToken>>} */
|
|
340
415
|
const tokenizeCharCodeOp = state => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "functionalscript",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.498",
|
|
4
4
|
"description": "FunctionalScript is a functional subset of JavaScript",
|
|
5
5
|
"main": "module.f.cjs",
|
|
6
6
|
"scripts": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
},
|
|
31
31
|
"homepage": "https://github.com/functionalscript/functionalscript#readme",
|
|
32
32
|
"devDependencies": {
|
|
33
|
-
"@types/node": "^18.11.
|
|
33
|
+
"@types/node": "^18.11.18",
|
|
34
34
|
"typescript": "^4.9.4"
|
|
35
35
|
}
|
|
36
36
|
}
|