grammatik 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +443 -0
- package/bin/grammatik.js +115 -0
- package/dist/cjs/index.js +3010 -0
- package/dist/cjs/package.json +3 -0
- package/dist/cjs/runtime.js +639 -0
- package/dist/esm/index.js +2959 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/runtime.js +605 -0
- package/dist/types/index.d.ts +214 -0
- package/dist/types/runtime.d.ts +40 -0
- package/package.json +65 -0
- package/src/analyze.d.ts +78 -0
- package/src/analyze.js +602 -0
- package/src/codegen.d.ts +36 -0
- package/src/codegen.js +635 -0
- package/src/definition.d.ts +133 -0
- package/src/definition.js +291 -0
- package/src/emit.d.ts +23 -0
- package/src/emit.js +374 -0
- package/src/firstchars.d.ts +6 -0
- package/src/firstchars.js +353 -0
- package/src/index.d.ts +7 -0
- package/src/index.js +15 -0
- package/src/lexer.d.ts +96 -0
- package/src/lexer.js +358 -0
- package/src/methods.d.ts +20 -0
- package/src/methods.js +61 -0
- package/src/methodsblock.js +197 -0
- package/src/parser.d.ts +130 -0
- package/src/parser.js +194 -0
- package/src/plan.d.ts +29 -0
- package/src/plan.js +938 -0
- package/src/runtime.js +16 -0
- package/src/state.d.ts +98 -0
- package/src/state.js +121 -0
- package/src/values.js +67 -0
package/src/lexer.js
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
'use strict'
|
|
3
|
+
|
|
4
|
+
import { firstCharSet } from './firstchars.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A Chevrotain-style lexer. Token definitions are plain JSON, so a whole
|
|
8
|
+
* grammar (tokens + rules) stays serializable.
|
|
9
|
+
*
|
|
10
|
+
* @typedef {object} TokenDef
|
|
11
|
+
* @property {string} name The token type produced.
|
|
12
|
+
* @property {string} [pattern] A regex source string, matched sticky at the current offset.
|
|
13
|
+
* @property {string} [literal] A literal string to match. Cheaper than a pattern; prefer it.
|
|
14
|
+
* @property {boolean} [skip] When true the match is consumed but produces no token (whitespace, comments).
|
|
15
|
+
* @property {string} [longerAlt] Name of another token to prefer when it matches a longer string
|
|
16
|
+
* at the same offset (the classic `if` / `identifier` problem).
|
|
17
|
+
* @property {boolean} [ignoreCase] Case-insensitive matching.
|
|
18
|
+
* @property {string} [mode] The lexer mode this token belongs to. Default `'default'`.
|
|
19
|
+
* @property {string[]} [modes] Several modes, when one token serves more than one.
|
|
20
|
+
* @property {string} [pushMode] Enter this mode after the token matches.
|
|
21
|
+
* @property {boolean} [popMode] Leave the current mode after the token matches.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @typedef {object} Token
|
|
26
|
+
* @property {string} type
|
|
27
|
+
* @property {string} image
|
|
28
|
+
* @property {number} start
|
|
29
|
+
* @property {number} end
|
|
30
|
+
* @property {number} line
|
|
31
|
+
* @property {number} col
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
export class LexError extends Error {
|
|
35
|
+
/**
|
|
36
|
+
* @param {string} message
|
|
37
|
+
* @param {{ offset: number, line: number, col: number }} position
|
|
38
|
+
*/
|
|
39
|
+
constructor (message, position) {
|
|
40
|
+
super(message)
|
|
41
|
+
this.name = 'LexError'
|
|
42
|
+
this.offset = position.offset
|
|
43
|
+
this.line = position.line
|
|
44
|
+
this.col = position.col
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const DEFAULT_MODE = 'default'
|
|
49
|
+
|
|
50
|
+
/** The modes one definition belongs to, however it spelled them. */
|
|
51
|
+
function modesOf (def) {
|
|
52
|
+
if (Array.isArray(def.modes) && def.modes.length > 0) return def.modes
|
|
53
|
+
if (def.mode) return [def.mode]
|
|
54
|
+
return [DEFAULT_MODE]
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Compiles token definitions into a fast sticky-regex lexer.
|
|
59
|
+
*
|
|
60
|
+
* Literal tokens are matched with `startsWith` rather than a regex, which is a
|
|
61
|
+
* large win for punctuation-heavy grammars (JSON is ~60% punctuation by token count).
|
|
62
|
+
*
|
|
63
|
+
* ## Modes
|
|
64
|
+
*
|
|
65
|
+
* A token belongs to one or more modes and only competes while one of them is
|
|
66
|
+
* current. `pushMode` enters a mode after the token matches, `popMode` leaves
|
|
67
|
+
* one, and the stack starts fresh at `'default'` on every `tokenize` call. This
|
|
68
|
+
* is how context-sensitive vocabularies are expressed as data: in Handlebars,
|
|
69
|
+
* whitespace is literal text outside `{{ }}` and skipped inside it, which is one
|
|
70
|
+
* token definition per mode rather than a hand-written scanner.
|
|
71
|
+
*
|
|
72
|
+
* Everything a mode needs — its candidate list and its first-character dispatch
|
|
73
|
+
* table — is compiled here, so the hot path only indexes by mode id. A grammar
|
|
74
|
+
* with a single mode never touches a mode stack at all.
|
|
75
|
+
*
|
|
76
|
+
* @param {TokenDef[]} defs
|
|
77
|
+
* @param {{ positions?: 'full' | 'offset' }} [options]
|
|
78
|
+
* `full` (default) records line and column on every token. `offset` records
|
|
79
|
+
* only `start`/`end`, which skips a scan of the entire input for newlines —
|
|
80
|
+
* worth taking when errors do not need to name a line, or when positions are
|
|
81
|
+
* resolved lazily from the offset afterwards.
|
|
82
|
+
* @returns {{ tokenize: (text: string) => Token[], tokenNames: string[], positions: string, modeNames: string[] }}
|
|
83
|
+
*/
|
|
84
|
+
export function createLexer (defs, options = {}) {
|
|
85
|
+
const positions = options.positions === 'offset' ? 'offset' : 'full'
|
|
86
|
+
const trackLines = positions === 'full'
|
|
87
|
+
if (!Array.isArray(defs) || defs.length === 0) throw new Error('createLexer: expected a non-empty array of token definitions')
|
|
88
|
+
|
|
89
|
+
const tokenNames = defs.map((d) => d.name)
|
|
90
|
+
const seen = new Set()
|
|
91
|
+
for (const name of tokenNames) {
|
|
92
|
+
if (seen.has(name)) throw new Error(`createLexer: duplicate token name '${name}'`)
|
|
93
|
+
seen.add(name)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ---- modes ----
|
|
97
|
+
// Mode ids are assigned before anything else so a definition can carry the
|
|
98
|
+
// numbers rather than the names, and the hot path never touches a string.
|
|
99
|
+
const modeIds = new Map([[DEFAULT_MODE, 0]])
|
|
100
|
+
const modeNames = [DEFAULT_MODE]
|
|
101
|
+
const idOf = (name) => {
|
|
102
|
+
let id = modeIds.get(name)
|
|
103
|
+
if (id === undefined) {
|
|
104
|
+
id = modeNames.length
|
|
105
|
+
modeIds.set(name, id)
|
|
106
|
+
modeNames.push(name)
|
|
107
|
+
}
|
|
108
|
+
return id
|
|
109
|
+
}
|
|
110
|
+
for (const def of defs) for (const mode of modesOf(def)) idOf(mode)
|
|
111
|
+
|
|
112
|
+
const compiled = defs.map((def) => {
|
|
113
|
+
if (!def.name) throw new Error('createLexer: every token definition needs a name')
|
|
114
|
+
if (!def.pattern && !def.literal) throw new Error(`createLexer: token '${def.name}' needs a pattern or a literal`)
|
|
115
|
+
if (def.pattern && def.literal) throw new Error(`createLexer: token '${def.name}' has both a pattern and a literal; pick one`)
|
|
116
|
+
if (def.longerAlt && !seen.has(def.longerAlt)) throw new Error(`createLexer: token '${def.name}' has longerAlt '${def.longerAlt}', which is not a defined token`)
|
|
117
|
+
if (def.pushMode && def.popMode) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`createLexer: token '${def.name}' both pushes '${def.pushMode}' and pops. ` +
|
|
120
|
+
'One token cannot do both — split it into two tokens, one per mode.'
|
|
121
|
+
)
|
|
122
|
+
}
|
|
123
|
+
// A mode nothing declares is a typo, and would otherwise strand the lexer in
|
|
124
|
+
// a mode where no token can ever match.
|
|
125
|
+
if (def.pushMode && !modeIds.has(def.pushMode)) {
|
|
126
|
+
throw new Error(`createLexer: token '${def.name}' pushes mode '${def.pushMode}', which no token belongs to`)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
name: def.name,
|
|
131
|
+
skip: !!def.skip,
|
|
132
|
+
literal: def.literal,
|
|
133
|
+
longerAlt: def.longerAlt || null,
|
|
134
|
+
regex: def.pattern ? new RegExp(def.pattern, def.ignoreCase ? 'iy' : 'y') : null,
|
|
135
|
+
modes: modesOf(def),
|
|
136
|
+
pushMode: def.pushMode ? idOf(def.pushMode) : -1,
|
|
137
|
+
popMode: !!def.popMode
|
|
138
|
+
}
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
const byName = new Map(compiled.map((c) => [c.name, c]))
|
|
142
|
+
// Resolve longerAlt names to the compiled definitions once, up front.
|
|
143
|
+
for (const c of compiled) if (c.longerAlt) c.longerAltDef = byName.get(c.longerAlt)
|
|
144
|
+
|
|
145
|
+
// A longerAlt can only win where it is itself active; otherwise a token would
|
|
146
|
+
// silently lose its alternative in one mode and keep it in another.
|
|
147
|
+
for (const c of compiled) {
|
|
148
|
+
if (!c.longerAltDef) continue
|
|
149
|
+
for (const mode of c.modes) {
|
|
150
|
+
if (c.longerAltDef.modes.includes(mode)) continue
|
|
151
|
+
throw new Error(
|
|
152
|
+
`createLexer: token '${c.name}' is active in mode '${mode}' with longerAlt '${c.longerAlt}', ` +
|
|
153
|
+
`which is not active there. Add '${mode}' to '${c.longerAlt}'.`
|
|
154
|
+
)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// ---- per-mode dispatch tables ----
|
|
159
|
+
// For each definition, work out which characters it can start with, so that a
|
|
160
|
+
// given offset only tries the handful of patterns that could match there
|
|
161
|
+
// rather than all of them in declaration order. Built once per mode, since a
|
|
162
|
+
// mode's candidate set never changes after the lexer is created.
|
|
163
|
+
for (const c of compiled) {
|
|
164
|
+
c.firstChars = c.literal !== undefined
|
|
165
|
+
? (c.literal.length > 0 ? new Set([c.literal.charCodeAt(0)]) : null)
|
|
166
|
+
: firstCharSet(c.regex.source, !!c.regex.ignoreCase)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const modes = modeNames.map((name) => {
|
|
170
|
+
const members = compiled.filter((c) => c.modes.includes(name))
|
|
171
|
+
// Definitions whose first characters could not be determined must be tried
|
|
172
|
+
// at every offset, so they belong in every bucket.
|
|
173
|
+
const alwaysTry = members.filter((c) => c.firstChars === null)
|
|
174
|
+
const candidatesFor = (code) => members.filter((c) => c.firstChars === null || c.firstChars.has(code))
|
|
175
|
+
|
|
176
|
+
const ascii = new Array(128)
|
|
177
|
+
for (let code = 0; code < 128; code++) ascii[code] = candidatesFor(code)
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
name,
|
|
181
|
+
members,
|
|
182
|
+
ascii,
|
|
183
|
+
wide: new Map(),
|
|
184
|
+
candidatesFor,
|
|
185
|
+
// If nothing could be analyzed, dispatch is pure overhead; skip it.
|
|
186
|
+
dispatch: alwaysTry.length < members.length
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
for (const mode of modes) {
|
|
191
|
+
if (mode.members.length === 0) throw new Error(`createLexer: mode '${mode.name}' has no tokens`)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function candidates (mode, code) {
|
|
195
|
+
if (code < 128) return mode.ascii[code]
|
|
196
|
+
let list = mode.wide.get(code)
|
|
197
|
+
if (list === undefined) {
|
|
198
|
+
list = mode.candidatesFor(code)
|
|
199
|
+
mode.wide.set(code, list)
|
|
200
|
+
}
|
|
201
|
+
return list
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// No transitions anywhere means no stack and no per-token mode checks — the
|
|
205
|
+
// loop this had before modes existed. A grammar can declare one mode and still
|
|
206
|
+
// pop out of it, which is always an underflow; that has to be reported, not
|
|
207
|
+
// skipped along with the rest of the machinery.
|
|
208
|
+
const multiMode = modes.length > 1 || compiled.some((c) => c.popMode || c.pushMode !== -1)
|
|
209
|
+
// Naming the mode in an error only helps when there is more than one.
|
|
210
|
+
const namesModes = modes.length > 1
|
|
211
|
+
const initial = modes[0]
|
|
212
|
+
|
|
213
|
+
/** The `longerAlt` match end, or -1. Only reached when a `longerAlt` is set. */
|
|
214
|
+
function longerAltEnd (def, text, offset) {
|
|
215
|
+
const alt = def.longerAltDef
|
|
216
|
+
if (alt.literal !== undefined) return text.startsWith(alt.literal, offset) ? offset + alt.literal.length : -1
|
|
217
|
+
alt.regex.lastIndex = offset
|
|
218
|
+
return alt.regex.test(text) ? alt.regex.lastIndex : -1
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* @param {string} text
|
|
223
|
+
* @returns {Token[]}
|
|
224
|
+
*/
|
|
225
|
+
function tokenize (text) {
|
|
226
|
+
if (typeof text !== 'string') throw new TypeError('tokenize: expected a string')
|
|
227
|
+
|
|
228
|
+
/** @type {Token[]} */
|
|
229
|
+
const tokens = []
|
|
230
|
+
let offset = 0
|
|
231
|
+
let line = 1
|
|
232
|
+
let lineStart = 0
|
|
233
|
+
const length = text.length
|
|
234
|
+
|
|
235
|
+
// The stack is local to this call, so a lexer is reusable and a failure
|
|
236
|
+
// partway through leaves nothing behind for the next one.
|
|
237
|
+
let mode = initial
|
|
238
|
+
const stack = multiMode ? [] : null
|
|
239
|
+
let depth = 0
|
|
240
|
+
|
|
241
|
+
outer: while (offset < length) {
|
|
242
|
+
// Inlining the ASCII table lookup here instead of calling `candidates` was
|
|
243
|
+
// measured and made no difference; the cost is in the patterns, not the
|
|
244
|
+
// dispatch.
|
|
245
|
+
const applicable = mode.dispatch ? candidates(mode, text.charCodeAt(offset)) : mode.members
|
|
246
|
+
|
|
247
|
+
for (let i = 0; i < applicable.length; i++) {
|
|
248
|
+
const def = applicable[i]
|
|
249
|
+
|
|
250
|
+
// Find where the match ends without necessarily building it. `test` on a
|
|
251
|
+
// sticky regex advances `lastIndex` just like `exec` but skips
|
|
252
|
+
// materializing the match array, and a skipped token's text is never
|
|
253
|
+
// read at all, so nothing is allocated for it.
|
|
254
|
+
let end = -1
|
|
255
|
+
let matched = null
|
|
256
|
+
if (def.literal !== undefined) {
|
|
257
|
+
if (text.startsWith(def.literal, offset)) end = offset + def.literal.length
|
|
258
|
+
} else if (def.skip) {
|
|
259
|
+
def.regex.lastIndex = offset
|
|
260
|
+
// A zero-length match would spin forever; treat it as no match.
|
|
261
|
+
if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex
|
|
262
|
+
} else {
|
|
263
|
+
def.regex.lastIndex = offset
|
|
264
|
+
const match = def.regex.exec(text)
|
|
265
|
+
if (match !== null && match[0].length > 0) matched = match[0]
|
|
266
|
+
if (matched !== null) end = offset + matched.length
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (end === -1) continue
|
|
270
|
+
|
|
271
|
+
// `longerAlt` lets a keyword defer to an identifier that matches more
|
|
272
|
+
// text: `iffy` is an identifier, not `if` followed by `fy`.
|
|
273
|
+
let winner = def
|
|
274
|
+
if (def.longerAltDef) {
|
|
275
|
+
const altEnd = longerAltEnd(def, text, offset)
|
|
276
|
+
if (altEnd > end) {
|
|
277
|
+
end = altEnd
|
|
278
|
+
winner = def.longerAltDef
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
if (!winner.skip) {
|
|
283
|
+
const image = winner.literal !== undefined
|
|
284
|
+
? winner.literal
|
|
285
|
+
: (winner === def && matched !== null ? matched : text.slice(offset, end))
|
|
286
|
+
// One object shape either way, so the parser's property loads stay
|
|
287
|
+
// monomorphic regardless of the position mode.
|
|
288
|
+
tokens.push({
|
|
289
|
+
type: winner.name,
|
|
290
|
+
image,
|
|
291
|
+
start: offset,
|
|
292
|
+
end,
|
|
293
|
+
line: trackLines ? line : 0,
|
|
294
|
+
col: trackLines ? offset - lineStart + 1 : 0
|
|
295
|
+
})
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// A bounded scan over this token's own span, so the whole tokenize costs
|
|
299
|
+
// one pass over the input — and nothing at all in `offset` mode.
|
|
300
|
+
//
|
|
301
|
+
// An `indexOf('\n')` cursor was tried instead and looks strictly better:
|
|
302
|
+
// native scanning, and no work when the input has no newlines. It made
|
|
303
|
+
// `tokenize` slightly faster in isolation and then made parsing
|
|
304
|
+
// freshly-lexed tokens *sixty times slower* (6.8 ms -> 361 ms),
|
|
305
|
+
// reproducibly, for reasons that did not survive investigation. Do not
|
|
306
|
+
// "optimize" this without measuring the full pipeline.
|
|
307
|
+
if (trackLines) {
|
|
308
|
+
for (let j = offset; j < end; j++) {
|
|
309
|
+
if (text.charCodeAt(j) !== 10) continue
|
|
310
|
+
line++
|
|
311
|
+
lineStart = j + 1
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// The transition belongs to whichever token was actually produced, so a
|
|
316
|
+
// `longerAlt` that wins brings its own mode change with it. Skipped
|
|
317
|
+
// tokens transition too: a comment can open a mode without being one.
|
|
318
|
+
if (multiMode) {
|
|
319
|
+
if (winner.popMode) {
|
|
320
|
+
if (depth === 0) {
|
|
321
|
+
throw new LexError(
|
|
322
|
+
`Token '${winner.name}' popped the initial lexer mode at ${describe(offset, line, lineStart)}`,
|
|
323
|
+
{ offset, line: trackLines ? line : 0, col: trackLines ? offset - lineStart + 1 : 0 }
|
|
324
|
+
)
|
|
325
|
+
}
|
|
326
|
+
mode = stack[--depth]
|
|
327
|
+
} else if (winner.pushMode !== -1) {
|
|
328
|
+
stack[depth++] = mode
|
|
329
|
+
mode = modes[winner.pushMode]
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
offset = end
|
|
334
|
+
continue outer
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const column = offset - lineStart + 1
|
|
338
|
+
throw new LexError(
|
|
339
|
+
trackLines
|
|
340
|
+
? `Unexpected character '${text[offset]}' at line ${line} column ${column}${inMode(mode)}`
|
|
341
|
+
: `Unexpected character '${text[offset]}' at offset ${offset}${inMode(mode)}`,
|
|
342
|
+
{ offset, line: trackLines ? line : 0, col: trackLines ? column : 0 }
|
|
343
|
+
)
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Reaching the end inside a pushed mode is not a lexer error: the tokens are
|
|
347
|
+
// all valid, and the parser gives a better message for the missing closer
|
|
348
|
+
// than "unexpected end of input" ever could.
|
|
349
|
+
return tokens
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Names the mode in an error, but only when there is more than one to confuse. */
|
|
353
|
+
const inMode = (mode) => namesModes ? ` (in lexer mode '${mode.name}')` : ''
|
|
354
|
+
const describe = (offset, line, lineStart) =>
|
|
355
|
+
trackLines ? `line ${line} column ${offset - lineStart + 1}` : `offset ${offset}`
|
|
356
|
+
|
|
357
|
+
return { tokenize, tokenNames, positions, modeNames }
|
|
358
|
+
}
|
package/src/methods.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The grammar combinators, as json-logic-engine methods.
|
|
3
|
+
*
|
|
4
|
+
* Every one is the same shape, because every one does the same thing: hand the
|
|
5
|
+
* node to the planner and run the closure it gives back. The per-operator
|
|
6
|
+
* behaviour lives in `plan.js` (interpreted) and `codegen.js` (compiled) —
|
|
7
|
+
* nothing here needs to know what `alt` or `manySep` mean.
|
|
8
|
+
*
|
|
9
|
+
* Two flags matter, and both are load-bearing:
|
|
10
|
+
*
|
|
11
|
+
* `lazy: true` — a combinator's arguments are alternatives and repetitions. The
|
|
12
|
+
* engine must not evaluate them before the combinator has decided whether, and
|
|
13
|
+
* how many times, each should run, so the method takes raw logic.
|
|
14
|
+
*
|
|
15
|
+
* `deterministic: false` — the optimizer inlines deterministic calls *at
|
|
16
|
+
* optimize time* (`optimizer.js`: `if (deterministic) return result()`), which
|
|
17
|
+
* for a parser would mean running it against no input before the parse begins.
|
|
18
|
+
*/
|
|
19
|
+
export const OPERATORS: string[];
|
|
20
|
+
export function createMethods(): Record<string, any>;
|
package/src/methods.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
'use strict'
|
|
3
|
+
|
|
4
|
+
import { declareSync } from './plan.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The grammar combinators, as json-logic-engine methods.
|
|
8
|
+
*
|
|
9
|
+
* Every one is the same shape, because every one does the same thing: hand the
|
|
10
|
+
* node to the planner and run the closure it gives back. The per-operator
|
|
11
|
+
* behaviour lives in `plan.js` (interpreted) and `codegen.js` (compiled) —
|
|
12
|
+
* nothing here needs to know what `alt` or `manySep` mean.
|
|
13
|
+
*
|
|
14
|
+
* Two flags matter, and both are load-bearing:
|
|
15
|
+
*
|
|
16
|
+
* `lazy: true` — a combinator's arguments are alternatives and repetitions. The
|
|
17
|
+
* engine must not evaluate them before the combinator has decided whether, and
|
|
18
|
+
* how many times, each should run, so the method takes raw logic.
|
|
19
|
+
*
|
|
20
|
+
* `deterministic: false` — the optimizer inlines deterministic calls *at
|
|
21
|
+
* optimize time* (`optimizer.js`: `if (deterministic) return result()`), which
|
|
22
|
+
* for a parser would mean running it against no input before the parse begins.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export const OPERATORS = [
|
|
26
|
+
'consume', 'epsilon', 'seq', 'alt', 'oneOf', 'eof',
|
|
27
|
+
'many', 'many1', 'option', 'manySep', 'many1Sep',
|
|
28
|
+
'infixLeft', 'infixRight', 'prefix', 'postfix', 'between', 'as',
|
|
29
|
+
'subrule', 'label', 'action', 'lookahead', 'negLookahead', 'text'
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
const plannerOf = (engine) => {
|
|
33
|
+
const context = engine.__jlGrammar
|
|
34
|
+
if (!context) throw new Error('grammar combinators require a parser built by createParser')
|
|
35
|
+
return context.planner
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The compiler calls this while generating source. Interpolating a *function*
|
|
40
|
+
* into `buildState.compile` puts it in the generated code's scope and emits a
|
|
41
|
+
* reference to it, so what comes back is a direct call into the planner's
|
|
42
|
+
* closure — no engine dispatch, no operator lookup, no per-node lookup at parse
|
|
43
|
+
* time. Returning `false` falls back to calling the method, which is what
|
|
44
|
+
* happens on an engine `createParser` never set up.
|
|
45
|
+
*/
|
|
46
|
+
const compileVia = (op) => (logic, buildState) => {
|
|
47
|
+
const context = buildState.engine.__jlGrammar
|
|
48
|
+
if (!context) return false
|
|
49
|
+
return buildState.compile`${declareSync(context.planner.planOp(op, logic))}(context)`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const combinator = (op) => ({
|
|
53
|
+
lazy: true,
|
|
54
|
+
deterministic: false,
|
|
55
|
+
compile: compileVia(op),
|
|
56
|
+
method: (logic, state, above, engine) => plannerOf(engine).planOp(op, logic)(state)
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
/** @returns {Record<string, any>} */
|
|
60
|
+
export const createMethods = () =>
|
|
61
|
+
Object.fromEntries(OPERATORS.map((op) => [op, combinator(op)]))
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
'use strict'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The `methods { ... }` block: a terminal section of a `.gram` file holding the
|
|
6
|
+
* JavaScript its semantic actions call.
|
|
7
|
+
*
|
|
8
|
+
* A grammar's actions name methods — `{"formula.number": {"val": "image"}}` —
|
|
9
|
+
* and those methods have to be real functions somewhere. Without this, "somewhere"
|
|
10
|
+
* is a second file the caller has to wire up by hand.
|
|
11
|
+
*
|
|
12
|
+
* The block does not compromise the grammar's data property. It is separated
|
|
13
|
+
* textually *before* the definition language sees the source, so what
|
|
14
|
+
* `parseDefinition` returns is the same JSON-serializable object it always was,
|
|
15
|
+
* with the block carried alongside as an opaque string. A `.gram` with no block
|
|
16
|
+
* parses exactly as it did.
|
|
17
|
+
*
|
|
18
|
+
* The block is a module body: its default export is the methods table, or, with
|
|
19
|
+
* no default export, its named exports become the table. Named exports are also
|
|
20
|
+
* re-exported by a generated parser, which is how a grammar can ship helpers
|
|
21
|
+
* that its host needs — see `useFunctionSet` in sheetlang.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** Matches the block header. Anchored to a line so a stray identifier cannot start one. */
|
|
25
|
+
const HEADER = /^[ \t]*methods[ \t]*\{/gm
|
|
26
|
+
|
|
27
|
+
/** `import` as a single statement, which is all the CommonJS transform handles. */
|
|
28
|
+
const IMPORT = /^[ \t]*import\b[^\n]*?['"][^'"]+['"][ \t]*;?[ \t]*$/gm
|
|
29
|
+
|
|
30
|
+
/** `export const x`, `export function x`, `export class x`. */
|
|
31
|
+
const DECLARED_EXPORT = /^[ \t]*export[ \t]+(?:const|let|var|function|class|async[ \t]+function)[ \t]+([A-Za-z_$][\w$]*)/gm
|
|
32
|
+
|
|
33
|
+
/** `export { a, b as c }` — a standalone list, not a re-export from a module. */
|
|
34
|
+
const EXPORT_LIST = /^[ \t]*export[ \t]*\{([^}]*)\}[ \t]*;?[ \t]*$/gm
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Splits definition source into the grammar text and the block body.
|
|
38
|
+
*
|
|
39
|
+
* The block is terminal by construction: it runs from its `{` to the last `}` in
|
|
40
|
+
* the file. That makes finding its end a search rather than a parse — no
|
|
41
|
+
* counting braces through string literals, comments and regex literals, which is
|
|
42
|
+
* where this kind of thing usually goes wrong.
|
|
43
|
+
*
|
|
44
|
+
* @param {string} source
|
|
45
|
+
* @returns {{ grammar: string, block: string | null }}
|
|
46
|
+
*/
|
|
47
|
+
export function splitMethodsBlock (source) {
|
|
48
|
+
HEADER.lastIndex = 0
|
|
49
|
+
let header = null
|
|
50
|
+
for (let match = HEADER.exec(source); match; match = HEADER.exec(source)) header = match
|
|
51
|
+
|
|
52
|
+
if (!header) return { grammar: source, block: null }
|
|
53
|
+
|
|
54
|
+
const open = header.index + header[0].length
|
|
55
|
+
const close = source.lastIndexOf('}')
|
|
56
|
+
if (close < open) throw new Error('methods block is not closed: no `}` after it')
|
|
57
|
+
|
|
58
|
+
const trailing = source.slice(close + 1).trim()
|
|
59
|
+
if (trailing) throw new Error(`the methods block must be the last thing in the file; found ${JSON.stringify(trailing.slice(0, 40))} after it`)
|
|
60
|
+
|
|
61
|
+
// Keep the grammar text the same length so parse errors still report true
|
|
62
|
+
// offsets; the block becomes whitespace rather than disappearing.
|
|
63
|
+
const blanked = source.slice(header.index, source.length).replace(/[^\n]/g, ' ')
|
|
64
|
+
return { grammar: source.slice(0, header.index) + blanked, block: source.slice(open, close) }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Reads a block body without executing it: which names it exports, whether it has
|
|
69
|
+
* a default export, and whether it imports anything.
|
|
70
|
+
*
|
|
71
|
+
* @param {string} body
|
|
72
|
+
*/
|
|
73
|
+
export function analyzeMethodsBlock (body) {
|
|
74
|
+
const names = []
|
|
75
|
+
for (const match of body.matchAll(DECLARED_EXPORT)) names.push(match[1])
|
|
76
|
+
|
|
77
|
+
// `export { a, b as c }`, including re-exports of things the block imported.
|
|
78
|
+
for (const match of body.matchAll(EXPORT_LIST)) {
|
|
79
|
+
for (const entry of match[1].split(',')) {
|
|
80
|
+
const name = entry.trim().split(/\s+as\s+/).pop()
|
|
81
|
+
if (name) names.push(name)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
names,
|
|
87
|
+
hasDefault: /^[ \t]*export[ \t]+default\b/m.test(body),
|
|
88
|
+
imports: (body.match(IMPORT) || []).map((line) => line.trim())
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The block body as a plain function body: `export` stripped, and a `return`
|
|
94
|
+
* yielding the methods table.
|
|
95
|
+
*
|
|
96
|
+
* @param {string} body
|
|
97
|
+
* @param {{ names: string[], hasDefault: boolean }} info
|
|
98
|
+
*/
|
|
99
|
+
function asFunctionBody (body, info) {
|
|
100
|
+
const stripped = body
|
|
101
|
+
.replace(/^([ \t]*)export[ \t]+default[ \t]+/m, '$1const __default = ')
|
|
102
|
+
// `export { a, b }` names things already bound above it; the list itself goes.
|
|
103
|
+
.replace(EXPORT_LIST, '')
|
|
104
|
+
.replace(/^([ \t]*)export[ \t]+/gm, '$1')
|
|
105
|
+
const table = info.hasDefault ? '__default' : `{ ${info.names.join(', ')} }`
|
|
106
|
+
return `${stripped}\nreturn { methods: ${table}, exports: { ${info.names.join(', ')} } }`
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Evaluates a block to its methods table.
|
|
111
|
+
*
|
|
112
|
+
* Needs `new Function`, so it is unavailable under a Content-Security-Policy
|
|
113
|
+
* without `unsafe-eval` — and unavailable to a block that imports anything,
|
|
114
|
+
* since a function body cannot. Both are build-time concerns: `emitModule`
|
|
115
|
+
* writes the block out as source and never runs it, and the CLI resolves an
|
|
116
|
+
* importing block through a real module.
|
|
117
|
+
*
|
|
118
|
+
* @param {string} body
|
|
119
|
+
* @returns {{ methods: Record<string, any>, exports: Record<string, any> }}
|
|
120
|
+
*/
|
|
121
|
+
export function evaluateMethodsBlock (body) {
|
|
122
|
+
const info = analyzeMethodsBlock(body)
|
|
123
|
+
if (info.imports.length > 0) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
'this methods block imports, so it cannot be evaluated in place: ' +
|
|
126
|
+
`${JSON.stringify(info.imports[0])}. Generate a parser file from it instead ` +
|
|
127
|
+
'(`grammatik generate`), or pass `methods` explicitly.'
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
if (!info.hasDefault && info.names.length === 0) {
|
|
131
|
+
throw new Error('methods block exports nothing; it needs `export default <table>` or named exports')
|
|
132
|
+
}
|
|
133
|
+
// eslint-disable-next-line no-new-func
|
|
134
|
+
return new Function(asFunctionBody(body, info))()
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Removes the indentation the block carried inside its braces, so it lands at
|
|
139
|
+
* the left margin of the generated module instead of two spaces in.
|
|
140
|
+
*/
|
|
141
|
+
function dedent (body) {
|
|
142
|
+
const lines = body.replace(/^\n+|\s+$/g, '').split('\n')
|
|
143
|
+
let common = Infinity
|
|
144
|
+
for (const line of lines) {
|
|
145
|
+
if (!line.trim()) continue
|
|
146
|
+
common = Math.min(common, line.length - line.trimStart().length)
|
|
147
|
+
}
|
|
148
|
+
if (!Number.isFinite(common) || common === 0) return lines.join('\n')
|
|
149
|
+
return lines.map((line) => line.slice(common)).join('\n')
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Rewrites the `import` forms a CommonJS file can express as `require`. */
|
|
153
|
+
function toRequire (line) {
|
|
154
|
+
const namespace = line.match(/^import[ \t]*\*[ \t]*as[ \t]+([\w$]+)[ \t]+from[ \t]*(['"][^'"]+['"])/)
|
|
155
|
+
if (namespace) return `const ${namespace[1]} = require(${namespace[2]})`
|
|
156
|
+
|
|
157
|
+
const named = line.match(/^import[ \t]*(\{[^}]*\})[ \t]*from[ \t]*(['"][^'"]+['"])/)
|
|
158
|
+
if (named) return `const ${named[1].replace(/\bas\b/g, ':')} = require(${named[2]})`
|
|
159
|
+
|
|
160
|
+
const fallback = line.match(/^import[ \t]+([\w$]+)[ \t]+from[ \t]*(['"][^'"]+['"])/)
|
|
161
|
+
if (fallback) return `const ${fallback[1]} = require(${fallback[2]}).default ?? require(${fallback[2]})`
|
|
162
|
+
|
|
163
|
+
const bare = line.match(/^import[ \t]*(['"][^'"]+['"])/)
|
|
164
|
+
if (bare) return `require(${bare[1]})`
|
|
165
|
+
|
|
166
|
+
throw new Error(`cannot express this import in CommonJS, move it or emit ES modules: ${JSON.stringify(line)}`)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The block as source for a generated module: `export` stripped so nothing
|
|
171
|
+
* escapes by accident, and the table bound to a name the file can register.
|
|
172
|
+
*
|
|
173
|
+
* @param {string} body
|
|
174
|
+
* @param {'esm'|'cjs'} format
|
|
175
|
+
* @returns {{ source: string, table: string, names: string[] }}
|
|
176
|
+
*/
|
|
177
|
+
export function emitMethodsBlock (body, format) {
|
|
178
|
+
const info = analyzeMethodsBlock(body)
|
|
179
|
+
if (!info.hasDefault && info.names.length === 0) {
|
|
180
|
+
throw new Error('methods block exports nothing; it needs `export default <table>` or named exports')
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
let source = dedent(body)
|
|
184
|
+
.replace(/^export[ \t]+default[ \t]+/m, 'const __methods = ')
|
|
185
|
+
.replace(EXPORT_LIST, '')
|
|
186
|
+
.replace(/^export[ \t]+/gm, '')
|
|
187
|
+
|
|
188
|
+
// ES modules take the imports as they are — an import declaration is legal
|
|
189
|
+
// anywhere at the top level of a module, so the block needs no hoisting.
|
|
190
|
+
if (format === 'cjs') source = source.replace(IMPORT, (line) => toRequire(line.trim()))
|
|
191
|
+
|
|
192
|
+
return {
|
|
193
|
+
source,
|
|
194
|
+
table: info.hasDefault ? '__methods' : `{ ${info.names.join(', ')} }`,
|
|
195
|
+
names: info.names
|
|
196
|
+
}
|
|
197
|
+
}
|