@tellaro/tql 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/tokenizer.js +88 -86
- package/package.json +2 -1
package/dist/src/tokenizer.js
CHANGED
|
@@ -191,16 +191,16 @@ class g {
|
|
|
191
191
|
line;
|
|
192
192
|
column;
|
|
193
193
|
tokens;
|
|
194
|
-
constructor(
|
|
195
|
-
this.input =
|
|
194
|
+
constructor(e) {
|
|
195
|
+
this.input = e ?? "", this.position = 0, this.line = 1, this.column = 1, this.tokens = [];
|
|
196
196
|
}
|
|
197
197
|
/**
|
|
198
198
|
* Tokenize the entire input string
|
|
199
199
|
*/
|
|
200
200
|
tokenize() {
|
|
201
201
|
for (this.tokens = [], this.position = 0, this.line = 1, this.column = 1; !this.isAtEnd(); ) {
|
|
202
|
-
const
|
|
203
|
-
|
|
202
|
+
const e = this.scanToken();
|
|
203
|
+
e && this.tokens.push(e);
|
|
204
204
|
}
|
|
205
205
|
return this.tokens;
|
|
206
206
|
}
|
|
@@ -210,42 +210,44 @@ class g {
|
|
|
210
210
|
scanToken() {
|
|
211
211
|
if (this.isWhitespace(this.peek()))
|
|
212
212
|
return this.scanWhitespace();
|
|
213
|
-
const
|
|
214
|
-
switch (
|
|
213
|
+
const e = this.position, n = this.line, i = this.column, t = this.advance();
|
|
214
|
+
switch (t) {
|
|
215
215
|
case "(":
|
|
216
216
|
case ")":
|
|
217
|
-
return this.makeToken("paren",
|
|
217
|
+
return this.makeToken("paren", t, e, n, i);
|
|
218
218
|
case "[":
|
|
219
219
|
case "]":
|
|
220
|
-
return this.makeToken("bracket",
|
|
220
|
+
return this.makeToken("bracket", t, e, n, i);
|
|
221
221
|
case ",":
|
|
222
|
-
return this.makeToken("comma",
|
|
222
|
+
return this.makeToken("comma", t, e, n, i);
|
|
223
223
|
case "|":
|
|
224
|
-
return this.makeToken("pipe",
|
|
224
|
+
return this.makeToken("pipe", t, e, n, i);
|
|
225
225
|
}
|
|
226
|
-
return
|
|
226
|
+
return t === "=" && this.peek() === ">" ? (this.advance(), this.makeToken("arrow", "=>", e, n, i)) : t === ":" && this.peek() === ":" ? (this.advance(), this.makeToken("doubleColon", "::", e, n, i)) : t === '"' || t === "'" ? this.scanString(t, e, n, i) : this.isDigit(t) || t === "-" && this.isDigit(this.peek()) ? this.scanNumber(e, n, i) : this.isOperatorChar(t) ? this.scanSymbolOperator(e, n, i) : this.isIdentifierStart(t) ? this.scanIdentifier(e, n, i) : this.makeToken("unknown", t, e, n, i);
|
|
227
227
|
}
|
|
228
228
|
/**
|
|
229
229
|
* Scan whitespace characters
|
|
230
230
|
*/
|
|
231
231
|
scanWhitespace() {
|
|
232
|
-
const
|
|
233
|
-
let
|
|
232
|
+
const e = this.position, n = this.line, i = this.column;
|
|
233
|
+
let t = "";
|
|
234
234
|
for (; !this.isAtEnd() && this.isWhitespace(this.peek()); ) {
|
|
235
235
|
const s = this.advance();
|
|
236
|
-
|
|
236
|
+
t += s;
|
|
237
237
|
}
|
|
238
|
-
return this.makeToken("whitespace",
|
|
238
|
+
return this.makeToken("whitespace", t, e, n, i);
|
|
239
239
|
}
|
|
240
240
|
/**
|
|
241
241
|
* Scan a string literal (single or double quoted)
|
|
242
242
|
*/
|
|
243
|
-
scanString(
|
|
243
|
+
scanString(e, n, i, t) {
|
|
244
244
|
let s = "";
|
|
245
|
-
const r =
|
|
246
|
-
for (; !this.isAtEnd() && this.peek() !==
|
|
247
|
-
if (this.peek() === "\\")
|
|
248
|
-
if (this.advance(),
|
|
245
|
+
const r = e;
|
|
246
|
+
for (; !this.isAtEnd() && this.peek() !== e; )
|
|
247
|
+
if (this.peek() === "\\")
|
|
248
|
+
if (this.advance(), this.isAtEnd())
|
|
249
|
+
s += "\\";
|
|
250
|
+
else {
|
|
249
251
|
const a = this.advance();
|
|
250
252
|
switch (a) {
|
|
251
253
|
case "n":
|
|
@@ -268,10 +270,10 @@ class g {
|
|
|
268
270
|
s += "'";
|
|
269
271
|
break;
|
|
270
272
|
default:
|
|
271
|
-
s += a;
|
|
273
|
+
s += "\\" + a;
|
|
272
274
|
}
|
|
273
275
|
}
|
|
274
|
-
|
|
276
|
+
else
|
|
275
277
|
s += this.advance();
|
|
276
278
|
return this.isAtEnd() ? {
|
|
277
279
|
type: "error",
|
|
@@ -279,7 +281,7 @@ class g {
|
|
|
279
281
|
start: n,
|
|
280
282
|
end: this.position,
|
|
281
283
|
line: i,
|
|
282
|
-
column:
|
|
284
|
+
column: t,
|
|
283
285
|
raw: r + s
|
|
284
286
|
} : (this.advance(), {
|
|
285
287
|
type: "string",
|
|
@@ -287,88 +289,88 @@ class g {
|
|
|
287
289
|
start: n,
|
|
288
290
|
end: this.position,
|
|
289
291
|
line: i,
|
|
290
|
-
column:
|
|
291
|
-
raw:
|
|
292
|
+
column: t,
|
|
293
|
+
raw: e + s + e
|
|
292
294
|
});
|
|
293
295
|
}
|
|
294
296
|
/**
|
|
295
297
|
* Scan a numeric literal
|
|
296
298
|
*/
|
|
297
|
-
scanNumber(
|
|
298
|
-
this.position =
|
|
299
|
-
let
|
|
300
|
-
for (this.peek() === "-" && (
|
|
301
|
-
|
|
299
|
+
scanNumber(e, n, i) {
|
|
300
|
+
this.position = e, this.column = i;
|
|
301
|
+
let t = "";
|
|
302
|
+
for (this.peek() === "-" && (t += this.advance()); !this.isAtEnd() && this.isDigit(this.peek()); )
|
|
303
|
+
t += this.advance();
|
|
302
304
|
if (this.peek() === "." && this.isDigit(this.peekNext()))
|
|
303
|
-
for (
|
|
304
|
-
|
|
305
|
+
for (t += this.advance(); !this.isAtEnd() && this.isDigit(this.peek()); )
|
|
306
|
+
t += this.advance();
|
|
305
307
|
if (this.peek() === "e" || this.peek() === "E")
|
|
306
|
-
for (
|
|
307
|
-
|
|
308
|
-
return this.makeToken("number",
|
|
308
|
+
for (t += this.advance(), (this.peek() === "+" || this.peek() === "-") && (t += this.advance()); !this.isAtEnd() && this.isDigit(this.peek()); )
|
|
309
|
+
t += this.advance();
|
|
310
|
+
return this.makeToken("number", t, e, n, i);
|
|
309
311
|
}
|
|
310
312
|
/**
|
|
311
313
|
* Scan symbol-based operators (==, !=, >, >=, <, <=, !)
|
|
312
314
|
*/
|
|
313
|
-
scanSymbolOperator(
|
|
314
|
-
this.position =
|
|
315
|
-
let
|
|
315
|
+
scanSymbolOperator(e, n, i) {
|
|
316
|
+
this.position = e, this.column = i;
|
|
317
|
+
let t = this.advance();
|
|
316
318
|
if (!this.isAtEnd()) {
|
|
317
|
-
const s =
|
|
319
|
+
const s = t + this.peek();
|
|
318
320
|
if (s === "=>")
|
|
319
|
-
return this.advance(), this.makeToken("arrow", "=>",
|
|
320
|
-
(s in h || s === "==" || s === "!=") && (
|
|
321
|
+
return this.advance(), this.makeToken("arrow", "=>", e, n, i);
|
|
322
|
+
(s in h || s === "==" || s === "!=") && (t = s, this.advance());
|
|
321
323
|
}
|
|
322
|
-
if (
|
|
324
|
+
if (t === "!") {
|
|
323
325
|
const s = this.position;
|
|
324
326
|
let r = "";
|
|
325
327
|
for (; !this.isAtEnd() && this.isIdentifierChar(this.peek()); )
|
|
326
328
|
r += this.advance();
|
|
327
329
|
if (r) {
|
|
328
330
|
if ("!" + r in u)
|
|
329
|
-
return this.makeToken("operator", "!" + r,
|
|
331
|
+
return this.makeToken("operator", "!" + r, e, n, i);
|
|
330
332
|
this.position = s, this.column = i + 1;
|
|
331
333
|
}
|
|
332
334
|
}
|
|
333
|
-
return
|
|
335
|
+
return t in h || t === "==" || t === "=" || t === "!=" || t === ">" || t === ">=" || t === "<" || t === "<=" ? this.makeToken("operator", t, e, n, i) : this.makeToken("unknown", t, e, n, i);
|
|
334
336
|
}
|
|
335
337
|
/**
|
|
336
338
|
* Scan an identifier (field name, operator keyword, etc.)
|
|
337
339
|
*/
|
|
338
|
-
scanIdentifier(
|
|
339
|
-
this.position =
|
|
340
|
-
let
|
|
340
|
+
scanIdentifier(e, n, i) {
|
|
341
|
+
this.position = e, this.column = i;
|
|
342
|
+
let t = "";
|
|
341
343
|
const s = this.advance();
|
|
342
|
-
if (
|
|
343
|
-
return this.makeToken("unknown",
|
|
344
|
+
if (t += s, s === "@" && this.peek() === "@")
|
|
345
|
+
return this.makeToken("unknown", t, e, n, i);
|
|
344
346
|
let r = s;
|
|
345
347
|
for (; !this.isAtEnd() && this.isIdentifierChar(this.peek()); ) {
|
|
346
348
|
const c = this.advance();
|
|
347
349
|
if (c === "." && r === ".")
|
|
348
|
-
return this.position--, this.column--,
|
|
349
|
-
|
|
350
|
+
return this.position--, this.column--, t = t.slice(0, -1), this.makeToken("unknown", t, e, n, i);
|
|
351
|
+
t += c, r = c;
|
|
350
352
|
}
|
|
351
|
-
|
|
352
|
-
const a = this.classifyIdentifier(
|
|
353
|
-
return this.makeToken(a,
|
|
353
|
+
t.endsWith(".") && (this.position--, this.column--, t = t.slice(0, -1));
|
|
354
|
+
const a = this.classifyIdentifier(t);
|
|
355
|
+
return this.makeToken(a, t, e, n, i);
|
|
354
356
|
}
|
|
355
357
|
/**
|
|
356
358
|
* Classify an identifier into the appropriate token type
|
|
357
359
|
*/
|
|
358
|
-
classifyIdentifier(
|
|
359
|
-
const n =
|
|
360
|
+
classifyIdentifier(e) {
|
|
361
|
+
const n = e.toUpperCase(), i = e.toLowerCase();
|
|
360
362
|
return d.includes(n) ? "logicalOp" : p.includes(n) ? "collectionOp" : i === "true" || i === "false" ? "boolean" : i === "null" ? "null" : i === "stats" ? "keyword" : l.includes(i) ? "operator" : _.includes(i) ? "aggregation" : k.includes(i) ? "mutator" : f.includes(i) ? "typeHint" : m.includes(i) ? "visualization" : w.includes(i) ? "keyword" : "field";
|
|
361
363
|
}
|
|
362
364
|
// ============================================================================
|
|
363
365
|
// Helper Methods
|
|
364
366
|
// ============================================================================
|
|
365
|
-
makeToken(
|
|
367
|
+
makeToken(e, n, i, t, s) {
|
|
366
368
|
return {
|
|
367
|
-
type:
|
|
369
|
+
type: e,
|
|
368
370
|
value: n,
|
|
369
371
|
start: i,
|
|
370
372
|
end: this.position,
|
|
371
|
-
line:
|
|
373
|
+
line: t,
|
|
372
374
|
column: s
|
|
373
375
|
};
|
|
374
376
|
}
|
|
@@ -382,64 +384,64 @@ class g {
|
|
|
382
384
|
return this.position + 1 >= this.input.length ? "\0" : this.input[this.position + 1];
|
|
383
385
|
}
|
|
384
386
|
advance() {
|
|
385
|
-
const
|
|
386
|
-
return this.position++,
|
|
387
|
-
` ? (this.line++, this.column = 1) : this.column++,
|
|
387
|
+
const e = this.input[this.position];
|
|
388
|
+
return this.position++, e === `
|
|
389
|
+
` ? (this.line++, this.column = 1) : this.column++, e;
|
|
388
390
|
}
|
|
389
|
-
isWhitespace(
|
|
390
|
-
return
|
|
391
|
-
` ||
|
|
391
|
+
isWhitespace(e) {
|
|
392
|
+
return e === " " || e === " " || e === `
|
|
393
|
+
` || e === "\r";
|
|
392
394
|
}
|
|
393
|
-
isDigit(
|
|
394
|
-
return
|
|
395
|
+
isDigit(e) {
|
|
396
|
+
return e >= "0" && e <= "9";
|
|
395
397
|
}
|
|
396
|
-
isIdentifierStart(
|
|
397
|
-
return
|
|
398
|
+
isIdentifierStart(e) {
|
|
399
|
+
return e >= "a" && e <= "z" || e >= "A" && e <= "Z" || e === "_" || e === "@";
|
|
398
400
|
}
|
|
399
401
|
/**
|
|
400
402
|
* Check if char can continue an identifier (NOT including @ which is only valid at start)
|
|
401
403
|
*/
|
|
402
|
-
isIdentifierChar(
|
|
403
|
-
return
|
|
404
|
-
|
|
404
|
+
isIdentifierChar(e) {
|
|
405
|
+
return e >= "a" && e <= "z" || e >= "A" && e <= "Z" || e === "_" || this.isDigit(e) || e === "." || // For nested field access
|
|
406
|
+
e === "-";
|
|
405
407
|
}
|
|
406
|
-
isOperatorChar(
|
|
407
|
-
return
|
|
408
|
+
isOperatorChar(e) {
|
|
409
|
+
return e === "=" || e === "!" || e === ">" || e === "<";
|
|
408
410
|
}
|
|
409
411
|
}
|
|
410
412
|
function T(o) {
|
|
411
413
|
return new g(o).tokenize();
|
|
412
414
|
}
|
|
413
415
|
function b(o) {
|
|
414
|
-
return T(o).filter((
|
|
416
|
+
return T(o).filter((e) => e.type !== "whitespace");
|
|
415
417
|
}
|
|
416
|
-
function v(o,
|
|
418
|
+
function v(o, e) {
|
|
417
419
|
for (const n of o)
|
|
418
|
-
if (
|
|
420
|
+
if (e >= n.start && e < n.end)
|
|
419
421
|
return n;
|
|
420
422
|
return null;
|
|
421
423
|
}
|
|
422
|
-
function O(o,
|
|
423
|
-
return o.filter((i) => i.start < n && i.end >
|
|
424
|
+
function O(o, e, n) {
|
|
425
|
+
return o.filter((i) => i.start < n && i.end > e);
|
|
424
426
|
}
|
|
425
|
-
function A(o,
|
|
427
|
+
function A(o, e) {
|
|
426
428
|
let n = null;
|
|
427
429
|
for (const i of o)
|
|
428
|
-
if (i.end <=
|
|
430
|
+
if (i.end <= e)
|
|
429
431
|
n = i;
|
|
430
432
|
else
|
|
431
433
|
break;
|
|
432
434
|
return n;
|
|
433
435
|
}
|
|
434
|
-
function E(o,
|
|
436
|
+
function E(o, e, n = !0) {
|
|
435
437
|
const i = n ? o.filter((s) => s.type !== "whitespace") : o;
|
|
436
|
-
let
|
|
438
|
+
let t = null;
|
|
437
439
|
for (const s of i)
|
|
438
|
-
if (s.end <=
|
|
439
|
-
|
|
440
|
+
if (s.end <= e)
|
|
441
|
+
t = s;
|
|
440
442
|
else
|
|
441
443
|
break;
|
|
442
|
-
return
|
|
444
|
+
return t;
|
|
443
445
|
}
|
|
444
446
|
export {
|
|
445
447
|
_ as AGGREGATION_FUNCTIONS,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tellaro/tql",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Tellaro Query Language (TQL) parser, validator, and autocomplete for JavaScript/TypeScript",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -120,6 +120,7 @@
|
|
|
120
120
|
"@microsoft/api-extractor": {
|
|
121
121
|
"minimatch": ">=10.2.3"
|
|
122
122
|
},
|
|
123
|
+
"esbuild": ">=0.28.1",
|
|
123
124
|
"flatted": ">=3.4.0",
|
|
124
125
|
"lodash": ">=4.18.0",
|
|
125
126
|
"postcss": ">=8.5.10"
|