@gooddata/maql-antlr4ts 0.7.2-567f11f5.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -35,8 +35,13 @@ const normalizedAst = normalizer.visit(ast);
|
|
|
35
35
|
```
|
|
36
36
|
|
|
37
37
|
The grammar declares the `caseInsensitive` option and the generated lexer honors it, so MAQL keywords may be
|
|
38
|
-
written in any letter case with a plain `CharStream`.
|
|
39
|
-
|
|
38
|
+
written in any letter case with a plain `CharStream`.
|
|
39
|
+
|
|
40
|
+
Use the exported `CaseInsensitiveCharStream` instead of `CharStream.fromString` when token positions have to
|
|
41
|
+
match JavaScript string offsets or LSP ranges. The antlr4ng stream indexes by Unicode code point, so a character
|
|
42
|
+
outside the Basic Multilingual Plane (an emoji, for example) shifts every following `start`, `stop` and `column`
|
|
43
|
+
by one against the source string. `CaseInsensitiveCharStream` indexes by UTF-16 code unit, so its positions are
|
|
44
|
+
string offsets.
|
|
40
45
|
|
|
41
46
|
## Local development
|
|
42
47
|
|
|
@@ -1,13 +1,35 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type CharStream, type Interval } from "antlr4ng";
|
|
2
2
|
/**
|
|
3
|
-
* ANTLR4 CharStream
|
|
3
|
+
* ANTLR4 CharStream over a JavaScript string that indexes by UTF-16 code units and lowercases lookahead.
|
|
4
|
+
*
|
|
5
|
+
* The antlr4ng `CharStream.fromString` stream stores Unicode code points, so a character outside the Basic
|
|
6
|
+
* Multilingual Plane (for example an emoji) takes one position there, while it takes two in a JavaScript string
|
|
7
|
+
* and in LSP ranges. Token `start`, `stop` and `column` values would then drift from the source text after such
|
|
8
|
+
* a character. This stream feeds the lexer UTF-16 code units instead, so every token position is directly a
|
|
9
|
+
* string offset. The MAQL grammar only matches such characters with `.` and `~[...]` sets (in strings, comments
|
|
10
|
+
* and object IDs), which accept the two surrogate halves one by one, so the tokens come out the same.
|
|
11
|
+
*
|
|
12
|
+
* The lexer also honors the grammar's `caseInsensitive` option, so lowercasing in `LA` is kept only for
|
|
13
|
+
* backward compatibility.
|
|
4
14
|
*/
|
|
5
|
-
export declare class CaseInsensitiveCharStream
|
|
6
|
-
|
|
15
|
+
export declare class CaseInsensitiveCharStream implements CharStream {
|
|
16
|
+
private readonly text;
|
|
17
|
+
name: string;
|
|
18
|
+
index: number;
|
|
19
|
+
constructor(text: string, sourceName?: string);
|
|
20
|
+
reset(): void;
|
|
21
|
+
consume(): void;
|
|
7
22
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* The input string is not modified.
|
|
23
|
+
* Lookahead by UTF-16 code unit. The unit is lowercased before it is passed to the caller;
|
|
24
|
+
* the input string is not modified.
|
|
11
25
|
*/
|
|
12
26
|
LA(offset: number): number;
|
|
27
|
+
mark(): number;
|
|
28
|
+
release(_marker: number): void;
|
|
29
|
+
seek(index: number): void;
|
|
30
|
+
get size(): number;
|
|
31
|
+
getSourceName(): string;
|
|
32
|
+
getTextFromRange(start: number, stop?: number): string;
|
|
33
|
+
getTextFromInterval(interval: Interval): string;
|
|
34
|
+
toString(): string;
|
|
13
35
|
}
|
|
@@ -1,25 +1,74 @@
|
|
|
1
1
|
// (C) 2022 GoodData Corporation
|
|
2
|
-
import {
|
|
2
|
+
import { IntStream, Token } from "antlr4ng";
|
|
3
3
|
/**
|
|
4
|
-
* ANTLR4 CharStream
|
|
4
|
+
* ANTLR4 CharStream over a JavaScript string that indexes by UTF-16 code units and lowercases lookahead.
|
|
5
|
+
*
|
|
6
|
+
* The antlr4ng `CharStream.fromString` stream stores Unicode code points, so a character outside the Basic
|
|
7
|
+
* Multilingual Plane (for example an emoji) takes one position there, while it takes two in a JavaScript string
|
|
8
|
+
* and in LSP ranges. Token `start`, `stop` and `column` values would then drift from the source text after such
|
|
9
|
+
* a character. This stream feeds the lexer UTF-16 code units instead, so every token position is directly a
|
|
10
|
+
* string offset. The MAQL grammar only matches such characters with `.` and `~[...]` sets (in strings, comments
|
|
11
|
+
* and object IDs), which accept the two surrogate halves one by one, so the tokens come out the same.
|
|
12
|
+
*
|
|
13
|
+
* The lexer also honors the grammar's `caseInsensitive` option, so lowercasing in `LA` is kept only for
|
|
14
|
+
* backward compatibility.
|
|
5
15
|
*/
|
|
6
|
-
export class CaseInsensitiveCharStream
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
16
|
+
export class CaseInsensitiveCharStream {
|
|
17
|
+
text;
|
|
18
|
+
name;
|
|
19
|
+
index = 0;
|
|
20
|
+
constructor(text, sourceName) {
|
|
21
|
+
this.text = text;
|
|
22
|
+
this.name = sourceName ?? IntStream.UNKNOWN_SOURCE_NAME;
|
|
23
|
+
}
|
|
24
|
+
reset() {
|
|
25
|
+
this.index = 0;
|
|
26
|
+
}
|
|
27
|
+
consume() {
|
|
28
|
+
if (this.index >= this.text.length) {
|
|
29
|
+
throw new Error("cannot consume EOF");
|
|
11
30
|
}
|
|
31
|
+
this.index += 1;
|
|
12
32
|
}
|
|
13
33
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* The input string is not modified.
|
|
34
|
+
* Lookahead by UTF-16 code unit. The unit is lowercased before it is passed to the caller;
|
|
35
|
+
* the input string is not modified.
|
|
17
36
|
*/
|
|
18
37
|
LA(offset) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
38
|
+
if (offset === 0) {
|
|
39
|
+
return 0;
|
|
40
|
+
}
|
|
41
|
+
if (offset < 0) {
|
|
42
|
+
offset += 1;
|
|
22
43
|
}
|
|
23
|
-
|
|
44
|
+
const pos = this.index + offset - 1;
|
|
45
|
+
if (pos < 0 || pos >= this.text.length) {
|
|
46
|
+
return Token.EOF;
|
|
47
|
+
}
|
|
48
|
+
const c = this.text.charCodeAt(pos);
|
|
49
|
+
return String.fromCharCode(c).toLowerCase().charCodeAt(0);
|
|
50
|
+
}
|
|
51
|
+
// mark/release do nothing; the entire buffer is in memory
|
|
52
|
+
mark() {
|
|
53
|
+
return -1;
|
|
54
|
+
}
|
|
55
|
+
release(_marker) { }
|
|
56
|
+
seek(index) {
|
|
57
|
+
this.index = Math.min(index, this.text.length);
|
|
58
|
+
}
|
|
59
|
+
get size() {
|
|
60
|
+
return this.text.length;
|
|
61
|
+
}
|
|
62
|
+
getSourceName() {
|
|
63
|
+
return this.name;
|
|
64
|
+
}
|
|
65
|
+
getTextFromRange(start, stop = this.text.length - 1) {
|
|
66
|
+
return this.text.substring(start, stop + 1);
|
|
67
|
+
}
|
|
68
|
+
getTextFromInterval(interval) {
|
|
69
|
+
return this.getTextFromRange(interval.start, interval.stop);
|
|
70
|
+
}
|
|
71
|
+
toString() {
|
|
72
|
+
return this.text;
|
|
24
73
|
}
|
|
25
74
|
}
|