@gooddata/maql-antlr4ts 0.7.2-567f11f5.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -35,8 +35,13 @@ const normalizedAst = normalizer.visit(ast);
35
35
  ```
36
36
 
37
37
  The grammar declares the `caseInsensitive` option and the generated lexer honors it, so MAQL keywords may be
38
- written in any letter case with a plain `CharStream`. The exported `CaseInsensitiveCharStream` is kept for
39
- backward compatibility and can still be used in place of `CharStream.fromString`.
38
+ written in any letter case with a plain `CharStream`.
39
+
40
+ Use the exported `CaseInsensitiveCharStream` instead of `CharStream.fromString` when token positions have to
41
+ match JavaScript string offsets or LSP ranges. The antlr4ng stream indexes by Unicode code point, so a character
42
+ outside the Basic Multilingual Plane (an emoji, for example) shifts every following `start`, `stop` and `column`
43
+ by one against the source string. `CaseInsensitiveCharStream` indexes by UTF-16 code unit, so its positions are
44
+ string offsets.
40
45
 
41
46
  ## Local development
42
47
 
@@ -1,13 +1,35 @@
1
- import { CharStreamImpl } from "antlr4ng";
1
+ import { type CharStream, type Interval } from "antlr4ng";
2
2
  /**
3
- * ANTLR4 CharStream extension that sees all characters as lowercase
3
+ * ANTLR4 CharStream over a JavaScript string that indexes by UTF-16 code units and lowercases lookahead.
4
+ *
5
+ * The antlr4ng `CharStream.fromString` stream stores Unicode code points, so a character outside the Basic
6
+ * Multilingual Plane (for example an emoji) takes one position there, while it takes two in a JavaScript string
7
+ * and in LSP ranges. Token `start`, `stop` and `column` values would then drift from the source text after such
8
+ * a character. This stream feeds the lexer UTF-16 code units instead, so every token position is directly a
9
+ * string offset. The MAQL grammar only matches such characters with `.` and `~[...]` sets (in strings, comments
10
+ * and object IDs), which accept the two surrogate halves one by one, so the tokens come out the same.
11
+ *
12
+ * The lexer also honors the grammar's `caseInsensitive` option, so lowercasing in `LA` is kept only for
13
+ * backward compatibility.
4
14
  */
5
- export declare class CaseInsensitiveCharStream extends CharStreamImpl {
6
- constructor(s: string, sourceName?: string);
15
+ export declare class CaseInsensitiveCharStream implements CharStream {
16
+ private readonly text;
17
+ name: string;
18
+ index: number;
19
+ constructor(text: string, sourceName?: string);
20
+ reset(): void;
21
+ consume(): void;
7
22
  /**
8
- * Modified lookahead method.
9
- * Transform the character to lowercase before passing it to the caller.
10
- * The input string is not modified.
23
+ * Lookahead by UTF-16 code unit. The unit is lowercased before it is passed to the caller;
24
+ * the input string is not modified.
11
25
  */
12
26
  LA(offset: number): number;
27
+ mark(): number;
28
+ release(_marker: number): void;
29
+ seek(index: number): void;
30
+ get size(): number;
31
+ getSourceName(): string;
32
+ getTextFromRange(start: number, stop?: number): string;
33
+ getTextFromInterval(interval: Interval): string;
34
+ toString(): string;
13
35
  }
@@ -1,25 +1,74 @@
1
1
  // (C) 2022 GoodData Corporation
2
- import { CharStreamImpl } from "antlr4ng";
2
+ import { IntStream, Token } from "antlr4ng";
3
3
  /**
4
- * ANTLR4 CharStream extension that sees all characters as lowercase
4
+ * ANTLR4 CharStream over a JavaScript string that indexes by UTF-16 code units and lowercases lookahead.
5
+ *
6
+ * The antlr4ng `CharStream.fromString` stream stores Unicode code points, so a character outside the Basic
7
+ * Multilingual Plane (for example an emoji) takes one position there, while it takes two in a JavaScript string
8
+ * and in LSP ranges. Token `start`, `stop` and `column` values would then drift from the source text after such
9
+ * a character. This stream feeds the lexer UTF-16 code units instead, so every token position is directly a
10
+ * string offset. The MAQL grammar only matches such characters with `.` and `~[...]` sets (in strings, comments
11
+ * and object IDs), which accept the two surrogate halves one by one, so the tokens come out the same.
12
+ *
13
+ * The lexer also honors the grammar's `caseInsensitive` option, so lowercasing in `LA` is kept only for
14
+ * backward compatibility.
5
15
  */
6
- export class CaseInsensitiveCharStream extends CharStreamImpl {
7
- constructor(s, sourceName) {
8
- super(s);
9
- if (sourceName !== undefined) {
10
- this.name = sourceName;
16
+ export class CaseInsensitiveCharStream {
17
+ text;
18
+ name;
19
+ index = 0;
20
+ constructor(text, sourceName) {
21
+ this.text = text;
22
+ this.name = sourceName ?? IntStream.UNKNOWN_SOURCE_NAME;
23
+ }
24
+ reset() {
25
+ this.index = 0;
26
+ }
27
+ consume() {
28
+ if (this.index >= this.text.length) {
29
+ throw new Error("cannot consume EOF");
11
30
  }
31
+ this.index += 1;
12
32
  }
13
33
  /**
14
- * Modified lookahead method.
15
- * Transform the character to lowercase before passing it to the caller.
16
- * The input string is not modified.
34
+ * Lookahead by UTF-16 code unit. The unit is lowercased before it is passed to the caller;
35
+ * the input string is not modified.
17
36
  */
18
37
  LA(offset) {
19
- const c = super.LA(offset);
20
- if (c < 0) {
21
- return c;
38
+ if (offset === 0) {
39
+ return 0;
40
+ }
41
+ if (offset < 0) {
42
+ offset += 1;
22
43
  }
23
- return String.fromCodePoint(c).toLowerCase().codePointAt(0) ?? c;
44
+ const pos = this.index + offset - 1;
45
+ if (pos < 0 || pos >= this.text.length) {
46
+ return Token.EOF;
47
+ }
48
+ const c = this.text.charCodeAt(pos);
49
+ return String.fromCharCode(c).toLowerCase().charCodeAt(0);
50
+ }
51
+ // mark/release do nothing; the entire buffer is in memory
52
+ mark() {
53
+ return -1;
54
+ }
55
+ release(_marker) { }
56
+ seek(index) {
57
+ this.index = Math.min(index, this.text.length);
58
+ }
59
+ get size() {
60
+ return this.text.length;
61
+ }
62
+ getSourceName() {
63
+ return this.name;
64
+ }
65
+ getTextFromRange(start, stop = this.text.length - 1) {
66
+ return this.text.substring(start, stop + 1);
67
+ }
68
+ getTextFromInterval(interval) {
69
+ return this.getTextFromRange(interval.start, interval.stop);
70
+ }
71
+ toString() {
72
+ return this.text;
24
73
  }
25
74
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gooddata/maql-antlr4ts",
3
- "version": "0.7.2-567f11f5.0",
3
+ "version": "0.7.2",
4
4
  "description": "TypeScript lexer and parser for MAQL",
5
5
  "license": "MIT",
6
6
  "repository": {