@willbooster/tree-sitter-bash 0.25.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -25
- package/grammar.js +200 -32
- package/package.json +19 -2
- package/prebuilds/darwin-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/darwin-x64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-x64/tree-sitter-bash.node +0 -0
- package/queries/highlights.scm +33 -12
- package/src/grammar.json +821 -60
- package/src/node-types.json +206 -66
- package/src/parser.c +219488 -88692
- package/src/scanner.c +651 -92
- package/tree-sitter.json +2 -2
package/src/scanner.c
CHANGED
|
@@ -22,18 +22,29 @@ enum TokenType {
|
|
|
22
22
|
BACKTICK_OPEN,
|
|
23
23
|
BACKTICK_CLOSE,
|
|
24
24
|
NEWLINE,
|
|
25
|
+
SUBSTITUTION_START,
|
|
26
|
+
SUBSTITUTION_END,
|
|
27
|
+
BRACE_SUBSTITUTION_END,
|
|
28
|
+
BRACKET_SUBSTITUTION_END,
|
|
29
|
+
EXTGLOB_PREFIX,
|
|
30
|
+
LINE_CONTINUATION,
|
|
31
|
+
BRACE_SUBSTITUTION_START,
|
|
32
|
+
BRACKET_SUBSTITUTION_START,
|
|
25
33
|
ERROR_RECOVERY,
|
|
26
34
|
};
|
|
27
35
|
|
|
28
|
-
typedef Array(
|
|
36
|
+
typedef Array(int32_t) CodePoints;
|
|
29
37
|
|
|
30
38
|
// A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
|
|
31
|
-
// (not one inside a string)
|
|
39
|
+
// (not one inside a string) at the substitution depth where the operator appeared, after the bodies
|
|
40
|
+
// of heredocs opened earlier at that depth: bash reads a substitution, including the bodies of the
|
|
41
|
+
// heredocs inside it, before the body of a heredoc whose operator precedes it on the same line.
|
|
32
42
|
typedef struct {
|
|
33
43
|
bool is_raw;
|
|
34
44
|
bool allows_indent;
|
|
35
45
|
bool started;
|
|
36
|
-
|
|
46
|
+
uint16_t depth;
|
|
47
|
+
CodePoints delimiter;
|
|
37
48
|
} Heredoc;
|
|
38
49
|
|
|
39
50
|
typedef struct {
|
|
@@ -41,8 +52,17 @@ typedef struct {
|
|
|
41
52
|
// Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
|
|
42
53
|
// whether a backquote opens or closes one.
|
|
43
54
|
uint8_t backtick_depth;
|
|
55
|
+
// The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
|
|
56
|
+
Array(uint8_t) closers;
|
|
57
|
+
// A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
|
|
58
|
+
// line began the body, so a terminator is still owed before the `}`.
|
|
59
|
+
bool terminator_owed;
|
|
44
60
|
} Scanner;
|
|
45
61
|
|
|
62
|
+
static inline uint16_t current_depth(Scanner *scanner) {
|
|
63
|
+
return scanner->backtick_depth + scanner->closers.size;
|
|
64
|
+
}
|
|
65
|
+
|
|
46
66
|
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
|
|
47
67
|
|
|
48
68
|
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
|
|
@@ -63,6 +83,60 @@ static inline bool starts_quoted_expansion(int32_t c) {
|
|
|
63
83
|
c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
|
|
64
84
|
}
|
|
65
85
|
|
|
86
|
+
// Code points are serialized as zigzag varints, one byte for code points below 64 and two for the
|
|
87
|
+
// rest of ASCII; every int32 round-trips, including the negative value the lexer reports for invalid
|
|
88
|
+
// UTF-8.
|
|
89
|
+
static unsigned varint_length(int32_t c) {
|
|
90
|
+
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
91
|
+
unsigned length = 1;
|
|
92
|
+
while (value >= 0x80) {
|
|
93
|
+
value >>= 7;
|
|
94
|
+
length++;
|
|
95
|
+
}
|
|
96
|
+
return length;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static unsigned encode_varint(int32_t c, char *out) {
|
|
100
|
+
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
101
|
+
unsigned length = 0;
|
|
102
|
+
while (value >= 0x80) {
|
|
103
|
+
out[length++] = (char)(0x80 | (value & 0x7F));
|
|
104
|
+
value >>= 7;
|
|
105
|
+
}
|
|
106
|
+
out[length++] = (char)value;
|
|
107
|
+
return length;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Returns 0 when the input ends before the varint does.
|
|
111
|
+
static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
|
|
112
|
+
uint32_t value = 0;
|
|
113
|
+
for (unsigned i = 0; i < available && i < 5; i++) {
|
|
114
|
+
value |= (uint32_t)((unsigned char)in[i] & 0x7F) << (7 * i);
|
|
115
|
+
if (((unsigned char)in[i] & 0x80) == 0) {
|
|
116
|
+
*c = (int32_t)((value >> 1) ^ (0u - (value & 1)));
|
|
117
|
+
return i + 1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return 0;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
|
|
124
|
+
// (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
|
|
125
|
+
// delimiter length (uint16), and the delimiter as varints.
|
|
126
|
+
#define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
|
|
127
|
+
|
|
128
|
+
static unsigned serialized_size(Scanner *scanner) {
|
|
129
|
+
unsigned size = 3 + sizeof(uint16_t) + scanner->closers.size;
|
|
130
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
131
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
132
|
+
size += HEREDOC_HEADER_SIZE;
|
|
133
|
+
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
134
|
+
size += varint_length(*array_get(&heredoc->delimiter, j));
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return size;
|
|
138
|
+
}
|
|
139
|
+
|
|
66
140
|
static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
|
|
67
141
|
|
|
68
142
|
static void scanner_reset(Scanner *scanner) {
|
|
@@ -71,18 +145,70 @@ static void scanner_reset(Scanner *scanner) {
|
|
|
71
145
|
}
|
|
72
146
|
array_clear(&scanner->heredocs);
|
|
73
147
|
scanner->backtick_depth = 0;
|
|
148
|
+
array_clear(&scanner->closers);
|
|
149
|
+
scanner->terminator_owed = false;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// The body being read: the most deeply nested started heredoc.
|
|
153
|
+
static int32_t active_heredoc_index(Scanner *scanner) {
|
|
154
|
+
int32_t active = -1;
|
|
155
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
156
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
157
|
+
if (heredoc->started && (active < 0 || heredoc->depth >= array_get(&scanner->heredocs, active)->depth)) {
|
|
158
|
+
active = (int32_t)i;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return active;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// The next body to start: the first unstarted heredoc opened at the current substitution depth.
|
|
165
|
+
static int32_t next_heredoc_index(Scanner *scanner) {
|
|
166
|
+
uint16_t depth = current_depth(scanner);
|
|
167
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
168
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
169
|
+
if (!heredoc->started && heredoc->depth == depth) {
|
|
170
|
+
return (int32_t)i;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return -1;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// An entry without a delimiter belongs to an operator whose delimiter was rejected.
|
|
177
|
+
static inline bool is_rejected(Heredoc *heredoc) { return heredoc->delimiter.size == 0 && !heredoc->is_raw; }
|
|
178
|
+
|
|
179
|
+
// Drops the unstarted heredocs at the current depth, which were all queued on the line just ended.
|
|
180
|
+
static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
|
|
181
|
+
uint16_t depth = current_depth(scanner);
|
|
182
|
+
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
183
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
184
|
+
if (!heredoc->started && heredoc->depth == depth) {
|
|
185
|
+
heredoc_delete(heredoc);
|
|
186
|
+
array_erase(&scanner->heredocs, i - 1);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
static void remove_heredoc(Scanner *scanner, uint32_t index) {
|
|
192
|
+
heredoc_delete(array_get(&scanner->heredocs, index));
|
|
193
|
+
array_erase(&scanner->heredocs, index);
|
|
74
194
|
}
|
|
75
195
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
196
|
+
// Like bash, drop the heredocs of a substitution that closes before their bodies started.
|
|
197
|
+
static void discard_unstarted_heredocs(Scanner *scanner) {
|
|
198
|
+
uint16_t depth = current_depth(scanner);
|
|
199
|
+
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
200
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
201
|
+
if (!heredoc->started && heredoc->depth > depth) {
|
|
202
|
+
remove_heredoc(scanner, i - 1);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
79
205
|
}
|
|
80
206
|
|
|
81
207
|
// Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
|
|
82
208
|
// line (or at the end of input, as bash does with a warning). An unquoted body stops before each
|
|
83
209
|
// expansion so that the parser can read it.
|
|
84
|
-
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
85
|
-
Heredoc *heredoc = array_get(&scanner->heredocs,
|
|
210
|
+
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
|
|
211
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, index);
|
|
86
212
|
bool did_advance = false;
|
|
87
213
|
bool at_line_start = lexer->get_column(lexer) == 0;
|
|
88
214
|
|
|
@@ -98,21 +224,29 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
98
224
|
}
|
|
99
225
|
}
|
|
100
226
|
uint32_t matched = 0;
|
|
101
|
-
|
|
227
|
+
// At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
|
|
228
|
+
while (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
|
|
229
|
+
lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
|
|
102
230
|
advance(lexer);
|
|
103
231
|
matched++;
|
|
104
232
|
}
|
|
105
233
|
if (lexer->lookahead == '\r') {
|
|
106
234
|
advance(lexer);
|
|
107
235
|
}
|
|
108
|
-
|
|
236
|
+
// Inside a substitution, bash also ends the body at a delimiter directly followed by the
|
|
237
|
+
// innermost substitution's closer (`)` or `}`) or a closing backquote.
|
|
238
|
+
bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
|
|
239
|
+
(scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
|
|
240
|
+
(lexer->lookahead == '`' && scanner->backtick_depth > 0);
|
|
241
|
+
if (matched == heredoc->delimiter.size && ends_line) {
|
|
109
242
|
if (did_advance) {
|
|
110
243
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
111
244
|
return true;
|
|
112
245
|
}
|
|
113
246
|
lexer->mark_end(lexer);
|
|
114
247
|
lexer->result_symbol = HEREDOC_END;
|
|
115
|
-
|
|
248
|
+
scanner->terminator_owed = lexer->lookahead == '}';
|
|
249
|
+
remove_heredoc(scanner, index);
|
|
116
250
|
return true;
|
|
117
251
|
}
|
|
118
252
|
// The characters read while trying the delimiter are content; an empty attempt must not
|
|
@@ -125,7 +259,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
125
259
|
lexer->mark_end(lexer);
|
|
126
260
|
lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
|
|
127
261
|
if (!did_advance) {
|
|
128
|
-
|
|
262
|
+
remove_heredoc(scanner, index);
|
|
129
263
|
}
|
|
130
264
|
return true;
|
|
131
265
|
}
|
|
@@ -141,7 +275,12 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
141
275
|
did_advance = true;
|
|
142
276
|
if (!heredoc->is_raw && !lexer->eof(lexer)) {
|
|
143
277
|
// A backslash-newline joins lines before the delimiter comparison.
|
|
144
|
-
|
|
278
|
+
if (lexer->lookahead == '\r') {
|
|
279
|
+
advance(lexer);
|
|
280
|
+
}
|
|
281
|
+
if (!lexer->eof(lexer)) {
|
|
282
|
+
advance(lexer);
|
|
283
|
+
}
|
|
145
284
|
}
|
|
146
285
|
break;
|
|
147
286
|
case '`':
|
|
@@ -179,6 +318,79 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
179
318
|
}
|
|
180
319
|
|
|
181
320
|
// The delimiter word undergoes quote removal; any quoting makes the body literal.
|
|
321
|
+
static inline int32_t hex_value(int32_t c) {
|
|
322
|
+
return iswdigit(c) ? c - '0' : (c >= 'a' && c <= 'f') ? c - 'a' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 : -1;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// Reads up to `max_digits` digits in `base` and returns their value, or -1 when there are none.
|
|
326
|
+
static int32_t read_number(TSLexer *lexer, int32_t base, int max_digits) {
|
|
327
|
+
int32_t value = 0;
|
|
328
|
+
int digits = 0;
|
|
329
|
+
while (digits < max_digits) {
|
|
330
|
+
int32_t digit = hex_value(lexer->lookahead);
|
|
331
|
+
if (digit < 0 || digit >= base) {
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
value = value * base + digit;
|
|
335
|
+
advance(lexer);
|
|
336
|
+
digits++;
|
|
337
|
+
}
|
|
338
|
+
return digits == 0 ? -1 : value;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Decodes the escape after a backslash in `$'...'` as bash does, appending it to `out`.
|
|
342
|
+
static void push_ansi_c_escape(TSLexer *lexer, CodePoints *out) {
|
|
343
|
+
int32_t c = lexer->lookahead;
|
|
344
|
+
int32_t value = -1;
|
|
345
|
+
switch (c) {
|
|
346
|
+
case 'a': value = 7; break;
|
|
347
|
+
case 'b': value = 8; break;
|
|
348
|
+
case 'e':
|
|
349
|
+
case 'E': value = 27; break;
|
|
350
|
+
case 'f': value = 12; break;
|
|
351
|
+
case 'n': value = '\n'; break;
|
|
352
|
+
case 'r': value = '\r'; break;
|
|
353
|
+
case 't': value = '\t'; break;
|
|
354
|
+
case 'v': value = 11; break;
|
|
355
|
+
case '\\':
|
|
356
|
+
case '\'':
|
|
357
|
+
case '"':
|
|
358
|
+
case '?': value = c; break;
|
|
359
|
+
default: break;
|
|
360
|
+
}
|
|
361
|
+
if (value >= 0) {
|
|
362
|
+
advance(lexer);
|
|
363
|
+
array_push(out, value);
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
if (c >= '0' && c <= '7') {
|
|
367
|
+
array_push(out, read_number(lexer, 8, 3));
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
int max_digits = c == 'x' ? 2 : c == 'u' ? 4 : c == 'U' ? 8 : 0;
|
|
371
|
+
if (max_digits > 0) {
|
|
372
|
+
advance(lexer);
|
|
373
|
+
value = read_number(lexer, 16, max_digits);
|
|
374
|
+
if (value < 0) {
|
|
375
|
+
array_push(out, '\\');
|
|
376
|
+
array_push(out, c);
|
|
377
|
+
} else {
|
|
378
|
+
array_push(out, value);
|
|
379
|
+
}
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (c == 'c') {
|
|
383
|
+
advance(lexer);
|
|
384
|
+
if (!lexer->eof(lexer)) {
|
|
385
|
+
array_push(out, lexer->lookahead & 0x1F);
|
|
386
|
+
advance(lexer);
|
|
387
|
+
}
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
// Bash keeps an unknown escape as written.
|
|
391
|
+
array_push(out, '\\');
|
|
392
|
+
}
|
|
393
|
+
|
|
182
394
|
static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
183
395
|
Heredoc *heredoc = array_back(&scanner->heredocs);
|
|
184
396
|
array_clear(&heredoc->delimiter);
|
|
@@ -189,6 +401,7 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
|
189
401
|
}
|
|
190
402
|
|
|
191
403
|
int32_t quote = 0;
|
|
404
|
+
bool ansi_c = false;
|
|
192
405
|
while (!lexer->eof(lexer)) {
|
|
193
406
|
int32_t c = lexer->lookahead;
|
|
194
407
|
if (quote) {
|
|
@@ -197,36 +410,80 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
|
197
410
|
quote = 0;
|
|
198
411
|
continue;
|
|
199
412
|
}
|
|
413
|
+
if (ansi_c && c == '\\') {
|
|
414
|
+
if (!lexer->eof(lexer)) {
|
|
415
|
+
push_ansi_c_escape(lexer, &heredoc->delimiter);
|
|
416
|
+
}
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
// Inside double quotes a backslash-newline still only continues the line.
|
|
420
|
+
if (quote == '"' && c == '\\' && (lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
|
|
421
|
+
if (lexer->lookahead == '\r') {
|
|
422
|
+
advance(lexer);
|
|
423
|
+
}
|
|
424
|
+
if (lexer->lookahead == '\n') {
|
|
425
|
+
advance(lexer);
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
array_push(&heredoc->delimiter, '\\');
|
|
429
|
+
array_push(&heredoc->delimiter, '\r');
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
200
432
|
if (quote == '"' && c == '\\' &&
|
|
201
433
|
(lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
|
|
202
434
|
lexer->lookahead == '\\')) {
|
|
203
435
|
c = lexer->lookahead;
|
|
204
436
|
advance(lexer);
|
|
205
437
|
}
|
|
206
|
-
array_push(&heredoc->delimiter,
|
|
438
|
+
array_push(&heredoc->delimiter, c);
|
|
207
439
|
continue;
|
|
208
440
|
}
|
|
209
|
-
|
|
441
|
+
// Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
|
|
442
|
+
// the word (`<<EO`true`F`).
|
|
443
|
+
if (iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
210
444
|
break;
|
|
211
445
|
}
|
|
212
446
|
advance(lexer);
|
|
213
|
-
if (c == '\'' ||
|
|
447
|
+
if (c == '$' && (lexer->lookahead == '\'' || lexer->lookahead == '"')) {
|
|
448
|
+
// `$'...'` and `$"..."` are quoting constructs, so their `$` is not part of the word.
|
|
449
|
+
c = lexer->lookahead;
|
|
450
|
+
advance(lexer);
|
|
214
451
|
quote = c;
|
|
452
|
+
ansi_c = c == '\'';
|
|
215
453
|
heredoc->is_raw = true;
|
|
216
|
-
} else if (c == '
|
|
454
|
+
} else if (c == '\'' || c == '"') {
|
|
455
|
+
quote = c;
|
|
456
|
+
ansi_c = false;
|
|
217
457
|
heredoc->is_raw = true;
|
|
458
|
+
} else if (c == '\\') {
|
|
459
|
+
// A backslash-newline only continues the line; any other backslash quotes the delimiter.
|
|
460
|
+
bool carriage_return = lexer->lookahead == '\r';
|
|
461
|
+
if (carriage_return) {
|
|
462
|
+
advance(lexer);
|
|
463
|
+
}
|
|
218
464
|
if (lexer->lookahead == '\n') {
|
|
219
465
|
advance(lexer);
|
|
466
|
+
} else if (carriage_return) {
|
|
467
|
+
heredoc->is_raw = true;
|
|
468
|
+
array_push(&heredoc->delimiter, '\r');
|
|
220
469
|
} else if (!lexer->eof(lexer)) {
|
|
221
|
-
|
|
470
|
+
heredoc->is_raw = true;
|
|
471
|
+
array_push(&heredoc->delimiter, lexer->lookahead);
|
|
222
472
|
advance(lexer);
|
|
223
473
|
}
|
|
224
474
|
} else {
|
|
225
|
-
|
|
475
|
+
// Bash counts a backquote in the delimiter word as quoting, so the body stays literal.
|
|
476
|
+
if (c == '`') {
|
|
477
|
+
heredoc->is_raw = true;
|
|
478
|
+
}
|
|
479
|
+
array_push(&heredoc->delimiter, c);
|
|
226
480
|
}
|
|
227
481
|
}
|
|
228
482
|
|
|
229
|
-
|
|
483
|
+
// A heredoc whose state cannot be serialized in full is rejected, so the parse shows an error
|
|
484
|
+
// rather than comparing against a partial delimiter.
|
|
485
|
+
if ((heredoc->delimiter.size == 0 && !heredoc->is_raw) ||
|
|
486
|
+
serialized_size(scanner) > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
230
487
|
return false;
|
|
231
488
|
}
|
|
232
489
|
lexer->mark_end(lexer);
|
|
@@ -250,46 +507,23 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
|
|
|
250
507
|
} else if (!valid_symbols[HEREDOC_ARROW]) {
|
|
251
508
|
return false;
|
|
252
509
|
}
|
|
510
|
+
if (scanner->heredocs.size >= UINT8_MAX ||
|
|
511
|
+
serialized_size(scanner) + HEREDOC_HEADER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
253
514
|
lexer->mark_end(lexer);
|
|
254
|
-
Heredoc heredoc = {
|
|
515
|
+
Heredoc heredoc = {
|
|
516
|
+
.is_raw = false,
|
|
517
|
+
.allows_indent = allows_indent,
|
|
518
|
+
.started = false,
|
|
519
|
+
.depth = current_depth(scanner),
|
|
520
|
+
.delimiter = array_new(),
|
|
521
|
+
};
|
|
255
522
|
array_push(&scanner->heredocs, heredoc);
|
|
256
523
|
lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
|
|
257
524
|
return true;
|
|
258
525
|
}
|
|
259
526
|
|
|
260
|
-
// A name followed by `=`, `+=`, or `[subscript]=` starts an assignment.
|
|
261
|
-
static bool scan_variable_name(TSLexer *lexer) {
|
|
262
|
-
while (is_name_char(lexer->lookahead)) {
|
|
263
|
-
advance(lexer);
|
|
264
|
-
}
|
|
265
|
-
lexer->mark_end(lexer);
|
|
266
|
-
if (lexer->lookahead == '+') {
|
|
267
|
-
advance(lexer);
|
|
268
|
-
return lexer->lookahead == '=';
|
|
269
|
-
}
|
|
270
|
-
if (lexer->lookahead == '[') {
|
|
271
|
-
uint32_t depth = 0;
|
|
272
|
-
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
|
273
|
-
if (lexer->lookahead == '[') {
|
|
274
|
-
depth++;
|
|
275
|
-
} else if (lexer->lookahead == ']' && --depth == 0) {
|
|
276
|
-
advance(lexer);
|
|
277
|
-
break;
|
|
278
|
-
} else if (lexer->lookahead == '\\') {
|
|
279
|
-
advance(lexer);
|
|
280
|
-
}
|
|
281
|
-
advance(lexer);
|
|
282
|
-
}
|
|
283
|
-
if (depth != 0) {
|
|
284
|
-
return false;
|
|
285
|
-
}
|
|
286
|
-
if (lexer->lookahead == '+') {
|
|
287
|
-
advance(lexer);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
return lexer->lookahead == '=';
|
|
291
|
-
}
|
|
292
|
-
|
|
293
527
|
// The right side of `=~` extends to unquoted whitespace outside parentheses.
|
|
294
528
|
static bool scan_regex(TSLexer *lexer) {
|
|
295
529
|
while (is_blank(lexer->lookahead)) {
|
|
@@ -298,10 +532,16 @@ static bool scan_regex(TSLexer *lexer) {
|
|
|
298
532
|
uint32_t depth = 0;
|
|
299
533
|
bool did_advance = false;
|
|
300
534
|
while (!lexer->eof(lexer)) {
|
|
535
|
+
lexer->mark_end(lexer);
|
|
301
536
|
int32_t c = lexer->lookahead;
|
|
302
537
|
if (depth == 0 && iswspace(c)) {
|
|
303
538
|
break;
|
|
304
539
|
}
|
|
540
|
+
// Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
|
|
541
|
+
// the regex even without surrounding blanks (`[[ a =~ a&&b ]]`).
|
|
542
|
+
if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
|
|
543
|
+
break;
|
|
544
|
+
}
|
|
305
545
|
if (c == '\\') {
|
|
306
546
|
advance(lexer);
|
|
307
547
|
} else if (c == '\'' || c == '"') {
|
|
@@ -323,7 +563,9 @@ static bool scan_regex(TSLexer *lexer) {
|
|
|
323
563
|
advance(lexer);
|
|
324
564
|
did_advance = true;
|
|
325
565
|
}
|
|
326
|
-
lexer->
|
|
566
|
+
if (lexer->eof(lexer)) {
|
|
567
|
+
lexer->mark_end(lexer);
|
|
568
|
+
}
|
|
327
569
|
lexer->result_symbol = REGEX;
|
|
328
570
|
return did_advance;
|
|
329
571
|
}
|
|
@@ -352,6 +594,186 @@ static bool scan_string_content(TSLexer *lexer) {
|
|
|
352
594
|
return did_advance;
|
|
353
595
|
}
|
|
354
596
|
|
|
597
|
+
static inline bool is_word_break(int32_t c) {
|
|
598
|
+
return iswspace(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
|
|
602
|
+
|
|
603
|
+
typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
|
|
604
|
+
|
|
605
|
+
// From the `(` of an extglob group, reads to its matching `)`: a group holding quotes or expansions
|
|
606
|
+
// is parsed as `extglob_pattern`, while the word token keeps literal ones.
|
|
607
|
+
// The nesting depth up to which grammar.js's word token lexes a literal group (`extglobGroup(…, 3)`);
|
|
608
|
+
// a deeper group is parsed as `extglob_pattern`.
|
|
609
|
+
#define EXTGLOB_LITERAL_DEPTH 3
|
|
610
|
+
|
|
611
|
+
static GroupKind scan_extglob_group(TSLexer *lexer) {
|
|
612
|
+
uint32_t depth = 0;
|
|
613
|
+
while (!lexer->eof(lexer)) {
|
|
614
|
+
int32_t c = lexer->lookahead;
|
|
615
|
+
if (c == '$' || c == '`' || c == '\'' || c == '"') {
|
|
616
|
+
return GROUP_STRUCTURED;
|
|
617
|
+
}
|
|
618
|
+
advance(lexer);
|
|
619
|
+
if (c == '\\') {
|
|
620
|
+
// The word token's literal groups cannot hold a line continuation.
|
|
621
|
+
if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
622
|
+
return lexer->eof(lexer) ? GROUP_UNSUPPORTED : GROUP_STRUCTURED;
|
|
623
|
+
}
|
|
624
|
+
advance(lexer);
|
|
625
|
+
} else if (c == '(') {
|
|
626
|
+
if (++depth > EXTGLOB_LITERAL_DEPTH) {
|
|
627
|
+
return GROUP_STRUCTURED;
|
|
628
|
+
}
|
|
629
|
+
} else if (c == ')' && --depth == 0) {
|
|
630
|
+
return GROUP_LITERAL;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return GROUP_UNSUPPORTED;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// The literal text of a word up to a structured extglob group (`foo` in `foo@($x)`), which the
|
|
637
|
+
// word token's longest match would otherwise run into.
|
|
638
|
+
static bool continue_extglob_prefix(TSLexer *lexer, bool consumed) {
|
|
639
|
+
while (!lexer->eof(lexer)) {
|
|
640
|
+
int32_t c = lexer->lookahead;
|
|
641
|
+
if (is_extglob_operator(c)) {
|
|
642
|
+
lexer->mark_end(lexer);
|
|
643
|
+
advance(lexer);
|
|
644
|
+
if (lexer->lookahead == '(') {
|
|
645
|
+
GroupKind kind = scan_extglob_group(lexer);
|
|
646
|
+
if (kind == GROUP_STRUCTURED) {
|
|
647
|
+
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
648
|
+
return consumed;
|
|
649
|
+
}
|
|
650
|
+
if (kind == GROUP_UNSUPPORTED) {
|
|
651
|
+
return false;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
} else if (c == '\\') {
|
|
655
|
+
advance(lexer);
|
|
656
|
+
// A backslash-newline continues the line; the word token handles that.
|
|
657
|
+
if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
advance(lexer);
|
|
661
|
+
} else if (is_word_break(c)) {
|
|
662
|
+
return false;
|
|
663
|
+
} else {
|
|
664
|
+
advance(lexer);
|
|
665
|
+
}
|
|
666
|
+
consumed = true;
|
|
667
|
+
}
|
|
668
|
+
return false;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// A leading `#` starts a comment except in an assignment value (`A=#@($x)`).
|
|
672
|
+
static bool scan_extglob_prefix(TSLexer *lexer, bool in_value) {
|
|
673
|
+
return (lexer->lookahead != '#' || in_value) && continue_extglob_prefix(lexer, false);
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// Skips a quoted span from its opening quote through its closing one.
|
|
677
|
+
static void skip_quoted(TSLexer *lexer) {
|
|
678
|
+
int32_t quote = lexer->lookahead;
|
|
679
|
+
advance(lexer);
|
|
680
|
+
while (!lexer->eof(lexer) && lexer->lookahead != quote) {
|
|
681
|
+
if (lexer->lookahead == '\\' && quote != '\'') {
|
|
682
|
+
advance(lexer);
|
|
683
|
+
if (lexer->eof(lexer)) {
|
|
684
|
+
return;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
advance(lexer);
|
|
688
|
+
}
|
|
689
|
+
advance(lexer);
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// Skips a parenthesized span such as a command substitution's, quotes included.
|
|
693
|
+
static void skip_parenthesized(TSLexer *lexer) {
|
|
694
|
+
uint32_t depth = 0;
|
|
695
|
+
while (!lexer->eof(lexer)) {
|
|
696
|
+
int32_t c = lexer->lookahead;
|
|
697
|
+
if (c == '\'' || c == '"' || c == '`') {
|
|
698
|
+
skip_quoted(lexer);
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
if (c == '\\') {
|
|
702
|
+
advance(lexer);
|
|
703
|
+
} else if (c == '(') {
|
|
704
|
+
depth++;
|
|
705
|
+
} else if (c == ')' && --depth == 0) {
|
|
706
|
+
advance(lexer);
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
advance(lexer);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// A name at a position where both an assignment and a word may start: `name=`, `name+=`, and
|
|
714
|
+
// `name[…]=` are assignments, and otherwise the name may begin the literal text before a structured
|
|
715
|
+
// extglob group (`foo@($x)`), in which `+(` is an extglob operator rather than an unfinished `+=`.
|
|
716
|
+
static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbols) {
|
|
717
|
+
while (is_name_char(lexer->lookahead)) {
|
|
718
|
+
advance(lexer);
|
|
719
|
+
}
|
|
720
|
+
lexer->mark_end(lexer);
|
|
721
|
+
lexer->result_symbol = VARIABLE_NAME;
|
|
722
|
+
if (lexer->lookahead == '=') {
|
|
723
|
+
return true;
|
|
724
|
+
}
|
|
725
|
+
if (lexer->lookahead == '+') {
|
|
726
|
+
advance(lexer);
|
|
727
|
+
if (lexer->lookahead == '=') {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
if (!valid_symbols[EXTGLOB_PREFIX]) {
|
|
731
|
+
return false;
|
|
732
|
+
}
|
|
733
|
+
if (lexer->lookahead == '(') {
|
|
734
|
+
GroupKind kind = scan_extglob_group(lexer);
|
|
735
|
+
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
736
|
+
return kind == GROUP_STRUCTURED || (kind == GROUP_LITERAL && continue_extglob_prefix(lexer, true));
|
|
737
|
+
}
|
|
738
|
+
return continue_extglob_prefix(lexer, true);
|
|
739
|
+
}
|
|
740
|
+
if (lexer->lookahead == '[') {
|
|
741
|
+
// Like bash, the subscript extends to the matching `]` across lines; without one the name
|
|
742
|
+
// still starts an assignment, so the parse reports the missing `]`.
|
|
743
|
+
uint32_t depth = 0;
|
|
744
|
+
while (!lexer->eof(lexer)) {
|
|
745
|
+
int32_t c = lexer->lookahead;
|
|
746
|
+
if (c == '[') {
|
|
747
|
+
depth++;
|
|
748
|
+
} else if (c == ']' && --depth == 0) {
|
|
749
|
+
advance(lexer);
|
|
750
|
+
break;
|
|
751
|
+
} else if (c == '\\') {
|
|
752
|
+
advance(lexer);
|
|
753
|
+
} else if (c == '\'' || c == '"' || c == '`') {
|
|
754
|
+
// Brackets inside quotes do not count (`A["]"]=x`).
|
|
755
|
+
skip_quoted(lexer);
|
|
756
|
+
continue;
|
|
757
|
+
} else if (c == '$') {
|
|
758
|
+
advance(lexer);
|
|
759
|
+
if (lexer->lookahead == '(') {
|
|
760
|
+
skip_parenthesized(lexer);
|
|
761
|
+
}
|
|
762
|
+
continue;
|
|
763
|
+
}
|
|
764
|
+
advance(lexer);
|
|
765
|
+
}
|
|
766
|
+
if (depth == 0 && lexer->lookahead == '+') {
|
|
767
|
+
advance(lexer);
|
|
768
|
+
}
|
|
769
|
+
if (depth != 0 || lexer->lookahead == '=') {
|
|
770
|
+
return true;
|
|
771
|
+
}
|
|
772
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
773
|
+
}
|
|
774
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
775
|
+
}
|
|
776
|
+
|
|
355
777
|
static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
356
778
|
bool error_recovery = valid_symbols[ERROR_RECOVERY];
|
|
357
779
|
// Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
|
|
@@ -359,22 +781,54 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
359
781
|
int32_t first = lexer->lookahead;
|
|
360
782
|
bool at_eof = lexer->eof(lexer);
|
|
361
783
|
|
|
784
|
+
if (scanner->terminator_owed) {
|
|
785
|
+
scanner->terminator_owed = false;
|
|
786
|
+
if (valid_symbols[NEWLINE] && lexer->lookahead == '}') {
|
|
787
|
+
lexer->result_symbol = NEWLINE;
|
|
788
|
+
return true;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
// Counted before a heredoc body may start, so a newline right after `$(` is inside it.
|
|
793
|
+
if (!error_recovery &&
|
|
794
|
+
(valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
|
|
795
|
+
valid_symbols[BRACKET_SUBSTITUTION_START])) {
|
|
796
|
+
// Nesting too deep for the serialized state is left to the parse as an error.
|
|
797
|
+
if (serialized_size(scanner) >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
798
|
+
return false;
|
|
799
|
+
}
|
|
800
|
+
lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
|
|
801
|
+
: valid_symbols[BRACE_SUBSTITUTION_START] ? BRACE_SUBSTITUTION_START
|
|
802
|
+
: BRACKET_SUBSTITUTION_START;
|
|
803
|
+
uint8_t closer = lexer->result_symbol == SUBSTITUTION_START ? ')'
|
|
804
|
+
: lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
|
|
805
|
+
: ']';
|
|
806
|
+
array_push(&scanner->closers, closer);
|
|
807
|
+
return true;
|
|
808
|
+
}
|
|
809
|
+
|
|
362
810
|
if (scanner->heredocs.size > 0) {
|
|
363
|
-
|
|
364
|
-
if (
|
|
365
|
-
!(lexer->lookahead == '`' && !
|
|
366
|
-
return scan_heredoc_content(scanner, lexer);
|
|
811
|
+
int32_t active = active_heredoc_index(scanner);
|
|
812
|
+
if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
|
|
813
|
+
!(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw)) {
|
|
814
|
+
return scan_heredoc_content(scanner, lexer, (uint32_t)active);
|
|
367
815
|
}
|
|
368
816
|
// A newline inside a string is part of it, so bodies start only outside strings.
|
|
369
|
-
|
|
370
|
-
|
|
817
|
+
int32_t next = next_heredoc_index(scanner);
|
|
818
|
+
if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && !valid_symbols[STRING_CONTENT]) {
|
|
371
819
|
while (is_blank(lexer->lookahead)) {
|
|
372
820
|
skip(lexer);
|
|
373
821
|
}
|
|
374
822
|
if (lexer->lookahead == '\n') {
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
823
|
+
// The line's heredocs cannot be located once one of them was rejected, so none gets a
|
|
824
|
+
// body; scanning continues so that the change is kept with the token found below.
|
|
825
|
+
if (is_rejected(array_get(&scanner->heredocs, next))) {
|
|
826
|
+
drop_unstarted_heredocs_at_depth(scanner);
|
|
827
|
+
} else {
|
|
828
|
+
array_get(&scanner->heredocs, next)->started = true;
|
|
829
|
+
lexer->result_symbol = HEREDOC_BODY_START;
|
|
830
|
+
return true;
|
|
831
|
+
}
|
|
378
832
|
}
|
|
379
833
|
}
|
|
380
834
|
}
|
|
@@ -393,8 +847,46 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
393
847
|
|
|
394
848
|
if (valid_symbols[CONCAT]) {
|
|
395
849
|
int32_t c = first;
|
|
396
|
-
if (
|
|
397
|
-
|
|
850
|
+
if (c == '\\' && lexer->lookahead == '\\') {
|
|
851
|
+
// A backslash-newline joins lines, so the word continues only if the next line does.
|
|
852
|
+
lexer->mark_end(lexer);
|
|
853
|
+
advance(lexer);
|
|
854
|
+
if (lexer->lookahead == '\r') {
|
|
855
|
+
advance(lexer);
|
|
856
|
+
}
|
|
857
|
+
// Bash drops a backslash at the end of input like a continuation.
|
|
858
|
+
if (lexer->eof(lexer)) {
|
|
859
|
+
lexer->mark_end(lexer);
|
|
860
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
861
|
+
return true;
|
|
862
|
+
}
|
|
863
|
+
if (lexer->lookahead != '\n') {
|
|
864
|
+
lexer->result_symbol = CONCAT;
|
|
865
|
+
return true;
|
|
866
|
+
}
|
|
867
|
+
advance(lexer);
|
|
868
|
+
lexer->mark_end(lexer);
|
|
869
|
+
c = lexer->lookahead;
|
|
870
|
+
// The word continues only if the next line does; otherwise the continuation itself is the
|
|
871
|
+
// token, so that whatever follows it is lexed as it would be without it.
|
|
872
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
873
|
+
if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
874
|
+
return true;
|
|
875
|
+
}
|
|
876
|
+
if (c == '\\') {
|
|
877
|
+
// Another continuation is whitespace too; an escaped character continues the word.
|
|
878
|
+
advance(lexer);
|
|
879
|
+
if (lexer->lookahead == '\r') {
|
|
880
|
+
advance(lexer);
|
|
881
|
+
}
|
|
882
|
+
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
883
|
+
return true;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
lexer->result_symbol = CONCAT;
|
|
887
|
+
return true;
|
|
888
|
+
}
|
|
889
|
+
if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
|
|
398
890
|
lexer->result_symbol = CONCAT;
|
|
399
891
|
return true;
|
|
400
892
|
}
|
|
@@ -402,6 +894,17 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
402
894
|
|
|
403
895
|
if (valid_symbols[EMPTY_VALUE]) {
|
|
404
896
|
int32_t c = first;
|
|
897
|
+
// A redirection may follow an empty value (`A=>f`), but `<(`/`>(` is a process substitution
|
|
898
|
+
// value; nothing else a value may start with begins with `<` or `>`.
|
|
899
|
+
if ((c == '<' || c == '>') && lexer->lookahead == c) {
|
|
900
|
+
lexer->mark_end(lexer);
|
|
901
|
+
advance(lexer);
|
|
902
|
+
if (lexer->lookahead == '(') {
|
|
903
|
+
return false;
|
|
904
|
+
}
|
|
905
|
+
lexer->result_symbol = EMPTY_VALUE;
|
|
906
|
+
return true;
|
|
907
|
+
}
|
|
405
908
|
if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
|
|
406
909
|
(c == '`' && scanner->backtick_depth > 0)) {
|
|
407
910
|
lexer->result_symbol = EMPTY_VALUE;
|
|
@@ -426,24 +929,49 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
426
929
|
}
|
|
427
930
|
skip(lexer);
|
|
428
931
|
} else if (lexer->lookahead == '\\') {
|
|
429
|
-
|
|
932
|
+
// A continuation is returned as a token, since tree-sitter's lexer cannot always resume
|
|
933
|
+
// after one; a backslash escaping a word's first character can only start an extglob prefix
|
|
934
|
+
// (`\foo@($x)`), and the word token handles it otherwise.
|
|
935
|
+
advance(lexer);
|
|
430
936
|
if (lexer->lookahead == '\r') {
|
|
431
|
-
|
|
937
|
+
advance(lexer);
|
|
432
938
|
}
|
|
433
|
-
if (lexer->lookahead
|
|
434
|
-
|
|
939
|
+
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
940
|
+
advance(lexer);
|
|
941
|
+
lexer->mark_end(lexer);
|
|
942
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
943
|
+
return true;
|
|
435
944
|
}
|
|
436
|
-
|
|
945
|
+
if (valid_symbols[EXTGLOB_PREFIX]) {
|
|
946
|
+
advance(lexer);
|
|
947
|
+
return continue_extglob_prefix(lexer, true);
|
|
948
|
+
}
|
|
949
|
+
return false;
|
|
437
950
|
} else {
|
|
438
951
|
break;
|
|
439
952
|
}
|
|
440
953
|
}
|
|
441
954
|
|
|
955
|
+
// Each closer has its own marker, so a `]` or `}` argument inside `$( )` does not end it.
|
|
956
|
+
enum TokenType end = valid_symbols[SUBSTITUTION_END] && lexer->lookahead == ')' ? SUBSTITUTION_END
|
|
957
|
+
: valid_symbols[BRACE_SUBSTITUTION_END] && lexer->lookahead == '}' ? BRACE_SUBSTITUTION_END
|
|
958
|
+
: valid_symbols[BRACKET_SUBSTITUTION_END] && lexer->lookahead == ']' ? BRACKET_SUBSTITUTION_END
|
|
959
|
+
: ERROR_RECOVERY;
|
|
960
|
+
if (end != ERROR_RECOVERY) {
|
|
961
|
+
if (scanner->closers.size > 0) {
|
|
962
|
+
array_pop(&scanner->closers);
|
|
963
|
+
}
|
|
964
|
+
discard_unstarted_heredocs(scanner);
|
|
965
|
+
lexer->result_symbol = end;
|
|
966
|
+
return true;
|
|
967
|
+
}
|
|
968
|
+
|
|
442
969
|
if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
|
|
443
970
|
if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
|
|
444
971
|
advance(lexer);
|
|
445
972
|
lexer->mark_end(lexer);
|
|
446
973
|
scanner->backtick_depth--;
|
|
974
|
+
discard_unstarted_heredocs(scanner);
|
|
447
975
|
lexer->result_symbol = BACKTICK_CLOSE;
|
|
448
976
|
return true;
|
|
449
977
|
}
|
|
@@ -460,18 +988,27 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
460
988
|
return scan_heredoc_arrow(scanner, lexer, valid_symbols);
|
|
461
989
|
}
|
|
462
990
|
|
|
991
|
+
// Only one check that consumes input can run, so the prefix scan leaves names before a possible
|
|
992
|
+
// assignment to their own check; digits are read once as a descriptor or a word prefix.
|
|
463
993
|
if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
|
|
464
994
|
while (iswdigit(lexer->lookahead)) {
|
|
465
995
|
advance(lexer);
|
|
466
996
|
}
|
|
467
997
|
lexer->mark_end(lexer);
|
|
468
998
|
lexer->result_symbol = FILE_DESCRIPTOR;
|
|
469
|
-
|
|
999
|
+
if (lexer->lookahead == '<' || lexer->lookahead == '>') {
|
|
1000
|
+
return true;
|
|
1001
|
+
}
|
|
1002
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
if (valid_symbols[EXTGLOB_PREFIX] && !(valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) &&
|
|
1006
|
+
!is_word_break(lexer->lookahead)) {
|
|
1007
|
+
return scan_extglob_prefix(lexer, valid_symbols[EMPTY_VALUE]);
|
|
470
1008
|
}
|
|
471
1009
|
|
|
472
1010
|
if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
|
|
473
|
-
lexer
|
|
474
|
-
return scan_variable_name(lexer);
|
|
1011
|
+
return scan_name_or_extglob_prefix(lexer, valid_symbols);
|
|
475
1012
|
}
|
|
476
1013
|
|
|
477
1014
|
if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
|
|
@@ -488,6 +1025,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
488
1025
|
void *tree_sitter_bash_external_scanner_create(void) {
|
|
489
1026
|
Scanner *scanner = ts_calloc(1, sizeof(Scanner));
|
|
490
1027
|
array_init(&scanner->heredocs);
|
|
1028
|
+
array_init(&scanner->closers);
|
|
491
1029
|
return scanner;
|
|
492
1030
|
}
|
|
493
1031
|
|
|
@@ -495,6 +1033,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
|
|
|
495
1033
|
Scanner *scanner = (Scanner *)payload;
|
|
496
1034
|
scanner_reset(scanner);
|
|
497
1035
|
array_delete(&scanner->heredocs);
|
|
1036
|
+
array_delete(&scanner->closers);
|
|
498
1037
|
ts_free(scanner);
|
|
499
1038
|
}
|
|
500
1039
|
|
|
@@ -506,51 +1045,71 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
|
|
|
506
1045
|
Scanner *scanner = (Scanner *)payload;
|
|
507
1046
|
unsigned size = 0;
|
|
508
1047
|
buffer[size++] = (char)scanner->backtick_depth;
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
1048
|
+
buffer[size++] = (char)scanner->terminator_owed;
|
|
1049
|
+
uint16_t closer_count = (uint16_t)scanner->closers.size;
|
|
1050
|
+
memcpy(&buffer[size], &closer_count, sizeof(closer_count));
|
|
1051
|
+
size += sizeof(closer_count);
|
|
1052
|
+
if (scanner->closers.size > 0) {
|
|
1053
|
+
memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
|
|
1054
|
+
size += scanner->closers.size;
|
|
1055
|
+
}
|
|
1056
|
+
buffer[size++] = (char)scanner->heredocs.size;
|
|
1057
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
512
1058
|
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
513
|
-
if (size + 3 + sizeof(uint16_t) + heredoc->delimiter.size > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
514
|
-
break;
|
|
515
|
-
}
|
|
516
1059
|
buffer[size++] = (char)heredoc->is_raw;
|
|
517
1060
|
buffer[size++] = (char)heredoc->allows_indent;
|
|
518
1061
|
buffer[size++] = (char)heredoc->started;
|
|
1062
|
+
memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
|
|
1063
|
+
size += sizeof(heredoc->depth);
|
|
519
1064
|
uint16_t length = (uint16_t)heredoc->delimiter.size;
|
|
520
1065
|
memcpy(&buffer[size], &length, sizeof(length));
|
|
521
1066
|
size += sizeof(length);
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
size += length;
|
|
1067
|
+
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
1068
|
+
size += encode_varint(*array_get(&heredoc->delimiter, j), &buffer[size]);
|
|
525
1069
|
}
|
|
526
|
-
count++;
|
|
527
1070
|
}
|
|
528
|
-
buffer[count_position] = (char)count;
|
|
529
1071
|
return size;
|
|
530
1072
|
}
|
|
531
1073
|
|
|
532
1074
|
void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
|
|
533
1075
|
Scanner *scanner = (Scanner *)payload;
|
|
534
1076
|
scanner_reset(scanner);
|
|
535
|
-
if (length
|
|
1077
|
+
if (length < 2 + sizeof(uint16_t)) {
|
|
536
1078
|
return;
|
|
537
1079
|
}
|
|
538
1080
|
unsigned size = 0;
|
|
539
1081
|
scanner->backtick_depth = (uint8_t)buffer[size++];
|
|
1082
|
+
scanner->terminator_owed = buffer[size++];
|
|
1083
|
+
uint16_t closer_count;
|
|
1084
|
+
memcpy(&closer_count, &buffer[size], sizeof(closer_count));
|
|
1085
|
+
size += sizeof(closer_count);
|
|
1086
|
+
if (size + closer_count >= length) {
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
array_reserve(&scanner->closers, closer_count);
|
|
1090
|
+
memcpy(scanner->closers.contents, &buffer[size], closer_count);
|
|
1091
|
+
scanner->closers.size = closer_count;
|
|
1092
|
+
size += closer_count;
|
|
540
1093
|
uint8_t count = (uint8_t)buffer[size++];
|
|
541
|
-
for (uint8_t i = 0; i < count; i++) {
|
|
1094
|
+
for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
|
|
542
1095
|
Heredoc heredoc = {.delimiter = array_new()};
|
|
543
1096
|
heredoc.is_raw = buffer[size++];
|
|
544
1097
|
heredoc.allows_indent = buffer[size++];
|
|
545
1098
|
heredoc.started = buffer[size++];
|
|
1099
|
+
memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
|
|
1100
|
+
size += sizeof(heredoc.depth);
|
|
546
1101
|
uint16_t delimiter_length;
|
|
547
1102
|
memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
|
|
548
1103
|
size += sizeof(delimiter_length);
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
1104
|
+
array_reserve(&heredoc.delimiter, delimiter_length);
|
|
1105
|
+
for (uint16_t j = 0; j < delimiter_length; j++) {
|
|
1106
|
+
int32_t c;
|
|
1107
|
+
unsigned consumed = decode_varint(&buffer[size], length - size, &c);
|
|
1108
|
+
if (consumed == 0) {
|
|
1109
|
+
break;
|
|
1110
|
+
}
|
|
1111
|
+
size += consumed;
|
|
1112
|
+
array_push(&heredoc.delimiter, c);
|
|
554
1113
|
}
|
|
555
1114
|
array_push(&scanner->heredocs, heredoc);
|
|
556
1115
|
}
|