@willbooster/tree-sitter-bash 1.1.3 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/grammar.js +33 -4
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/darwin-x64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-x64/tree-sitter-bash.node +0 -0
- package/src/grammar.json +171 -14
- package/src/node-types.json +84 -13
- package/src/parser.c +170184 -133779
- package/src/scanner.c +466 -110
- package/tree-sitter-bash.wasm +0 -0
- package/tree-sitter.json +1 -1
package/src/scanner.c
CHANGED
|
@@ -30,6 +30,13 @@ enum TokenType {
|
|
|
30
30
|
LINE_CONTINUATION,
|
|
31
31
|
BRACE_SUBSTITUTION_START,
|
|
32
32
|
BRACKET_SUBSTITUTION_START,
|
|
33
|
+
REGEX_START,
|
|
34
|
+
REGEX_CONCAT,
|
|
35
|
+
CLOSER_LINE_RAW_STRING_START,
|
|
36
|
+
CLOSER_LINE_ANSI_C_STRING_START,
|
|
37
|
+
CLOSER_LINE_RAW_STRING_CONTENT,
|
|
38
|
+
CLOSER_LINE_ANSI_C_STRING_CONTENT,
|
|
39
|
+
JOINED_COMMENT,
|
|
33
40
|
ERROR_RECOVERY,
|
|
34
41
|
};
|
|
35
42
|
|
|
@@ -43,6 +50,9 @@ typedef struct {
|
|
|
43
50
|
bool is_raw;
|
|
44
51
|
bool allows_indent;
|
|
45
52
|
bool started;
|
|
53
|
+
// Queued on the same line as a heredoc whose body ended at a line holding the substitution's `)`:
|
|
54
|
+
// bash then reads this body right after that line, wherever the line ends.
|
|
55
|
+
bool deferred;
|
|
46
56
|
uint16_t depth;
|
|
47
57
|
CodePoints delimiter;
|
|
48
58
|
} Heredoc;
|
|
@@ -54,6 +64,8 @@ typedef struct {
|
|
|
54
64
|
uint8_t backtick_depth;
|
|
55
65
|
// The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
|
|
56
66
|
Array(uint8_t) closers;
|
|
67
|
+
// The regex group depth when each open substitution started, restored when it ends.
|
|
68
|
+
Array(uint16_t) closer_regex_depths;
|
|
57
69
|
// How many closers were open when the backquotes opened; the ones after them are nested inside.
|
|
58
70
|
uint16_t backtick_closers;
|
|
59
71
|
// A heredoc body ended before the end of its line (at a `${ list; }` closer, or at a delimiter
|
|
@@ -63,6 +75,20 @@ typedef struct {
|
|
|
63
75
|
// The last token of this scanner was a line continuation that ended a line; nothing concatenates at
|
|
64
76
|
// the start of the next one.
|
|
65
77
|
bool after_line_break;
|
|
78
|
+
// The rest of a line that ended a heredoc body at the substitution's `)` is being read: bash reads it
|
|
79
|
+
// with its line continuations removed, even inside single quotes and comments.
|
|
80
|
+
bool joined_line;
|
|
81
|
+
// A line continuation was split before its newline, where a deferred body started; its newline is still
|
|
82
|
+
// to be read, at the end of a word (`in_word`, of a regex if `in_regex`) or between words.
|
|
83
|
+
bool split_continuation;
|
|
84
|
+
bool split_continuation_in_word;
|
|
85
|
+
bool split_continuation_in_regex;
|
|
86
|
+
// A quoted string on a line that ended a body at the substitution's `)` has just opened; its first text
|
|
87
|
+
// token may be empty, so that it always has one and joining them gives its value.
|
|
88
|
+
bool closer_line_string_opened;
|
|
89
|
+
// The number of unclosed parentheses in the regex being read, which hold blanks and metacharacters.
|
|
90
|
+
uint16_t regex_depth;
|
|
91
|
+
uint16_t backtick_regex_depth;
|
|
66
92
|
} Scanner;
|
|
67
93
|
|
|
68
94
|
static inline uint16_t current_depth(Scanner *scanner) {
|
|
@@ -130,14 +156,17 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
|
|
|
130
156
|
return 0;
|
|
131
157
|
}
|
|
132
158
|
|
|
133
|
-
// Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator
|
|
134
|
-
// line-break flags, the
|
|
135
|
-
// (uint16
|
|
136
|
-
//
|
|
159
|
+
// Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator,
|
|
160
|
+
// line-break, joined-line, and split-continuation flags, the regex depth and the one saved when the backquotes
|
|
161
|
+
// opened (uint16 each), the closers of the open substitutions (uint16 count, then one byte each, then their
|
|
162
|
+
// saved regex depths as uint16), and heredoc count, then per heredoc its flags, depth (uint16), delimiter
|
|
163
|
+
// length (uint16), and the delimiter as varints.
|
|
164
|
+
#define SCANNER_HEADER_SIZE (2 + 4 * sizeof(uint16_t))
|
|
165
|
+
#define CLOSER_SIZE (1 + sizeof(uint16_t))
|
|
137
166
|
#define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
|
|
138
167
|
|
|
139
168
|
static unsigned serialized_size(Scanner *scanner) {
|
|
140
|
-
unsigned size =
|
|
169
|
+
unsigned size = SCANNER_HEADER_SIZE + CLOSER_SIZE * scanner->closers.size + 1;
|
|
141
170
|
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
142
171
|
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
143
172
|
size += HEREDOC_HEADER_SIZE;
|
|
@@ -157,9 +186,17 @@ static void scanner_reset(Scanner *scanner) {
|
|
|
157
186
|
array_clear(&scanner->heredocs);
|
|
158
187
|
scanner->backtick_depth = 0;
|
|
159
188
|
array_clear(&scanner->closers);
|
|
189
|
+
array_clear(&scanner->closer_regex_depths);
|
|
160
190
|
scanner->backtick_closers = 0;
|
|
161
191
|
scanner->terminator_owed = false;
|
|
162
192
|
scanner->after_line_break = false;
|
|
193
|
+
scanner->joined_line = false;
|
|
194
|
+
scanner->split_continuation = false;
|
|
195
|
+
scanner->split_continuation_in_word = false;
|
|
196
|
+
scanner->split_continuation_in_regex = false;
|
|
197
|
+
scanner->closer_line_string_opened = false;
|
|
198
|
+
scanner->regex_depth = 0;
|
|
199
|
+
scanner->backtick_regex_depth = 0;
|
|
163
200
|
}
|
|
164
201
|
|
|
165
202
|
// The body being read: the most deeply nested started heredoc.
|
|
@@ -174,16 +211,24 @@ static int32_t active_heredoc_index(Scanner *scanner) {
|
|
|
174
211
|
return active;
|
|
175
212
|
}
|
|
176
213
|
|
|
177
|
-
// The next body to start: the first unstarted heredoc opened at the
|
|
214
|
+
// The next body to start: the first deferred heredoc, or else the first unstarted heredoc opened at the
|
|
215
|
+
// current substitution depth.
|
|
178
216
|
static int32_t next_heredoc_index(Scanner *scanner) {
|
|
179
217
|
uint16_t depth = current_depth(scanner);
|
|
218
|
+
int32_t next = -1;
|
|
180
219
|
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
181
220
|
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
182
|
-
if (
|
|
221
|
+
if (heredoc->started) {
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
if (heredoc->deferred) {
|
|
183
225
|
return (int32_t)i;
|
|
184
226
|
}
|
|
227
|
+
if (next < 0 && heredoc->depth == depth) {
|
|
228
|
+
next = (int32_t)i;
|
|
229
|
+
}
|
|
185
230
|
}
|
|
186
|
-
return
|
|
231
|
+
return next;
|
|
187
232
|
}
|
|
188
233
|
|
|
189
234
|
// An entry without a delimiter belongs to an operator whose delimiter was rejected.
|
|
@@ -194,7 +239,7 @@ static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
|
|
|
194
239
|
uint16_t depth = current_depth(scanner);
|
|
195
240
|
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
196
241
|
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
197
|
-
if (!heredoc->started && heredoc->depth == depth) {
|
|
242
|
+
if (!heredoc->started && !heredoc->deferred && heredoc->depth == depth) {
|
|
198
243
|
heredoc_delete(heredoc);
|
|
199
244
|
array_erase(&scanner->heredocs, i - 1);
|
|
200
245
|
}
|
|
@@ -211,17 +256,47 @@ static void discard_unstarted_heredocs(Scanner *scanner) {
|
|
|
211
256
|
uint16_t depth = current_depth(scanner);
|
|
212
257
|
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
213
258
|
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
214
|
-
if (!heredoc->started && heredoc->depth > depth) {
|
|
259
|
+
if (!heredoc->started && !heredoc->deferred && heredoc->depth > depth) {
|
|
215
260
|
remove_heredoc(scanner, i - 1);
|
|
216
261
|
}
|
|
217
262
|
}
|
|
218
263
|
}
|
|
219
264
|
|
|
265
|
+
// Ends a body at a line that holds the substitution's `)`. Like bash, the bodies still queued on the line
|
|
266
|
+
// that began this one are then read right after that line, and the rest of the line is read without its
|
|
267
|
+
// line continuations unless the delimiter was quoted.
|
|
268
|
+
static void end_heredoc_at_closer_line(Scanner *scanner, uint32_t index) {
|
|
269
|
+
Heredoc *ended = array_get(&scanner->heredocs, index);
|
|
270
|
+
scanner->joined_line = !ended->is_raw;
|
|
271
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
272
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
273
|
+
if (!heredoc->started && heredoc->depth == ended->depth) {
|
|
274
|
+
heredoc->deferred = true;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
remove_heredoc(scanner, index);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// Whether a deferred body waits for the end of the line that ended a body at the substitution's `)`. That
|
|
281
|
+
// line ends at its first newline even after a backslash or inside quotes, unless it is joined.
|
|
282
|
+
static bool awaits_deferred_body(Scanner *scanner) {
|
|
283
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
284
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
285
|
+
if (heredoc->deferred && !heredoc->started) {
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
|
|
220
292
|
// Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
|
|
221
293
|
// line (or at the end of input, as bash does with a warning). An unquoted body stops before each
|
|
222
294
|
// expansion so that the parser can read it.
|
|
223
295
|
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
|
|
224
296
|
Heredoc *heredoc = array_get(&scanner->heredocs, index);
|
|
297
|
+
// Bash reads a deferred body from the input after its line, never from the text of backquotes that the
|
|
298
|
+
// line opened, so a backquote in it is literal.
|
|
299
|
+
bool in_backquotes = scanner->backtick_depth > 0 && !heredoc->deferred;
|
|
225
300
|
bool did_advance = false;
|
|
226
301
|
bool at_line_start = lexer->get_column(lexer) == 0;
|
|
227
302
|
|
|
@@ -239,7 +314,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
239
314
|
// code. Such a line ends the content token before it and the end token after the delimiter.
|
|
240
315
|
// Where the backquotes are innermost, bash fixes their extent before it reads the body, so
|
|
241
316
|
// the rule is off.
|
|
242
|
-
uint16_t outside_backquotes =
|
|
317
|
+
uint16_t outside_backquotes = in_backquotes ? scanner->backtick_closers : 0;
|
|
243
318
|
bool in_parenthesized =
|
|
244
319
|
scanner->closers.size > outside_backquotes && *array_back(&scanner->closers) == ')';
|
|
245
320
|
bool prefix_marked = false;
|
|
@@ -286,7 +361,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
286
361
|
// innermost substitution's closer (`)` or `}`) or a closing backquote.
|
|
287
362
|
bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
|
|
288
363
|
(scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
|
|
289
|
-
(lexer->lookahead == '`' &&
|
|
364
|
+
(lexer->lookahead == '`' && in_backquotes);
|
|
290
365
|
if (!escaped && matched == heredoc->delimiter.size && ends_line) {
|
|
291
366
|
if (did_advance) {
|
|
292
367
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
@@ -295,22 +370,27 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
295
370
|
lexer->mark_end(lexer);
|
|
296
371
|
lexer->result_symbol = HEREDOC_END;
|
|
297
372
|
scanner->terminator_owed = lexer->lookahead == '}';
|
|
298
|
-
|
|
373
|
+
if (in_parenthesized && lexer->lookahead == ')') {
|
|
374
|
+
end_heredoc_at_closer_line(scanner, index);
|
|
375
|
+
} else {
|
|
376
|
+
remove_heredoc(scanner, index);
|
|
377
|
+
}
|
|
299
378
|
return true;
|
|
300
379
|
}
|
|
301
380
|
if (prefix_marked) {
|
|
302
|
-
// The line is read after joining its backslash-newlines, as for the delimiter
|
|
381
|
+
// The line is read after joining its backslash-newlines, as for the delimiter; any other
|
|
382
|
+
// escaped character is kept with its backslash, so a newline after it ends the line.
|
|
303
383
|
while (!lexer->eof(lexer) && lexer->lookahead != '\n' && lexer->lookahead != ')') {
|
|
304
384
|
bool backslash = lexer->lookahead == '\\' && !heredoc->is_raw;
|
|
305
385
|
advance(lexer);
|
|
306
|
-
if (backslash && lexer->lookahead
|
|
386
|
+
if (backslash && !lexer->eof(lexer) && lexer->lookahead != ')') {
|
|
307
387
|
advance(lexer);
|
|
308
388
|
}
|
|
309
389
|
}
|
|
310
390
|
if (escaped_after_prefix == ')' || lexer->lookahead == ')') {
|
|
311
391
|
lexer->result_symbol = HEREDOC_END;
|
|
312
392
|
scanner->terminator_owed = true;
|
|
313
|
-
|
|
393
|
+
end_heredoc_at_closer_line(scanner, index);
|
|
314
394
|
} else {
|
|
315
395
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
316
396
|
}
|
|
@@ -336,13 +416,14 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
336
416
|
advance(lexer);
|
|
337
417
|
did_advance = true;
|
|
338
418
|
at_line_start = true;
|
|
419
|
+
scanner->joined_line = false;
|
|
339
420
|
break;
|
|
340
421
|
case '\\':
|
|
341
422
|
advance(lexer);
|
|
342
423
|
did_advance = true;
|
|
343
424
|
// Inside backquotes a backslash also hides a backquote from a quoted body's end.
|
|
344
425
|
if (!lexer->eof(lexer) &&
|
|
345
|
-
(!heredoc->is_raw || (
|
|
426
|
+
(!heredoc->is_raw || (in_backquotes && lexer->lookahead == '`'))) {
|
|
346
427
|
// A backslash-newline joins lines before the delimiter comparison, while a
|
|
347
428
|
// backslash-CR escapes only the CR, so the LF after it still ends the line.
|
|
348
429
|
advance(lexer);
|
|
@@ -351,7 +432,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
351
432
|
case '`':
|
|
352
433
|
// Bash ends backquotes at the first unescaped backquote, even inside a heredoc body,
|
|
353
434
|
// and ends the body there.
|
|
354
|
-
if (
|
|
435
|
+
if (in_backquotes) {
|
|
355
436
|
lexer->mark_end(lexer);
|
|
356
437
|
lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
|
|
357
438
|
if (!did_advance) {
|
|
@@ -359,7 +440,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
359
440
|
}
|
|
360
441
|
return true;
|
|
361
442
|
}
|
|
362
|
-
if (!heredoc->is_raw) {
|
|
443
|
+
if (!heredoc->is_raw && scanner->backtick_depth == 0) {
|
|
363
444
|
lexer->mark_end(lexer);
|
|
364
445
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
365
446
|
return did_advance;
|
|
@@ -485,7 +566,17 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
|
485
566
|
quote = 0;
|
|
486
567
|
continue;
|
|
487
568
|
}
|
|
488
|
-
if (
|
|
569
|
+
if (scanner->joined_line && c == '\\' && lexer->lookahead == '\n') {
|
|
570
|
+
advance(lexer);
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
if (scanner->joined_line && quote == '\'' && !ansi_c && c == '\\' && lexer->lookahead != '\'' &&
|
|
574
|
+
!lexer->eof(lexer)) {
|
|
575
|
+
// The character after a backslash is kept as it is, so it cannot begin a line continuation.
|
|
576
|
+
array_push(&heredoc->delimiter, c);
|
|
577
|
+
c = lexer->lookahead;
|
|
578
|
+
advance(lexer);
|
|
579
|
+
} else if (ansi_c && c == '\\') {
|
|
489
580
|
if (!lexer->eof(lexer)) {
|
|
490
581
|
push_ansi_c_escape(lexer, &heredoc->delimiter);
|
|
491
582
|
}
|
|
@@ -589,53 +680,65 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
|
|
|
589
680
|
return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
|
|
590
681
|
}
|
|
591
682
|
|
|
592
|
-
// The
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
683
|
+
// The literal text of the right side of `=~`, which, like bash, reads `(` and `|` as regex characters
|
|
684
|
+
// and a parenthesized group, blanks and metacharacters included, as part of the word up to its matching
|
|
685
|
+
// `)`. Quotes and expansions are separate parts, so the group depth is kept for the text after them.
|
|
686
|
+
static bool scan_regex(Scanner *scanner, TSLexer *lexer) {
|
|
687
|
+
uint32_t depth = scanner->regex_depth;
|
|
688
|
+
if (depth == 0) {
|
|
689
|
+
while (is_blank(lexer->lookahead)) {
|
|
690
|
+
skip(lexer);
|
|
691
|
+
}
|
|
596
692
|
}
|
|
597
|
-
uint32_t depth = 0;
|
|
598
693
|
bool did_advance = false;
|
|
599
|
-
|
|
694
|
+
for (;;) {
|
|
600
695
|
lexer->mark_end(lexer);
|
|
696
|
+
if (lexer->eof(lexer)) {
|
|
697
|
+
break;
|
|
698
|
+
}
|
|
601
699
|
int32_t c = lexer->lookahead;
|
|
602
|
-
if (
|
|
700
|
+
if (c == '\'' || c == '"' || c == '`' ||
|
|
701
|
+
(depth == 0 && (is_separator(c) || c == '&' || c == ';' || c == ')'))) {
|
|
603
702
|
break;
|
|
604
703
|
}
|
|
605
|
-
//
|
|
606
|
-
|
|
607
|
-
if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
|
|
704
|
+
// A deferred body starts at the newline that ends the line, even inside a group.
|
|
705
|
+
if (c == '\n' && awaits_deferred_body(scanner)) {
|
|
608
706
|
break;
|
|
609
707
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
708
|
+
advance(lexer);
|
|
709
|
+
if (c == '\\' && lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
710
|
+
// The newline ends the line, after which a deferred body starts; a concatenation splits it.
|
|
711
|
+
break;
|
|
712
|
+
}
|
|
713
|
+
if (c == '$' || c == '<' || c == '>') {
|
|
714
|
+
int32_t next = lexer->lookahead;
|
|
715
|
+
bool starts_part = c == '$' ? starts_quoted_expansion(next) || next == '\'' || next == '"' : next == '(';
|
|
716
|
+
if (starts_part || (c != '$' && depth == 0)) {
|
|
717
|
+
break;
|
|
718
|
+
}
|
|
719
|
+
} else if (c == '\\') {
|
|
720
|
+
if (!lexer->eof(lexer)) {
|
|
618
721
|
advance(lexer);
|
|
619
722
|
}
|
|
620
723
|
} else if (c == '(') {
|
|
621
724
|
depth++;
|
|
622
725
|
} else if (c == ')') {
|
|
623
|
-
if (depth == 0) {
|
|
624
|
-
break;
|
|
625
|
-
}
|
|
626
726
|
depth--;
|
|
727
|
+
} else if (c == '\n') {
|
|
728
|
+
scanner->joined_line = false;
|
|
627
729
|
}
|
|
628
|
-
advance(lexer);
|
|
629
730
|
did_advance = true;
|
|
630
731
|
}
|
|
631
|
-
if (
|
|
632
|
-
|
|
732
|
+
if (!did_advance || depth > UINT16_MAX) {
|
|
733
|
+
return false;
|
|
633
734
|
}
|
|
735
|
+
scanner->regex_depth = (uint16_t)depth;
|
|
634
736
|
lexer->result_symbol = REGEX;
|
|
635
|
-
return
|
|
737
|
+
return true;
|
|
636
738
|
}
|
|
637
739
|
|
|
638
|
-
static bool scan_string_content(TSLexer *lexer) {
|
|
740
|
+
static bool scan_string_content(Scanner *scanner, TSLexer *lexer) {
|
|
741
|
+
bool line_breaks = !scanner->joined_line && awaits_deferred_body(scanner);
|
|
639
742
|
bool did_advance = false;
|
|
640
743
|
for (;;) {
|
|
641
744
|
lexer->mark_end(lexer);
|
|
@@ -646,11 +749,18 @@ static bool scan_string_content(TSLexer *lexer) {
|
|
|
646
749
|
if (c == '"' || c == '`') {
|
|
647
750
|
break;
|
|
648
751
|
}
|
|
752
|
+
if (c == '\n' && (scanner->joined_line || line_breaks)) {
|
|
753
|
+
// A deferred body starts right before the newline that ends the line.
|
|
754
|
+
if (did_advance) {
|
|
755
|
+
break;
|
|
756
|
+
}
|
|
757
|
+
scanner->joined_line = false;
|
|
758
|
+
}
|
|
649
759
|
advance(lexer);
|
|
650
760
|
if (c == '$' && starts_quoted_expansion(lexer->lookahead)) {
|
|
651
761
|
break;
|
|
652
762
|
}
|
|
653
|
-
if (c == '\\' && !lexer->eof(lexer)) {
|
|
763
|
+
if (c == '\\' && !lexer->eof(lexer) && !(line_breaks && lexer->lookahead == '\n')) {
|
|
654
764
|
advance(lexer);
|
|
655
765
|
}
|
|
656
766
|
did_advance = true;
|
|
@@ -663,6 +773,11 @@ static inline bool is_word_break(int32_t c) {
|
|
|
663
773
|
return is_separator(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
|
|
664
774
|
}
|
|
665
775
|
|
|
776
|
+
// Whether `c` ends a word; `(` and `|` are regex characters in the right side of `=~`.
|
|
777
|
+
static inline bool ends_word(int32_t c, bool regex) {
|
|
778
|
+
return is_separator(c) || (is_metacharacter(c) && !(regex && (c == '(' || c == '|')));
|
|
779
|
+
}
|
|
780
|
+
|
|
666
781
|
static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
|
|
667
782
|
|
|
668
783
|
typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
|
|
@@ -873,6 +988,122 @@ static bool at_closing_reserved_word(TSLexer *lexer) {
|
|
|
873
988
|
return false;
|
|
874
989
|
}
|
|
875
990
|
|
|
991
|
+
// After a line continuation inside a word: bash removes every backslash-newline before it splits words, so
|
|
992
|
+
// further continuations right after this one are part of the token (`a\` + `\` + `b` is `ab`), which is a
|
|
993
|
+
// concatenation if the word continues on the next line. Otherwise the continuations themselves are the token,
|
|
994
|
+
// so that whatever follows them is lexed as it would be without them.
|
|
995
|
+
static bool finish_word_continuations(Scanner *scanner, TSLexer *lexer, enum TokenType concat, bool regex) {
|
|
996
|
+
int32_t c;
|
|
997
|
+
for (;;) {
|
|
998
|
+
c = lexer->lookahead;
|
|
999
|
+
if (c != '\\') {
|
|
1000
|
+
break;
|
|
1001
|
+
}
|
|
1002
|
+
advance(lexer);
|
|
1003
|
+
if (lexer->eof(lexer)) {
|
|
1004
|
+
lexer->mark_end(lexer);
|
|
1005
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1006
|
+
return true;
|
|
1007
|
+
}
|
|
1008
|
+
if (lexer->lookahead != '\n') {
|
|
1009
|
+
// An escaped character continues the word.
|
|
1010
|
+
lexer->result_symbol = concat;
|
|
1011
|
+
return true;
|
|
1012
|
+
}
|
|
1013
|
+
advance(lexer);
|
|
1014
|
+
lexer->mark_end(lexer);
|
|
1015
|
+
}
|
|
1016
|
+
if ((c == '<' || c == '>')) {
|
|
1017
|
+
advance(lexer);
|
|
1018
|
+
if (lexer->lookahead == '(') {
|
|
1019
|
+
lexer->result_symbol = concat;
|
|
1020
|
+
return true;
|
|
1021
|
+
}
|
|
1022
|
+
scanner->after_line_break = true;
|
|
1023
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1024
|
+
} else if (lexer->eof(lexer) || (ends_word(c, regex) && !(regex && scanner->regex_depth > 0)) ||
|
|
1025
|
+
(c == '`' && scanner->backtick_depth > 0)) {
|
|
1026
|
+
scanner->after_line_break = true;
|
|
1027
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1028
|
+
} else {
|
|
1029
|
+
lexer->result_symbol = concat;
|
|
1030
|
+
}
|
|
1031
|
+
return true;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
// Ends a line continuation's token at its backslash, since its newline ends the line after which a deferred
|
|
1035
|
+
// body starts; the newline is read after the body.
|
|
1036
|
+
static void split_continuation(Scanner *scanner, TSLexer *lexer, bool in_word, bool in_regex) {
|
|
1037
|
+
lexer->mark_end(lexer);
|
|
1038
|
+
scanner->split_continuation = true;
|
|
1039
|
+
scanner->split_continuation_in_word = in_word;
|
|
1040
|
+
scanner->split_continuation_in_regex = in_regex;
|
|
1041
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
// After `$`, a bare dollar sign unless it begins an expansion or a quote.
|
|
1045
|
+
static bool finish_bare_dollar(TSLexer *lexer) {
|
|
1046
|
+
lexer->mark_end(lexer);
|
|
1047
|
+
int32_t c = lexer->lookahead;
|
|
1048
|
+
lexer->result_symbol = BARE_DOLLAR;
|
|
1049
|
+
return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
// The text of a single-quoted string on a line that ended a body at the substitution's `)`. Where the line
|
|
1053
|
+
// is joined, the text is split at its line continuations, which are tokens of their own, and, as when bash
|
|
1054
|
+
// joins the line, the character after any other backslash is kept with it; in a raw string that backslash
|
|
1055
|
+
// is literal, so a quote after it still ends the string. The text also ends before the newline that ends
|
|
1056
|
+
// the line, where a deferred body starts.
|
|
1057
|
+
static bool scan_closer_line_string_content(Scanner *scanner, TSLexer *lexer, bool ansi_c) {
|
|
1058
|
+
bool line_open = scanner->joined_line || awaits_deferred_body(scanner);
|
|
1059
|
+
bool did_advance = scanner->closer_line_string_opened;
|
|
1060
|
+
scanner->closer_line_string_opened = false;
|
|
1061
|
+
for (;;) {
|
|
1062
|
+
lexer->mark_end(lexer);
|
|
1063
|
+
if (lexer->eof(lexer) || lexer->lookahead == '\'') {
|
|
1064
|
+
break;
|
|
1065
|
+
}
|
|
1066
|
+
int32_t c = lexer->lookahead;
|
|
1067
|
+
// A deferred body starts right before the newline that ends the line.
|
|
1068
|
+
if (c == '\n' && line_open && did_advance) {
|
|
1069
|
+
break;
|
|
1070
|
+
}
|
|
1071
|
+
advance(lexer);
|
|
1072
|
+
if (c == '\\' && scanner->joined_line && lexer->lookahead == '\n') {
|
|
1073
|
+
if (did_advance) {
|
|
1074
|
+
break;
|
|
1075
|
+
}
|
|
1076
|
+
advance(lexer);
|
|
1077
|
+
lexer->mark_end(lexer);
|
|
1078
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1079
|
+
return true;
|
|
1080
|
+
}
|
|
1081
|
+
if (c == '\\' && !lexer->eof(lexer) && lexer->lookahead != '\n' && (ansi_c || lexer->lookahead != '\'')) {
|
|
1082
|
+
advance(lexer);
|
|
1083
|
+
} else if (c == '\n') {
|
|
1084
|
+
scanner->joined_line = false;
|
|
1085
|
+
line_open = false;
|
|
1086
|
+
}
|
|
1087
|
+
did_advance = true;
|
|
1088
|
+
}
|
|
1089
|
+
lexer->result_symbol = ansi_c ? CLOSER_LINE_ANSI_C_STRING_CONTENT : CLOSER_LINE_RAW_STRING_CONTENT;
|
|
1090
|
+
return did_advance;
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
// A comment on a joined line, which continues past each line continuation.
|
|
1094
|
+
static bool scan_joined_comment(TSLexer *lexer) {
|
|
1095
|
+
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
|
1096
|
+
bool backslash = lexer->lookahead == '\\';
|
|
1097
|
+
advance(lexer);
|
|
1098
|
+
if (backslash && !lexer->eof(lexer)) {
|
|
1099
|
+
advance(lexer);
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
lexer->mark_end(lexer);
|
|
1103
|
+
lexer->result_symbol = JOINED_COMMENT;
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
876
1107
|
static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
877
1108
|
bool error_recovery = valid_symbols[ERROR_RECOVERY];
|
|
878
1109
|
// Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
|
|
@@ -896,7 +1127,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
896
1127
|
(valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
|
|
897
1128
|
valid_symbols[BRACKET_SUBSTITUTION_START])) {
|
|
898
1129
|
// Nesting too deep for the serialized state is left to the parse as an error.
|
|
899
|
-
if (serialized_size(scanner)
|
|
1130
|
+
if (serialized_size(scanner) + CLOSER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
900
1131
|
return false;
|
|
901
1132
|
}
|
|
902
1133
|
lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
|
|
@@ -906,29 +1137,50 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
906
1137
|
: lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
|
|
907
1138
|
: ']';
|
|
908
1139
|
array_push(&scanner->closers, closer);
|
|
1140
|
+
array_push(&scanner->closer_regex_depths, scanner->regex_depth);
|
|
1141
|
+
return true;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
if (!error_recovery && valid_symbols[REGEX_START]) {
|
|
1145
|
+
scanner->regex_depth = 0;
|
|
1146
|
+
lexer->result_symbol = REGEX_START;
|
|
909
1147
|
return true;
|
|
910
1148
|
}
|
|
911
1149
|
|
|
1150
|
+
// Blanks, metacharacters, and newlines inside a regex group are part of the regex.
|
|
1151
|
+
bool in_regex_group = scanner->regex_depth > 0 && (valid_symbols[REGEX] || valid_symbols[REGEX_CONCAT]);
|
|
1152
|
+
|
|
912
1153
|
if (scanner->heredocs.size > 0) {
|
|
1154
|
+
int32_t next = next_heredoc_index(scanner);
|
|
1155
|
+
// Only another deferred body starts at the newline of a split line continuation.
|
|
1156
|
+
bool deferred = next >= 0 && array_get(&scanner->heredocs, next)->deferred;
|
|
1157
|
+
// A deferred body starts at the end of its line even inside the body of an enclosing heredoc, but
|
|
1158
|
+
// the deferred bodies of one line follow one another.
|
|
913
1159
|
int32_t active = active_heredoc_index(scanner);
|
|
914
1160
|
if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
|
|
1161
|
+
!(deferred && !array_get(&scanner->heredocs, active)->deferred && lexer->lookahead == '\n' &&
|
|
1162
|
+
valid_symbols[HEREDOC_BODY_START]) &&
|
|
915
1163
|
!(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw &&
|
|
916
1164
|
scanner->backtick_depth == 0)) {
|
|
917
1165
|
return scan_heredoc_content(scanner, lexer, (uint32_t)active);
|
|
918
1166
|
}
|
|
919
|
-
// A newline inside a string is part of it, so bodies start only outside strings
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1167
|
+
// A newline inside a string is part of it, so bodies start only outside strings, except for a
|
|
1168
|
+
// deferred body, which bash reads right after its line wherever that line ends.
|
|
1169
|
+
bool in_quotes = valid_symbols[STRING_CONTENT] || valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] ||
|
|
1170
|
+
valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT];
|
|
1171
|
+
if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && (!in_regex_group || deferred) &&
|
|
1172
|
+
(deferred || (!in_quotes && !scanner->split_continuation))) {
|
|
1173
|
+
while (is_blank(lexer->lookahead) && !in_quotes) {
|
|
923
1174
|
skip(lexer);
|
|
924
1175
|
}
|
|
925
1176
|
if (lexer->lookahead == '\n') {
|
|
926
1177
|
// The line's heredocs cannot be located once one of them was rejected, so none gets a
|
|
927
1178
|
// body; scanning continues so that the change is kept with the token found below.
|
|
928
|
-
if (is_rejected(array_get(&scanner->heredocs, next))) {
|
|
1179
|
+
if (is_rejected(array_get(&scanner->heredocs, next)) && !deferred) {
|
|
929
1180
|
drop_unstarted_heredocs_at_depth(scanner);
|
|
930
1181
|
} else {
|
|
931
1182
|
array_get(&scanner->heredocs, next)->started = true;
|
|
1183
|
+
scanner->joined_line = false;
|
|
932
1184
|
lexer->result_symbol = HEREDOC_BODY_START;
|
|
933
1185
|
return true;
|
|
934
1186
|
}
|
|
@@ -940,26 +1192,58 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
940
1192
|
return false;
|
|
941
1193
|
}
|
|
942
1194
|
|
|
1195
|
+
// The right side of `=~` joins its parts with a concatenation of its own, since `(` and `|` continue
|
|
1196
|
+
// it and a group continues it up to its matching `)`.
|
|
1197
|
+
bool regex = valid_symbols[REGEX_CONCAT] && !valid_symbols[CONCAT];
|
|
1198
|
+
enum TokenType concat = regex ? REGEX_CONCAT : CONCAT;
|
|
1199
|
+
bool can_concat = valid_symbols[CONCAT] || regex;
|
|
1200
|
+
|
|
1201
|
+
// The word goes on after the body even where the parse state no longer allows a concatenation: that
|
|
1202
|
+
// state comes from a reparse that reused the words before the body as a complete command, which the
|
|
1203
|
+
// concatenation then breaks down again.
|
|
1204
|
+
if (scanner->split_continuation && lexer->lookahead == '\n') {
|
|
1205
|
+
scanner->split_continuation = false;
|
|
1206
|
+
advance(lexer);
|
|
1207
|
+
lexer->mark_end(lexer);
|
|
1208
|
+
if (scanner->split_continuation_in_word) {
|
|
1209
|
+
bool in_regex = scanner->split_continuation_in_regex;
|
|
1210
|
+
return finish_word_continuations(scanner, lexer, in_regex ? REGEX_CONCAT : CONCAT, in_regex);
|
|
1211
|
+
}
|
|
1212
|
+
scanner->after_line_break = true;
|
|
1213
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1214
|
+
return true;
|
|
1215
|
+
}
|
|
1216
|
+
|
|
943
1217
|
if (valid_symbols[STRING_CONTENT] && lexer->lookahead != '`') {
|
|
944
|
-
return scan_string_content(lexer);
|
|
1218
|
+
return scan_string_content(scanner, lexer);
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
if (valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] || valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]) {
|
|
1222
|
+
return scan_closer_line_string_content(scanner, lexer, valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]);
|
|
945
1223
|
}
|
|
946
1224
|
|
|
947
1225
|
if (valid_symbols[HEREDOC_START] && scanner->heredocs.size > 0) {
|
|
948
1226
|
return scan_heredoc_start(scanner, lexer);
|
|
949
1227
|
}
|
|
950
1228
|
|
|
1229
|
+
if (regex && scanner->regex_depth > 0 && !at_eof && first != '\\') {
|
|
1230
|
+
lexer->result_symbol = REGEX_CONCAT;
|
|
1231
|
+
return true;
|
|
1232
|
+
}
|
|
1233
|
+
|
|
951
1234
|
// A line continuation after a blank separates words, so nothing concatenates at the start of the
|
|
952
1235
|
// next line. The column costs a scan back to the line start, so it is computed only right after
|
|
953
1236
|
// such a continuation and only where a concatenation could start; a concatenation then clears the
|
|
954
1237
|
// flag, which bounds the scans to one per continuation.
|
|
1238
|
+
|
|
955
1239
|
// Like bash, `a<(cmd)` is one word. `<` and `>` end a word part otherwise, so the concatenation
|
|
956
1240
|
// needs a look at the next character, and a `<<` read along the way is scanned as a heredoc arrow.
|
|
957
|
-
if (
|
|
1241
|
+
if (can_concat && (first == '<' || first == '>') &&
|
|
958
1242
|
!(after_line_break && lexer->get_column(lexer) == 0)) {
|
|
959
1243
|
lexer->mark_end(lexer);
|
|
960
1244
|
advance(lexer);
|
|
961
1245
|
if (lexer->lookahead == '(') {
|
|
962
|
-
lexer->result_symbol =
|
|
1246
|
+
lexer->result_symbol = concat;
|
|
963
1247
|
return true;
|
|
964
1248
|
}
|
|
965
1249
|
if (first == '<' && (valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH])) {
|
|
@@ -968,7 +1252,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
968
1252
|
return false;
|
|
969
1253
|
}
|
|
970
1254
|
|
|
971
|
-
if (
|
|
1255
|
+
if (can_concat && !at_eof && (!ends_word(first, regex) || (regex && scanner->regex_depth > 0)) &&
|
|
972
1256
|
!(after_line_break && lexer->get_column(lexer) == 0)) {
|
|
973
1257
|
int32_t c = first;
|
|
974
1258
|
if (c == '\\' && lexer->lookahead == '\\') {
|
|
@@ -982,53 +1266,19 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
982
1266
|
return true;
|
|
983
1267
|
}
|
|
984
1268
|
if (lexer->lookahead != '\n') {
|
|
985
|
-
lexer->result_symbol =
|
|
1269
|
+
lexer->result_symbol = concat;
|
|
1270
|
+
return true;
|
|
1271
|
+
}
|
|
1272
|
+
if (!scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
1273
|
+
split_continuation(scanner, lexer, true, regex);
|
|
986
1274
|
return true;
|
|
987
1275
|
}
|
|
988
1276
|
advance(lexer);
|
|
989
1277
|
lexer->mark_end(lexer);
|
|
990
|
-
|
|
991
|
-
// right after this one are part of the token (`a\` + `\` + `b` is `ab`).
|
|
992
|
-
for (;;) {
|
|
993
|
-
c = lexer->lookahead;
|
|
994
|
-
if (c != '\\') {
|
|
995
|
-
break;
|
|
996
|
-
}
|
|
997
|
-
advance(lexer);
|
|
998
|
-
if (lexer->eof(lexer)) {
|
|
999
|
-
lexer->mark_end(lexer);
|
|
1000
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1001
|
-
return true;
|
|
1002
|
-
}
|
|
1003
|
-
if (lexer->lookahead != '\n') {
|
|
1004
|
-
// An escaped character continues the word.
|
|
1005
|
-
lexer->result_symbol = CONCAT;
|
|
1006
|
-
return true;
|
|
1007
|
-
}
|
|
1008
|
-
advance(lexer);
|
|
1009
|
-
lexer->mark_end(lexer);
|
|
1010
|
-
}
|
|
1011
|
-
// The word continues only if the next line does; otherwise the continuations themselves are
|
|
1012
|
-
// the token, so that whatever follows them is lexed as it would be without them.
|
|
1013
|
-
if ((c == '<' || c == '>')) {
|
|
1014
|
-
advance(lexer);
|
|
1015
|
-
if (lexer->lookahead == '(') {
|
|
1016
|
-
lexer->result_symbol = CONCAT;
|
|
1017
|
-
return true;
|
|
1018
|
-
}
|
|
1019
|
-
scanner->after_line_break = true;
|
|
1020
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1021
|
-
} else if (lexer->eof(lexer) || is_separator(c) || is_metacharacter(c) ||
|
|
1022
|
-
(c == '`' && scanner->backtick_depth > 0)) {
|
|
1023
|
-
scanner->after_line_break = true;
|
|
1024
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1025
|
-
} else {
|
|
1026
|
-
lexer->result_symbol = CONCAT;
|
|
1027
|
-
}
|
|
1028
|
-
return true;
|
|
1278
|
+
return finish_word_continuations(scanner, lexer, concat, regex);
|
|
1029
1279
|
}
|
|
1030
1280
|
if (c != '`' || scanner->backtick_depth == 0) {
|
|
1031
|
-
lexer->result_symbol =
|
|
1281
|
+
lexer->result_symbol = concat;
|
|
1032
1282
|
return true;
|
|
1033
1283
|
}
|
|
1034
1284
|
}
|
|
@@ -1053,8 +1303,70 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1053
1303
|
}
|
|
1054
1304
|
}
|
|
1055
1305
|
|
|
1306
|
+
// The rest of a line that ended a body at the substitution's `)` scans its single-quoted strings itself,
|
|
1307
|
+
// to remove their line continuations where the line is joined, and to end the line inside them. Inside
|
|
1308
|
+
// backquotes, the next backquote ends a quote, as handled below, but a newline still ends the line.
|
|
1309
|
+
if ((scanner->joined_line || awaits_deferred_body(scanner)) && !error_recovery) {
|
|
1310
|
+
while (is_blank(lexer->lookahead) && !in_regex_group) {
|
|
1311
|
+
skip(lexer);
|
|
1312
|
+
}
|
|
1313
|
+
if (lexer->lookahead == '\n' && scanner->joined_line && !in_regex_group) {
|
|
1314
|
+
// The joined line ends here; a token is returned even where the newline is only whitespace,
|
|
1315
|
+
// so that the line's end is kept.
|
|
1316
|
+
advance(lexer);
|
|
1317
|
+
lexer->mark_end(lexer);
|
|
1318
|
+
scanner->joined_line = false;
|
|
1319
|
+
lexer->result_symbol = valid_symbols[NEWLINE] ? NEWLINE : LINE_CONTINUATION;
|
|
1320
|
+
return true;
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 && !error_recovery) {
|
|
1324
|
+
if (lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_RAW_STRING_START]) {
|
|
1325
|
+
advance(lexer);
|
|
1326
|
+
lexer->mark_end(lexer);
|
|
1327
|
+
scanner->closer_line_string_opened = true;
|
|
1328
|
+
lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
|
|
1329
|
+
return true;
|
|
1330
|
+
}
|
|
1331
|
+
// A `#` begins a word, and so a comment, after a blank or an operator. Right after a word part, where
|
|
1332
|
+
// another part may follow, it continues the word; a command may also start there, as after `;`, but
|
|
1333
|
+
// a reparse may reuse the statements before it as a complete list, where no part may follow.
|
|
1334
|
+
bool in_word = valid_symbols[BARE_DOLLAR] && !valid_symbols[VARIABLE_NAME];
|
|
1335
|
+
if (lexer->lookahead == '#' && (is_blank(first) || !in_word) && scanner->joined_line &&
|
|
1336
|
+
valid_symbols[JOINED_COMMENT]) {
|
|
1337
|
+
return scan_joined_comment(lexer);
|
|
1338
|
+
}
|
|
1339
|
+
// A regex reads `$` as text unless it begins a part, so it looks for `$'` itself.
|
|
1340
|
+
if (lexer->lookahead == '$' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START] && !valid_symbols[REGEX]) {
|
|
1341
|
+
advance(lexer);
|
|
1342
|
+
if (lexer->lookahead == '\'') {
|
|
1343
|
+
advance(lexer);
|
|
1344
|
+
lexer->mark_end(lexer);
|
|
1345
|
+
scanner->closer_line_string_opened = true;
|
|
1346
|
+
lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
|
|
1347
|
+
return true;
|
|
1348
|
+
}
|
|
1349
|
+
return valid_symbols[BARE_DOLLAR] && finish_bare_dollar(lexer);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1056
1353
|
if (valid_symbols[REGEX]) {
|
|
1057
|
-
|
|
1354
|
+
if (scan_regex(scanner, lexer)) {
|
|
1355
|
+
return true;
|
|
1356
|
+
}
|
|
1357
|
+
// Having found no text, the scan stopped before a quote, or after the `$` of `$'`.
|
|
1358
|
+
if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 &&
|
|
1359
|
+
lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START]) {
|
|
1360
|
+
advance(lexer);
|
|
1361
|
+
lexer->mark_end(lexer);
|
|
1362
|
+
scanner->closer_line_string_opened = true;
|
|
1363
|
+
lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
|
|
1364
|
+
return true;
|
|
1365
|
+
}
|
|
1366
|
+
// Backquotes are the only other part that the scanner opens.
|
|
1367
|
+
if (lexer->lookahead != '`') {
|
|
1368
|
+
return false;
|
|
1369
|
+
}
|
|
1058
1370
|
}
|
|
1059
1371
|
|
|
1060
1372
|
// A newline ends a statement where a terminator may follow and is whitespace elsewhere.
|
|
@@ -1074,6 +1386,14 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1074
1386
|
// after one; a backslash escaping a word's first character can only start an extglob prefix
|
|
1075
1387
|
// (`\foo@($x)`), and the word token handles it otherwise.
|
|
1076
1388
|
advance(lexer);
|
|
1389
|
+
if (lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
1390
|
+
// A reparse may reuse the words before the backslash as a complete command, where no
|
|
1391
|
+
// concatenation is valid, so a backslash right after the previous token is taken to end a
|
|
1392
|
+
// word unless a command or an assignment value may start there.
|
|
1393
|
+
split_continuation(scanner, lexer,
|
|
1394
|
+
first == '\\' && !valid_symbols[VARIABLE_NAME] && !valid_symbols[EMPTY_VALUE], false);
|
|
1395
|
+
return true;
|
|
1396
|
+
}
|
|
1077
1397
|
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
1078
1398
|
advance(lexer);
|
|
1079
1399
|
lexer->mark_end(lexer);
|
|
@@ -1099,6 +1419,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1099
1419
|
if (end != ERROR_RECOVERY) {
|
|
1100
1420
|
if (scanner->closers.size > 0) {
|
|
1101
1421
|
array_pop(&scanner->closers);
|
|
1422
|
+
scanner->regex_depth = array_pop(&scanner->closer_regex_depths);
|
|
1102
1423
|
}
|
|
1103
1424
|
discard_unstarted_heredocs(scanner);
|
|
1104
1425
|
lexer->result_symbol = end;
|
|
@@ -1111,7 +1432,11 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1111
1432
|
// up to the backquote, which then cannot be parsed; the quote never hides the text after it.
|
|
1112
1433
|
if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
|
|
1113
1434
|
advance(lexer);
|
|
1114
|
-
|
|
1435
|
+
lexer->mark_end(lexer);
|
|
1436
|
+
// A deferred body starts at the newline that ends the line, so what follows it is read later.
|
|
1437
|
+
bool breaks_line = awaits_deferred_body(scanner) && valid_symbols[CLOSER_LINE_RAW_STRING_START];
|
|
1438
|
+
while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`' &&
|
|
1439
|
+
!(breaks_line && lexer->lookahead == '\n')) {
|
|
1115
1440
|
// A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
|
|
1116
1441
|
// quote itself stays literal, so `'\'` still ends at its second quote.
|
|
1117
1442
|
bool backslash = lexer->lookahead == '\\';
|
|
@@ -1121,12 +1446,20 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1121
1446
|
}
|
|
1122
1447
|
}
|
|
1123
1448
|
if (lexer->lookahead != '`') {
|
|
1449
|
+
// A quote that the backquotes do not cut short scans its text itself where a deferred body may
|
|
1450
|
+
// start inside it.
|
|
1451
|
+
if (!lexer->eof(lexer) && breaks_line) {
|
|
1452
|
+
scanner->closer_line_string_opened = true;
|
|
1453
|
+
lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
|
|
1454
|
+
return true;
|
|
1455
|
+
}
|
|
1124
1456
|
return false;
|
|
1125
1457
|
}
|
|
1126
1458
|
if (valid_symbols[BACKTICK_CLOSE]) {
|
|
1127
1459
|
advance(lexer);
|
|
1128
1460
|
lexer->mark_end(lexer);
|
|
1129
1461
|
scanner->backtick_depth--;
|
|
1462
|
+
scanner->regex_depth = scanner->backtick_regex_depth;
|
|
1130
1463
|
discard_unstarted_heredocs(scanner);
|
|
1131
1464
|
lexer->result_symbol = BACKTICK_CLOSE;
|
|
1132
1465
|
return true;
|
|
@@ -1141,6 +1474,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1141
1474
|
advance(lexer);
|
|
1142
1475
|
lexer->mark_end(lexer);
|
|
1143
1476
|
scanner->backtick_depth--;
|
|
1477
|
+
scanner->regex_depth = scanner->backtick_regex_depth;
|
|
1144
1478
|
discard_unstarted_heredocs(scanner);
|
|
1145
1479
|
lexer->result_symbol = BACKTICK_CLOSE;
|
|
1146
1480
|
return true;
|
|
@@ -1150,6 +1484,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1150
1484
|
lexer->mark_end(lexer);
|
|
1151
1485
|
scanner->backtick_depth++;
|
|
1152
1486
|
scanner->backtick_closers = (uint16_t)scanner->closers.size;
|
|
1487
|
+
scanner->backtick_regex_depth = scanner->regex_depth;
|
|
1153
1488
|
lexer->result_symbol = BACKTICK_OPEN;
|
|
1154
1489
|
return true;
|
|
1155
1490
|
}
|
|
@@ -1186,10 +1521,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1186
1521
|
|
|
1187
1522
|
if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
|
|
1188
1523
|
advance(lexer);
|
|
1189
|
-
|
|
1190
|
-
int32_t c = lexer->lookahead;
|
|
1191
|
-
lexer->result_symbol = BARE_DOLLAR;
|
|
1192
|
-
return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
|
|
1524
|
+
return finish_bare_dollar(lexer);
|
|
1193
1525
|
}
|
|
1194
1526
|
|
|
1195
1527
|
// Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
|
|
@@ -1208,6 +1540,7 @@ void *tree_sitter_bash_external_scanner_create(void) {
|
|
|
1208
1540
|
Scanner *scanner = ts_calloc(1, sizeof(Scanner));
|
|
1209
1541
|
array_init(&scanner->heredocs);
|
|
1210
1542
|
array_init(&scanner->closers);
|
|
1543
|
+
array_init(&scanner->closer_regex_depths);
|
|
1211
1544
|
return scanner;
|
|
1212
1545
|
}
|
|
1213
1546
|
|
|
@@ -1216,6 +1549,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
|
|
|
1216
1549
|
scanner_reset(scanner);
|
|
1217
1550
|
array_delete(&scanner->heredocs);
|
|
1218
1551
|
array_delete(&scanner->closers);
|
|
1552
|
+
array_delete(&scanner->closer_regex_depths);
|
|
1219
1553
|
ts_free(scanner);
|
|
1220
1554
|
}
|
|
1221
1555
|
|
|
@@ -1229,20 +1563,28 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
|
|
|
1229
1563
|
buffer[size++] = (char)scanner->backtick_depth;
|
|
1230
1564
|
memcpy(&buffer[size], &scanner->backtick_closers, sizeof(scanner->backtick_closers));
|
|
1231
1565
|
size += sizeof(scanner->backtick_closers);
|
|
1232
|
-
buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1
|
|
1566
|
+
buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1 | scanner->joined_line << 2 |
|
|
1567
|
+
scanner->split_continuation << 3 | scanner->split_continuation_in_word << 4 |
|
|
1568
|
+
scanner->split_continuation_in_regex << 5 | scanner->closer_line_string_opened << 6);
|
|
1569
|
+
memcpy(&buffer[size], &scanner->regex_depth, sizeof(scanner->regex_depth));
|
|
1570
|
+
size += sizeof(scanner->regex_depth);
|
|
1571
|
+
memcpy(&buffer[size], &scanner->backtick_regex_depth, sizeof(scanner->backtick_regex_depth));
|
|
1572
|
+
size += sizeof(scanner->backtick_regex_depth);
|
|
1233
1573
|
uint16_t closer_count = (uint16_t)scanner->closers.size;
|
|
1234
1574
|
memcpy(&buffer[size], &closer_count, sizeof(closer_count));
|
|
1235
1575
|
size += sizeof(closer_count);
|
|
1236
1576
|
if (scanner->closers.size > 0) {
|
|
1237
1577
|
memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
|
|
1238
1578
|
size += scanner->closers.size;
|
|
1579
|
+
memcpy(&buffer[size], scanner->closer_regex_depths.contents, closer_count * sizeof(uint16_t));
|
|
1580
|
+
size += closer_count * sizeof(uint16_t);
|
|
1239
1581
|
}
|
|
1240
1582
|
buffer[size++] = (char)scanner->heredocs.size;
|
|
1241
1583
|
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
1242
1584
|
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
1243
1585
|
buffer[size++] = (char)heredoc->is_raw;
|
|
1244
1586
|
buffer[size++] = (char)heredoc->allows_indent;
|
|
1245
|
-
buffer[size++] = (char)heredoc->started;
|
|
1587
|
+
buffer[size++] = (char)(heredoc->started | heredoc->deferred << 1);
|
|
1246
1588
|
memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
|
|
1247
1589
|
size += sizeof(heredoc->depth);
|
|
1248
1590
|
uint16_t length = (uint16_t)heredoc->delimiter.size;
|
|
@@ -1258,7 +1600,7 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
|
|
|
1258
1600
|
void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
|
|
1259
1601
|
Scanner *scanner = (Scanner *)payload;
|
|
1260
1602
|
scanner_reset(scanner);
|
|
1261
|
-
if (length <
|
|
1603
|
+
if (length < SCANNER_HEADER_SIZE) {
|
|
1262
1604
|
return;
|
|
1263
1605
|
}
|
|
1264
1606
|
unsigned size = 0;
|
|
@@ -1266,23 +1608,37 @@ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *bu
|
|
|
1266
1608
|
memcpy(&scanner->backtick_closers, &buffer[size], sizeof(scanner->backtick_closers));
|
|
1267
1609
|
size += sizeof(scanner->backtick_closers);
|
|
1268
1610
|
scanner->terminator_owed = buffer[size] & 1;
|
|
1269
|
-
scanner->after_line_break = (buffer[size
|
|
1611
|
+
scanner->after_line_break = (buffer[size] & 2) != 0;
|
|
1612
|
+
scanner->joined_line = (buffer[size] & 4) != 0;
|
|
1613
|
+
scanner->split_continuation = (buffer[size] & 8) != 0;
|
|
1614
|
+
scanner->split_continuation_in_word = (buffer[size] & 16) != 0;
|
|
1615
|
+
scanner->split_continuation_in_regex = (buffer[size] & 32) != 0;
|
|
1616
|
+
scanner->closer_line_string_opened = (buffer[size++] & 64) != 0;
|
|
1617
|
+
memcpy(&scanner->regex_depth, &buffer[size], sizeof(scanner->regex_depth));
|
|
1618
|
+
size += sizeof(scanner->regex_depth);
|
|
1619
|
+
memcpy(&scanner->backtick_regex_depth, &buffer[size], sizeof(scanner->backtick_regex_depth));
|
|
1620
|
+
size += sizeof(scanner->backtick_regex_depth);
|
|
1270
1621
|
uint16_t closer_count;
|
|
1271
1622
|
memcpy(&closer_count, &buffer[size], sizeof(closer_count));
|
|
1272
1623
|
size += sizeof(closer_count);
|
|
1273
|
-
if (size + closer_count >= length) {
|
|
1624
|
+
if (size + CLOSER_SIZE * closer_count >= length) {
|
|
1274
1625
|
return;
|
|
1275
1626
|
}
|
|
1276
1627
|
array_reserve(&scanner->closers, closer_count);
|
|
1277
1628
|
memcpy(scanner->closers.contents, &buffer[size], closer_count);
|
|
1278
1629
|
scanner->closers.size = closer_count;
|
|
1279
1630
|
size += closer_count;
|
|
1631
|
+
array_reserve(&scanner->closer_regex_depths, closer_count);
|
|
1632
|
+
memcpy(scanner->closer_regex_depths.contents, &buffer[size], closer_count * sizeof(uint16_t));
|
|
1633
|
+
scanner->closer_regex_depths.size = closer_count;
|
|
1634
|
+
size += closer_count * sizeof(uint16_t);
|
|
1280
1635
|
uint8_t count = (uint8_t)buffer[size++];
|
|
1281
1636
|
for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
|
|
1282
1637
|
Heredoc heredoc = {.delimiter = array_new()};
|
|
1283
1638
|
heredoc.is_raw = buffer[size++];
|
|
1284
1639
|
heredoc.allows_indent = buffer[size++];
|
|
1285
|
-
heredoc.started = buffer[size
|
|
1640
|
+
heredoc.started = buffer[size] & 1;
|
|
1641
|
+
heredoc.deferred = (buffer[size++] & 2) != 0;
|
|
1286
1642
|
memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
|
|
1287
1643
|
size += sizeof(heredoc.depth);
|
|
1288
1644
|
uint16_t delimiter_length;
|