@willbooster/tree-sitter-bash 1.1.3 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -30,6 +30,13 @@ enum TokenType {
30
30
  LINE_CONTINUATION,
31
31
  BRACE_SUBSTITUTION_START,
32
32
  BRACKET_SUBSTITUTION_START,
33
+ REGEX_START,
34
+ REGEX_CONCAT,
35
+ CLOSER_LINE_RAW_STRING_START,
36
+ CLOSER_LINE_ANSI_C_STRING_START,
37
+ CLOSER_LINE_RAW_STRING_CONTENT,
38
+ CLOSER_LINE_ANSI_C_STRING_CONTENT,
39
+ JOINED_COMMENT,
33
40
  ERROR_RECOVERY,
34
41
  };
35
42
 
@@ -43,6 +50,9 @@ typedef struct {
43
50
  bool is_raw;
44
51
  bool allows_indent;
45
52
  bool started;
53
+ // Queued on the same line as a heredoc whose body ended at a line holding the substitution's `)`:
54
+ // bash then reads this body right after that line, wherever the line ends.
55
+ bool deferred;
46
56
  uint16_t depth;
47
57
  CodePoints delimiter;
48
58
  } Heredoc;
@@ -54,6 +64,8 @@ typedef struct {
54
64
  uint8_t backtick_depth;
55
65
  // The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
56
66
  Array(uint8_t) closers;
67
+ // The regex group depth when each open substitution started, restored when it ends.
68
+ Array(uint16_t) closer_regex_depths;
57
69
  // How many closers were open when the backquotes opened; the ones after them are nested inside.
58
70
  uint16_t backtick_closers;
59
71
  // A heredoc body ended before the end of its line (at a `${ list; }` closer, or at a delimiter
@@ -63,6 +75,20 @@ typedef struct {
63
75
  // The last token of this scanner was a line continuation that ended a line; nothing concatenates at
64
76
  // the start of the next one.
65
77
  bool after_line_break;
78
+ // The rest of a line that ended a heredoc body at the substitution's `)` is being read: bash reads it
79
+ // with its line continuations removed, even inside single quotes and comments.
80
+ bool joined_line;
81
+ // A line continuation was split before its newline, where a deferred body started; its newline is still
82
+ // to be read, at the end of a word (`in_word`, of a regex if `in_regex`) or between words.
83
+ bool split_continuation;
84
+ bool split_continuation_in_word;
85
+ bool split_continuation_in_regex;
86
+ // A quoted string on a line that ended a body at the substitution's `)` has just opened; its first text
87
+ // token may be empty, so that it always has one and joining them gives its value.
88
+ bool closer_line_string_opened;
89
+ // The number of unclosed parentheses in the regex being read, which hold blanks and metacharacters.
90
+ uint16_t regex_depth;
91
+ uint16_t backtick_regex_depth;
66
92
  } Scanner;
67
93
 
68
94
  static inline uint16_t current_depth(Scanner *scanner) {
@@ -130,14 +156,17 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
130
156
  return 0;
131
157
  }
132
158
 
133
- // Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator and
134
- // line-break flags, the closers of the open substitutions
135
- // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
136
- // delimiter length (uint16), and the delimiter as varints.
159
+ // Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator,
160
+ // line-break, joined-line, and split-continuation flags, the regex depth and the one saved when the backquotes
161
+ // opened (uint16 each), the closers of the open substitutions (uint16 count, then one byte each, then their
162
+ // saved regex depths as uint16), and heredoc count, then per heredoc its flags, depth (uint16), delimiter
163
+ // length (uint16), and the delimiter as varints.
164
+ #define SCANNER_HEADER_SIZE (2 + 4 * sizeof(uint16_t))
165
+ #define CLOSER_SIZE (1 + sizeof(uint16_t))
137
166
  #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
138
167
 
139
168
  static unsigned serialized_size(Scanner *scanner) {
140
- unsigned size = 3 + 2 * sizeof(uint16_t) + scanner->closers.size;
169
+ unsigned size = SCANNER_HEADER_SIZE + CLOSER_SIZE * scanner->closers.size + 1;
141
170
  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
142
171
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
143
172
  size += HEREDOC_HEADER_SIZE;
@@ -157,9 +186,17 @@ static void scanner_reset(Scanner *scanner) {
157
186
  array_clear(&scanner->heredocs);
158
187
  scanner->backtick_depth = 0;
159
188
  array_clear(&scanner->closers);
189
+ array_clear(&scanner->closer_regex_depths);
160
190
  scanner->backtick_closers = 0;
161
191
  scanner->terminator_owed = false;
162
192
  scanner->after_line_break = false;
193
+ scanner->joined_line = false;
194
+ scanner->split_continuation = false;
195
+ scanner->split_continuation_in_word = false;
196
+ scanner->split_continuation_in_regex = false;
197
+ scanner->closer_line_string_opened = false;
198
+ scanner->regex_depth = 0;
199
+ scanner->backtick_regex_depth = 0;
163
200
  }
164
201
 
165
202
  // The body being read: the most deeply nested started heredoc.
@@ -174,16 +211,24 @@ static int32_t active_heredoc_index(Scanner *scanner) {
174
211
  return active;
175
212
  }
176
213
 
177
- // The next body to start: the first unstarted heredoc opened at the current substitution depth.
214
+ // The next body to start: the first deferred heredoc, or else the first unstarted heredoc opened at the
215
+ // current substitution depth.
178
216
  static int32_t next_heredoc_index(Scanner *scanner) {
179
217
  uint16_t depth = current_depth(scanner);
218
+ int32_t next = -1;
180
219
  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
181
220
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
182
- if (!heredoc->started && heredoc->depth == depth) {
221
+ if (heredoc->started) {
222
+ continue;
223
+ }
224
+ if (heredoc->deferred) {
183
225
  return (int32_t)i;
184
226
  }
227
+ if (next < 0 && heredoc->depth == depth) {
228
+ next = (int32_t)i;
229
+ }
185
230
  }
186
- return -1;
231
+ return next;
187
232
  }
188
233
 
189
234
  // An entry without a delimiter belongs to an operator whose delimiter was rejected.
@@ -194,7 +239,7 @@ static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
194
239
  uint16_t depth = current_depth(scanner);
195
240
  for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
196
241
  Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
197
- if (!heredoc->started && heredoc->depth == depth) {
242
+ if (!heredoc->started && !heredoc->deferred && heredoc->depth == depth) {
198
243
  heredoc_delete(heredoc);
199
244
  array_erase(&scanner->heredocs, i - 1);
200
245
  }
@@ -211,17 +256,47 @@ static void discard_unstarted_heredocs(Scanner *scanner) {
211
256
  uint16_t depth = current_depth(scanner);
212
257
  for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
213
258
  Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
214
- if (!heredoc->started && heredoc->depth > depth) {
259
+ if (!heredoc->started && !heredoc->deferred && heredoc->depth > depth) {
215
260
  remove_heredoc(scanner, i - 1);
216
261
  }
217
262
  }
218
263
  }
219
264
 
265
+ // Ends a body at a line that holds the substitution's `)`. Like bash, the bodies still queued on the line
266
+ // that began this one are then read right after that line, and the rest of the line is read without its
267
+ // line continuations unless the delimiter was quoted.
268
+ static void end_heredoc_at_closer_line(Scanner *scanner, uint32_t index) {
269
+ Heredoc *ended = array_get(&scanner->heredocs, index);
270
+ scanner->joined_line = !ended->is_raw;
271
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
272
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
273
+ if (!heredoc->started && heredoc->depth == ended->depth) {
274
+ heredoc->deferred = true;
275
+ }
276
+ }
277
+ remove_heredoc(scanner, index);
278
+ }
279
+
280
+ // Whether a deferred body waits for the end of the line that ended a body at the substitution's `)`. That
281
+ // line ends at its first newline even after a backslash or inside quotes, unless it is joined.
282
+ static bool awaits_deferred_body(Scanner *scanner) {
283
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
284
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
285
+ if (heredoc->deferred && !heredoc->started) {
286
+ return true;
287
+ }
288
+ }
289
+ return false;
290
+ }
291
+
220
292
  // Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
221
293
  // line (or at the end of input, as bash does with a warning). An unquoted body stops before each
222
294
  // expansion so that the parser can read it.
223
295
  static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
224
296
  Heredoc *heredoc = array_get(&scanner->heredocs, index);
297
+ // Bash reads a deferred body from the input after its line, never from the text of backquotes that the
298
+ // line opened, so a backquote in it is literal.
299
+ bool in_backquotes = scanner->backtick_depth > 0 && !heredoc->deferred;
225
300
  bool did_advance = false;
226
301
  bool at_line_start = lexer->get_column(lexer) == 0;
227
302
 
@@ -239,7 +314,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
239
314
  // code. Such a line ends the content token before it and the end token after the delimiter.
240
315
  // Where the backquotes are innermost, bash fixes their extent before it reads the body, so
241
316
  // the rule is off.
242
- uint16_t outside_backquotes = scanner->backtick_depth > 0 ? scanner->backtick_closers : 0;
317
+ uint16_t outside_backquotes = in_backquotes ? scanner->backtick_closers : 0;
243
318
  bool in_parenthesized =
244
319
  scanner->closers.size > outside_backquotes && *array_back(&scanner->closers) == ')';
245
320
  bool prefix_marked = false;
@@ -286,7 +361,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
286
361
  // innermost substitution's closer (`)` or `}`) or a closing backquote.
287
362
  bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
288
363
  (scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
289
- (lexer->lookahead == '`' && scanner->backtick_depth > 0);
364
+ (lexer->lookahead == '`' && in_backquotes);
290
365
  if (!escaped && matched == heredoc->delimiter.size && ends_line) {
291
366
  if (did_advance) {
292
367
  lexer->result_symbol = HEREDOC_CONTENT;
@@ -295,22 +370,27 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
295
370
  lexer->mark_end(lexer);
296
371
  lexer->result_symbol = HEREDOC_END;
297
372
  scanner->terminator_owed = lexer->lookahead == '}';
298
- remove_heredoc(scanner, index);
373
+ if (in_parenthesized && lexer->lookahead == ')') {
374
+ end_heredoc_at_closer_line(scanner, index);
375
+ } else {
376
+ remove_heredoc(scanner, index);
377
+ }
299
378
  return true;
300
379
  }
301
380
  if (prefix_marked) {
302
- // The line is read after joining its backslash-newlines, as for the delimiter.
381
+ // The line is read after joining its backslash-newlines, as for the delimiter; any other
382
+ // escaped character is kept with its backslash, so a newline after it ends the line.
303
383
  while (!lexer->eof(lexer) && lexer->lookahead != '\n' && lexer->lookahead != ')') {
304
384
  bool backslash = lexer->lookahead == '\\' && !heredoc->is_raw;
305
385
  advance(lexer);
306
- if (backslash && lexer->lookahead == '\n') {
386
+ if (backslash && !lexer->eof(lexer) && lexer->lookahead != ')') {
307
387
  advance(lexer);
308
388
  }
309
389
  }
310
390
  if (escaped_after_prefix == ')' || lexer->lookahead == ')') {
311
391
  lexer->result_symbol = HEREDOC_END;
312
392
  scanner->terminator_owed = true;
313
- remove_heredoc(scanner, index);
393
+ end_heredoc_at_closer_line(scanner, index);
314
394
  } else {
315
395
  lexer->result_symbol = HEREDOC_CONTENT;
316
396
  }
@@ -336,13 +416,14 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
336
416
  advance(lexer);
337
417
  did_advance = true;
338
418
  at_line_start = true;
419
+ scanner->joined_line = false;
339
420
  break;
340
421
  case '\\':
341
422
  advance(lexer);
342
423
  did_advance = true;
343
424
  // Inside backquotes a backslash also hides a backquote from a quoted body's end.
344
425
  if (!lexer->eof(lexer) &&
345
- (!heredoc->is_raw || (scanner->backtick_depth > 0 && lexer->lookahead == '`'))) {
426
+ (!heredoc->is_raw || (in_backquotes && lexer->lookahead == '`'))) {
346
427
  // A backslash-newline joins lines before the delimiter comparison, while a
347
428
  // backslash-CR escapes only the CR, so the LF after it still ends the line.
348
429
  advance(lexer);
@@ -351,7 +432,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
351
432
  case '`':
352
433
  // Bash ends backquotes at the first unescaped backquote, even inside a heredoc body,
353
434
  // and ends the body there.
354
- if (scanner->backtick_depth > 0) {
435
+ if (in_backquotes) {
355
436
  lexer->mark_end(lexer);
356
437
  lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
357
438
  if (!did_advance) {
@@ -359,7 +440,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
359
440
  }
360
441
  return true;
361
442
  }
362
- if (!heredoc->is_raw) {
443
+ if (!heredoc->is_raw && scanner->backtick_depth == 0) {
363
444
  lexer->mark_end(lexer);
364
445
  lexer->result_symbol = HEREDOC_CONTENT;
365
446
  return did_advance;
@@ -485,7 +566,17 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
485
566
  quote = 0;
486
567
  continue;
487
568
  }
488
- if (ansi_c && c == '\\') {
569
+ if (scanner->joined_line && c == '\\' && lexer->lookahead == '\n') {
570
+ advance(lexer);
571
+ continue;
572
+ }
573
+ if (scanner->joined_line && quote == '\'' && !ansi_c && c == '\\' && lexer->lookahead != '\'' &&
574
+ !lexer->eof(lexer)) {
575
+ // The character after a backslash is kept as it is, so it cannot begin a line continuation.
576
+ array_push(&heredoc->delimiter, c);
577
+ c = lexer->lookahead;
578
+ advance(lexer);
579
+ } else if (ansi_c && c == '\\') {
489
580
  if (!lexer->eof(lexer)) {
490
581
  push_ansi_c_escape(lexer, &heredoc->delimiter);
491
582
  }
@@ -589,53 +680,65 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
589
680
  return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
590
681
  }
591
682
 
592
- // The right side of `=~` extends to an unquoted space, tab, or newline outside parentheses.
593
- static bool scan_regex(TSLexer *lexer) {
594
- while (is_blank(lexer->lookahead)) {
595
- skip(lexer);
683
+ // The literal text of the right side of `=~`, which, like bash, reads `(` and `|` as regex characters
684
+ // and a parenthesized group, blanks and metacharacters included, as part of the word up to its matching
685
+ // `)`. Quotes and expansions are separate parts, so the group depth is kept for the text after them.
686
+ static bool scan_regex(Scanner *scanner, TSLexer *lexer) {
687
+ uint32_t depth = scanner->regex_depth;
688
+ if (depth == 0) {
689
+ while (is_blank(lexer->lookahead)) {
690
+ skip(lexer);
691
+ }
596
692
  }
597
- uint32_t depth = 0;
598
693
  bool did_advance = false;
599
- while (!lexer->eof(lexer)) {
694
+ for (;;) {
600
695
  lexer->mark_end(lexer);
696
+ if (lexer->eof(lexer)) {
697
+ break;
698
+ }
601
699
  int32_t c = lexer->lookahead;
602
- if (depth == 0 && is_separator(c)) {
700
+ if (c == '\'' || c == '"' || c == '`' ||
701
+ (depth == 0 && (is_separator(c) || c == '&' || c == ';' || c == ')'))) {
603
702
  break;
604
703
  }
605
- // Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
606
- // the regex even without surrounding blanks (`[[ a =~ a&&b ]]`).
607
- if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
704
+ // A deferred body starts at the newline that ends the line, even inside a group.
705
+ if (c == '\n' && awaits_deferred_body(scanner)) {
608
706
  break;
609
707
  }
610
- if (c == '\\') {
611
- advance(lexer);
612
- } else if (c == '\'' || c == '"') {
613
- advance(lexer);
614
- while (!lexer->eof(lexer) && lexer->lookahead != c) {
615
- if (c == '"' && lexer->lookahead == '\\') {
616
- advance(lexer);
617
- }
708
+ advance(lexer);
709
+ if (c == '\\' && lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
710
+ // The newline ends the line, after which a deferred body starts; a concatenation splits it.
711
+ break;
712
+ }
713
+ if (c == '$' || c == '<' || c == '>') {
714
+ int32_t next = lexer->lookahead;
715
+ bool starts_part = c == '$' ? starts_quoted_expansion(next) || next == '\'' || next == '"' : next == '(';
716
+ if (starts_part || (c != '$' && depth == 0)) {
717
+ break;
718
+ }
719
+ } else if (c == '\\') {
720
+ if (!lexer->eof(lexer)) {
618
721
  advance(lexer);
619
722
  }
620
723
  } else if (c == '(') {
621
724
  depth++;
622
725
  } else if (c == ')') {
623
- if (depth == 0) {
624
- break;
625
- }
626
726
  depth--;
727
+ } else if (c == '\n') {
728
+ scanner->joined_line = false;
627
729
  }
628
- advance(lexer);
629
730
  did_advance = true;
630
731
  }
631
- if (lexer->eof(lexer)) {
632
- lexer->mark_end(lexer);
732
+ if (!did_advance || depth > UINT16_MAX) {
733
+ return false;
633
734
  }
735
+ scanner->regex_depth = (uint16_t)depth;
634
736
  lexer->result_symbol = REGEX;
635
- return did_advance;
737
+ return true;
636
738
  }
637
739
 
638
- static bool scan_string_content(TSLexer *lexer) {
740
+ static bool scan_string_content(Scanner *scanner, TSLexer *lexer) {
741
+ bool line_breaks = !scanner->joined_line && awaits_deferred_body(scanner);
639
742
  bool did_advance = false;
640
743
  for (;;) {
641
744
  lexer->mark_end(lexer);
@@ -646,11 +749,18 @@ static bool scan_string_content(TSLexer *lexer) {
646
749
  if (c == '"' || c == '`') {
647
750
  break;
648
751
  }
752
+ if (c == '\n' && (scanner->joined_line || line_breaks)) {
753
+ // A deferred body starts right before the newline that ends the line.
754
+ if (did_advance) {
755
+ break;
756
+ }
757
+ scanner->joined_line = false;
758
+ }
649
759
  advance(lexer);
650
760
  if (c == '$' && starts_quoted_expansion(lexer->lookahead)) {
651
761
  break;
652
762
  }
653
- if (c == '\\' && !lexer->eof(lexer)) {
763
+ if (c == '\\' && !lexer->eof(lexer) && !(line_breaks && lexer->lookahead == '\n')) {
654
764
  advance(lexer);
655
765
  }
656
766
  did_advance = true;
@@ -663,6 +773,11 @@ static inline bool is_word_break(int32_t c) {
663
773
  return is_separator(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
664
774
  }
665
775
 
776
+ // Whether `c` ends a word; `(` and `|` are regex characters in the right side of `=~`.
777
+ static inline bool ends_word(int32_t c, bool regex) {
778
+ return is_separator(c) || (is_metacharacter(c) && !(regex && (c == '(' || c == '|')));
779
+ }
780
+
666
781
  static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
667
782
 
668
783
  typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
@@ -873,6 +988,122 @@ static bool at_closing_reserved_word(TSLexer *lexer) {
873
988
  return false;
874
989
  }
875
990
 
991
+ // After a line continuation inside a word: bash removes every backslash-newline before it splits words, so
992
+ // further continuations right after this one are part of the token (`a\` + `\` + `b` is `ab`), which is a
993
+ // concatenation if the word continues on the next line. Otherwise the continuations themselves are the token,
994
+ // so that whatever follows them is lexed as it would be without them.
995
+ static bool finish_word_continuations(Scanner *scanner, TSLexer *lexer, enum TokenType concat, bool regex) {
996
+ int32_t c;
997
+ for (;;) {
998
+ c = lexer->lookahead;
999
+ if (c != '\\') {
1000
+ break;
1001
+ }
1002
+ advance(lexer);
1003
+ if (lexer->eof(lexer)) {
1004
+ lexer->mark_end(lexer);
1005
+ lexer->result_symbol = LINE_CONTINUATION;
1006
+ return true;
1007
+ }
1008
+ if (lexer->lookahead != '\n') {
1009
+ // An escaped character continues the word.
1010
+ lexer->result_symbol = concat;
1011
+ return true;
1012
+ }
1013
+ advance(lexer);
1014
+ lexer->mark_end(lexer);
1015
+ }
1016
+ if ((c == '<' || c == '>')) {
1017
+ advance(lexer);
1018
+ if (lexer->lookahead == '(') {
1019
+ lexer->result_symbol = concat;
1020
+ return true;
1021
+ }
1022
+ scanner->after_line_break = true;
1023
+ lexer->result_symbol = LINE_CONTINUATION;
1024
+ } else if (lexer->eof(lexer) || (ends_word(c, regex) && !(regex && scanner->regex_depth > 0)) ||
1025
+ (c == '`' && scanner->backtick_depth > 0)) {
1026
+ scanner->after_line_break = true;
1027
+ lexer->result_symbol = LINE_CONTINUATION;
1028
+ } else {
1029
+ lexer->result_symbol = concat;
1030
+ }
1031
+ return true;
1032
+ }
1033
+
1034
+ // Ends a line continuation's token at its backslash, since its newline ends the line after which a deferred
1035
+ // body starts; the newline is read after the body.
1036
+ static void split_continuation(Scanner *scanner, TSLexer *lexer, bool in_word, bool in_regex) {
1037
+ lexer->mark_end(lexer);
1038
+ scanner->split_continuation = true;
1039
+ scanner->split_continuation_in_word = in_word;
1040
+ scanner->split_continuation_in_regex = in_regex;
1041
+ lexer->result_symbol = LINE_CONTINUATION;
1042
+ }
1043
+
1044
+ // After `$`, a bare dollar sign unless it begins an expansion or a quote.
1045
+ static bool finish_bare_dollar(TSLexer *lexer) {
1046
+ lexer->mark_end(lexer);
1047
+ int32_t c = lexer->lookahead;
1048
+ lexer->result_symbol = BARE_DOLLAR;
1049
+ return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
1050
+ }
1051
+
1052
+ // The text of a single-quoted string on a line that ended a body at the substitution's `)`. Where the line
1053
+ // is joined, the text is split at its line continuations, which are tokens of their own, and, as when bash
1054
+ // joins the line, the character after any other backslash is kept with it; in a raw string that backslash
1055
+ // is literal, so a quote after it still ends the string. The text also ends before the newline that ends
1056
+ // the line, where a deferred body starts.
1057
+ static bool scan_closer_line_string_content(Scanner *scanner, TSLexer *lexer, bool ansi_c) {
1058
+ bool line_open = scanner->joined_line || awaits_deferred_body(scanner);
1059
+ bool did_advance = scanner->closer_line_string_opened;
1060
+ scanner->closer_line_string_opened = false;
1061
+ for (;;) {
1062
+ lexer->mark_end(lexer);
1063
+ if (lexer->eof(lexer) || lexer->lookahead == '\'') {
1064
+ break;
1065
+ }
1066
+ int32_t c = lexer->lookahead;
1067
+ // A deferred body starts right before the newline that ends the line.
1068
+ if (c == '\n' && line_open && did_advance) {
1069
+ break;
1070
+ }
1071
+ advance(lexer);
1072
+ if (c == '\\' && scanner->joined_line && lexer->lookahead == '\n') {
1073
+ if (did_advance) {
1074
+ break;
1075
+ }
1076
+ advance(lexer);
1077
+ lexer->mark_end(lexer);
1078
+ lexer->result_symbol = LINE_CONTINUATION;
1079
+ return true;
1080
+ }
1081
+ if (c == '\\' && !lexer->eof(lexer) && lexer->lookahead != '\n' && (ansi_c || lexer->lookahead != '\'')) {
1082
+ advance(lexer);
1083
+ } else if (c == '\n') {
1084
+ scanner->joined_line = false;
1085
+ line_open = false;
1086
+ }
1087
+ did_advance = true;
1088
+ }
1089
+ lexer->result_symbol = ansi_c ? CLOSER_LINE_ANSI_C_STRING_CONTENT : CLOSER_LINE_RAW_STRING_CONTENT;
1090
+ return did_advance;
1091
+ }
1092
+
1093
+ // A comment on a joined line, which continues past each line continuation.
1094
+ static bool scan_joined_comment(TSLexer *lexer) {
1095
+ while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
1096
+ bool backslash = lexer->lookahead == '\\';
1097
+ advance(lexer);
1098
+ if (backslash && !lexer->eof(lexer)) {
1099
+ advance(lexer);
1100
+ }
1101
+ }
1102
+ lexer->mark_end(lexer);
1103
+ lexer->result_symbol = JOINED_COMMENT;
1104
+ return true;
1105
+ }
1106
+
876
1107
  static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
877
1108
  bool error_recovery = valid_symbols[ERROR_RECOVERY];
878
1109
  // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
@@ -896,7 +1127,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
896
1127
  (valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
897
1128
  valid_symbols[BRACKET_SUBSTITUTION_START])) {
898
1129
  // Nesting too deep for the serialized state is left to the parse as an error.
899
- if (serialized_size(scanner) >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
1130
+ if (serialized_size(scanner) + CLOSER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
900
1131
  return false;
901
1132
  }
902
1133
  lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
@@ -906,29 +1137,50 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
906
1137
  : lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
907
1138
  : ']';
908
1139
  array_push(&scanner->closers, closer);
1140
+ array_push(&scanner->closer_regex_depths, scanner->regex_depth);
1141
+ return true;
1142
+ }
1143
+
1144
+ if (!error_recovery && valid_symbols[REGEX_START]) {
1145
+ scanner->regex_depth = 0;
1146
+ lexer->result_symbol = REGEX_START;
909
1147
  return true;
910
1148
  }
911
1149
 
1150
+ // Blanks, metacharacters, and newlines inside a regex group are part of the regex.
1151
+ bool in_regex_group = scanner->regex_depth > 0 && (valid_symbols[REGEX] || valid_symbols[REGEX_CONCAT]);
1152
+
912
1153
  if (scanner->heredocs.size > 0) {
1154
+ int32_t next = next_heredoc_index(scanner);
1155
+ // Only another deferred body starts at the newline of a split line continuation.
1156
+ bool deferred = next >= 0 && array_get(&scanner->heredocs, next)->deferred;
1157
+ // A deferred body starts at the end of its line even inside the body of an enclosing heredoc, but
1158
+ // the deferred bodies of one line follow one another.
913
1159
  int32_t active = active_heredoc_index(scanner);
914
1160
  if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
1161
+ !(deferred && !array_get(&scanner->heredocs, active)->deferred && lexer->lookahead == '\n' &&
1162
+ valid_symbols[HEREDOC_BODY_START]) &&
915
1163
  !(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw &&
916
1164
  scanner->backtick_depth == 0)) {
917
1165
  return scan_heredoc_content(scanner, lexer, (uint32_t)active);
918
1166
  }
919
- // A newline inside a string is part of it, so bodies start only outside strings.
920
- int32_t next = next_heredoc_index(scanner);
921
- if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && !valid_symbols[STRING_CONTENT]) {
922
- while (is_blank(lexer->lookahead)) {
1167
+ // A newline inside a string is part of it, so bodies start only outside strings, except for a
1168
+ // deferred body, which bash reads right after its line wherever that line ends.
1169
+ bool in_quotes = valid_symbols[STRING_CONTENT] || valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] ||
1170
+ valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT];
1171
+ if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && (!in_regex_group || deferred) &&
1172
+ (deferred || (!in_quotes && !scanner->split_continuation))) {
1173
+ while (is_blank(lexer->lookahead) && !in_quotes) {
923
1174
  skip(lexer);
924
1175
  }
925
1176
  if (lexer->lookahead == '\n') {
926
1177
  // The line's heredocs cannot be located once one of them was rejected, so none gets a
927
1178
  // body; scanning continues so that the change is kept with the token found below.
928
- if (is_rejected(array_get(&scanner->heredocs, next))) {
1179
+ if (is_rejected(array_get(&scanner->heredocs, next)) && !deferred) {
929
1180
  drop_unstarted_heredocs_at_depth(scanner);
930
1181
  } else {
931
1182
  array_get(&scanner->heredocs, next)->started = true;
1183
+ scanner->joined_line = false;
932
1184
  lexer->result_symbol = HEREDOC_BODY_START;
933
1185
  return true;
934
1186
  }
@@ -940,26 +1192,58 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
940
1192
  return false;
941
1193
  }
942
1194
 
1195
+ // The right side of `=~` joins its parts with a concatenation of its own, since `(` and `|` continue
1196
+ // it and a group continues it up to its matching `)`.
1197
+ bool regex = valid_symbols[REGEX_CONCAT] && !valid_symbols[CONCAT];
1198
+ enum TokenType concat = regex ? REGEX_CONCAT : CONCAT;
1199
+ bool can_concat = valid_symbols[CONCAT] || regex;
1200
+
1201
+ // The word goes on after the body even where the parse state no longer allows a concatenation: that
1202
+ // state comes from a reparse that reused the words before the body as a complete command, which the
1203
+ // concatenation then breaks down again.
1204
+ if (scanner->split_continuation && lexer->lookahead == '\n') {
1205
+ scanner->split_continuation = false;
1206
+ advance(lexer);
1207
+ lexer->mark_end(lexer);
1208
+ if (scanner->split_continuation_in_word) {
1209
+ bool in_regex = scanner->split_continuation_in_regex;
1210
+ return finish_word_continuations(scanner, lexer, in_regex ? REGEX_CONCAT : CONCAT, in_regex);
1211
+ }
1212
+ scanner->after_line_break = true;
1213
+ lexer->result_symbol = LINE_CONTINUATION;
1214
+ return true;
1215
+ }
1216
+
943
1217
  if (valid_symbols[STRING_CONTENT] && lexer->lookahead != '`') {
944
- return scan_string_content(lexer);
1218
+ return scan_string_content(scanner, lexer);
1219
+ }
1220
+
1221
+ if (valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] || valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]) {
1222
+ return scan_closer_line_string_content(scanner, lexer, valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]);
945
1223
  }
946
1224
 
947
1225
  if (valid_symbols[HEREDOC_START] && scanner->heredocs.size > 0) {
948
1226
  return scan_heredoc_start(scanner, lexer);
949
1227
  }
950
1228
 
1229
+ if (regex && scanner->regex_depth > 0 && !at_eof && first != '\\') {
1230
+ lexer->result_symbol = REGEX_CONCAT;
1231
+ return true;
1232
+ }
1233
+
951
1234
  // A line continuation after a blank separates words, so nothing concatenates at the start of the
952
1235
  // next line. The column costs a scan back to the line start, so it is computed only right after
953
1236
  // such a continuation and only where a concatenation could start; a concatenation then clears the
954
1237
  // flag, which bounds the scans to one per continuation.
1238
+
955
1239
  // Like bash, `a<(cmd)` is one word. `<` and `>` end a word part otherwise, so the concatenation
956
1240
  // needs a look at the next character, and a `<<` read along the way is scanned as a heredoc arrow.
957
- if (valid_symbols[CONCAT] && (first == '<' || first == '>') &&
1241
+ if (can_concat && (first == '<' || first == '>') &&
958
1242
  !(after_line_break && lexer->get_column(lexer) == 0)) {
959
1243
  lexer->mark_end(lexer);
960
1244
  advance(lexer);
961
1245
  if (lexer->lookahead == '(') {
962
- lexer->result_symbol = CONCAT;
1246
+ lexer->result_symbol = concat;
963
1247
  return true;
964
1248
  }
965
1249
  if (first == '<' && (valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH])) {
@@ -968,7 +1252,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
968
1252
  return false;
969
1253
  }
970
1254
 
971
- if (valid_symbols[CONCAT] && !at_eof && !is_separator(first) && !is_metacharacter(first) &&
1255
+ if (can_concat && !at_eof && (!ends_word(first, regex) || (regex && scanner->regex_depth > 0)) &&
972
1256
  !(after_line_break && lexer->get_column(lexer) == 0)) {
973
1257
  int32_t c = first;
974
1258
  if (c == '\\' && lexer->lookahead == '\\') {
@@ -982,53 +1266,19 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
982
1266
  return true;
983
1267
  }
984
1268
  if (lexer->lookahead != '\n') {
985
- lexer->result_symbol = CONCAT;
1269
+ lexer->result_symbol = concat;
1270
+ return true;
1271
+ }
1272
+ if (!scanner->joined_line && awaits_deferred_body(scanner)) {
1273
+ split_continuation(scanner, lexer, true, regex);
986
1274
  return true;
987
1275
  }
988
1276
  advance(lexer);
989
1277
  lexer->mark_end(lexer);
990
- // Bash removes every backslash-newline before it splits words, so further continuations
991
- // right after this one are part of the token (`a\` + `\` + `b` is `ab`).
992
- for (;;) {
993
- c = lexer->lookahead;
994
- if (c != '\\') {
995
- break;
996
- }
997
- advance(lexer);
998
- if (lexer->eof(lexer)) {
999
- lexer->mark_end(lexer);
1000
- lexer->result_symbol = LINE_CONTINUATION;
1001
- return true;
1002
- }
1003
- if (lexer->lookahead != '\n') {
1004
- // An escaped character continues the word.
1005
- lexer->result_symbol = CONCAT;
1006
- return true;
1007
- }
1008
- advance(lexer);
1009
- lexer->mark_end(lexer);
1010
- }
1011
- // The word continues only if the next line does; otherwise the continuations themselves are
1012
- // the token, so that whatever follows them is lexed as it would be without them.
1013
- if ((c == '<' || c == '>')) {
1014
- advance(lexer);
1015
- if (lexer->lookahead == '(') {
1016
- lexer->result_symbol = CONCAT;
1017
- return true;
1018
- }
1019
- scanner->after_line_break = true;
1020
- lexer->result_symbol = LINE_CONTINUATION;
1021
- } else if (lexer->eof(lexer) || is_separator(c) || is_metacharacter(c) ||
1022
- (c == '`' && scanner->backtick_depth > 0)) {
1023
- scanner->after_line_break = true;
1024
- lexer->result_symbol = LINE_CONTINUATION;
1025
- } else {
1026
- lexer->result_symbol = CONCAT;
1027
- }
1028
- return true;
1278
+ return finish_word_continuations(scanner, lexer, concat, regex);
1029
1279
  }
1030
1280
  if (c != '`' || scanner->backtick_depth == 0) {
1031
- lexer->result_symbol = CONCAT;
1281
+ lexer->result_symbol = concat;
1032
1282
  return true;
1033
1283
  }
1034
1284
  }
@@ -1053,8 +1303,70 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1053
1303
  }
1054
1304
  }
1055
1305
 
1306
+ // The rest of a line that ended a body at the substitution's `)` scans its single-quoted strings itself,
1307
+ // to remove their line continuations where the line is joined, and to end the line inside them. Inside
1308
+ // backquotes, the next backquote ends a quote, as handled below, but a newline still ends the line.
1309
+ if ((scanner->joined_line || awaits_deferred_body(scanner)) && !error_recovery) {
1310
+ while (is_blank(lexer->lookahead) && !in_regex_group) {
1311
+ skip(lexer);
1312
+ }
1313
+ if (lexer->lookahead == '\n' && scanner->joined_line && !in_regex_group) {
1314
+ // The joined line ends here; a token is returned even where the newline is only whitespace,
1315
+ // so that the line's end is kept.
1316
+ advance(lexer);
1317
+ lexer->mark_end(lexer);
1318
+ scanner->joined_line = false;
1319
+ lexer->result_symbol = valid_symbols[NEWLINE] ? NEWLINE : LINE_CONTINUATION;
1320
+ return true;
1321
+ }
1322
+ }
1323
+ if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 && !error_recovery) {
1324
+ if (lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_RAW_STRING_START]) {
1325
+ advance(lexer);
1326
+ lexer->mark_end(lexer);
1327
+ scanner->closer_line_string_opened = true;
1328
+ lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
1329
+ return true;
1330
+ }
1331
+ // A `#` begins a word, and so a comment, after a blank or an operator. Right after a word part, where
1332
+ // another part may follow, it continues the word; a command may also start there, as after `;`, but
1333
+ // a reparse may reuse the statements before it as a complete list, where no part may follow.
1334
+ bool in_word = valid_symbols[BARE_DOLLAR] && !valid_symbols[VARIABLE_NAME];
1335
+ if (lexer->lookahead == '#' && (is_blank(first) || !in_word) && scanner->joined_line &&
1336
+ valid_symbols[JOINED_COMMENT]) {
1337
+ return scan_joined_comment(lexer);
1338
+ }
1339
+ // A regex reads `$` as text unless it begins a part, so it looks for `$'` itself.
1340
+ if (lexer->lookahead == '$' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START] && !valid_symbols[REGEX]) {
1341
+ advance(lexer);
1342
+ if (lexer->lookahead == '\'') {
1343
+ advance(lexer);
1344
+ lexer->mark_end(lexer);
1345
+ scanner->closer_line_string_opened = true;
1346
+ lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
1347
+ return true;
1348
+ }
1349
+ return valid_symbols[BARE_DOLLAR] && finish_bare_dollar(lexer);
1350
+ }
1351
+ }
1352
+
1056
1353
  if (valid_symbols[REGEX]) {
1057
- return scan_regex(lexer);
1354
+ if (scan_regex(scanner, lexer)) {
1355
+ return true;
1356
+ }
1357
+ // Having found no text, the scan stopped before a quote, or after the `$` of `$'`.
1358
+ if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 &&
1359
+ lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START]) {
1360
+ advance(lexer);
1361
+ lexer->mark_end(lexer);
1362
+ scanner->closer_line_string_opened = true;
1363
+ lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
1364
+ return true;
1365
+ }
1366
+ // Backquotes are the only other part that the scanner opens.
1367
+ if (lexer->lookahead != '`') {
1368
+ return false;
1369
+ }
1058
1370
  }
1059
1371
 
1060
1372
  // A newline ends a statement where a terminator may follow and is whitespace elsewhere.
@@ -1074,6 +1386,14 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1074
1386
  // after one; a backslash escaping a word's first character can only start an extglob prefix
1075
1387
  // (`\foo@($x)`), and the word token handles it otherwise.
1076
1388
  advance(lexer);
1389
+ if (lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
1390
+ // A reparse may reuse the words before the backslash as a complete command, where no
1391
+ // concatenation is valid, so a backslash right after the previous token is taken to end a
1392
+ // word unless a command or an assignment value may start there.
1393
+ split_continuation(scanner, lexer,
1394
+ first == '\\' && !valid_symbols[VARIABLE_NAME] && !valid_symbols[EMPTY_VALUE], false);
1395
+ return true;
1396
+ }
1077
1397
  if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
1078
1398
  advance(lexer);
1079
1399
  lexer->mark_end(lexer);
@@ -1099,6 +1419,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1099
1419
  if (end != ERROR_RECOVERY) {
1100
1420
  if (scanner->closers.size > 0) {
1101
1421
  array_pop(&scanner->closers);
1422
+ scanner->regex_depth = array_pop(&scanner->closer_regex_depths);
1102
1423
  }
1103
1424
  discard_unstarted_heredocs(scanner);
1104
1425
  lexer->result_symbol = end;
@@ -1111,7 +1432,11 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1111
1432
  // up to the backquote, which then cannot be parsed; the quote never hides the text after it.
1112
1433
  if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
1113
1434
  advance(lexer);
1114
- while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`') {
1435
+ lexer->mark_end(lexer);
1436
+ // A deferred body starts at the newline that ends the line, so what follows it is read later.
1437
+ bool breaks_line = awaits_deferred_body(scanner) && valid_symbols[CLOSER_LINE_RAW_STRING_START];
1438
+ while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`' &&
1439
+ !(breaks_line && lexer->lookahead == '\n')) {
1115
1440
  // A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
1116
1441
  // quote itself stays literal, so `'\'` still ends at its second quote.
1117
1442
  bool backslash = lexer->lookahead == '\\';
@@ -1121,12 +1446,20 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1121
1446
  }
1122
1447
  }
1123
1448
  if (lexer->lookahead != '`') {
1449
+ // A quote that the backquotes do not cut short scans its text itself where a deferred body may
1450
+ // start inside it.
1451
+ if (!lexer->eof(lexer) && breaks_line) {
1452
+ scanner->closer_line_string_opened = true;
1453
+ lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
1454
+ return true;
1455
+ }
1124
1456
  return false;
1125
1457
  }
1126
1458
  if (valid_symbols[BACKTICK_CLOSE]) {
1127
1459
  advance(lexer);
1128
1460
  lexer->mark_end(lexer);
1129
1461
  scanner->backtick_depth--;
1462
+ scanner->regex_depth = scanner->backtick_regex_depth;
1130
1463
  discard_unstarted_heredocs(scanner);
1131
1464
  lexer->result_symbol = BACKTICK_CLOSE;
1132
1465
  return true;
@@ -1141,6 +1474,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1141
1474
  advance(lexer);
1142
1475
  lexer->mark_end(lexer);
1143
1476
  scanner->backtick_depth--;
1477
+ scanner->regex_depth = scanner->backtick_regex_depth;
1144
1478
  discard_unstarted_heredocs(scanner);
1145
1479
  lexer->result_symbol = BACKTICK_CLOSE;
1146
1480
  return true;
@@ -1150,6 +1484,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1150
1484
  lexer->mark_end(lexer);
1151
1485
  scanner->backtick_depth++;
1152
1486
  scanner->backtick_closers = (uint16_t)scanner->closers.size;
1487
+ scanner->backtick_regex_depth = scanner->regex_depth;
1153
1488
  lexer->result_symbol = BACKTICK_OPEN;
1154
1489
  return true;
1155
1490
  }
@@ -1186,10 +1521,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1186
1521
 
1187
1522
  if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
1188
1523
  advance(lexer);
1189
- lexer->mark_end(lexer);
1190
- int32_t c = lexer->lookahead;
1191
- lexer->result_symbol = BARE_DOLLAR;
1192
- return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
1524
+ return finish_bare_dollar(lexer);
1193
1525
  }
1194
1526
 
1195
1527
  // Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
@@ -1208,6 +1540,7 @@ void *tree_sitter_bash_external_scanner_create(void) {
1208
1540
  Scanner *scanner = ts_calloc(1, sizeof(Scanner));
1209
1541
  array_init(&scanner->heredocs);
1210
1542
  array_init(&scanner->closers);
1543
+ array_init(&scanner->closer_regex_depths);
1211
1544
  return scanner;
1212
1545
  }
1213
1546
 
@@ -1216,6 +1549,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
1216
1549
  scanner_reset(scanner);
1217
1550
  array_delete(&scanner->heredocs);
1218
1551
  array_delete(&scanner->closers);
1552
+ array_delete(&scanner->closer_regex_depths);
1219
1553
  ts_free(scanner);
1220
1554
  }
1221
1555
 
@@ -1229,20 +1563,28 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1229
1563
  buffer[size++] = (char)scanner->backtick_depth;
1230
1564
  memcpy(&buffer[size], &scanner->backtick_closers, sizeof(scanner->backtick_closers));
1231
1565
  size += sizeof(scanner->backtick_closers);
1232
- buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1);
1566
+ buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1 | scanner->joined_line << 2 |
1567
+ scanner->split_continuation << 3 | scanner->split_continuation_in_word << 4 |
1568
+ scanner->split_continuation_in_regex << 5 | scanner->closer_line_string_opened << 6);
1569
+ memcpy(&buffer[size], &scanner->regex_depth, sizeof(scanner->regex_depth));
1570
+ size += sizeof(scanner->regex_depth);
1571
+ memcpy(&buffer[size], &scanner->backtick_regex_depth, sizeof(scanner->backtick_regex_depth));
1572
+ size += sizeof(scanner->backtick_regex_depth);
1233
1573
  uint16_t closer_count = (uint16_t)scanner->closers.size;
1234
1574
  memcpy(&buffer[size], &closer_count, sizeof(closer_count));
1235
1575
  size += sizeof(closer_count);
1236
1576
  if (scanner->closers.size > 0) {
1237
1577
  memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
1238
1578
  size += scanner->closers.size;
1579
+ memcpy(&buffer[size], scanner->closer_regex_depths.contents, closer_count * sizeof(uint16_t));
1580
+ size += closer_count * sizeof(uint16_t);
1239
1581
  }
1240
1582
  buffer[size++] = (char)scanner->heredocs.size;
1241
1583
  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
1242
1584
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
1243
1585
  buffer[size++] = (char)heredoc->is_raw;
1244
1586
  buffer[size++] = (char)heredoc->allows_indent;
1245
- buffer[size++] = (char)heredoc->started;
1587
+ buffer[size++] = (char)(heredoc->started | heredoc->deferred << 1);
1246
1588
  memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
1247
1589
  size += sizeof(heredoc->depth);
1248
1590
  uint16_t length = (uint16_t)heredoc->delimiter.size;
@@ -1258,7 +1600,7 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1258
1600
  void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
1259
1601
  Scanner *scanner = (Scanner *)payload;
1260
1602
  scanner_reset(scanner);
1261
- if (length < 2 + 2 * sizeof(uint16_t)) {
1603
+ if (length < SCANNER_HEADER_SIZE) {
1262
1604
  return;
1263
1605
  }
1264
1606
  unsigned size = 0;
@@ -1266,23 +1608,37 @@ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *bu
1266
1608
  memcpy(&scanner->backtick_closers, &buffer[size], sizeof(scanner->backtick_closers));
1267
1609
  size += sizeof(scanner->backtick_closers);
1268
1610
  scanner->terminator_owed = buffer[size] & 1;
1269
- scanner->after_line_break = (buffer[size++] & 2) != 0;
1611
+ scanner->after_line_break = (buffer[size] & 2) != 0;
1612
+ scanner->joined_line = (buffer[size] & 4) != 0;
1613
+ scanner->split_continuation = (buffer[size] & 8) != 0;
1614
+ scanner->split_continuation_in_word = (buffer[size] & 16) != 0;
1615
+ scanner->split_continuation_in_regex = (buffer[size] & 32) != 0;
1616
+ scanner->closer_line_string_opened = (buffer[size++] & 64) != 0;
1617
+ memcpy(&scanner->regex_depth, &buffer[size], sizeof(scanner->regex_depth));
1618
+ size += sizeof(scanner->regex_depth);
1619
+ memcpy(&scanner->backtick_regex_depth, &buffer[size], sizeof(scanner->backtick_regex_depth));
1620
+ size += sizeof(scanner->backtick_regex_depth);
1270
1621
  uint16_t closer_count;
1271
1622
  memcpy(&closer_count, &buffer[size], sizeof(closer_count));
1272
1623
  size += sizeof(closer_count);
1273
- if (size + closer_count >= length) {
1624
+ if (size + CLOSER_SIZE * closer_count >= length) {
1274
1625
  return;
1275
1626
  }
1276
1627
  array_reserve(&scanner->closers, closer_count);
1277
1628
  memcpy(scanner->closers.contents, &buffer[size], closer_count);
1278
1629
  scanner->closers.size = closer_count;
1279
1630
  size += closer_count;
1631
+ array_reserve(&scanner->closer_regex_depths, closer_count);
1632
+ memcpy(scanner->closer_regex_depths.contents, &buffer[size], closer_count * sizeof(uint16_t));
1633
+ scanner->closer_regex_depths.size = closer_count;
1634
+ size += closer_count * sizeof(uint16_t);
1280
1635
  uint8_t count = (uint8_t)buffer[size++];
1281
1636
  for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
1282
1637
  Heredoc heredoc = {.delimiter = array_new()};
1283
1638
  heredoc.is_raw = buffer[size++];
1284
1639
  heredoc.allows_indent = buffer[size++];
1285
- heredoc.started = buffer[size++];
1640
+ heredoc.started = buffer[size] & 1;
1641
+ heredoc.deferred = (buffer[size++] & 2) != 0;
1286
1642
  memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
1287
1643
  size += sizeof(heredoc.depth);
1288
1644
  uint16_t delimiter_length;