@willbooster/tree-sitter-bash 0.25.1 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -22,18 +22,29 @@ enum TokenType {
22
22
  BACKTICK_OPEN,
23
23
  BACKTICK_CLOSE,
24
24
  NEWLINE,
25
+ SUBSTITUTION_START,
26
+ SUBSTITUTION_END,
27
+ BRACE_SUBSTITUTION_END,
28
+ BRACKET_SUBSTITUTION_END,
29
+ EXTGLOB_PREFIX,
30
+ LINE_CONTINUATION,
31
+ BRACE_SUBSTITUTION_START,
32
+ BRACKET_SUBSTITUTION_START,
25
33
  ERROR_RECOVERY,
26
34
  };
27
35
 
28
- typedef Array(char) String;
36
+ typedef Array(int32_t) CodePoints;
29
37
 
30
38
  // A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
31
- // (not one inside a string), after the bodies of heredocs opened earlier.
39
+ // (not one inside a string) at the substitution depth where the operator appeared, after the bodies
40
+ // of heredocs opened earlier at that depth: bash reads a substitution, including the bodies of the
41
+ // heredocs inside it, before the body of a heredoc whose operator precedes it on the same line.
32
42
  typedef struct {
33
43
  bool is_raw;
34
44
  bool allows_indent;
35
45
  bool started;
36
- String delimiter;
46
+ uint16_t depth;
47
+ CodePoints delimiter;
37
48
  } Heredoc;
38
49
 
39
50
  typedef struct {
@@ -41,8 +52,17 @@ typedef struct {
41
52
  // Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
42
53
  // whether a backquote opens or closes one.
43
54
  uint8_t backtick_depth;
55
+ // The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
56
+ Array(uint8_t) closers;
57
+ // A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
58
+ // line began the body, so a terminator is still owed before the `}`.
59
+ bool terminator_owed;
44
60
  } Scanner;
45
61
 
62
+ static inline uint16_t current_depth(Scanner *scanner) {
63
+ return scanner->backtick_depth + scanner->closers.size;
64
+ }
65
+
46
66
  static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
47
67
 
48
68
  static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
@@ -63,6 +83,60 @@ static inline bool starts_quoted_expansion(int32_t c) {
63
83
  c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
64
84
  }
65
85
 
86
+ // Code points are serialized as zigzag varints, one byte for code points below 64 and two for the
87
+ // rest of ASCII; every int32 round-trips, including the negative value the lexer reports for invalid
88
+ // UTF-8.
89
+ static unsigned varint_length(int32_t c) {
90
+ uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
91
+ unsigned length = 1;
92
+ while (value >= 0x80) {
93
+ value >>= 7;
94
+ length++;
95
+ }
96
+ return length;
97
+ }
98
+
99
+ static unsigned encode_varint(int32_t c, char *out) {
100
+ uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
101
+ unsigned length = 0;
102
+ while (value >= 0x80) {
103
+ out[length++] = (char)(0x80 | (value & 0x7F));
104
+ value >>= 7;
105
+ }
106
+ out[length++] = (char)value;
107
+ return length;
108
+ }
109
+
110
+ // Returns 0 when the input ends before the varint does.
111
+ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
112
+ uint32_t value = 0;
113
+ for (unsigned i = 0; i < available && i < 5; i++) {
114
+ value |= (uint32_t)((unsigned char)in[i] & 0x7F) << (7 * i);
115
+ if (((unsigned char)in[i] & 0x80) == 0) {
116
+ *c = (int32_t)((value >> 1) ^ (0u - (value & 1)));
117
+ return i + 1;
118
+ }
119
+ }
120
+ return 0;
121
+ }
122
+
123
+ // Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
124
+ // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
125
+ // delimiter length (uint16), and the delimiter as varints.
126
+ #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
127
+
128
+ static unsigned serialized_size(Scanner *scanner) {
129
+ unsigned size = 3 + sizeof(uint16_t) + scanner->closers.size;
130
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
131
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
132
+ size += HEREDOC_HEADER_SIZE;
133
+ for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
134
+ size += varint_length(*array_get(&heredoc->delimiter, j));
135
+ }
136
+ }
137
+ return size;
138
+ }
139
+
66
140
  static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
67
141
 
68
142
  static void scanner_reset(Scanner *scanner) {
@@ -71,18 +145,70 @@ static void scanner_reset(Scanner *scanner) {
71
145
  }
72
146
  array_clear(&scanner->heredocs);
73
147
  scanner->backtick_depth = 0;
148
+ array_clear(&scanner->closers);
149
+ scanner->terminator_owed = false;
74
150
  }
75
151
 
76
- static void pop_front_heredoc(Scanner *scanner) {
77
- heredoc_delete(array_get(&scanner->heredocs, 0));
78
- array_erase(&scanner->heredocs, 0);
152
+ // The body being read: the most deeply nested started heredoc.
153
+ static int32_t active_heredoc_index(Scanner *scanner) {
154
+ int32_t active = -1;
155
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
156
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
157
+ if (heredoc->started && (active < 0 || heredoc->depth >= array_get(&scanner->heredocs, active)->depth)) {
158
+ active = (int32_t)i;
159
+ }
160
+ }
161
+ return active;
162
+ }
163
+
164
+ // The next body to start: the first unstarted heredoc opened at the current substitution depth.
165
+ static int32_t next_heredoc_index(Scanner *scanner) {
166
+ uint16_t depth = current_depth(scanner);
167
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
168
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
169
+ if (!heredoc->started && heredoc->depth == depth) {
170
+ return (int32_t)i;
171
+ }
172
+ }
173
+ return -1;
174
+ }
175
+
176
+ // An entry without a delimiter belongs to an operator whose delimiter was rejected.
177
+ static inline bool is_rejected(Heredoc *heredoc) { return heredoc->delimiter.size == 0 && !heredoc->is_raw; }
178
+
179
+ // Drops the unstarted heredocs at the current depth, which were all queued on the line just ended.
180
+ static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
181
+ uint16_t depth = current_depth(scanner);
182
+ for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
183
+ Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
184
+ if (!heredoc->started && heredoc->depth == depth) {
185
+ heredoc_delete(heredoc);
186
+ array_erase(&scanner->heredocs, i - 1);
187
+ }
188
+ }
189
+ }
190
+
191
+ static void remove_heredoc(Scanner *scanner, uint32_t index) {
192
+ heredoc_delete(array_get(&scanner->heredocs, index));
193
+ array_erase(&scanner->heredocs, index);
194
+ }
195
+
196
+ // Like bash, drop the heredocs of a substitution that closes before their bodies started.
197
+ static void discard_unstarted_heredocs(Scanner *scanner) {
198
+ uint16_t depth = current_depth(scanner);
199
+ for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
200
+ Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
201
+ if (!heredoc->started && heredoc->depth > depth) {
202
+ remove_heredoc(scanner, i - 1);
203
+ }
204
+ }
79
205
  }
80
206
 
81
207
  // Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
82
208
  // line (or at the end of input, as bash does with a warning). An unquoted body stops before each
83
209
  // expansion so that the parser can read it.
84
- static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
85
- Heredoc *heredoc = array_get(&scanner->heredocs, 0);
210
+ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
211
+ Heredoc *heredoc = array_get(&scanner->heredocs, index);
86
212
  bool did_advance = false;
87
213
  bool at_line_start = lexer->get_column(lexer) == 0;
88
214
 
@@ -90,34 +216,64 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
90
216
  if (at_line_start) {
91
217
  at_line_start = false;
92
218
  lexer->mark_end(lexer);
93
- bool consumed_tabs = false;
94
- if (heredoc->allows_indent) {
95
- while (lexer->lookahead == '\t') {
219
+ // Like bash, compare the delimiter with the line after joining backslash-newlines and
220
+ // then stripping the leading tabs of `<<-`. As elsewhere in the grammar, CRLF counts as a
221
+ // newline.
222
+ bool consumed = false;
223
+ bool escaped = false;
224
+ uint32_t matched = 0;
225
+ // At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
226
+ for (;;) {
227
+ if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
228
+ advance(lexer);
229
+ consumed = true;
230
+ } else if (lexer->lookahead == '\\' && !heredoc->is_raw) {
231
+ advance(lexer);
232
+ consumed = true;
233
+ bool escaped_cr = lexer->lookahead == '\r';
234
+ if (escaped_cr) {
235
+ advance(lexer);
236
+ }
237
+ if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
238
+ // An escape rather than a line continuation: the escaped character, which is
239
+ // the CR itself when no LF follows it, is content.
240
+ if (!escaped_cr && !lexer->eof(lexer)) {
241
+ advance(lexer);
242
+ }
243
+ escaped = true;
244
+ break;
245
+ }
246
+ advance(lexer);
247
+ } else if (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
248
+ lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
96
249
  advance(lexer);
97
- consumed_tabs = true;
250
+ matched++;
251
+ } else {
252
+ break;
98
253
  }
99
254
  }
100
- uint32_t matched = 0;
101
- while (matched < heredoc->delimiter.size && lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
102
- advance(lexer);
103
- matched++;
104
- }
105
- if (lexer->lookahead == '\r') {
255
+ if (!escaped && lexer->lookahead == '\r') {
106
256
  advance(lexer);
107
257
  }
108
- if (matched == heredoc->delimiter.size && (lexer->lookahead == '\n' || lexer->eof(lexer))) {
258
+ // Inside a substitution, bash also ends the body at a delimiter directly followed by the
259
+ // innermost substitution's closer (`)` or `}`) or a closing backquote.
260
+ bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
261
+ (scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
262
+ (lexer->lookahead == '`' && scanner->backtick_depth > 0);
263
+ if (!escaped && matched == heredoc->delimiter.size && ends_line) {
109
264
  if (did_advance) {
110
265
  lexer->result_symbol = HEREDOC_CONTENT;
111
266
  return true;
112
267
  }
113
268
  lexer->mark_end(lexer);
114
269
  lexer->result_symbol = HEREDOC_END;
115
- pop_front_heredoc(scanner);
270
+ scanner->terminator_owed = lexer->lookahead == '}';
271
+ remove_heredoc(scanner, index);
116
272
  return true;
117
273
  }
118
274
  // The characters read while trying the delimiter are content; an empty attempt must not
119
275
  // count, or a zero-width content token would repeat forever.
120
- did_advance = did_advance || consumed_tabs || matched > 0;
276
+ did_advance = did_advance || consumed || matched > 0;
121
277
  continue;
122
278
  }
123
279
 
@@ -125,7 +281,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
125
281
  lexer->mark_end(lexer);
126
282
  lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
127
283
  if (!did_advance) {
128
- pop_front_heredoc(scanner);
284
+ remove_heredoc(scanner, index);
129
285
  }
130
286
  return true;
131
287
  }
@@ -140,8 +296,16 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
140
296
  advance(lexer);
141
297
  did_advance = true;
142
298
  if (!heredoc->is_raw && !lexer->eof(lexer)) {
143
- // A backslash-newline joins lines before the delimiter comparison.
144
- advance(lexer);
299
+ // A backslash-newline joins lines before the delimiter comparison. A CR without an
300
+ // LF after it is the escaped character itself.
301
+ if (lexer->lookahead == '\r') {
302
+ advance(lexer);
303
+ if (lexer->lookahead == '\n') {
304
+ advance(lexer);
305
+ }
306
+ } else if (!lexer->eof(lexer)) {
307
+ advance(lexer);
308
+ }
145
309
  }
146
310
  break;
147
311
  case '`':
@@ -179,6 +343,79 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
179
343
  }
180
344
 
181
345
  // The delimiter word undergoes quote removal; any quoting makes the body literal.
346
+ static inline int32_t hex_value(int32_t c) {
347
+ return iswdigit(c) ? c - '0' : (c >= 'a' && c <= 'f') ? c - 'a' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 : -1;
348
+ }
349
+
350
+ // Reads up to `max_digits` digits in `base` and returns their value, or -1 when there are none.
351
+ static int32_t read_number(TSLexer *lexer, int32_t base, int max_digits) {
352
+ int32_t value = 0;
353
+ int digits = 0;
354
+ while (digits < max_digits) {
355
+ int32_t digit = hex_value(lexer->lookahead);
356
+ if (digit < 0 || digit >= base) {
357
+ break;
358
+ }
359
+ value = value * base + digit;
360
+ advance(lexer);
361
+ digits++;
362
+ }
363
+ return digits == 0 ? -1 : value;
364
+ }
365
+
366
+ // Decodes the escape after a backslash in `$'...'` as bash does, appending it to `out`.
367
+ static void push_ansi_c_escape(TSLexer *lexer, CodePoints *out) {
368
+ int32_t c = lexer->lookahead;
369
+ int32_t value = -1;
370
+ switch (c) {
371
+ case 'a': value = 7; break;
372
+ case 'b': value = 8; break;
373
+ case 'e':
374
+ case 'E': value = 27; break;
375
+ case 'f': value = 12; break;
376
+ case 'n': value = '\n'; break;
377
+ case 'r': value = '\r'; break;
378
+ case 't': value = '\t'; break;
379
+ case 'v': value = 11; break;
380
+ case '\\':
381
+ case '\'':
382
+ case '"':
383
+ case '?': value = c; break;
384
+ default: break;
385
+ }
386
+ if (value >= 0) {
387
+ advance(lexer);
388
+ array_push(out, value);
389
+ return;
390
+ }
391
+ if (c >= '0' && c <= '7') {
392
+ array_push(out, read_number(lexer, 8, 3));
393
+ return;
394
+ }
395
+ int max_digits = c == 'x' ? 2 : c == 'u' ? 4 : c == 'U' ? 8 : 0;
396
+ if (max_digits > 0) {
397
+ advance(lexer);
398
+ value = read_number(lexer, 16, max_digits);
399
+ if (value < 0) {
400
+ array_push(out, '\\');
401
+ array_push(out, c);
402
+ } else {
403
+ array_push(out, value);
404
+ }
405
+ return;
406
+ }
407
+ if (c == 'c') {
408
+ advance(lexer);
409
+ if (!lexer->eof(lexer)) {
410
+ array_push(out, lexer->lookahead & 0x1F);
411
+ advance(lexer);
412
+ }
413
+ return;
414
+ }
415
+ // Bash keeps an unknown escape as written.
416
+ array_push(out, '\\');
417
+ }
418
+
182
419
  static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
183
420
  Heredoc *heredoc = array_back(&scanner->heredocs);
184
421
  array_clear(&heredoc->delimiter);
@@ -189,6 +426,7 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
189
426
  }
190
427
 
191
428
  int32_t quote = 0;
429
+ bool ansi_c = false;
192
430
  while (!lexer->eof(lexer)) {
193
431
  int32_t c = lexer->lookahead;
194
432
  if (quote) {
@@ -197,36 +435,80 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
197
435
  quote = 0;
198
436
  continue;
199
437
  }
438
+ if (ansi_c && c == '\\') {
439
+ if (!lexer->eof(lexer)) {
440
+ push_ansi_c_escape(lexer, &heredoc->delimiter);
441
+ }
442
+ continue;
443
+ }
444
+ // Inside double quotes a backslash-newline still only continues the line.
445
+ if (quote == '"' && c == '\\' && (lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
446
+ if (lexer->lookahead == '\r') {
447
+ advance(lexer);
448
+ }
449
+ if (lexer->lookahead == '\n') {
450
+ advance(lexer);
451
+ continue;
452
+ }
453
+ array_push(&heredoc->delimiter, '\\');
454
+ array_push(&heredoc->delimiter, '\r');
455
+ continue;
456
+ }
200
457
  if (quote == '"' && c == '\\' &&
201
458
  (lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
202
459
  lexer->lookahead == '\\')) {
203
460
  c = lexer->lookahead;
204
461
  advance(lexer);
205
462
  }
206
- array_push(&heredoc->delimiter, (char)c);
463
+ array_push(&heredoc->delimiter, c);
207
464
  continue;
208
465
  }
209
- if (iswspace(c) || is_metacharacter(c)) {
466
+ // Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
467
+ // the word (`<<EO`true`F`).
468
+ if (iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
210
469
  break;
211
470
  }
212
471
  advance(lexer);
213
- if (c == '\'' || c == '"') {
472
+ if (c == '$' && (lexer->lookahead == '\'' || lexer->lookahead == '"')) {
473
+ // `$'...'` and `$"..."` are quoting constructs, so their `$` is not part of the word.
474
+ c = lexer->lookahead;
475
+ advance(lexer);
214
476
  quote = c;
477
+ ansi_c = c == '\'';
215
478
  heredoc->is_raw = true;
216
- } else if (c == '\\') {
479
+ } else if (c == '\'' || c == '"') {
480
+ quote = c;
481
+ ansi_c = false;
217
482
  heredoc->is_raw = true;
483
+ } else if (c == '\\') {
484
+ // A backslash-newline only continues the line; any other backslash quotes the delimiter.
485
+ bool carriage_return = lexer->lookahead == '\r';
486
+ if (carriage_return) {
487
+ advance(lexer);
488
+ }
218
489
  if (lexer->lookahead == '\n') {
219
490
  advance(lexer);
491
+ } else if (carriage_return) {
492
+ heredoc->is_raw = true;
493
+ array_push(&heredoc->delimiter, '\r');
220
494
  } else if (!lexer->eof(lexer)) {
221
- array_push(&heredoc->delimiter, (char)lexer->lookahead);
495
+ heredoc->is_raw = true;
496
+ array_push(&heredoc->delimiter, lexer->lookahead);
222
497
  advance(lexer);
223
498
  }
224
499
  } else {
225
- array_push(&heredoc->delimiter, (char)c);
500
+ // Bash counts a backquote in the delimiter word as quoting, so the body stays literal.
501
+ if (c == '`') {
502
+ heredoc->is_raw = true;
503
+ }
504
+ array_push(&heredoc->delimiter, c);
226
505
  }
227
506
  }
228
507
 
229
- if (heredoc->delimiter.size == 0 && !heredoc->is_raw) {
508
+ // A heredoc whose state cannot be serialized in full is rejected, so the parse shows an error
509
+ // rather than comparing against a partial delimiter.
510
+ if ((heredoc->delimiter.size == 0 && !heredoc->is_raw) ||
511
+ serialized_size(scanner) > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
230
512
  return false;
231
513
  }
232
514
  lexer->mark_end(lexer);
@@ -250,46 +532,23 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
250
532
  } else if (!valid_symbols[HEREDOC_ARROW]) {
251
533
  return false;
252
534
  }
535
+ if (scanner->heredocs.size >= UINT8_MAX ||
536
+ serialized_size(scanner) + HEREDOC_HEADER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
537
+ return false;
538
+ }
253
539
  lexer->mark_end(lexer);
254
- Heredoc heredoc = {.is_raw = false, .allows_indent = allows_indent, .started = false, .delimiter = array_new()};
540
+ Heredoc heredoc = {
541
+ .is_raw = false,
542
+ .allows_indent = allows_indent,
543
+ .started = false,
544
+ .depth = current_depth(scanner),
545
+ .delimiter = array_new(),
546
+ };
255
547
  array_push(&scanner->heredocs, heredoc);
256
548
  lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
257
549
  return true;
258
550
  }
259
551
 
260
- // A name followed by `=`, `+=`, or `[subscript]=` starts an assignment.
261
- static bool scan_variable_name(TSLexer *lexer) {
262
- while (is_name_char(lexer->lookahead)) {
263
- advance(lexer);
264
- }
265
- lexer->mark_end(lexer);
266
- if (lexer->lookahead == '+') {
267
- advance(lexer);
268
- return lexer->lookahead == '=';
269
- }
270
- if (lexer->lookahead == '[') {
271
- uint32_t depth = 0;
272
- while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
273
- if (lexer->lookahead == '[') {
274
- depth++;
275
- } else if (lexer->lookahead == ']' && --depth == 0) {
276
- advance(lexer);
277
- break;
278
- } else if (lexer->lookahead == '\\') {
279
- advance(lexer);
280
- }
281
- advance(lexer);
282
- }
283
- if (depth != 0) {
284
- return false;
285
- }
286
- if (lexer->lookahead == '+') {
287
- advance(lexer);
288
- }
289
- }
290
- return lexer->lookahead == '=';
291
- }
292
-
293
552
  // The right side of `=~` extends to unquoted whitespace outside parentheses.
294
553
  static bool scan_regex(TSLexer *lexer) {
295
554
  while (is_blank(lexer->lookahead)) {
@@ -298,10 +557,16 @@ static bool scan_regex(TSLexer *lexer) {
298
557
  uint32_t depth = 0;
299
558
  bool did_advance = false;
300
559
  while (!lexer->eof(lexer)) {
560
+ lexer->mark_end(lexer);
301
561
  int32_t c = lexer->lookahead;
302
562
  if (depth == 0 && iswspace(c)) {
303
563
  break;
304
564
  }
565
+ // Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
566
+ // the regex even without surrounding blanks (`[[ a =~ a&&b ]]`).
567
+ if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
568
+ break;
569
+ }
305
570
  if (c == '\\') {
306
571
  advance(lexer);
307
572
  } else if (c == '\'' || c == '"') {
@@ -323,7 +588,9 @@ static bool scan_regex(TSLexer *lexer) {
323
588
  advance(lexer);
324
589
  did_advance = true;
325
590
  }
326
- lexer->mark_end(lexer);
591
+ if (lexer->eof(lexer)) {
592
+ lexer->mark_end(lexer);
593
+ }
327
594
  lexer->result_symbol = REGEX;
328
595
  return did_advance;
329
596
  }
@@ -352,6 +619,186 @@ static bool scan_string_content(TSLexer *lexer) {
352
619
  return did_advance;
353
620
  }
354
621
 
622
+ static inline bool is_word_break(int32_t c) {
623
+ return iswspace(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
624
+ }
625
+
626
+ static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
627
+
628
+ typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
629
+
630
+ // From the `(` of an extglob group, reads to its matching `)`: a group holding quotes or expansions
631
+ // is parsed as `extglob_pattern`, while the word token keeps literal ones.
632
+ // The nesting depth up to which grammar.js's word token lexes a literal group (`extglobGroup(…, 3)`);
633
+ // a deeper group is parsed as `extglob_pattern`.
634
+ #define EXTGLOB_LITERAL_DEPTH 3
635
+
636
+ static GroupKind scan_extglob_group(TSLexer *lexer) {
637
+ uint32_t depth = 0;
638
+ while (!lexer->eof(lexer)) {
639
+ int32_t c = lexer->lookahead;
640
+ if (c == '$' || c == '`' || c == '\'' || c == '"') {
641
+ return GROUP_STRUCTURED;
642
+ }
643
+ advance(lexer);
644
+ if (c == '\\') {
645
+ // The word token's literal groups cannot hold a line continuation.
646
+ if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
647
+ return lexer->eof(lexer) ? GROUP_UNSUPPORTED : GROUP_STRUCTURED;
648
+ }
649
+ advance(lexer);
650
+ } else if (c == '(') {
651
+ if (++depth > EXTGLOB_LITERAL_DEPTH) {
652
+ return GROUP_STRUCTURED;
653
+ }
654
+ } else if (c == ')' && --depth == 0) {
655
+ return GROUP_LITERAL;
656
+ }
657
+ }
658
+ return GROUP_UNSUPPORTED;
659
+ }
660
+
661
+ // The literal text of a word up to a structured extglob group (`foo` in `foo@($x)`), which the
662
+ // word token's longest match would otherwise run into.
663
+ static bool continue_extglob_prefix(TSLexer *lexer, bool consumed) {
664
+ while (!lexer->eof(lexer)) {
665
+ int32_t c = lexer->lookahead;
666
+ if (is_extglob_operator(c)) {
667
+ lexer->mark_end(lexer);
668
+ advance(lexer);
669
+ if (lexer->lookahead == '(') {
670
+ GroupKind kind = scan_extglob_group(lexer);
671
+ if (kind == GROUP_STRUCTURED) {
672
+ lexer->result_symbol = EXTGLOB_PREFIX;
673
+ return consumed;
674
+ }
675
+ if (kind == GROUP_UNSUPPORTED) {
676
+ return false;
677
+ }
678
+ }
679
+ } else if (c == '\\') {
680
+ advance(lexer);
681
+ // A backslash-newline continues the line; the word token handles that.
682
+ if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
683
+ return false;
684
+ }
685
+ advance(lexer);
686
+ } else if (is_word_break(c)) {
687
+ return false;
688
+ } else {
689
+ advance(lexer);
690
+ }
691
+ consumed = true;
692
+ }
693
+ return false;
694
+ }
695
+
696
+ // A leading `#` starts a comment except in an assignment value (`A=#@($x)`).
697
+ static bool scan_extglob_prefix(TSLexer *lexer, bool in_value) {
698
+ return (lexer->lookahead != '#' || in_value) && continue_extglob_prefix(lexer, false);
699
+ }
700
+
701
+ // Skips a quoted span from its opening quote through its closing one.
702
+ static void skip_quoted(TSLexer *lexer) {
703
+ int32_t quote = lexer->lookahead;
704
+ advance(lexer);
705
+ while (!lexer->eof(lexer) && lexer->lookahead != quote) {
706
+ if (lexer->lookahead == '\\' && quote != '\'') {
707
+ advance(lexer);
708
+ if (lexer->eof(lexer)) {
709
+ return;
710
+ }
711
+ }
712
+ advance(lexer);
713
+ }
714
+ advance(lexer);
715
+ }
716
+
717
+ // Skips a parenthesized span such as a command substitution's, quotes included.
718
+ static void skip_parenthesized(TSLexer *lexer) {
719
+ uint32_t depth = 0;
720
+ while (!lexer->eof(lexer)) {
721
+ int32_t c = lexer->lookahead;
722
+ if (c == '\'' || c == '"' || c == '`') {
723
+ skip_quoted(lexer);
724
+ continue;
725
+ }
726
+ if (c == '\\') {
727
+ advance(lexer);
728
+ } else if (c == '(') {
729
+ depth++;
730
+ } else if (c == ')' && --depth == 0) {
731
+ advance(lexer);
732
+ return;
733
+ }
734
+ advance(lexer);
735
+ }
736
+ }
737
+
738
+ // A name at a position where both an assignment and a word may start: `name=`, `name+=`, and
739
+ // `name[…]=` are assignments, and otherwise the name may begin the literal text before a structured
740
+ // extglob group (`foo@($x)`), in which `+(` is an extglob operator rather than an unfinished `+=`.
741
+ static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbols) {
742
+ while (is_name_char(lexer->lookahead)) {
743
+ advance(lexer);
744
+ }
745
+ lexer->mark_end(lexer);
746
+ lexer->result_symbol = VARIABLE_NAME;
747
+ if (lexer->lookahead == '=') {
748
+ return true;
749
+ }
750
+ if (lexer->lookahead == '+') {
751
+ advance(lexer);
752
+ if (lexer->lookahead == '=') {
753
+ return true;
754
+ }
755
+ if (!valid_symbols[EXTGLOB_PREFIX]) {
756
+ return false;
757
+ }
758
+ if (lexer->lookahead == '(') {
759
+ GroupKind kind = scan_extglob_group(lexer);
760
+ lexer->result_symbol = EXTGLOB_PREFIX;
761
+ return kind == GROUP_STRUCTURED || (kind == GROUP_LITERAL && continue_extglob_prefix(lexer, true));
762
+ }
763
+ return continue_extglob_prefix(lexer, true);
764
+ }
765
+ if (lexer->lookahead == '[') {
766
+ // Like bash, the subscript extends to the matching `]` across lines; without one the name
767
+ // still starts an assignment, so the parse reports the missing `]`.
768
+ uint32_t depth = 0;
769
+ while (!lexer->eof(lexer)) {
770
+ int32_t c = lexer->lookahead;
771
+ if (c == '[') {
772
+ depth++;
773
+ } else if (c == ']' && --depth == 0) {
774
+ advance(lexer);
775
+ break;
776
+ } else if (c == '\\') {
777
+ advance(lexer);
778
+ } else if (c == '\'' || c == '"' || c == '`') {
779
+ // Brackets inside quotes do not count (`A["]"]=x`).
780
+ skip_quoted(lexer);
781
+ continue;
782
+ } else if (c == '$') {
783
+ advance(lexer);
784
+ if (lexer->lookahead == '(') {
785
+ skip_parenthesized(lexer);
786
+ }
787
+ continue;
788
+ }
789
+ advance(lexer);
790
+ }
791
+ if (depth == 0 && lexer->lookahead == '+') {
792
+ advance(lexer);
793
+ }
794
+ if (depth != 0 || lexer->lookahead == '=') {
795
+ return true;
796
+ }
797
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
798
+ }
799
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
800
+ }
801
+
355
802
  static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
356
803
  bool error_recovery = valid_symbols[ERROR_RECOVERY];
357
804
  // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
@@ -359,22 +806,54 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
359
806
  int32_t first = lexer->lookahead;
360
807
  bool at_eof = lexer->eof(lexer);
361
808
 
809
+ if (scanner->terminator_owed) {
810
+ scanner->terminator_owed = false;
811
+ if (valid_symbols[NEWLINE] && lexer->lookahead == '}') {
812
+ lexer->result_symbol = NEWLINE;
813
+ return true;
814
+ }
815
+ }
816
+
817
+ // Counted before a heredoc body may start, so a newline right after `$(` is inside it.
818
+ if (!error_recovery &&
819
+ (valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
820
+ valid_symbols[BRACKET_SUBSTITUTION_START])) {
821
+ // Nesting too deep for the serialized state is left to the parse as an error.
822
+ if (serialized_size(scanner) >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
823
+ return false;
824
+ }
825
+ lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
826
+ : valid_symbols[BRACE_SUBSTITUTION_START] ? BRACE_SUBSTITUTION_START
827
+ : BRACKET_SUBSTITUTION_START;
828
+ uint8_t closer = lexer->result_symbol == SUBSTITUTION_START ? ')'
829
+ : lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
830
+ : ']';
831
+ array_push(&scanner->closers, closer);
832
+ return true;
833
+ }
834
+
362
835
  if (scanner->heredocs.size > 0) {
363
- Heredoc *front = array_get(&scanner->heredocs, 0);
364
- if (front->started && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
365
- !(lexer->lookahead == '`' && !front->is_raw)) {
366
- return scan_heredoc_content(scanner, lexer);
836
+ int32_t active = active_heredoc_index(scanner);
837
+ if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
838
+ !(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw)) {
839
+ return scan_heredoc_content(scanner, lexer, (uint32_t)active);
367
840
  }
368
841
  // A newline inside a string is part of it, so bodies start only outside strings.
369
- if (!front->started && (front->delimiter.size > 0 || front->is_raw) && valid_symbols[HEREDOC_BODY_START] &&
370
- !valid_symbols[STRING_CONTENT]) {
842
+ int32_t next = next_heredoc_index(scanner);
843
+ if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && !valid_symbols[STRING_CONTENT]) {
371
844
  while (is_blank(lexer->lookahead)) {
372
845
  skip(lexer);
373
846
  }
374
847
  if (lexer->lookahead == '\n') {
375
- front->started = true;
376
- lexer->result_symbol = HEREDOC_BODY_START;
377
- return true;
848
+ // The line's heredocs cannot be located once one of them was rejected, so none gets a
849
+ // body; scanning continues so that the change is kept with the token found below.
850
+ if (is_rejected(array_get(&scanner->heredocs, next))) {
851
+ drop_unstarted_heredocs_at_depth(scanner);
852
+ } else {
853
+ array_get(&scanner->heredocs, next)->started = true;
854
+ lexer->result_symbol = HEREDOC_BODY_START;
855
+ return true;
856
+ }
378
857
  }
379
858
  }
380
859
  }
@@ -393,8 +872,46 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
393
872
 
394
873
  if (valid_symbols[CONCAT]) {
395
874
  int32_t c = first;
396
- if (!at_eof && !iswspace(c) && !is_metacharacter(c) &&
397
- !(c == '`' && scanner->backtick_depth > 0)) {
875
+ if (c == '\\' && lexer->lookahead == '\\') {
876
+ // A backslash-newline joins lines, so the word continues only if the next line does.
877
+ lexer->mark_end(lexer);
878
+ advance(lexer);
879
+ if (lexer->lookahead == '\r') {
880
+ advance(lexer);
881
+ }
882
+ // Bash drops a backslash at the end of input like a continuation.
883
+ if (lexer->eof(lexer)) {
884
+ lexer->mark_end(lexer);
885
+ lexer->result_symbol = LINE_CONTINUATION;
886
+ return true;
887
+ }
888
+ if (lexer->lookahead != '\n') {
889
+ lexer->result_symbol = CONCAT;
890
+ return true;
891
+ }
892
+ advance(lexer);
893
+ lexer->mark_end(lexer);
894
+ c = lexer->lookahead;
895
+ // The word continues only if the next line does; otherwise the continuation itself is the
896
+ // token, so that whatever follows it is lexed as it would be without it.
897
+ lexer->result_symbol = LINE_CONTINUATION;
898
+ if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
899
+ return true;
900
+ }
901
+ if (c == '\\') {
902
+ // Another continuation is whitespace too; an escaped character continues the word.
903
+ advance(lexer);
904
+ if (lexer->lookahead == '\r') {
905
+ advance(lexer);
906
+ }
907
+ if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
908
+ return true;
909
+ }
910
+ }
911
+ lexer->result_symbol = CONCAT;
912
+ return true;
913
+ }
914
+ if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
398
915
  lexer->result_symbol = CONCAT;
399
916
  return true;
400
917
  }
@@ -402,6 +919,17 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
402
919
 
403
920
  if (valid_symbols[EMPTY_VALUE]) {
404
921
  int32_t c = first;
922
+ // A redirection may follow an empty value (`A=>f`), but `<(`/`>(` is a process substitution
923
+ // value; nothing else a value may start with begins with `<` or `>`.
924
+ if ((c == '<' || c == '>') && lexer->lookahead == c) {
925
+ lexer->mark_end(lexer);
926
+ advance(lexer);
927
+ if (lexer->lookahead == '(') {
928
+ return false;
929
+ }
930
+ lexer->result_symbol = EMPTY_VALUE;
931
+ return true;
932
+ }
405
933
  if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
406
934
  (c == '`' && scanner->backtick_depth > 0)) {
407
935
  lexer->result_symbol = EMPTY_VALUE;
@@ -426,24 +954,49 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
426
954
  }
427
955
  skip(lexer);
428
956
  } else if (lexer->lookahead == '\\') {
429
- skip(lexer);
957
+ // A continuation is returned as a token, since tree-sitter's lexer cannot always resume
958
+ // after one; a backslash escaping a word's first character can only start an extglob prefix
959
+ // (`\foo@($x)`), and the word token handles it otherwise.
960
+ advance(lexer);
430
961
  if (lexer->lookahead == '\r') {
431
- skip(lexer);
962
+ advance(lexer);
432
963
  }
433
- if (lexer->lookahead != '\n') {
434
- return false;
964
+ if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
965
+ advance(lexer);
966
+ lexer->mark_end(lexer);
967
+ lexer->result_symbol = LINE_CONTINUATION;
968
+ return true;
435
969
  }
436
- skip(lexer);
970
+ if (valid_symbols[EXTGLOB_PREFIX]) {
971
+ advance(lexer);
972
+ return continue_extglob_prefix(lexer, true);
973
+ }
974
+ return false;
437
975
  } else {
438
976
  break;
439
977
  }
440
978
  }
441
979
 
980
+ // Each closer has its own marker, so a `]` or `}` argument inside `$( )` does not end it.
981
+ enum TokenType end = valid_symbols[SUBSTITUTION_END] && lexer->lookahead == ')' ? SUBSTITUTION_END
982
+ : valid_symbols[BRACE_SUBSTITUTION_END] && lexer->lookahead == '}' ? BRACE_SUBSTITUTION_END
983
+ : valid_symbols[BRACKET_SUBSTITUTION_END] && lexer->lookahead == ']' ? BRACKET_SUBSTITUTION_END
984
+ : ERROR_RECOVERY;
985
+ if (end != ERROR_RECOVERY) {
986
+ if (scanner->closers.size > 0) {
987
+ array_pop(&scanner->closers);
988
+ }
989
+ discard_unstarted_heredocs(scanner);
990
+ lexer->result_symbol = end;
991
+ return true;
992
+ }
993
+
442
994
  if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
443
995
  if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
444
996
  advance(lexer);
445
997
  lexer->mark_end(lexer);
446
998
  scanner->backtick_depth--;
999
+ discard_unstarted_heredocs(scanner);
447
1000
  lexer->result_symbol = BACKTICK_CLOSE;
448
1001
  return true;
449
1002
  }
@@ -460,18 +1013,27 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
460
1013
  return scan_heredoc_arrow(scanner, lexer, valid_symbols);
461
1014
  }
462
1015
 
1016
+ // Only one check that consumes input can run, so the prefix scan leaves names before a possible
1017
+ // assignment to their own check; digits are read once as a descriptor or a word prefix.
463
1018
  if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
464
1019
  while (iswdigit(lexer->lookahead)) {
465
1020
  advance(lexer);
466
1021
  }
467
1022
  lexer->mark_end(lexer);
468
1023
  lexer->result_symbol = FILE_DESCRIPTOR;
469
- return lexer->lookahead == '<' || lexer->lookahead == '>';
1024
+ if (lexer->lookahead == '<' || lexer->lookahead == '>') {
1025
+ return true;
1026
+ }
1027
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
1028
+ }
1029
+
1030
+ if (valid_symbols[EXTGLOB_PREFIX] && !(valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) &&
1031
+ !is_word_break(lexer->lookahead)) {
1032
+ return scan_extglob_prefix(lexer, valid_symbols[EMPTY_VALUE]);
470
1033
  }
471
1034
 
472
1035
  if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
473
- lexer->result_symbol = VARIABLE_NAME;
474
- return scan_variable_name(lexer);
1036
+ return scan_name_or_extglob_prefix(lexer, valid_symbols);
475
1037
  }
476
1038
 
477
1039
  if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
@@ -488,6 +1050,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
488
1050
  void *tree_sitter_bash_external_scanner_create(void) {
489
1051
  Scanner *scanner = ts_calloc(1, sizeof(Scanner));
490
1052
  array_init(&scanner->heredocs);
1053
+ array_init(&scanner->closers);
491
1054
  return scanner;
492
1055
  }
493
1056
 
@@ -495,6 +1058,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
495
1058
  Scanner *scanner = (Scanner *)payload;
496
1059
  scanner_reset(scanner);
497
1060
  array_delete(&scanner->heredocs);
1061
+ array_delete(&scanner->closers);
498
1062
  ts_free(scanner);
499
1063
  }
500
1064
 
@@ -506,51 +1070,71 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
506
1070
  Scanner *scanner = (Scanner *)payload;
507
1071
  unsigned size = 0;
508
1072
  buffer[size++] = (char)scanner->backtick_depth;
509
- unsigned count_position = size++;
510
- uint8_t count = 0;
511
- for (uint32_t i = 0; i < scanner->heredocs.size && count < UINT8_MAX; i++) {
1073
+ buffer[size++] = (char)scanner->terminator_owed;
1074
+ uint16_t closer_count = (uint16_t)scanner->closers.size;
1075
+ memcpy(&buffer[size], &closer_count, sizeof(closer_count));
1076
+ size += sizeof(closer_count);
1077
+ if (scanner->closers.size > 0) {
1078
+ memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
1079
+ size += scanner->closers.size;
1080
+ }
1081
+ buffer[size++] = (char)scanner->heredocs.size;
1082
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
512
1083
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
513
- if (size + 3 + sizeof(uint16_t) + heredoc->delimiter.size > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
514
- break;
515
- }
516
1084
  buffer[size++] = (char)heredoc->is_raw;
517
1085
  buffer[size++] = (char)heredoc->allows_indent;
518
1086
  buffer[size++] = (char)heredoc->started;
1087
+ memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
1088
+ size += sizeof(heredoc->depth);
519
1089
  uint16_t length = (uint16_t)heredoc->delimiter.size;
520
1090
  memcpy(&buffer[size], &length, sizeof(length));
521
1091
  size += sizeof(length);
522
- if (length > 0) {
523
- memcpy(&buffer[size], heredoc->delimiter.contents, length);
524
- size += length;
1092
+ for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
1093
+ size += encode_varint(*array_get(&heredoc->delimiter, j), &buffer[size]);
525
1094
  }
526
- count++;
527
1095
  }
528
- buffer[count_position] = (char)count;
529
1096
  return size;
530
1097
  }
531
1098
 
532
1099
  void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
533
1100
  Scanner *scanner = (Scanner *)payload;
534
1101
  scanner_reset(scanner);
535
- if (length == 0) {
1102
+ if (length < 2 + sizeof(uint16_t)) {
536
1103
  return;
537
1104
  }
538
1105
  unsigned size = 0;
539
1106
  scanner->backtick_depth = (uint8_t)buffer[size++];
1107
+ scanner->terminator_owed = buffer[size++];
1108
+ uint16_t closer_count;
1109
+ memcpy(&closer_count, &buffer[size], sizeof(closer_count));
1110
+ size += sizeof(closer_count);
1111
+ if (size + closer_count >= length) {
1112
+ return;
1113
+ }
1114
+ array_reserve(&scanner->closers, closer_count);
1115
+ memcpy(scanner->closers.contents, &buffer[size], closer_count);
1116
+ scanner->closers.size = closer_count;
1117
+ size += closer_count;
540
1118
  uint8_t count = (uint8_t)buffer[size++];
541
- for (uint8_t i = 0; i < count; i++) {
1119
+ for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
542
1120
  Heredoc heredoc = {.delimiter = array_new()};
543
1121
  heredoc.is_raw = buffer[size++];
544
1122
  heredoc.allows_indent = buffer[size++];
545
1123
  heredoc.started = buffer[size++];
1124
+ memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
1125
+ size += sizeof(heredoc.depth);
546
1126
  uint16_t delimiter_length;
547
1127
  memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
548
1128
  size += sizeof(delimiter_length);
549
- if (delimiter_length > 0) {
550
- array_reserve(&heredoc.delimiter, delimiter_length);
551
- memcpy(heredoc.delimiter.contents, &buffer[size], delimiter_length);
552
- heredoc.delimiter.size = delimiter_length;
553
- size += delimiter_length;
1129
+ array_reserve(&heredoc.delimiter, delimiter_length);
1130
+ for (uint16_t j = 0; j < delimiter_length; j++) {
1131
+ int32_t c;
1132
+ unsigned consumed = decode_varint(&buffer[size], length - size, &c);
1133
+ if (consumed == 0) {
1134
+ break;
1135
+ }
1136
+ size += consumed;
1137
+ array_push(&heredoc.delimiter, c);
554
1138
  }
555
1139
  array_push(&scanner->heredocs, heredoc);
556
1140
  }