@willbooster/tree-sitter-bash 0.25.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -22,18 +22,29 @@ enum TokenType {
22
22
  BACKTICK_OPEN,
23
23
  BACKTICK_CLOSE,
24
24
  NEWLINE,
25
+ SUBSTITUTION_START,
26
+ SUBSTITUTION_END,
27
+ BRACE_SUBSTITUTION_END,
28
+ BRACKET_SUBSTITUTION_END,
29
+ EXTGLOB_PREFIX,
30
+ LINE_CONTINUATION,
31
+ BRACE_SUBSTITUTION_START,
32
+ BRACKET_SUBSTITUTION_START,
25
33
  ERROR_RECOVERY,
26
34
  };
27
35
 
28
- typedef Array(char) String;
36
+ typedef Array(int32_t) CodePoints;
29
37
 
30
38
  // A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
31
- // (not one inside a string), after the bodies of heredocs opened earlier.
39
+ // (not one inside a string) at the substitution depth where the operator appeared, after the bodies
40
+ // of heredocs opened earlier at that depth: bash reads a substitution, including the bodies of the
41
+ // heredocs inside it, before the body of a heredoc whose operator precedes it on the same line.
32
42
  typedef struct {
33
43
  bool is_raw;
34
44
  bool allows_indent;
35
45
  bool started;
36
- String delimiter;
46
+ uint16_t depth;
47
+ CodePoints delimiter;
37
48
  } Heredoc;
38
49
 
39
50
  typedef struct {
@@ -41,8 +52,17 @@ typedef struct {
41
52
  // Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
42
53
  // whether a backquote opens or closes one.
43
54
  uint8_t backtick_depth;
55
+ // The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
56
+ Array(uint8_t) closers;
57
+ // A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
58
+ // line began the body, so a terminator is still owed before the `}`.
59
+ bool terminator_owed;
44
60
  } Scanner;
45
61
 
62
+ static inline uint16_t current_depth(Scanner *scanner) {
63
+ return scanner->backtick_depth + scanner->closers.size;
64
+ }
65
+
46
66
  static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
47
67
 
48
68
  static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
@@ -63,6 +83,60 @@ static inline bool starts_quoted_expansion(int32_t c) {
63
83
  c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
64
84
  }
65
85
 
86
+ // Code points are serialized as zigzag varints, one byte for code points below 64 and two for the
87
+ // rest of ASCII; every int32 round-trips, including the negative value the lexer reports for invalid
88
+ // UTF-8.
89
+ static unsigned varint_length(int32_t c) {
90
+ uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
91
+ unsigned length = 1;
92
+ while (value >= 0x80) {
93
+ value >>= 7;
94
+ length++;
95
+ }
96
+ return length;
97
+ }
98
+
99
+ static unsigned encode_varint(int32_t c, char *out) {
100
+ uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
101
+ unsigned length = 0;
102
+ while (value >= 0x80) {
103
+ out[length++] = (char)(0x80 | (value & 0x7F));
104
+ value >>= 7;
105
+ }
106
+ out[length++] = (char)value;
107
+ return length;
108
+ }
109
+
110
+ // Returns 0 when the input ends before the varint does.
111
+ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
112
+ uint32_t value = 0;
113
+ for (unsigned i = 0; i < available && i < 5; i++) {
114
+ value |= (uint32_t)((unsigned char)in[i] & 0x7F) << (7 * i);
115
+ if (((unsigned char)in[i] & 0x80) == 0) {
116
+ *c = (int32_t)((value >> 1) ^ (0u - (value & 1)));
117
+ return i + 1;
118
+ }
119
+ }
120
+ return 0;
121
+ }
122
+
123
+ // Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
124
+ // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
125
+ // delimiter length (uint16), and the delimiter as varints.
126
+ #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
127
+
128
+ static unsigned serialized_size(Scanner *scanner) {
129
+ unsigned size = 3 + sizeof(uint16_t) + scanner->closers.size;
130
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
131
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
132
+ size += HEREDOC_HEADER_SIZE;
133
+ for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
134
+ size += varint_length(*array_get(&heredoc->delimiter, j));
135
+ }
136
+ }
137
+ return size;
138
+ }
139
+
66
140
  static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
67
141
 
68
142
  static void scanner_reset(Scanner *scanner) {
@@ -71,18 +145,70 @@ static void scanner_reset(Scanner *scanner) {
71
145
  }
72
146
  array_clear(&scanner->heredocs);
73
147
  scanner->backtick_depth = 0;
148
+ array_clear(&scanner->closers);
149
+ scanner->terminator_owed = false;
150
+ }
151
+
152
+ // The body being read: the most deeply nested started heredoc.
153
+ static int32_t active_heredoc_index(Scanner *scanner) {
154
+ int32_t active = -1;
155
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
156
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
157
+ if (heredoc->started && (active < 0 || heredoc->depth >= array_get(&scanner->heredocs, active)->depth)) {
158
+ active = (int32_t)i;
159
+ }
160
+ }
161
+ return active;
162
+ }
163
+
164
+ // The next body to start: the first unstarted heredoc opened at the current substitution depth.
165
+ static int32_t next_heredoc_index(Scanner *scanner) {
166
+ uint16_t depth = current_depth(scanner);
167
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
168
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
169
+ if (!heredoc->started && heredoc->depth == depth) {
170
+ return (int32_t)i;
171
+ }
172
+ }
173
+ return -1;
174
+ }
175
+
176
+ // An entry without a delimiter belongs to an operator whose delimiter was rejected.
177
+ static inline bool is_rejected(Heredoc *heredoc) { return heredoc->delimiter.size == 0 && !heredoc->is_raw; }
178
+
179
+ // Drops the unstarted heredocs at the current depth, which were all queued on the line just ended.
180
+ static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
181
+ uint16_t depth = current_depth(scanner);
182
+ for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
183
+ Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
184
+ if (!heredoc->started && heredoc->depth == depth) {
185
+ heredoc_delete(heredoc);
186
+ array_erase(&scanner->heredocs, i - 1);
187
+ }
188
+ }
189
+ }
190
+
191
+ static void remove_heredoc(Scanner *scanner, uint32_t index) {
192
+ heredoc_delete(array_get(&scanner->heredocs, index));
193
+ array_erase(&scanner->heredocs, index);
74
194
  }
75
195
 
76
- static void pop_front_heredoc(Scanner *scanner) {
77
- heredoc_delete(array_get(&scanner->heredocs, 0));
78
- array_erase(&scanner->heredocs, 0);
196
+ // Like bash, drop the heredocs of a substitution that closes before their bodies started.
197
+ static void discard_unstarted_heredocs(Scanner *scanner) {
198
+ uint16_t depth = current_depth(scanner);
199
+ for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
200
+ Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
201
+ if (!heredoc->started && heredoc->depth > depth) {
202
+ remove_heredoc(scanner, i - 1);
203
+ }
204
+ }
79
205
  }
80
206
 
81
207
  // Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
82
208
  // line (or at the end of input, as bash does with a warning). An unquoted body stops before each
83
209
  // expansion so that the parser can read it.
84
- static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
85
- Heredoc *heredoc = array_get(&scanner->heredocs, 0);
210
+ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
211
+ Heredoc *heredoc = array_get(&scanner->heredocs, index);
86
212
  bool did_advance = false;
87
213
  bool at_line_start = lexer->get_column(lexer) == 0;
88
214
 
@@ -98,21 +224,29 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
98
224
  }
99
225
  }
100
226
  uint32_t matched = 0;
101
- while (matched < heredoc->delimiter.size && lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
227
+ // At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
228
+ while (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
229
+ lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
102
230
  advance(lexer);
103
231
  matched++;
104
232
  }
105
233
  if (lexer->lookahead == '\r') {
106
234
  advance(lexer);
107
235
  }
108
- if (matched == heredoc->delimiter.size && (lexer->lookahead == '\n' || lexer->eof(lexer))) {
236
+ // Inside a substitution, bash also ends the body at a delimiter directly followed by the
237
+ // innermost substitution's closer (`)` or `}`) or a closing backquote.
238
+ bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
239
+ (scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
240
+ (lexer->lookahead == '`' && scanner->backtick_depth > 0);
241
+ if (matched == heredoc->delimiter.size && ends_line) {
109
242
  if (did_advance) {
110
243
  lexer->result_symbol = HEREDOC_CONTENT;
111
244
  return true;
112
245
  }
113
246
  lexer->mark_end(lexer);
114
247
  lexer->result_symbol = HEREDOC_END;
115
- pop_front_heredoc(scanner);
248
+ scanner->terminator_owed = lexer->lookahead == '}';
249
+ remove_heredoc(scanner, index);
116
250
  return true;
117
251
  }
118
252
  // The characters read while trying the delimiter are content; an empty attempt must not
@@ -125,7 +259,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
125
259
  lexer->mark_end(lexer);
126
260
  lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
127
261
  if (!did_advance) {
128
- pop_front_heredoc(scanner);
262
+ remove_heredoc(scanner, index);
129
263
  }
130
264
  return true;
131
265
  }
@@ -141,7 +275,12 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
141
275
  did_advance = true;
142
276
  if (!heredoc->is_raw && !lexer->eof(lexer)) {
143
277
  // A backslash-newline joins lines before the delimiter comparison.
144
- advance(lexer);
278
+ if (lexer->lookahead == '\r') {
279
+ advance(lexer);
280
+ }
281
+ if (!lexer->eof(lexer)) {
282
+ advance(lexer);
283
+ }
145
284
  }
146
285
  break;
147
286
  case '`':
@@ -179,6 +318,79 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
179
318
  }
180
319
 
181
320
  // The delimiter word undergoes quote removal; any quoting makes the body literal.
321
+ static inline int32_t hex_value(int32_t c) {
322
+ return iswdigit(c) ? c - '0' : (c >= 'a' && c <= 'f') ? c - 'a' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 : -1;
323
+ }
324
+
325
+ // Reads up to `max_digits` digits in `base` and returns their value, or -1 when there are none.
326
+ static int32_t read_number(TSLexer *lexer, int32_t base, int max_digits) {
327
+ int32_t value = 0;
328
+ int digits = 0;
329
+ while (digits < max_digits) {
330
+ int32_t digit = hex_value(lexer->lookahead);
331
+ if (digit < 0 || digit >= base) {
332
+ break;
333
+ }
334
+ value = value * base + digit;
335
+ advance(lexer);
336
+ digits++;
337
+ }
338
+ return digits == 0 ? -1 : value;
339
+ }
340
+
341
+ // Decodes the escape after a backslash in `$'...'` as bash does, appending it to `out`.
342
+ static void push_ansi_c_escape(TSLexer *lexer, CodePoints *out) {
343
+ int32_t c = lexer->lookahead;
344
+ int32_t value = -1;
345
+ switch (c) {
346
+ case 'a': value = 7; break;
347
+ case 'b': value = 8; break;
348
+ case 'e':
349
+ case 'E': value = 27; break;
350
+ case 'f': value = 12; break;
351
+ case 'n': value = '\n'; break;
352
+ case 'r': value = '\r'; break;
353
+ case 't': value = '\t'; break;
354
+ case 'v': value = 11; break;
355
+ case '\\':
356
+ case '\'':
357
+ case '"':
358
+ case '?': value = c; break;
359
+ default: break;
360
+ }
361
+ if (value >= 0) {
362
+ advance(lexer);
363
+ array_push(out, value);
364
+ return;
365
+ }
366
+ if (c >= '0' && c <= '7') {
367
+ array_push(out, read_number(lexer, 8, 3));
368
+ return;
369
+ }
370
+ int max_digits = c == 'x' ? 2 : c == 'u' ? 4 : c == 'U' ? 8 : 0;
371
+ if (max_digits > 0) {
372
+ advance(lexer);
373
+ value = read_number(lexer, 16, max_digits);
374
+ if (value < 0) {
375
+ array_push(out, '\\');
376
+ array_push(out, c);
377
+ } else {
378
+ array_push(out, value);
379
+ }
380
+ return;
381
+ }
382
+ if (c == 'c') {
383
+ advance(lexer);
384
+ if (!lexer->eof(lexer)) {
385
+ array_push(out, lexer->lookahead & 0x1F);
386
+ advance(lexer);
387
+ }
388
+ return;
389
+ }
390
+ // Bash keeps an unknown escape as written.
391
+ array_push(out, '\\');
392
+ }
393
+
182
394
  static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
183
395
  Heredoc *heredoc = array_back(&scanner->heredocs);
184
396
  array_clear(&heredoc->delimiter);
@@ -189,6 +401,7 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
189
401
  }
190
402
 
191
403
  int32_t quote = 0;
404
+ bool ansi_c = false;
192
405
  while (!lexer->eof(lexer)) {
193
406
  int32_t c = lexer->lookahead;
194
407
  if (quote) {
@@ -197,36 +410,80 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
197
410
  quote = 0;
198
411
  continue;
199
412
  }
413
+ if (ansi_c && c == '\\') {
414
+ if (!lexer->eof(lexer)) {
415
+ push_ansi_c_escape(lexer, &heredoc->delimiter);
416
+ }
417
+ continue;
418
+ }
419
+ // Inside double quotes a backslash-newline still only continues the line.
420
+ if (quote == '"' && c == '\\' && (lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
421
+ if (lexer->lookahead == '\r') {
422
+ advance(lexer);
423
+ }
424
+ if (lexer->lookahead == '\n') {
425
+ advance(lexer);
426
+ continue;
427
+ }
428
+ array_push(&heredoc->delimiter, '\\');
429
+ array_push(&heredoc->delimiter, '\r');
430
+ continue;
431
+ }
200
432
  if (quote == '"' && c == '\\' &&
201
433
  (lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
202
434
  lexer->lookahead == '\\')) {
203
435
  c = lexer->lookahead;
204
436
  advance(lexer);
205
437
  }
206
- array_push(&heredoc->delimiter, (char)c);
438
+ array_push(&heredoc->delimiter, c);
207
439
  continue;
208
440
  }
209
- if (iswspace(c) || is_metacharacter(c)) {
441
+ // Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
442
+ // the word (`<<EO`true`F`).
443
+ if (iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
210
444
  break;
211
445
  }
212
446
  advance(lexer);
213
- if (c == '\'' || c == '"') {
447
+ if (c == '$' && (lexer->lookahead == '\'' || lexer->lookahead == '"')) {
448
+ // `$'...'` and `$"..."` are quoting constructs, so their `$` is not part of the word.
449
+ c = lexer->lookahead;
450
+ advance(lexer);
214
451
  quote = c;
452
+ ansi_c = c == '\'';
215
453
  heredoc->is_raw = true;
216
- } else if (c == '\\') {
454
+ } else if (c == '\'' || c == '"') {
455
+ quote = c;
456
+ ansi_c = false;
217
457
  heredoc->is_raw = true;
458
+ } else if (c == '\\') {
459
+ // A backslash-newline only continues the line; any other backslash quotes the delimiter.
460
+ bool carriage_return = lexer->lookahead == '\r';
461
+ if (carriage_return) {
462
+ advance(lexer);
463
+ }
218
464
  if (lexer->lookahead == '\n') {
219
465
  advance(lexer);
466
+ } else if (carriage_return) {
467
+ heredoc->is_raw = true;
468
+ array_push(&heredoc->delimiter, '\r');
220
469
  } else if (!lexer->eof(lexer)) {
221
- array_push(&heredoc->delimiter, (char)lexer->lookahead);
470
+ heredoc->is_raw = true;
471
+ array_push(&heredoc->delimiter, lexer->lookahead);
222
472
  advance(lexer);
223
473
  }
224
474
  } else {
225
- array_push(&heredoc->delimiter, (char)c);
475
+ // Bash counts a backquote in the delimiter word as quoting, so the body stays literal.
476
+ if (c == '`') {
477
+ heredoc->is_raw = true;
478
+ }
479
+ array_push(&heredoc->delimiter, c);
226
480
  }
227
481
  }
228
482
 
229
- if (heredoc->delimiter.size == 0 && !heredoc->is_raw) {
483
+ // A heredoc whose state cannot be serialized in full is rejected, so the parse shows an error
484
+ // rather than comparing against a partial delimiter.
485
+ if ((heredoc->delimiter.size == 0 && !heredoc->is_raw) ||
486
+ serialized_size(scanner) > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
230
487
  return false;
231
488
  }
232
489
  lexer->mark_end(lexer);
@@ -250,46 +507,23 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
250
507
  } else if (!valid_symbols[HEREDOC_ARROW]) {
251
508
  return false;
252
509
  }
510
+ if (scanner->heredocs.size >= UINT8_MAX ||
511
+ serialized_size(scanner) + HEREDOC_HEADER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
512
+ return false;
513
+ }
253
514
  lexer->mark_end(lexer);
254
- Heredoc heredoc = {.is_raw = false, .allows_indent = allows_indent, .started = false, .delimiter = array_new()};
515
+ Heredoc heredoc = {
516
+ .is_raw = false,
517
+ .allows_indent = allows_indent,
518
+ .started = false,
519
+ .depth = current_depth(scanner),
520
+ .delimiter = array_new(),
521
+ };
255
522
  array_push(&scanner->heredocs, heredoc);
256
523
  lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
257
524
  return true;
258
525
  }
259
526
 
260
- // A name followed by `=`, `+=`, or `[subscript]=` starts an assignment.
261
- static bool scan_variable_name(TSLexer *lexer) {
262
- while (is_name_char(lexer->lookahead)) {
263
- advance(lexer);
264
- }
265
- lexer->mark_end(lexer);
266
- if (lexer->lookahead == '+') {
267
- advance(lexer);
268
- return lexer->lookahead == '=';
269
- }
270
- if (lexer->lookahead == '[') {
271
- uint32_t depth = 0;
272
- while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
273
- if (lexer->lookahead == '[') {
274
- depth++;
275
- } else if (lexer->lookahead == ']' && --depth == 0) {
276
- advance(lexer);
277
- break;
278
- } else if (lexer->lookahead == '\\') {
279
- advance(lexer);
280
- }
281
- advance(lexer);
282
- }
283
- if (depth != 0) {
284
- return false;
285
- }
286
- if (lexer->lookahead == '+') {
287
- advance(lexer);
288
- }
289
- }
290
- return lexer->lookahead == '=';
291
- }
292
-
293
527
  // The right side of `=~` extends to unquoted whitespace outside parentheses.
294
528
  static bool scan_regex(TSLexer *lexer) {
295
529
  while (is_blank(lexer->lookahead)) {
@@ -298,10 +532,16 @@ static bool scan_regex(TSLexer *lexer) {
298
532
  uint32_t depth = 0;
299
533
  bool did_advance = false;
300
534
  while (!lexer->eof(lexer)) {
535
+ lexer->mark_end(lexer);
301
536
  int32_t c = lexer->lookahead;
302
537
  if (depth == 0 && iswspace(c)) {
303
538
  break;
304
539
  }
540
+ // Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
541
+ // the regex even without surrounding blanks (`[[ a =~ a&&b ]]`).
542
+ if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
543
+ break;
544
+ }
305
545
  if (c == '\\') {
306
546
  advance(lexer);
307
547
  } else if (c == '\'' || c == '"') {
@@ -323,7 +563,9 @@ static bool scan_regex(TSLexer *lexer) {
323
563
  advance(lexer);
324
564
  did_advance = true;
325
565
  }
326
- lexer->mark_end(lexer);
566
+ if (lexer->eof(lexer)) {
567
+ lexer->mark_end(lexer);
568
+ }
327
569
  lexer->result_symbol = REGEX;
328
570
  return did_advance;
329
571
  }
@@ -352,6 +594,186 @@ static bool scan_string_content(TSLexer *lexer) {
352
594
  return did_advance;
353
595
  }
354
596
 
597
+ static inline bool is_word_break(int32_t c) {
598
+ return iswspace(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
599
+ }
600
+
601
+ static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
602
+
603
+ typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
604
+
605
+ // From the `(` of an extglob group, reads to its matching `)`: a group holding quotes or expansions
606
+ // is parsed as `extglob_pattern`, while the word token keeps literal ones.
607
+ // The nesting depth up to which grammar.js's word token lexes a literal group (`extglobGroup(…, 3)`);
608
+ // a deeper group is parsed as `extglob_pattern`.
609
+ #define EXTGLOB_LITERAL_DEPTH 3
610
+
611
+ static GroupKind scan_extglob_group(TSLexer *lexer) {
612
+ uint32_t depth = 0;
613
+ while (!lexer->eof(lexer)) {
614
+ int32_t c = lexer->lookahead;
615
+ if (c == '$' || c == '`' || c == '\'' || c == '"') {
616
+ return GROUP_STRUCTURED;
617
+ }
618
+ advance(lexer);
619
+ if (c == '\\') {
620
+ // The word token's literal groups cannot hold a line continuation.
621
+ if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
622
+ return lexer->eof(lexer) ? GROUP_UNSUPPORTED : GROUP_STRUCTURED;
623
+ }
624
+ advance(lexer);
625
+ } else if (c == '(') {
626
+ if (++depth > EXTGLOB_LITERAL_DEPTH) {
627
+ return GROUP_STRUCTURED;
628
+ }
629
+ } else if (c == ')' && --depth == 0) {
630
+ return GROUP_LITERAL;
631
+ }
632
+ }
633
+ return GROUP_UNSUPPORTED;
634
+ }
635
+
636
+ // The literal text of a word up to a structured extglob group (`foo` in `foo@($x)`), which the
637
+ // word token's longest match would otherwise run into.
638
+ static bool continue_extglob_prefix(TSLexer *lexer, bool consumed) {
639
+ while (!lexer->eof(lexer)) {
640
+ int32_t c = lexer->lookahead;
641
+ if (is_extglob_operator(c)) {
642
+ lexer->mark_end(lexer);
643
+ advance(lexer);
644
+ if (lexer->lookahead == '(') {
645
+ GroupKind kind = scan_extglob_group(lexer);
646
+ if (kind == GROUP_STRUCTURED) {
647
+ lexer->result_symbol = EXTGLOB_PREFIX;
648
+ return consumed;
649
+ }
650
+ if (kind == GROUP_UNSUPPORTED) {
651
+ return false;
652
+ }
653
+ }
654
+ } else if (c == '\\') {
655
+ advance(lexer);
656
+ // A backslash-newline continues the line; the word token handles that.
657
+ if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
658
+ return false;
659
+ }
660
+ advance(lexer);
661
+ } else if (is_word_break(c)) {
662
+ return false;
663
+ } else {
664
+ advance(lexer);
665
+ }
666
+ consumed = true;
667
+ }
668
+ return false;
669
+ }
670
+
671
+ // A leading `#` starts a comment except in an assignment value (`A=#@($x)`).
672
+ static bool scan_extglob_prefix(TSLexer *lexer, bool in_value) {
673
+ return (lexer->lookahead != '#' || in_value) && continue_extglob_prefix(lexer, false);
674
+ }
675
+
676
+ // Skips a quoted span from its opening quote through its closing one.
677
+ static void skip_quoted(TSLexer *lexer) {
678
+ int32_t quote = lexer->lookahead;
679
+ advance(lexer);
680
+ while (!lexer->eof(lexer) && lexer->lookahead != quote) {
681
+ if (lexer->lookahead == '\\' && quote != '\'') {
682
+ advance(lexer);
683
+ if (lexer->eof(lexer)) {
684
+ return;
685
+ }
686
+ }
687
+ advance(lexer);
688
+ }
689
+ advance(lexer);
690
+ }
691
+
692
+ // Skips a parenthesized span such as a command substitution's, quotes included.
693
+ static void skip_parenthesized(TSLexer *lexer) {
694
+ uint32_t depth = 0;
695
+ while (!lexer->eof(lexer)) {
696
+ int32_t c = lexer->lookahead;
697
+ if (c == '\'' || c == '"' || c == '`') {
698
+ skip_quoted(lexer);
699
+ continue;
700
+ }
701
+ if (c == '\\') {
702
+ advance(lexer);
703
+ } else if (c == '(') {
704
+ depth++;
705
+ } else if (c == ')' && --depth == 0) {
706
+ advance(lexer);
707
+ return;
708
+ }
709
+ advance(lexer);
710
+ }
711
+ }
712
+
713
+ // A name at a position where both an assignment and a word may start: `name=`, `name+=`, and
714
+ // `name[…]=` are assignments, and otherwise the name may begin the literal text before a structured
715
+ // extglob group (`foo@($x)`), in which `+(` is an extglob operator rather than an unfinished `+=`.
716
+ static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbols) {
717
+ while (is_name_char(lexer->lookahead)) {
718
+ advance(lexer);
719
+ }
720
+ lexer->mark_end(lexer);
721
+ lexer->result_symbol = VARIABLE_NAME;
722
+ if (lexer->lookahead == '=') {
723
+ return true;
724
+ }
725
+ if (lexer->lookahead == '+') {
726
+ advance(lexer);
727
+ if (lexer->lookahead == '=') {
728
+ return true;
729
+ }
730
+ if (!valid_symbols[EXTGLOB_PREFIX]) {
731
+ return false;
732
+ }
733
+ if (lexer->lookahead == '(') {
734
+ GroupKind kind = scan_extglob_group(lexer);
735
+ lexer->result_symbol = EXTGLOB_PREFIX;
736
+ return kind == GROUP_STRUCTURED || (kind == GROUP_LITERAL && continue_extglob_prefix(lexer, true));
737
+ }
738
+ return continue_extglob_prefix(lexer, true);
739
+ }
740
+ if (lexer->lookahead == '[') {
741
+ // Like bash, the subscript extends to the matching `]` across lines; without one the name
742
+ // still starts an assignment, so the parse reports the missing `]`.
743
+ uint32_t depth = 0;
744
+ while (!lexer->eof(lexer)) {
745
+ int32_t c = lexer->lookahead;
746
+ if (c == '[') {
747
+ depth++;
748
+ } else if (c == ']' && --depth == 0) {
749
+ advance(lexer);
750
+ break;
751
+ } else if (c == '\\') {
752
+ advance(lexer);
753
+ } else if (c == '\'' || c == '"' || c == '`') {
754
+ // Brackets inside quotes do not count (`A["]"]=x`).
755
+ skip_quoted(lexer);
756
+ continue;
757
+ } else if (c == '$') {
758
+ advance(lexer);
759
+ if (lexer->lookahead == '(') {
760
+ skip_parenthesized(lexer);
761
+ }
762
+ continue;
763
+ }
764
+ advance(lexer);
765
+ }
766
+ if (depth == 0 && lexer->lookahead == '+') {
767
+ advance(lexer);
768
+ }
769
+ if (depth != 0 || lexer->lookahead == '=') {
770
+ return true;
771
+ }
772
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
773
+ }
774
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
775
+ }
776
+
355
777
  static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
356
778
  bool error_recovery = valid_symbols[ERROR_RECOVERY];
357
779
  // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
@@ -359,22 +781,54 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
359
781
  int32_t first = lexer->lookahead;
360
782
  bool at_eof = lexer->eof(lexer);
361
783
 
784
+ if (scanner->terminator_owed) {
785
+ scanner->terminator_owed = false;
786
+ if (valid_symbols[NEWLINE] && lexer->lookahead == '}') {
787
+ lexer->result_symbol = NEWLINE;
788
+ return true;
789
+ }
790
+ }
791
+
792
+ // Counted before a heredoc body may start, so a newline right after `$(` is inside it.
793
+ if (!error_recovery &&
794
+ (valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
795
+ valid_symbols[BRACKET_SUBSTITUTION_START])) {
796
+ // Nesting too deep for the serialized state is left to the parse as an error.
797
+ if (serialized_size(scanner) >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
798
+ return false;
799
+ }
800
+ lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
801
+ : valid_symbols[BRACE_SUBSTITUTION_START] ? BRACE_SUBSTITUTION_START
802
+ : BRACKET_SUBSTITUTION_START;
803
+ uint8_t closer = lexer->result_symbol == SUBSTITUTION_START ? ')'
804
+ : lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
805
+ : ']';
806
+ array_push(&scanner->closers, closer);
807
+ return true;
808
+ }
809
+
362
810
  if (scanner->heredocs.size > 0) {
363
- Heredoc *front = array_get(&scanner->heredocs, 0);
364
- if (front->started && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
365
- !(lexer->lookahead == '`' && !front->is_raw)) {
366
- return scan_heredoc_content(scanner, lexer);
811
+ int32_t active = active_heredoc_index(scanner);
812
+ if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
813
+ !(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw)) {
814
+ return scan_heredoc_content(scanner, lexer, (uint32_t)active);
367
815
  }
368
816
  // A newline inside a string is part of it, so bodies start only outside strings.
369
- if (!front->started && (front->delimiter.size > 0 || front->is_raw) && valid_symbols[HEREDOC_BODY_START] &&
370
- !valid_symbols[STRING_CONTENT]) {
817
+ int32_t next = next_heredoc_index(scanner);
818
+ if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && !valid_symbols[STRING_CONTENT]) {
371
819
  while (is_blank(lexer->lookahead)) {
372
820
  skip(lexer);
373
821
  }
374
822
  if (lexer->lookahead == '\n') {
375
- front->started = true;
376
- lexer->result_symbol = HEREDOC_BODY_START;
377
- return true;
823
+ // The line's heredocs cannot be located once one of them was rejected, so none gets a
824
+ // body; scanning continues so that the change is kept with the token found below.
825
+ if (is_rejected(array_get(&scanner->heredocs, next))) {
826
+ drop_unstarted_heredocs_at_depth(scanner);
827
+ } else {
828
+ array_get(&scanner->heredocs, next)->started = true;
829
+ lexer->result_symbol = HEREDOC_BODY_START;
830
+ return true;
831
+ }
378
832
  }
379
833
  }
380
834
  }
@@ -393,8 +847,46 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
393
847
 
394
848
  if (valid_symbols[CONCAT]) {
395
849
  int32_t c = first;
396
- if (!at_eof && !iswspace(c) && !is_metacharacter(c) &&
397
- !(c == '`' && scanner->backtick_depth > 0)) {
850
+ if (c == '\\' && lexer->lookahead == '\\') {
851
+ // A backslash-newline joins lines, so the word continues only if the next line does.
852
+ lexer->mark_end(lexer);
853
+ advance(lexer);
854
+ if (lexer->lookahead == '\r') {
855
+ advance(lexer);
856
+ }
857
+ // Bash drops a backslash at the end of input like a continuation.
858
+ if (lexer->eof(lexer)) {
859
+ lexer->mark_end(lexer);
860
+ lexer->result_symbol = LINE_CONTINUATION;
861
+ return true;
862
+ }
863
+ if (lexer->lookahead != '\n') {
864
+ lexer->result_symbol = CONCAT;
865
+ return true;
866
+ }
867
+ advance(lexer);
868
+ lexer->mark_end(lexer);
869
+ c = lexer->lookahead;
870
+ // The word continues only if the next line does; otherwise the continuation itself is the
871
+ // token, so that whatever follows it is lexed as it would be without it.
872
+ lexer->result_symbol = LINE_CONTINUATION;
873
+ if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
874
+ return true;
875
+ }
876
+ if (c == '\\') {
877
+ // Another continuation is whitespace too; an escaped character continues the word.
878
+ advance(lexer);
879
+ if (lexer->lookahead == '\r') {
880
+ advance(lexer);
881
+ }
882
+ if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
883
+ return true;
884
+ }
885
+ }
886
+ lexer->result_symbol = CONCAT;
887
+ return true;
888
+ }
889
+ if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
398
890
  lexer->result_symbol = CONCAT;
399
891
  return true;
400
892
  }
@@ -402,6 +894,17 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
402
894
 
403
895
  if (valid_symbols[EMPTY_VALUE]) {
404
896
  int32_t c = first;
897
+ // A redirection may follow an empty value (`A=>f`), but `<(`/`>(` is a process substitution
898
+ // value; nothing else a value may start with begins with `<` or `>`.
899
+ if ((c == '<' || c == '>') && lexer->lookahead == c) {
900
+ lexer->mark_end(lexer);
901
+ advance(lexer);
902
+ if (lexer->lookahead == '(') {
903
+ return false;
904
+ }
905
+ lexer->result_symbol = EMPTY_VALUE;
906
+ return true;
907
+ }
405
908
  if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
406
909
  (c == '`' && scanner->backtick_depth > 0)) {
407
910
  lexer->result_symbol = EMPTY_VALUE;
@@ -426,24 +929,49 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
426
929
  }
427
930
  skip(lexer);
428
931
  } else if (lexer->lookahead == '\\') {
429
- skip(lexer);
932
+ // A continuation is returned as a token, since tree-sitter's lexer cannot always resume
933
+ // after one; a backslash escaping a word's first character can only start an extglob prefix
934
+ // (`\foo@($x)`), and the word token handles it otherwise.
935
+ advance(lexer);
430
936
  if (lexer->lookahead == '\r') {
431
- skip(lexer);
937
+ advance(lexer);
432
938
  }
433
- if (lexer->lookahead != '\n') {
434
- return false;
939
+ if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
940
+ advance(lexer);
941
+ lexer->mark_end(lexer);
942
+ lexer->result_symbol = LINE_CONTINUATION;
943
+ return true;
435
944
  }
436
- skip(lexer);
945
+ if (valid_symbols[EXTGLOB_PREFIX]) {
946
+ advance(lexer);
947
+ return continue_extglob_prefix(lexer, true);
948
+ }
949
+ return false;
437
950
  } else {
438
951
  break;
439
952
  }
440
953
  }
441
954
 
955
+ // Each closer has its own marker, so a `]` or `}` argument inside `$( )` does not end it.
956
+ enum TokenType end = valid_symbols[SUBSTITUTION_END] && lexer->lookahead == ')' ? SUBSTITUTION_END
957
+ : valid_symbols[BRACE_SUBSTITUTION_END] && lexer->lookahead == '}' ? BRACE_SUBSTITUTION_END
958
+ : valid_symbols[BRACKET_SUBSTITUTION_END] && lexer->lookahead == ']' ? BRACKET_SUBSTITUTION_END
959
+ : ERROR_RECOVERY;
960
+ if (end != ERROR_RECOVERY) {
961
+ if (scanner->closers.size > 0) {
962
+ array_pop(&scanner->closers);
963
+ }
964
+ discard_unstarted_heredocs(scanner);
965
+ lexer->result_symbol = end;
966
+ return true;
967
+ }
968
+
442
969
  if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
443
970
  if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
444
971
  advance(lexer);
445
972
  lexer->mark_end(lexer);
446
973
  scanner->backtick_depth--;
974
+ discard_unstarted_heredocs(scanner);
447
975
  lexer->result_symbol = BACKTICK_CLOSE;
448
976
  return true;
449
977
  }
@@ -460,18 +988,27 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
460
988
  return scan_heredoc_arrow(scanner, lexer, valid_symbols);
461
989
  }
462
990
 
991
+ // Only one check that consumes input can run, so the prefix scan leaves names before a possible
992
+ // assignment to their own check; digits are read once as a descriptor or a word prefix.
463
993
  if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
464
994
  while (iswdigit(lexer->lookahead)) {
465
995
  advance(lexer);
466
996
  }
467
997
  lexer->mark_end(lexer);
468
998
  lexer->result_symbol = FILE_DESCRIPTOR;
469
- return lexer->lookahead == '<' || lexer->lookahead == '>';
999
+ if (lexer->lookahead == '<' || lexer->lookahead == '>') {
1000
+ return true;
1001
+ }
1002
+ return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
1003
+ }
1004
+
1005
+ if (valid_symbols[EXTGLOB_PREFIX] && !(valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) &&
1006
+ !is_word_break(lexer->lookahead)) {
1007
+ return scan_extglob_prefix(lexer, valid_symbols[EMPTY_VALUE]);
470
1008
  }
471
1009
 
472
1010
  if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
473
- lexer->result_symbol = VARIABLE_NAME;
474
- return scan_variable_name(lexer);
1011
+ return scan_name_or_extglob_prefix(lexer, valid_symbols);
475
1012
  }
476
1013
 
477
1014
  if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
@@ -488,6 +1025,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
488
1025
  void *tree_sitter_bash_external_scanner_create(void) {
489
1026
  Scanner *scanner = ts_calloc(1, sizeof(Scanner));
490
1027
  array_init(&scanner->heredocs);
1028
+ array_init(&scanner->closers);
491
1029
  return scanner;
492
1030
  }
493
1031
 
@@ -495,6 +1033,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
495
1033
  Scanner *scanner = (Scanner *)payload;
496
1034
  scanner_reset(scanner);
497
1035
  array_delete(&scanner->heredocs);
1036
+ array_delete(&scanner->closers);
498
1037
  ts_free(scanner);
499
1038
  }
500
1039
 
@@ -506,51 +1045,71 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
506
1045
  Scanner *scanner = (Scanner *)payload;
507
1046
  unsigned size = 0;
508
1047
  buffer[size++] = (char)scanner->backtick_depth;
509
- unsigned count_position = size++;
510
- uint8_t count = 0;
511
- for (uint32_t i = 0; i < scanner->heredocs.size && count < UINT8_MAX; i++) {
1048
+ buffer[size++] = (char)scanner->terminator_owed;
1049
+ uint16_t closer_count = (uint16_t)scanner->closers.size;
1050
+ memcpy(&buffer[size], &closer_count, sizeof(closer_count));
1051
+ size += sizeof(closer_count);
1052
+ if (scanner->closers.size > 0) {
1053
+ memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
1054
+ size += scanner->closers.size;
1055
+ }
1056
+ buffer[size++] = (char)scanner->heredocs.size;
1057
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
512
1058
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
513
- if (size + 3 + sizeof(uint16_t) + heredoc->delimiter.size > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
514
- break;
515
- }
516
1059
  buffer[size++] = (char)heredoc->is_raw;
517
1060
  buffer[size++] = (char)heredoc->allows_indent;
518
1061
  buffer[size++] = (char)heredoc->started;
1062
+ memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
1063
+ size += sizeof(heredoc->depth);
519
1064
  uint16_t length = (uint16_t)heredoc->delimiter.size;
520
1065
  memcpy(&buffer[size], &length, sizeof(length));
521
1066
  size += sizeof(length);
522
- if (length > 0) {
523
- memcpy(&buffer[size], heredoc->delimiter.contents, length);
524
- size += length;
1067
+ for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
1068
+ size += encode_varint(*array_get(&heredoc->delimiter, j), &buffer[size]);
525
1069
  }
526
- count++;
527
1070
  }
528
- buffer[count_position] = (char)count;
529
1071
  return size;
530
1072
  }
531
1073
 
532
1074
  void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
533
1075
  Scanner *scanner = (Scanner *)payload;
534
1076
  scanner_reset(scanner);
535
- if (length == 0) {
1077
+ if (length < 2 + sizeof(uint16_t)) {
536
1078
  return;
537
1079
  }
538
1080
  unsigned size = 0;
539
1081
  scanner->backtick_depth = (uint8_t)buffer[size++];
1082
+ scanner->terminator_owed = buffer[size++];
1083
+ uint16_t closer_count;
1084
+ memcpy(&closer_count, &buffer[size], sizeof(closer_count));
1085
+ size += sizeof(closer_count);
1086
+ if (size + closer_count >= length) {
1087
+ return;
1088
+ }
1089
+ array_reserve(&scanner->closers, closer_count);
1090
+ memcpy(scanner->closers.contents, &buffer[size], closer_count);
1091
+ scanner->closers.size = closer_count;
1092
+ size += closer_count;
540
1093
  uint8_t count = (uint8_t)buffer[size++];
541
- for (uint8_t i = 0; i < count; i++) {
1094
+ for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
542
1095
  Heredoc heredoc = {.delimiter = array_new()};
543
1096
  heredoc.is_raw = buffer[size++];
544
1097
  heredoc.allows_indent = buffer[size++];
545
1098
  heredoc.started = buffer[size++];
1099
+ memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
1100
+ size += sizeof(heredoc.depth);
546
1101
  uint16_t delimiter_length;
547
1102
  memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
548
1103
  size += sizeof(delimiter_length);
549
- if (delimiter_length > 0) {
550
- array_reserve(&heredoc.delimiter, delimiter_length);
551
- memcpy(heredoc.delimiter.contents, &buffer[size], delimiter_length);
552
- heredoc.delimiter.size = delimiter_length;
553
- size += delimiter_length;
1104
+ array_reserve(&heredoc.delimiter, delimiter_length);
1105
+ for (uint16_t j = 0; j < delimiter_length; j++) {
1106
+ int32_t c;
1107
+ unsigned consumed = decode_varint(&buffer[size], length - size, &c);
1108
+ if (consumed == 0) {
1109
+ break;
1110
+ }
1111
+ size += consumed;
1112
+ array_push(&heredoc.delimiter, c);
554
1113
  }
555
1114
  array_push(&scanner->heredocs, heredoc);
556
1115
  }