@willbooster/tree-sitter-bash 1.1.1 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -54,8 +54,11 @@ typedef struct {
54
54
  uint8_t backtick_depth;
55
55
  // The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
56
56
  Array(uint8_t) closers;
57
- // A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
58
- // line began the body, so a terminator is still owed before the `}`.
57
+ // How many closers were open when the backquotes opened; the ones after them are nested inside.
58
+ uint16_t backtick_closers;
59
+ // A heredoc body ended before the end of its line (at a `${ list; }` closer, or at a delimiter
60
+ // followed by more code); the newline that ended the command's line began the body, so a
61
+ // terminator is still owed before what follows.
59
62
  bool terminator_owed;
60
63
  // The last token of this scanner was a line continuation that ended a line; nothing concatenates at
61
64
  // the start of the next one.
@@ -74,7 +77,11 @@ static inline bool is_metacharacter(int32_t c) {
74
77
  return c == '|' || c == '&' || c == ';' || c == '(' || c == ')' || c == '<' || c == '>';
75
78
  }
76
79
 
77
- static inline bool is_blank(int32_t c) { return c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'; }
80
+ // Like bash, only a space and a tab are blanks; every other whitespace character, CR included, is part
81
+ // of a word.
82
+ static inline bool is_blank(int32_t c) { return c == ' ' || c == '\t'; }
83
+
84
+ static inline bool is_separator(int32_t c) { return is_blank(c) || c == '\n'; }
78
85
 
79
86
  static inline bool is_name_start(int32_t c) { return iswalpha(c) || c == '_'; }
80
87
 
@@ -123,13 +130,14 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
123
130
  return 0;
124
131
  }
125
132
 
126
- // Serialized layout: backtick depth, the owed-terminator and line-break flags, the closers of the open substitutions
133
+ // Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator and
134
+ // line-break flags, the closers of the open substitutions
127
135
  // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
128
136
  // delimiter length (uint16), and the delimiter as varints.
129
137
  #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
130
138
 
131
139
  static unsigned serialized_size(Scanner *scanner) {
132
- unsigned size = 3 + sizeof(uint16_t) + scanner->closers.size;
140
+ unsigned size = 3 + 2 * sizeof(uint16_t) + scanner->closers.size;
133
141
  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
134
142
  Heredoc *heredoc = array_get(&scanner->heredocs, i);
135
143
  size += HEREDOC_HEADER_SIZE;
@@ -149,6 +157,7 @@ static void scanner_reset(Scanner *scanner) {
149
157
  array_clear(&scanner->heredocs);
150
158
  scanner->backtick_depth = 0;
151
159
  array_clear(&scanner->closers);
160
+ scanner->backtick_closers = 0;
152
161
  scanner->terminator_owed = false;
153
162
  scanner->after_line_break = false;
154
163
  }
@@ -221,13 +230,22 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
221
230
  at_line_start = false;
222
231
  lexer->mark_end(lexer);
223
232
  // Like bash, compare the delimiter with the line after joining backslash-newlines and
224
- // then stripping the leading tabs of `<<-`. As elsewhere in the grammar, CRLF counts as a
225
- // newline.
233
+ // then stripping the leading tabs of `<<-`.
226
234
  bool consumed = false;
227
235
  bool escaped = false;
228
236
  uint32_t matched = 0;
237
+ // Inside `$( )`, `<( )`, and `>( )`, bash also ends the body at a line that merely starts with
238
+ // the delimiter when a `)` follows anywhere on that line, and reads the rest of the line as
239
+ // code. Such a line ends the content token before it and the end token after the delimiter.
240
+ // Where the backquotes are innermost, bash fixes their extent before it reads the body, so
241
+ // the rule is off.
242
+ uint16_t outside_backquotes = scanner->backtick_depth > 0 ? scanner->backtick_closers : 0;
243
+ bool in_parenthesized =
244
+ scanner->closers.size > outside_backquotes && *array_back(&scanner->closers) == ')';
245
+ bool prefix_marked = false;
246
+ int32_t escaped_after_prefix = 0;
229
247
  // At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
230
- // Bash compares one line at a time, so a delimiter holding a newline (`<<"E\` + CRLF +
248
+ // Bash compares one line at a time, so a delimiter holding a newline (`<<"E` + newline +
231
249
  // `OF"`) never matches.
232
250
  for (;;) {
233
251
  if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
@@ -237,9 +255,11 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
237
255
  advance(lexer);
238
256
  consumed = true;
239
257
  if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
240
- // An escape rather than a line continuation: the escaped character, which is
241
- // the CR itself before a CRLF line ending as in bash, is content.
258
+ // An escape rather than a line continuation: the escaped character is content.
242
259
  if (!lexer->eof(lexer)) {
260
+ if (prefix_marked) {
261
+ escaped_after_prefix = lexer->lookahead;
262
+ }
243
263
  advance(lexer);
244
264
  }
245
265
  escaped = true;
@@ -250,13 +270,18 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
250
270
  lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
251
271
  advance(lexer);
252
272
  matched++;
273
+ if (matched == heredoc->delimiter.size && in_parenthesized) {
274
+ if (did_advance) {
275
+ lexer->result_symbol = HEREDOC_CONTENT;
276
+ return true;
277
+ }
278
+ lexer->mark_end(lexer);
279
+ prefix_marked = true;
280
+ }
253
281
  } else {
254
282
  break;
255
283
  }
256
284
  }
257
- if (!escaped && lexer->lookahead == '\r') {
258
- advance(lexer);
259
- }
260
285
  // Inside a substitution, bash also ends the body at a delimiter directly followed by the
261
286
  // innermost substitution's closer (`)` or `}`) or a closing backquote.
262
287
  bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
@@ -273,6 +298,24 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
273
298
  remove_heredoc(scanner, index);
274
299
  return true;
275
300
  }
301
+ if (prefix_marked) {
302
+ // The line is read after joining its backslash-newlines, as for the delimiter.
303
+ while (!lexer->eof(lexer) && lexer->lookahead != '\n' && lexer->lookahead != ')') {
304
+ bool backslash = lexer->lookahead == '\\' && !heredoc->is_raw;
305
+ advance(lexer);
306
+ if (backslash && lexer->lookahead == '\n') {
307
+ advance(lexer);
308
+ }
309
+ }
310
+ if (escaped_after_prefix == ')' || lexer->lookahead == ')') {
311
+ lexer->result_symbol = HEREDOC_END;
312
+ scanner->terminator_owed = true;
313
+ remove_heredoc(scanner, index);
314
+ } else {
315
+ lexer->result_symbol = HEREDOC_CONTENT;
316
+ }
317
+ return true;
318
+ }
276
319
  // The characters read while trying the delimiter are content; an empty attempt must not
277
320
  // count, or a zero-width content token would repeat forever.
278
321
  did_advance = did_advance || consumed || matched > 0;
@@ -297,13 +340,25 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
297
340
  case '\\':
298
341
  advance(lexer);
299
342
  did_advance = true;
300
- if (!heredoc->is_raw && !lexer->eof(lexer)) {
343
+ // Inside backquotes a backslash also hides a backquote from a quoted body's end.
344
+ if (!lexer->eof(lexer) &&
345
+ (!heredoc->is_raw || (scanner->backtick_depth > 0 && lexer->lookahead == '`'))) {
301
346
  // A backslash-newline joins lines before the delimiter comparison, while a
302
347
  // backslash-CR escapes only the CR, so the LF after it still ends the line.
303
348
  advance(lexer);
304
349
  }
305
350
  break;
306
351
  case '`':
352
+ // Bash ends backquotes at the first unescaped backquote, even inside a heredoc body,
353
+ // and ends the body there.
354
+ if (scanner->backtick_depth > 0) {
355
+ lexer->mark_end(lexer);
356
+ lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
357
+ if (!did_advance) {
358
+ remove_heredoc(scanner, index);
359
+ }
360
+ return true;
361
+ }
307
362
  if (!heredoc->is_raw) {
308
363
  lexer->mark_end(lexer);
309
364
  lexer->result_symbol = HEREDOC_CONTENT;
@@ -452,7 +507,7 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
452
507
  }
453
508
  // Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
454
509
  // the word (`<<EO`true`F`).
455
- if (iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
510
+ if (is_separator(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
456
511
  break;
457
512
  }
458
513
  advance(lexer);
@@ -496,8 +551,8 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
496
551
  return true;
497
552
  }
498
553
 
499
- static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
500
- advance(lexer);
554
+ // Scans `<<` or `<<-` after its first `<` has been read.
555
+ static bool scan_heredoc_arrow_after_first(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
501
556
  if (lexer->lookahead != '<') {
502
557
  return false;
503
558
  }
@@ -529,7 +584,12 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
529
584
  return true;
530
585
  }
531
586
 
532
- // The right side of `=~` extends to unquoted whitespace outside parentheses.
587
+ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
588
+ advance(lexer);
589
+ return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
590
+ }
591
+
592
+ // The right side of `=~` extends to an unquoted space, tab, or newline outside parentheses.
533
593
  static bool scan_regex(TSLexer *lexer) {
534
594
  while (is_blank(lexer->lookahead)) {
535
595
  skip(lexer);
@@ -539,7 +599,7 @@ static bool scan_regex(TSLexer *lexer) {
539
599
  while (!lexer->eof(lexer)) {
540
600
  lexer->mark_end(lexer);
541
601
  int32_t c = lexer->lookahead;
542
- if (depth == 0 && iswspace(c)) {
602
+ if (depth == 0 && is_separator(c)) {
543
603
  break;
544
604
  }
545
605
  // Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
@@ -600,7 +660,7 @@ static bool scan_string_content(TSLexer *lexer) {
600
660
  }
601
661
 
602
662
  static inline bool is_word_break(int32_t c) {
603
- return iswspace(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
663
+ return is_separator(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
604
664
  }
605
665
 
606
666
  static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
@@ -802,7 +862,7 @@ static bool at_closing_reserved_word(TSLexer *lexer) {
802
862
  }
803
863
  }
804
864
  if (length == 0 || length == sizeof(word) ||
805
- !(lexer->eof(lexer) || iswspace(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
865
+ !(lexer->eof(lexer) || is_separator(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
806
866
  return false;
807
867
  }
808
868
  for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
@@ -825,7 +885,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
825
885
 
826
886
  if (scanner->terminator_owed) {
827
887
  scanner->terminator_owed = false;
828
- if (valid_symbols[NEWLINE] && lexer->lookahead == '}') {
888
+ if (valid_symbols[NEWLINE]) {
829
889
  lexer->result_symbol = NEWLINE;
830
890
  return true;
831
891
  }
@@ -852,7 +912,8 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
852
912
  if (scanner->heredocs.size > 0) {
853
913
  int32_t active = active_heredoc_index(scanner);
854
914
  if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
855
- !(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw)) {
915
+ !(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw &&
916
+ scanner->backtick_depth == 0)) {
856
917
  return scan_heredoc_content(scanner, lexer, (uint32_t)active);
857
918
  }
858
919
  // A newline inside a string is part of it, so bodies start only outside strings.
@@ -891,7 +952,23 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
891
952
  // next line. The column costs a scan back to the line start, so it is computed only right after
892
953
  // such a continuation and only where a concatenation could start; a concatenation then clears the
893
954
  // flag, which bounds the scans to one per continuation.
894
- if (valid_symbols[CONCAT] && !at_eof && !iswspace(first) && !is_metacharacter(first) &&
955
+ // Like bash, `a<(cmd)` is one word. `<` and `>` end a word part otherwise, so the concatenation
956
+ // needs a look at the next character, and a `<<` read along the way is scanned as a heredoc arrow.
957
+ if (valid_symbols[CONCAT] && (first == '<' || first == '>') &&
958
+ !(after_line_break && lexer->get_column(lexer) == 0)) {
959
+ lexer->mark_end(lexer);
960
+ advance(lexer);
961
+ if (lexer->lookahead == '(') {
962
+ lexer->result_symbol = CONCAT;
963
+ return true;
964
+ }
965
+ if (first == '<' && (valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH])) {
966
+ return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
967
+ }
968
+ return false;
969
+ }
970
+
971
+ if (valid_symbols[CONCAT] && !at_eof && !is_separator(first) && !is_metacharacter(first) &&
895
972
  !(after_line_break && lexer->get_column(lexer) == 0)) {
896
973
  int32_t c = first;
897
974
  if (c == '\\' && lexer->lookahead == '\\') {
@@ -933,7 +1010,16 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
933
1010
  }
934
1011
  // The word continues only if the next line does; otherwise the continuations themselves are
935
1012
  // the token, so that whatever follows them is lexed as it would be without them.
936
- if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
1013
+ if ((c == '<' || c == '>')) {
1014
+ advance(lexer);
1015
+ if (lexer->lookahead == '(') {
1016
+ lexer->result_symbol = CONCAT;
1017
+ return true;
1018
+ }
1019
+ scanner->after_line_break = true;
1020
+ lexer->result_symbol = LINE_CONTINUATION;
1021
+ } else if (lexer->eof(lexer) || is_separator(c) || is_metacharacter(c) ||
1022
+ (c == '`' && scanner->backtick_depth > 0)) {
937
1023
  scanner->after_line_break = true;
938
1024
  lexer->result_symbol = LINE_CONTINUATION;
939
1025
  } else {
@@ -960,7 +1046,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
960
1046
  lexer->result_symbol = EMPTY_VALUE;
961
1047
  return true;
962
1048
  }
963
- if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
1049
+ if (at_eof || is_separator(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
964
1050
  (c == '`' && scanner->backtick_depth > 0)) {
965
1051
  lexer->result_symbol = EMPTY_VALUE;
966
1052
  return true;
@@ -1063,6 +1149,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1063
1149
  advance(lexer);
1064
1150
  lexer->mark_end(lexer);
1065
1151
  scanner->backtick_depth++;
1152
+ scanner->backtick_closers = (uint16_t)scanner->closers.size;
1066
1153
  lexer->result_symbol = BACKTICK_OPEN;
1067
1154
  return true;
1068
1155
  }
@@ -1081,7 +1168,9 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1081
1168
  lexer->mark_end(lexer);
1082
1169
  lexer->result_symbol = FILE_DESCRIPTOR;
1083
1170
  if (lexer->lookahead == '<' || lexer->lookahead == '>') {
1084
- return true;
1171
+ // Like bash, digits before `<(` or `>(` are a word joined with the process substitution.
1172
+ advance(lexer);
1173
+ return lexer->lookahead != '(';
1085
1174
  }
1086
1175
  return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
1087
1176
  }
@@ -1138,6 +1227,8 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1138
1227
  Scanner *scanner = (Scanner *)payload;
1139
1228
  unsigned size = 0;
1140
1229
  buffer[size++] = (char)scanner->backtick_depth;
1230
+ memcpy(&buffer[size], &scanner->backtick_closers, sizeof(scanner->backtick_closers));
1231
+ size += sizeof(scanner->backtick_closers);
1141
1232
  buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1);
1142
1233
  uint16_t closer_count = (uint16_t)scanner->closers.size;
1143
1234
  memcpy(&buffer[size], &closer_count, sizeof(closer_count));
@@ -1167,11 +1258,13 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1167
1258
  void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
1168
1259
  Scanner *scanner = (Scanner *)payload;
1169
1260
  scanner_reset(scanner);
1170
- if (length < 2 + sizeof(uint16_t)) {
1261
+ if (length < 2 + 2 * sizeof(uint16_t)) {
1171
1262
  return;
1172
1263
  }
1173
1264
  unsigned size = 0;
1174
1265
  scanner->backtick_depth = (uint8_t)buffer[size++];
1266
+ memcpy(&scanner->backtick_closers, &buffer[size], sizeof(scanner->backtick_closers));
1267
+ size += sizeof(scanner->backtick_closers);
1175
1268
  scanner->terminator_owed = buffer[size] & 1;
1176
1269
  scanner->after_line_break = (buffer[size++] & 2) != 0;
1177
1270
  uint16_t closer_count;
Binary file
package/tree-sitter.json CHANGED
@@ -18,7 +18,7 @@
18
18
  }
19
19
  ],
20
20
  "metadata": {
21
- "version": "1.1.1",
21
+ "version": "1.1.2",
22
22
  "license": "MIT",
23
23
  "description": "Bash grammar for tree-sitter",
24
24
  "authors": [