@willbooster/tree-sitter-bash 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -57,6 +57,9 @@ typedef struct {
57
57
  // A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
58
58
  // line began the body, so a terminator is still owed before the `}`.
59
59
  bool terminator_owed;
60
+ // The last token of this scanner was a line continuation that ended a line; nothing concatenates at
61
+ // the start of the next one.
62
+ bool after_line_break;
60
63
  } Scanner;
61
64
 
62
65
  static inline uint16_t current_depth(Scanner *scanner) {
@@ -120,7 +123,7 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
120
123
  return 0;
121
124
  }
122
125
 
123
- // Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
126
+ // Serialized layout: backtick depth, the owed-terminator and line-break flags, the closers of the open substitutions
124
127
  // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
125
128
  // delimiter length (uint16), and the delimiter as varints.
126
129
  #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
@@ -147,6 +150,7 @@ static void scanner_reset(Scanner *scanner) {
147
150
  scanner->backtick_depth = 0;
148
151
  array_clear(&scanner->closers);
149
152
  scanner->terminator_owed = false;
153
+ scanner->after_line_break = false;
150
154
  }
151
155
 
152
156
  // The body being read: the most deeply nested started heredoc.
@@ -216,21 +220,43 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
216
220
  if (at_line_start) {
217
221
  at_line_start = false;
218
222
  lexer->mark_end(lexer);
219
- bool consumed_tabs = false;
220
- if (heredoc->allows_indent) {
221
- while (lexer->lookahead == '\t') {
222
- advance(lexer);
223
- consumed_tabs = true;
224
- }
225
- }
223
+ // Like bash, compare the delimiter with the line after joining backslash-newlines and
224
+ // then stripping the leading tabs of `<<-`. As elsewhere in the grammar, CRLF counts as a
225
+ // newline.
226
+ bool consumed = false;
227
+ bool escaped = false;
226
228
  uint32_t matched = 0;
227
229
  // At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
228
- while (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
229
- lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
230
- advance(lexer);
231
- matched++;
230
+ for (;;) {
231
+ if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
232
+ advance(lexer);
233
+ consumed = true;
234
+ } else if (lexer->lookahead == '\\' && !heredoc->is_raw) {
235
+ advance(lexer);
236
+ consumed = true;
237
+ bool escaped_cr = lexer->lookahead == '\r';
238
+ if (escaped_cr) {
239
+ advance(lexer);
240
+ }
241
+ if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
242
+ // An escape rather than a line continuation: the escaped character, which is
243
+ // the CR itself when no LF follows it, is content.
244
+ if (!escaped_cr && !lexer->eof(lexer)) {
245
+ advance(lexer);
246
+ }
247
+ escaped = true;
248
+ break;
249
+ }
250
+ advance(lexer);
251
+ } else if (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
252
+ lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
253
+ advance(lexer);
254
+ matched++;
255
+ } else {
256
+ break;
257
+ }
232
258
  }
233
- if (lexer->lookahead == '\r') {
259
+ if (!escaped && lexer->lookahead == '\r') {
234
260
  advance(lexer);
235
261
  }
236
262
  // Inside a substitution, bash also ends the body at a delimiter directly followed by the
@@ -238,7 +264,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
238
264
  bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
239
265
  (scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
240
266
  (lexer->lookahead == '`' && scanner->backtick_depth > 0);
241
- if (matched == heredoc->delimiter.size && ends_line) {
267
+ if (!escaped && matched == heredoc->delimiter.size && ends_line) {
242
268
  if (did_advance) {
243
269
  lexer->result_symbol = HEREDOC_CONTENT;
244
270
  return true;
@@ -251,7 +277,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
251
277
  }
252
278
  // The characters read while trying the delimiter are content; an empty attempt must not
253
279
  // count, or a zero-width content token would repeat forever.
254
- did_advance = did_advance || consumed_tabs || matched > 0;
280
+ did_advance = did_advance || consumed || matched > 0;
255
281
  continue;
256
282
  }
257
283
 
@@ -274,11 +300,14 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
274
300
  advance(lexer);
275
301
  did_advance = true;
276
302
  if (!heredoc->is_raw && !lexer->eof(lexer)) {
277
- // A backslash-newline joins lines before the delimiter comparison.
303
+ // A backslash-newline joins lines before the delimiter comparison. A CR without an
304
+ // LF after it is the escaped character itself.
278
305
  if (lexer->lookahead == '\r') {
279
306
  advance(lexer);
280
- }
281
- if (!lexer->eof(lexer)) {
307
+ if (lexer->lookahead == '\n') {
308
+ advance(lexer);
309
+ }
310
+ } else if (!lexer->eof(lexer)) {
282
311
  advance(lexer);
283
312
  }
284
313
  }
@@ -774,12 +803,52 @@ static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbol
774
803
  return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
775
804
  }
776
805
 
806
+ // Marks the current position and reports whether a reserved word that ends or continues a compound
807
+ // command (`then`, `do`, `done`, `fi`, `esac`, `else`, `elif`, `}`) starts here.
808
+ static bool at_closing_reserved_word(TSLexer *lexer) {
809
+ static const char *const words[] = {"then", "do", "done", "fi", "esac", "else", "elif", "}"};
810
+ lexer->mark_end(lexer);
811
+ char word[5];
812
+ uint32_t length = 0;
813
+ for (;;) {
814
+ if (lexer->lookahead == '\\') {
815
+ // Bash joins a backslash-newline before it checks where the word ends (`then\` + newline).
816
+ advance(lexer);
817
+ if (lexer->lookahead == '\r') {
818
+ advance(lexer);
819
+ }
820
+ if (lexer->lookahead != '\n') {
821
+ return false;
822
+ }
823
+ advance(lexer);
824
+ } else if (length < sizeof(word) && (iswlower(lexer->lookahead) || lexer->lookahead == '}')) {
825
+ word[length++] = (char)lexer->lookahead;
826
+ advance(lexer);
827
+ } else {
828
+ break;
829
+ }
830
+ }
831
+ if (length == 0 || length == sizeof(word) ||
832
+ !(lexer->eof(lexer) || iswspace(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
833
+ return false;
834
+ }
835
+ for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
836
+ if (strlen(words[i]) == length && strncmp(words[i], word, length) == 0) {
837
+ return true;
838
+ }
839
+ }
840
+ return false;
841
+ }
842
+
777
843
  static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
778
844
  bool error_recovery = valid_symbols[ERROR_RECOVERY];
779
845
  // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
780
846
  // by the character right after it, even when blanks are skipped below to find a newline.
781
847
  int32_t first = lexer->lookahead;
782
848
  bool at_eof = lexer->eof(lexer);
849
+ // Kept only if this call returns a token, so the flag lasts until the scanner's next token.
850
+ bool after_line_break = scanner->after_line_break;
851
+ scanner->after_line_break = false;
783
852
 
784
853
  if (scanner->terminator_owed) {
785
854
  scanner->terminator_owed = false;
@@ -845,7 +914,12 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
845
914
  return scan_heredoc_start(scanner, lexer);
846
915
  }
847
916
 
848
- if (valid_symbols[CONCAT]) {
917
+ // A line continuation after a blank separates words, so nothing concatenates at the start of the
918
+ // next line. The column costs a scan back to the line start, so it is computed only right after
919
+ // such a continuation and only where a concatenation could start; a concatenation then clears the
920
+ // flag, which bounds the scans to one per continuation.
921
+ if (valid_symbols[CONCAT] && !at_eof && !iswspace(first) && !is_metacharacter(first) &&
922
+ !(after_line_break && lexer->get_column(lexer) == 0)) {
849
923
  int32_t c = first;
850
924
  if (c == '\\' && lexer->lookahead == '\\') {
851
925
  // A backslash-newline joins lines, so the word continues only if the next line does.
@@ -866,27 +940,41 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
866
940
  }
867
941
  advance(lexer);
868
942
  lexer->mark_end(lexer);
869
- c = lexer->lookahead;
870
- // The word continues only if the next line does; otherwise the continuation itself is the
871
- // token, so that whatever follows it is lexed as it would be without it.
872
- lexer->result_symbol = LINE_CONTINUATION;
873
- if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
874
- return true;
875
- }
876
- if (c == '\\') {
877
- // Another continuation is whitespace too; an escaped character continues the word.
943
+ // Bash removes every backslash-newline before it splits words, so further continuations
944
+ // right after this one are part of the token (`a\` + `\` + `b` is `ab`).
945
+ for (;;) {
946
+ c = lexer->lookahead;
947
+ if (c != '\\') {
948
+ break;
949
+ }
878
950
  advance(lexer);
879
951
  if (lexer->lookahead == '\r') {
880
952
  advance(lexer);
881
953
  }
882
- if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
954
+ if (lexer->eof(lexer)) {
955
+ lexer->mark_end(lexer);
956
+ lexer->result_symbol = LINE_CONTINUATION;
957
+ return true;
958
+ }
959
+ if (lexer->lookahead != '\n') {
960
+ // An escaped character continues the word.
961
+ lexer->result_symbol = CONCAT;
883
962
  return true;
884
963
  }
964
+ advance(lexer);
965
+ lexer->mark_end(lexer);
966
+ }
967
+ // The word continues only if the next line does; otherwise the continuations themselves are
968
+ // the token, so that whatever follows them is lexed as it would be without them.
969
+ if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
970
+ scanner->after_line_break = true;
971
+ lexer->result_symbol = LINE_CONTINUATION;
972
+ } else {
973
+ lexer->result_symbol = CONCAT;
885
974
  }
886
- lexer->result_symbol = CONCAT;
887
975
  return true;
888
976
  }
889
- if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
977
+ if (c != '`' || scanner->backtick_depth == 0) {
890
978
  lexer->result_symbol = CONCAT;
891
979
  return true;
892
980
  }
@@ -939,6 +1027,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
939
1027
  if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
940
1028
  advance(lexer);
941
1029
  lexer->mark_end(lexer);
1030
+ scanner->after_line_break = true;
942
1031
  lexer->result_symbol = LINE_CONTINUATION;
943
1032
  return true;
944
1033
  }
@@ -966,6 +1055,37 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
966
1055
  return true;
967
1056
  }
968
1057
 
1058
+ // Bash ends backquotes at the first unescaped backquote, even inside single quotes (`echo '`),
1059
+ // and then reports the unterminated quote when it runs the substitution. So a quote that the next
1060
+ // backquote cuts short closes the substitution there when it may close, and otherwise is skipped
1061
+ // up to the backquote, which then cannot be parsed; the quote never hides the text after it.
1062
+ if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
1063
+ advance(lexer);
1064
+ while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`') {
1065
+ // A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
1066
+ // quote itself stays literal, so `'\'` still ends at its second quote.
1067
+ bool backslash = lexer->lookahead == '\\';
1068
+ advance(lexer);
1069
+ if (backslash && (lexer->lookahead == '`' || lexer->lookahead == '\\')) {
1070
+ advance(lexer);
1071
+ }
1072
+ }
1073
+ if (lexer->lookahead != '`') {
1074
+ return false;
1075
+ }
1076
+ if (valid_symbols[BACKTICK_CLOSE]) {
1077
+ advance(lexer);
1078
+ lexer->mark_end(lexer);
1079
+ scanner->backtick_depth--;
1080
+ discard_unstarted_heredocs(scanner);
1081
+ lexer->result_symbol = BACKTICK_CLOSE;
1082
+ return true;
1083
+ }
1084
+ lexer->mark_end(lexer);
1085
+ lexer->result_symbol = LINE_CONTINUATION;
1086
+ return true;
1087
+ }
1088
+
969
1089
  if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
970
1090
  if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
971
1091
  advance(lexer);
@@ -1019,6 +1139,15 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1019
1139
  return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
1020
1140
  }
1021
1141
 
1142
+ // Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
1143
+ // `{ (y) }`), so a zero-width terminator is returned before one there. After a word, the reserved
1144
+ // word would be an argument instead; a possible concatenation shows that a word just ended. This
1145
+ // check comes last because it consumes input even when it finds no reserved word.
1146
+ if (valid_symbols[NEWLINE] && !valid_symbols[CONCAT] && at_closing_reserved_word(lexer)) {
1147
+ lexer->result_symbol = NEWLINE;
1148
+ return true;
1149
+ }
1150
+
1022
1151
  return false;
1023
1152
  }
1024
1153
 
@@ -1045,7 +1174,7 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1045
1174
  Scanner *scanner = (Scanner *)payload;
1046
1175
  unsigned size = 0;
1047
1176
  buffer[size++] = (char)scanner->backtick_depth;
1048
- buffer[size++] = (char)scanner->terminator_owed;
1177
+ buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1);
1049
1178
  uint16_t closer_count = (uint16_t)scanner->closers.size;
1050
1179
  memcpy(&buffer[size], &closer_count, sizeof(closer_count));
1051
1180
  size += sizeof(closer_count);
@@ -1079,7 +1208,8 @@ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *bu
1079
1208
  }
1080
1209
  unsigned size = 0;
1081
1210
  scanner->backtick_depth = (uint8_t)buffer[size++];
1082
- scanner->terminator_owed = buffer[size++];
1211
+ scanner->terminator_owed = buffer[size] & 1;
1212
+ scanner->after_line_break = (buffer[size++] & 2) != 0;
1083
1213
  uint16_t closer_count;
1084
1214
  memcpy(&closer_count, &buffer[size], sizeof(closer_count));
1085
1215
  size += sizeof(closer_count);
package/tree-sitter.json CHANGED
@@ -18,7 +18,7 @@
18
18
  }
19
19
  ],
20
20
  "metadata": {
21
- "version": "1.0.0",
21
+ "version": "1.0.2",
22
22
  "license": "MIT",
23
23
  "description": "Bash grammar for tree-sitter",
24
24
  "authors": [
@@ -40,7 +40,7 @@
40
40
  "go": true,
41
41
  "node": true,
42
42
  "python": true,
43
- "rust": true,
43
+ "rust": false,
44
44
  "swift": true
45
45
  }
46
46
  }