@willbooster/tree-sitter-bash 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c CHANGED
@@ -57,6 +57,9 @@ typedef struct {
57
57
  // A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
58
58
  // line began the body, so a terminator is still owed before the `}`.
59
59
  bool terminator_owed;
60
+ // The last token of this scanner was a line continuation that ended a line; nothing concatenates at
61
+ // the start of the next one.
62
+ bool after_line_break;
60
63
  } Scanner;
61
64
 
62
65
  static inline uint16_t current_depth(Scanner *scanner) {
@@ -120,7 +123,7 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
120
123
  return 0;
121
124
  }
122
125
 
123
- // Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
126
+ // Serialized layout: backtick depth, the owed-terminator and line-break flags, the closers of the open substitutions
124
127
  // (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
125
128
  // delimiter length (uint16), and the delimiter as varints.
126
129
  #define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
@@ -147,6 +150,7 @@ static void scanner_reset(Scanner *scanner) {
147
150
  scanner->backtick_depth = 0;
148
151
  array_clear(&scanner->closers);
149
152
  scanner->terminator_owed = false;
153
+ scanner->after_line_break = false;
150
154
  }
151
155
 
152
156
  // The body being read: the most deeply nested started heredoc.
@@ -799,12 +803,52 @@ static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbol
799
803
  return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
800
804
  }
801
805
 
806
+ // Marks the current position and reports whether a reserved word that ends or continues a compound
807
+ // command (`then`, `do`, `done`, `fi`, `esac`, `else`, `elif`, `}`) starts here.
808
+ static bool at_closing_reserved_word(TSLexer *lexer) {
809
+ static const char *const words[] = {"then", "do", "done", "fi", "esac", "else", "elif", "}"};
810
+ lexer->mark_end(lexer);
811
+ char word[5];
812
+ uint32_t length = 0;
813
+ for (;;) {
814
+ if (lexer->lookahead == '\\') {
815
+ // Bash joins a backslash-newline before it checks where the word ends (`then\` + newline).
816
+ advance(lexer);
817
+ if (lexer->lookahead == '\r') {
818
+ advance(lexer);
819
+ }
820
+ if (lexer->lookahead != '\n') {
821
+ return false;
822
+ }
823
+ advance(lexer);
824
+ } else if (length < sizeof(word) && (iswlower(lexer->lookahead) || lexer->lookahead == '}')) {
825
+ word[length++] = (char)lexer->lookahead;
826
+ advance(lexer);
827
+ } else {
828
+ break;
829
+ }
830
+ }
831
+ if (length == 0 || length == sizeof(word) ||
832
+ !(lexer->eof(lexer) || iswspace(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
833
+ return false;
834
+ }
835
+ for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
836
+ if (strlen(words[i]) == length && strncmp(words[i], word, length) == 0) {
837
+ return true;
838
+ }
839
+ }
840
+ return false;
841
+ }
842
+
802
843
  static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
803
844
  bool error_recovery = valid_symbols[ERROR_RECOVERY];
804
845
  // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
805
846
  // by the character right after it, even when blanks are skipped below to find a newline.
806
847
  int32_t first = lexer->lookahead;
807
848
  bool at_eof = lexer->eof(lexer);
849
+ // Kept only if this call returns a token, so the flag lasts until the scanner's next token.
850
+ bool after_line_break = scanner->after_line_break;
851
+ scanner->after_line_break = false;
808
852
 
809
853
  if (scanner->terminator_owed) {
810
854
  scanner->terminator_owed = false;
@@ -870,7 +914,12 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
870
914
  return scan_heredoc_start(scanner, lexer);
871
915
  }
872
916
 
873
- if (valid_symbols[CONCAT]) {
917
+ // A line continuation after a blank separates words, so nothing concatenates at the start of the
918
+ // next line. The column costs a scan back to the line start, so it is computed only right after
919
+ // such a continuation and only where a concatenation could start; a concatenation then clears the
920
+ // flag, which bounds the scans to one per continuation.
921
+ if (valid_symbols[CONCAT] && !at_eof && !iswspace(first) && !is_metacharacter(first) &&
922
+ !(after_line_break && lexer->get_column(lexer) == 0)) {
874
923
  int32_t c = first;
875
924
  if (c == '\\' && lexer->lookahead == '\\') {
876
925
  // A backslash-newline joins lines, so the word continues only if the next line does.
@@ -891,27 +940,41 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
891
940
  }
892
941
  advance(lexer);
893
942
  lexer->mark_end(lexer);
894
- c = lexer->lookahead;
895
- // The word continues only if the next line does; otherwise the continuation itself is the
896
- // token, so that whatever follows it is lexed as it would be without it.
897
- lexer->result_symbol = LINE_CONTINUATION;
898
- if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
899
- return true;
900
- }
901
- if (c == '\\') {
902
- // Another continuation is whitespace too; an escaped character continues the word.
943
+ // Bash removes every backslash-newline before it splits words, so further continuations
944
+ // right after this one are part of the token (`a\` + `\` + `b` is `ab`).
945
+ for (;;) {
946
+ c = lexer->lookahead;
947
+ if (c != '\\') {
948
+ break;
949
+ }
903
950
  advance(lexer);
904
951
  if (lexer->lookahead == '\r') {
905
952
  advance(lexer);
906
953
  }
907
- if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
954
+ if (lexer->eof(lexer)) {
955
+ lexer->mark_end(lexer);
956
+ lexer->result_symbol = LINE_CONTINUATION;
957
+ return true;
958
+ }
959
+ if (lexer->lookahead != '\n') {
960
+ // An escaped character continues the word.
961
+ lexer->result_symbol = CONCAT;
908
962
  return true;
909
963
  }
964
+ advance(lexer);
965
+ lexer->mark_end(lexer);
966
+ }
967
+ // The word continues only if the next line does; otherwise the continuations themselves are
968
+ // the token, so that whatever follows them is lexed as it would be without them.
969
+ if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
970
+ scanner->after_line_break = true;
971
+ lexer->result_symbol = LINE_CONTINUATION;
972
+ } else {
973
+ lexer->result_symbol = CONCAT;
910
974
  }
911
- lexer->result_symbol = CONCAT;
912
975
  return true;
913
976
  }
914
- if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
977
+ if (c != '`' || scanner->backtick_depth == 0) {
915
978
  lexer->result_symbol = CONCAT;
916
979
  return true;
917
980
  }
@@ -964,6 +1027,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
964
1027
  if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
965
1028
  advance(lexer);
966
1029
  lexer->mark_end(lexer);
1030
+ scanner->after_line_break = true;
967
1031
  lexer->result_symbol = LINE_CONTINUATION;
968
1032
  return true;
969
1033
  }
@@ -991,6 +1055,37 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
991
1055
  return true;
992
1056
  }
993
1057
 
1058
+ // Bash ends backquotes at the first unescaped backquote, even inside single quotes (`echo '`),
1059
+ // and then reports the unterminated quote when it runs the substitution. So a quote that the next
1060
+ // backquote cuts short closes the substitution there when it may close, and otherwise is skipped
1061
+ // up to the backquote, which then cannot be parsed; the quote never hides the text after it.
1062
+ if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
1063
+ advance(lexer);
1064
+ while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`') {
1065
+ // A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
1066
+ // quote itself stays literal, so `'\'` still ends at its second quote.
1067
+ bool backslash = lexer->lookahead == '\\';
1068
+ advance(lexer);
1069
+ if (backslash && (lexer->lookahead == '`' || lexer->lookahead == '\\')) {
1070
+ advance(lexer);
1071
+ }
1072
+ }
1073
+ if (lexer->lookahead != '`') {
1074
+ return false;
1075
+ }
1076
+ if (valid_symbols[BACKTICK_CLOSE]) {
1077
+ advance(lexer);
1078
+ lexer->mark_end(lexer);
1079
+ scanner->backtick_depth--;
1080
+ discard_unstarted_heredocs(scanner);
1081
+ lexer->result_symbol = BACKTICK_CLOSE;
1082
+ return true;
1083
+ }
1084
+ lexer->mark_end(lexer);
1085
+ lexer->result_symbol = LINE_CONTINUATION;
1086
+ return true;
1087
+ }
1088
+
994
1089
  if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
995
1090
  if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
996
1091
  advance(lexer);
@@ -1044,6 +1139,15 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1044
1139
  return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
1045
1140
  }
1046
1141
 
1142
+ // Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
1143
+ // `{ (y) }`), so a zero-width terminator is returned before one there. After a word, the reserved
1144
+ // word would be an argument instead; a possible concatenation shows that a word just ended. This
1145
+ // check comes last because it consumes input even when it finds no reserved word.
1146
+ if (valid_symbols[NEWLINE] && !valid_symbols[CONCAT] && at_closing_reserved_word(lexer)) {
1147
+ lexer->result_symbol = NEWLINE;
1148
+ return true;
1149
+ }
1150
+
1047
1151
  return false;
1048
1152
  }
1049
1153
 
@@ -1070,7 +1174,7 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
1070
1174
  Scanner *scanner = (Scanner *)payload;
1071
1175
  unsigned size = 0;
1072
1176
  buffer[size++] = (char)scanner->backtick_depth;
1073
- buffer[size++] = (char)scanner->terminator_owed;
1177
+ buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1);
1074
1178
  uint16_t closer_count = (uint16_t)scanner->closers.size;
1075
1179
  memcpy(&buffer[size], &closer_count, sizeof(closer_count));
1076
1180
  size += sizeof(closer_count);
@@ -1104,7 +1208,8 @@ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *bu
1104
1208
  }
1105
1209
  unsigned size = 0;
1106
1210
  scanner->backtick_depth = (uint8_t)buffer[size++];
1107
- scanner->terminator_owed = buffer[size++];
1211
+ scanner->terminator_owed = buffer[size] & 1;
1212
+ scanner->after_line_break = (buffer[size++] & 2) != 0;
1108
1213
  uint16_t closer_count;
1109
1214
  memcpy(&closer_count, &buffer[size], sizeof(closer_count));
1110
1215
  size += sizeof(closer_count);
package/tree-sitter.json CHANGED
@@ -18,7 +18,7 @@
18
18
  }
19
19
  ],
20
20
  "metadata": {
21
- "version": "1.0.1",
21
+ "version": "1.0.2",
22
22
  "license": "MIT",
23
23
  "description": "Bash grammar for tree-sitter",
24
24
  "authors": [