@willbooster/tree-sitter-bash 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -10
- package/grammar.js +8 -2
- package/package.json +32 -35
- package/prebuilds/darwin-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/darwin-x64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-x64/tree-sitter-bash.node +0 -0
- package/src/grammar.json +7 -2
- package/src/parser.c +109530 -134537
- package/src/scanner.c +163 -33
- package/tree-sitter.json +2 -2
package/src/scanner.c
CHANGED
|
@@ -57,6 +57,9 @@ typedef struct {
|
|
|
57
57
|
// A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
|
|
58
58
|
// line began the body, so a terminator is still owed before the `}`.
|
|
59
59
|
bool terminator_owed;
|
|
60
|
+
// The last token of this scanner was a line continuation that ended a line; nothing concatenates at
|
|
61
|
+
// the start of the next one.
|
|
62
|
+
bool after_line_break;
|
|
60
63
|
} Scanner;
|
|
61
64
|
|
|
62
65
|
static inline uint16_t current_depth(Scanner *scanner) {
|
|
@@ -120,7 +123,7 @@ static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
|
|
|
120
123
|
return 0;
|
|
121
124
|
}
|
|
122
125
|
|
|
123
|
-
// Serialized layout: backtick depth, the owed-terminator
|
|
126
|
+
// Serialized layout: backtick depth, the owed-terminator and line-break flags, the closers of the open substitutions
|
|
124
127
|
// (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
|
|
125
128
|
// delimiter length (uint16), and the delimiter as varints.
|
|
126
129
|
#define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
|
|
@@ -147,6 +150,7 @@ static void scanner_reset(Scanner *scanner) {
|
|
|
147
150
|
scanner->backtick_depth = 0;
|
|
148
151
|
array_clear(&scanner->closers);
|
|
149
152
|
scanner->terminator_owed = false;
|
|
153
|
+
scanner->after_line_break = false;
|
|
150
154
|
}
|
|
151
155
|
|
|
152
156
|
// The body being read: the most deeply nested started heredoc.
|
|
@@ -216,21 +220,43 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
216
220
|
if (at_line_start) {
|
|
217
221
|
at_line_start = false;
|
|
218
222
|
lexer->mark_end(lexer);
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
}
|
|
225
|
-
}
|
|
223
|
+
// Like bash, compare the delimiter with the line after joining backslash-newlines and
|
|
224
|
+
// then stripping the leading tabs of `<<-`. As elsewhere in the grammar, CRLF counts as a
|
|
225
|
+
// newline.
|
|
226
|
+
bool consumed = false;
|
|
227
|
+
bool escaped = false;
|
|
226
228
|
uint32_t matched = 0;
|
|
227
229
|
// At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
230
|
+
for (;;) {
|
|
231
|
+
if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
|
|
232
|
+
advance(lexer);
|
|
233
|
+
consumed = true;
|
|
234
|
+
} else if (lexer->lookahead == '\\' && !heredoc->is_raw) {
|
|
235
|
+
advance(lexer);
|
|
236
|
+
consumed = true;
|
|
237
|
+
bool escaped_cr = lexer->lookahead == '\r';
|
|
238
|
+
if (escaped_cr) {
|
|
239
|
+
advance(lexer);
|
|
240
|
+
}
|
|
241
|
+
if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
|
|
242
|
+
// An escape rather than a line continuation: the escaped character, which is
|
|
243
|
+
// the CR itself when no LF follows it, is content.
|
|
244
|
+
if (!escaped_cr && !lexer->eof(lexer)) {
|
|
245
|
+
advance(lexer);
|
|
246
|
+
}
|
|
247
|
+
escaped = true;
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
advance(lexer);
|
|
251
|
+
} else if (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
|
|
252
|
+
lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
|
|
253
|
+
advance(lexer);
|
|
254
|
+
matched++;
|
|
255
|
+
} else {
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
232
258
|
}
|
|
233
|
-
if (lexer->lookahead == '\r') {
|
|
259
|
+
if (!escaped && lexer->lookahead == '\r') {
|
|
234
260
|
advance(lexer);
|
|
235
261
|
}
|
|
236
262
|
// Inside a substitution, bash also ends the body at a delimiter directly followed by the
|
|
@@ -238,7 +264,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
238
264
|
bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
|
|
239
265
|
(scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
|
|
240
266
|
(lexer->lookahead == '`' && scanner->backtick_depth > 0);
|
|
241
|
-
if (matched == heredoc->delimiter.size && ends_line) {
|
|
267
|
+
if (!escaped && matched == heredoc->delimiter.size && ends_line) {
|
|
242
268
|
if (did_advance) {
|
|
243
269
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
244
270
|
return true;
|
|
@@ -251,7 +277,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
251
277
|
}
|
|
252
278
|
// The characters read while trying the delimiter are content; an empty attempt must not
|
|
253
279
|
// count, or a zero-width content token would repeat forever.
|
|
254
|
-
did_advance = did_advance ||
|
|
280
|
+
did_advance = did_advance || consumed || matched > 0;
|
|
255
281
|
continue;
|
|
256
282
|
}
|
|
257
283
|
|
|
@@ -274,11 +300,14 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t inde
|
|
|
274
300
|
advance(lexer);
|
|
275
301
|
did_advance = true;
|
|
276
302
|
if (!heredoc->is_raw && !lexer->eof(lexer)) {
|
|
277
|
-
// A backslash-newline joins lines before the delimiter comparison.
|
|
303
|
+
// A backslash-newline joins lines before the delimiter comparison. A CR without an
|
|
304
|
+
// LF after it is the escaped character itself.
|
|
278
305
|
if (lexer->lookahead == '\r') {
|
|
279
306
|
advance(lexer);
|
|
280
|
-
|
|
281
|
-
|
|
307
|
+
if (lexer->lookahead == '\n') {
|
|
308
|
+
advance(lexer);
|
|
309
|
+
}
|
|
310
|
+
} else if (!lexer->eof(lexer)) {
|
|
282
311
|
advance(lexer);
|
|
283
312
|
}
|
|
284
313
|
}
|
|
@@ -774,12 +803,52 @@ static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbol
|
|
|
774
803
|
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
775
804
|
}
|
|
776
805
|
|
|
806
|
+
// Marks the current position and reports whether a reserved word that ends or continues a compound
|
|
807
|
+
// command (`then`, `do`, `done`, `fi`, `esac`, `else`, `elif`, `}`) starts here.
|
|
808
|
+
static bool at_closing_reserved_word(TSLexer *lexer) {
|
|
809
|
+
static const char *const words[] = {"then", "do", "done", "fi", "esac", "else", "elif", "}"};
|
|
810
|
+
lexer->mark_end(lexer);
|
|
811
|
+
char word[5];
|
|
812
|
+
uint32_t length = 0;
|
|
813
|
+
for (;;) {
|
|
814
|
+
if (lexer->lookahead == '\\') {
|
|
815
|
+
// Bash joins a backslash-newline before it checks where the word ends (`then\` + newline).
|
|
816
|
+
advance(lexer);
|
|
817
|
+
if (lexer->lookahead == '\r') {
|
|
818
|
+
advance(lexer);
|
|
819
|
+
}
|
|
820
|
+
if (lexer->lookahead != '\n') {
|
|
821
|
+
return false;
|
|
822
|
+
}
|
|
823
|
+
advance(lexer);
|
|
824
|
+
} else if (length < sizeof(word) && (iswlower(lexer->lookahead) || lexer->lookahead == '}')) {
|
|
825
|
+
word[length++] = (char)lexer->lookahead;
|
|
826
|
+
advance(lexer);
|
|
827
|
+
} else {
|
|
828
|
+
break;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
if (length == 0 || length == sizeof(word) ||
|
|
832
|
+
!(lexer->eof(lexer) || iswspace(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
|
|
833
|
+
return false;
|
|
834
|
+
}
|
|
835
|
+
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
|
|
836
|
+
if (strlen(words[i]) == length && strncmp(words[i], word, length) == 0) {
|
|
837
|
+
return true;
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
return false;
|
|
841
|
+
}
|
|
842
|
+
|
|
777
843
|
static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
778
844
|
bool error_recovery = valid_symbols[ERROR_RECOVERY];
|
|
779
845
|
// Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
|
|
780
846
|
// by the character right after it, even when blanks are skipped below to find a newline.
|
|
781
847
|
int32_t first = lexer->lookahead;
|
|
782
848
|
bool at_eof = lexer->eof(lexer);
|
|
849
|
+
// Kept only if this call returns a token, so the flag lasts until the scanner's next token.
|
|
850
|
+
bool after_line_break = scanner->after_line_break;
|
|
851
|
+
scanner->after_line_break = false;
|
|
783
852
|
|
|
784
853
|
if (scanner->terminator_owed) {
|
|
785
854
|
scanner->terminator_owed = false;
|
|
@@ -845,7 +914,12 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
845
914
|
return scan_heredoc_start(scanner, lexer);
|
|
846
915
|
}
|
|
847
916
|
|
|
848
|
-
|
|
917
|
+
// A line continuation after a blank separates words, so nothing concatenates at the start of the
|
|
918
|
+
// next line. The column costs a scan back to the line start, so it is computed only right after
|
|
919
|
+
// such a continuation and only where a concatenation could start; a concatenation then clears the
|
|
920
|
+
// flag, which bounds the scans to one per continuation.
|
|
921
|
+
if (valid_symbols[CONCAT] && !at_eof && !iswspace(first) && !is_metacharacter(first) &&
|
|
922
|
+
!(after_line_break && lexer->get_column(lexer) == 0)) {
|
|
849
923
|
int32_t c = first;
|
|
850
924
|
if (c == '\\' && lexer->lookahead == '\\') {
|
|
851
925
|
// A backslash-newline joins lines, so the word continues only if the next line does.
|
|
@@ -866,27 +940,41 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
866
940
|
}
|
|
867
941
|
advance(lexer);
|
|
868
942
|
lexer->mark_end(lexer);
|
|
869
|
-
|
|
870
|
-
//
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
if (c == '\\') {
|
|
877
|
-
// Another continuation is whitespace too; an escaped character continues the word.
|
|
943
|
+
// Bash removes every backslash-newline before it splits words, so further continuations
|
|
944
|
+
// right after this one are part of the token (`a\` + `\` + `b` is `ab`).
|
|
945
|
+
for (;;) {
|
|
946
|
+
c = lexer->lookahead;
|
|
947
|
+
if (c != '\\') {
|
|
948
|
+
break;
|
|
949
|
+
}
|
|
878
950
|
advance(lexer);
|
|
879
951
|
if (lexer->lookahead == '\r') {
|
|
880
952
|
advance(lexer);
|
|
881
953
|
}
|
|
882
|
-
if (lexer->
|
|
954
|
+
if (lexer->eof(lexer)) {
|
|
955
|
+
lexer->mark_end(lexer);
|
|
956
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
957
|
+
return true;
|
|
958
|
+
}
|
|
959
|
+
if (lexer->lookahead != '\n') {
|
|
960
|
+
// An escaped character continues the word.
|
|
961
|
+
lexer->result_symbol = CONCAT;
|
|
883
962
|
return true;
|
|
884
963
|
}
|
|
964
|
+
advance(lexer);
|
|
965
|
+
lexer->mark_end(lexer);
|
|
966
|
+
}
|
|
967
|
+
// The word continues only if the next line does; otherwise the continuations themselves are
|
|
968
|
+
// the token, so that whatever follows them is lexed as it would be without them.
|
|
969
|
+
if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
970
|
+
scanner->after_line_break = true;
|
|
971
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
972
|
+
} else {
|
|
973
|
+
lexer->result_symbol = CONCAT;
|
|
885
974
|
}
|
|
886
|
-
lexer->result_symbol = CONCAT;
|
|
887
975
|
return true;
|
|
888
976
|
}
|
|
889
|
-
if (
|
|
977
|
+
if (c != '`' || scanner->backtick_depth == 0) {
|
|
890
978
|
lexer->result_symbol = CONCAT;
|
|
891
979
|
return true;
|
|
892
980
|
}
|
|
@@ -939,6 +1027,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
939
1027
|
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
940
1028
|
advance(lexer);
|
|
941
1029
|
lexer->mark_end(lexer);
|
|
1030
|
+
scanner->after_line_break = true;
|
|
942
1031
|
lexer->result_symbol = LINE_CONTINUATION;
|
|
943
1032
|
return true;
|
|
944
1033
|
}
|
|
@@ -966,6 +1055,37 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
966
1055
|
return true;
|
|
967
1056
|
}
|
|
968
1057
|
|
|
1058
|
+
// Bash ends backquotes at the first unescaped backquote, even inside single quotes (`echo '`),
|
|
1059
|
+
// and then reports the unterminated quote when it runs the substitution. So a quote that the next
|
|
1060
|
+
// backquote cuts short closes the substitution there when it may close, and otherwise is skipped
|
|
1061
|
+
// up to the backquote, which then cannot be parsed; the quote never hides the text after it.
|
|
1062
|
+
if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
|
|
1063
|
+
advance(lexer);
|
|
1064
|
+
while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`') {
|
|
1065
|
+
// A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
|
|
1066
|
+
// quote itself stays literal, so `'\'` still ends at its second quote.
|
|
1067
|
+
bool backslash = lexer->lookahead == '\\';
|
|
1068
|
+
advance(lexer);
|
|
1069
|
+
if (backslash && (lexer->lookahead == '`' || lexer->lookahead == '\\')) {
|
|
1070
|
+
advance(lexer);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
if (lexer->lookahead != '`') {
|
|
1074
|
+
return false;
|
|
1075
|
+
}
|
|
1076
|
+
if (valid_symbols[BACKTICK_CLOSE]) {
|
|
1077
|
+
advance(lexer);
|
|
1078
|
+
lexer->mark_end(lexer);
|
|
1079
|
+
scanner->backtick_depth--;
|
|
1080
|
+
discard_unstarted_heredocs(scanner);
|
|
1081
|
+
lexer->result_symbol = BACKTICK_CLOSE;
|
|
1082
|
+
return true;
|
|
1083
|
+
}
|
|
1084
|
+
lexer->mark_end(lexer);
|
|
1085
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
1086
|
+
return true;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
969
1089
|
if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
|
|
970
1090
|
if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
|
|
971
1091
|
advance(lexer);
|
|
@@ -1019,6 +1139,15 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
1019
1139
|
return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
|
|
1020
1140
|
}
|
|
1021
1141
|
|
|
1142
|
+
// Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
|
|
1143
|
+
// `{ (y) }`), so a zero-width terminator is returned before one there. After a word, the reserved
|
|
1144
|
+
// word would be an argument instead; a possible concatenation shows that a word just ended. This
|
|
1145
|
+
// check comes last because it consumes input even when it finds no reserved word.
|
|
1146
|
+
if (valid_symbols[NEWLINE] && !valid_symbols[CONCAT] && at_closing_reserved_word(lexer)) {
|
|
1147
|
+
lexer->result_symbol = NEWLINE;
|
|
1148
|
+
return true;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1022
1151
|
return false;
|
|
1023
1152
|
}
|
|
1024
1153
|
|
|
@@ -1045,7 +1174,7 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
|
|
|
1045
1174
|
Scanner *scanner = (Scanner *)payload;
|
|
1046
1175
|
unsigned size = 0;
|
|
1047
1176
|
buffer[size++] = (char)scanner->backtick_depth;
|
|
1048
|
-
buffer[size++] = (char)scanner->terminator_owed;
|
|
1177
|
+
buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1);
|
|
1049
1178
|
uint16_t closer_count = (uint16_t)scanner->closers.size;
|
|
1050
1179
|
memcpy(&buffer[size], &closer_count, sizeof(closer_count));
|
|
1051
1180
|
size += sizeof(closer_count);
|
|
@@ -1079,7 +1208,8 @@ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *bu
|
|
|
1079
1208
|
}
|
|
1080
1209
|
unsigned size = 0;
|
|
1081
1210
|
scanner->backtick_depth = (uint8_t)buffer[size++];
|
|
1082
|
-
scanner->terminator_owed = buffer[size
|
|
1211
|
+
scanner->terminator_owed = buffer[size] & 1;
|
|
1212
|
+
scanner->after_line_break = (buffer[size++] & 2) != 0;
|
|
1083
1213
|
uint16_t closer_count;
|
|
1084
1214
|
memcpy(&closer_count, &buffer[size], sizeof(closer_count));
|
|
1085
1215
|
size += sizeof(closer_count);
|
package/tree-sitter.json
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
}
|
|
19
19
|
],
|
|
20
20
|
"metadata": {
|
|
21
|
-
"version": "1.0.
|
|
21
|
+
"version": "1.0.2",
|
|
22
22
|
"license": "MIT",
|
|
23
23
|
"description": "Bash grammar for tree-sitter",
|
|
24
24
|
"authors": [
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"go": true,
|
|
41
41
|
"node": true,
|
|
42
42
|
"python": true,
|
|
43
|
-
"rust":
|
|
43
|
+
"rust": false,
|
|
44
44
|
"swift": true
|
|
45
45
|
}
|
|
46
46
|
}
|