@willbooster/tree-sitter-bash 0.25.1 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -33
- package/grammar.js +200 -32
- package/package.json +37 -23
- package/prebuilds/darwin-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/darwin-x64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-arm64/tree-sitter-bash.node +0 -0
- package/prebuilds/linux-x64/tree-sitter-bash.node +0 -0
- package/queries/highlights.scm +33 -12
- package/src/grammar.json +821 -60
- package/src/node-types.json +206 -66
- package/src/parser.c +219487 -88691
- package/src/scanner.c +687 -103
- package/tree-sitter.json +3 -3
package/src/scanner.c
CHANGED
|
@@ -22,18 +22,29 @@ enum TokenType {
|
|
|
22
22
|
BACKTICK_OPEN,
|
|
23
23
|
BACKTICK_CLOSE,
|
|
24
24
|
NEWLINE,
|
|
25
|
+
SUBSTITUTION_START,
|
|
26
|
+
SUBSTITUTION_END,
|
|
27
|
+
BRACE_SUBSTITUTION_END,
|
|
28
|
+
BRACKET_SUBSTITUTION_END,
|
|
29
|
+
EXTGLOB_PREFIX,
|
|
30
|
+
LINE_CONTINUATION,
|
|
31
|
+
BRACE_SUBSTITUTION_START,
|
|
32
|
+
BRACKET_SUBSTITUTION_START,
|
|
25
33
|
ERROR_RECOVERY,
|
|
26
34
|
};
|
|
27
35
|
|
|
28
|
-
typedef Array(
|
|
36
|
+
typedef Array(int32_t) CodePoints;
|
|
29
37
|
|
|
30
38
|
// A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
|
|
31
|
-
// (not one inside a string)
|
|
39
|
+
// (not one inside a string) at the substitution depth where the operator appeared, after the bodies
|
|
40
|
+
// of heredocs opened earlier at that depth: bash reads a substitution, including the bodies of the
|
|
41
|
+
// heredocs inside it, before the body of a heredoc whose operator precedes it on the same line.
|
|
32
42
|
typedef struct {
|
|
33
43
|
bool is_raw;
|
|
34
44
|
bool allows_indent;
|
|
35
45
|
bool started;
|
|
36
|
-
|
|
46
|
+
uint16_t depth;
|
|
47
|
+
CodePoints delimiter;
|
|
37
48
|
} Heredoc;
|
|
38
49
|
|
|
39
50
|
typedef struct {
|
|
@@ -41,8 +52,17 @@ typedef struct {
|
|
|
41
52
|
// Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
|
|
42
53
|
// whether a backquote opens or closes one.
|
|
43
54
|
uint8_t backtick_depth;
|
|
55
|
+
// The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
|
|
56
|
+
Array(uint8_t) closers;
|
|
57
|
+
// A heredoc body ended right before a `${ list; }` closer; the newline that ended the command's
|
|
58
|
+
// line began the body, so a terminator is still owed before the `}`.
|
|
59
|
+
bool terminator_owed;
|
|
44
60
|
} Scanner;
|
|
45
61
|
|
|
62
|
+
static inline uint16_t current_depth(Scanner *scanner) {
|
|
63
|
+
return scanner->backtick_depth + scanner->closers.size;
|
|
64
|
+
}
|
|
65
|
+
|
|
46
66
|
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
|
|
47
67
|
|
|
48
68
|
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
|
|
@@ -63,6 +83,60 @@ static inline bool starts_quoted_expansion(int32_t c) {
|
|
|
63
83
|
c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
|
|
64
84
|
}
|
|
65
85
|
|
|
86
|
+
// Code points are serialized as zigzag varints, one byte for code points below 64 and two for the
|
|
87
|
+
// rest of ASCII; every int32 round-trips, including the negative value the lexer reports for invalid
|
|
88
|
+
// UTF-8.
|
|
89
|
+
static unsigned varint_length(int32_t c) {
|
|
90
|
+
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
91
|
+
unsigned length = 1;
|
|
92
|
+
while (value >= 0x80) {
|
|
93
|
+
value >>= 7;
|
|
94
|
+
length++;
|
|
95
|
+
}
|
|
96
|
+
return length;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static unsigned encode_varint(int32_t c, char *out) {
|
|
100
|
+
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
101
|
+
unsigned length = 0;
|
|
102
|
+
while (value >= 0x80) {
|
|
103
|
+
out[length++] = (char)(0x80 | (value & 0x7F));
|
|
104
|
+
value >>= 7;
|
|
105
|
+
}
|
|
106
|
+
out[length++] = (char)value;
|
|
107
|
+
return length;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Returns 0 when the input ends before the varint does.
|
|
111
|
+
static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
|
|
112
|
+
uint32_t value = 0;
|
|
113
|
+
for (unsigned i = 0; i < available && i < 5; i++) {
|
|
114
|
+
value |= (uint32_t)((unsigned char)in[i] & 0x7F) << (7 * i);
|
|
115
|
+
if (((unsigned char)in[i] & 0x80) == 0) {
|
|
116
|
+
*c = (int32_t)((value >> 1) ^ (0u - (value & 1)));
|
|
117
|
+
return i + 1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return 0;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Serialized layout: backtick depth, the owed-terminator flag, the closers of the open substitutions
|
|
124
|
+
// (uint16 count, then one byte each), and heredoc count, then per heredoc its flags, depth (uint16),
|
|
125
|
+
// delimiter length (uint16), and the delimiter as varints.
|
|
126
|
+
#define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
|
|
127
|
+
|
|
128
|
+
static unsigned serialized_size(Scanner *scanner) {
|
|
129
|
+
unsigned size = 3 + sizeof(uint16_t) + scanner->closers.size;
|
|
130
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
131
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
132
|
+
size += HEREDOC_HEADER_SIZE;
|
|
133
|
+
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
134
|
+
size += varint_length(*array_get(&heredoc->delimiter, j));
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return size;
|
|
138
|
+
}
|
|
139
|
+
|
|
66
140
|
static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
|
|
67
141
|
|
|
68
142
|
static void scanner_reset(Scanner *scanner) {
|
|
@@ -71,18 +145,70 @@ static void scanner_reset(Scanner *scanner) {
|
|
|
71
145
|
}
|
|
72
146
|
array_clear(&scanner->heredocs);
|
|
73
147
|
scanner->backtick_depth = 0;
|
|
148
|
+
array_clear(&scanner->closers);
|
|
149
|
+
scanner->terminator_owed = false;
|
|
74
150
|
}
|
|
75
151
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
152
|
+
// The body being read: the most deeply nested started heredoc.
|
|
153
|
+
static int32_t active_heredoc_index(Scanner *scanner) {
|
|
154
|
+
int32_t active = -1;
|
|
155
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
156
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
157
|
+
if (heredoc->started && (active < 0 || heredoc->depth >= array_get(&scanner->heredocs, active)->depth)) {
|
|
158
|
+
active = (int32_t)i;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return active;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// The next body to start: the first unstarted heredoc opened at the current substitution depth.
|
|
165
|
+
static int32_t next_heredoc_index(Scanner *scanner) {
|
|
166
|
+
uint16_t depth = current_depth(scanner);
|
|
167
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
168
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
169
|
+
if (!heredoc->started && heredoc->depth == depth) {
|
|
170
|
+
return (int32_t)i;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return -1;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// An entry without a delimiter belongs to an operator whose delimiter was rejected.
|
|
177
|
+
static inline bool is_rejected(Heredoc *heredoc) { return heredoc->delimiter.size == 0 && !heredoc->is_raw; }
|
|
178
|
+
|
|
179
|
+
// Drops the unstarted heredocs at the current depth, which were all queued on the line just ended.
|
|
180
|
+
static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
|
|
181
|
+
uint16_t depth = current_depth(scanner);
|
|
182
|
+
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
183
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
184
|
+
if (!heredoc->started && heredoc->depth == depth) {
|
|
185
|
+
heredoc_delete(heredoc);
|
|
186
|
+
array_erase(&scanner->heredocs, i - 1);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
static void remove_heredoc(Scanner *scanner, uint32_t index) {
|
|
192
|
+
heredoc_delete(array_get(&scanner->heredocs, index));
|
|
193
|
+
array_erase(&scanner->heredocs, index);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Like bash, drop the heredocs of a substitution that closes before their bodies started.
|
|
197
|
+
static void discard_unstarted_heredocs(Scanner *scanner) {
|
|
198
|
+
uint16_t depth = current_depth(scanner);
|
|
199
|
+
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
200
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
201
|
+
if (!heredoc->started && heredoc->depth > depth) {
|
|
202
|
+
remove_heredoc(scanner, i - 1);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
79
205
|
}
|
|
80
206
|
|
|
81
207
|
// Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
|
|
82
208
|
// line (or at the end of input, as bash does with a warning). An unquoted body stops before each
|
|
83
209
|
// expansion so that the parser can read it.
|
|
84
|
-
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
85
|
-
Heredoc *heredoc = array_get(&scanner->heredocs,
|
|
210
|
+
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
|
|
211
|
+
Heredoc *heredoc = array_get(&scanner->heredocs, index);
|
|
86
212
|
bool did_advance = false;
|
|
87
213
|
bool at_line_start = lexer->get_column(lexer) == 0;
|
|
88
214
|
|
|
@@ -90,34 +216,64 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
90
216
|
if (at_line_start) {
|
|
91
217
|
at_line_start = false;
|
|
92
218
|
lexer->mark_end(lexer);
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
219
|
+
// Like bash, compare the delimiter with the line after joining backslash-newlines and
|
|
220
|
+
// then stripping the leading tabs of `<<-`. As elsewhere in the grammar, CRLF counts as a
|
|
221
|
+
// newline.
|
|
222
|
+
bool consumed = false;
|
|
223
|
+
bool escaped = false;
|
|
224
|
+
uint32_t matched = 0;
|
|
225
|
+
// At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
|
|
226
|
+
for (;;) {
|
|
227
|
+
if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
|
|
228
|
+
advance(lexer);
|
|
229
|
+
consumed = true;
|
|
230
|
+
} else if (lexer->lookahead == '\\' && !heredoc->is_raw) {
|
|
231
|
+
advance(lexer);
|
|
232
|
+
consumed = true;
|
|
233
|
+
bool escaped_cr = lexer->lookahead == '\r';
|
|
234
|
+
if (escaped_cr) {
|
|
235
|
+
advance(lexer);
|
|
236
|
+
}
|
|
237
|
+
if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
|
|
238
|
+
// An escape rather than a line continuation: the escaped character, which is
|
|
239
|
+
// the CR itself when no LF follows it, is content.
|
|
240
|
+
if (!escaped_cr && !lexer->eof(lexer)) {
|
|
241
|
+
advance(lexer);
|
|
242
|
+
}
|
|
243
|
+
escaped = true;
|
|
244
|
+
break;
|
|
245
|
+
}
|
|
246
|
+
advance(lexer);
|
|
247
|
+
} else if (matched < heredoc->delimiter.size && !lexer->eof(lexer) &&
|
|
248
|
+
lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
|
|
96
249
|
advance(lexer);
|
|
97
|
-
|
|
250
|
+
matched++;
|
|
251
|
+
} else {
|
|
252
|
+
break;
|
|
98
253
|
}
|
|
99
254
|
}
|
|
100
|
-
|
|
101
|
-
while (matched < heredoc->delimiter.size && lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
|
|
102
|
-
advance(lexer);
|
|
103
|
-
matched++;
|
|
104
|
-
}
|
|
105
|
-
if (lexer->lookahead == '\r') {
|
|
255
|
+
if (!escaped && lexer->lookahead == '\r') {
|
|
106
256
|
advance(lexer);
|
|
107
257
|
}
|
|
108
|
-
|
|
258
|
+
// Inside a substitution, bash also ends the body at a delimiter directly followed by the
|
|
259
|
+
// innermost substitution's closer (`)` or `}`) or a closing backquote.
|
|
260
|
+
bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
|
|
261
|
+
(scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
|
|
262
|
+
(lexer->lookahead == '`' && scanner->backtick_depth > 0);
|
|
263
|
+
if (!escaped && matched == heredoc->delimiter.size && ends_line) {
|
|
109
264
|
if (did_advance) {
|
|
110
265
|
lexer->result_symbol = HEREDOC_CONTENT;
|
|
111
266
|
return true;
|
|
112
267
|
}
|
|
113
268
|
lexer->mark_end(lexer);
|
|
114
269
|
lexer->result_symbol = HEREDOC_END;
|
|
115
|
-
|
|
270
|
+
scanner->terminator_owed = lexer->lookahead == '}';
|
|
271
|
+
remove_heredoc(scanner, index);
|
|
116
272
|
return true;
|
|
117
273
|
}
|
|
118
274
|
// The characters read while trying the delimiter are content; an empty attempt must not
|
|
119
275
|
// count, or a zero-width content token would repeat forever.
|
|
120
|
-
did_advance = did_advance ||
|
|
276
|
+
did_advance = did_advance || consumed || matched > 0;
|
|
121
277
|
continue;
|
|
122
278
|
}
|
|
123
279
|
|
|
@@ -125,7 +281,7 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
125
281
|
lexer->mark_end(lexer);
|
|
126
282
|
lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
|
|
127
283
|
if (!did_advance) {
|
|
128
|
-
|
|
284
|
+
remove_heredoc(scanner, index);
|
|
129
285
|
}
|
|
130
286
|
return true;
|
|
131
287
|
}
|
|
@@ -140,8 +296,16 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
140
296
|
advance(lexer);
|
|
141
297
|
did_advance = true;
|
|
142
298
|
if (!heredoc->is_raw && !lexer->eof(lexer)) {
|
|
143
|
-
// A backslash-newline joins lines before the delimiter comparison.
|
|
144
|
-
|
|
299
|
+
// A backslash-newline joins lines before the delimiter comparison. A CR without an
|
|
300
|
+
// LF after it is the escaped character itself.
|
|
301
|
+
if (lexer->lookahead == '\r') {
|
|
302
|
+
advance(lexer);
|
|
303
|
+
if (lexer->lookahead == '\n') {
|
|
304
|
+
advance(lexer);
|
|
305
|
+
}
|
|
306
|
+
} else if (!lexer->eof(lexer)) {
|
|
307
|
+
advance(lexer);
|
|
308
|
+
}
|
|
145
309
|
}
|
|
146
310
|
break;
|
|
147
311
|
case '`':
|
|
@@ -179,6 +343,79 @@ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
|
|
|
179
343
|
}
|
|
180
344
|
|
|
181
345
|
// The delimiter word undergoes quote removal; any quoting makes the body literal.
|
|
346
|
+
static inline int32_t hex_value(int32_t c) {
|
|
347
|
+
return iswdigit(c) ? c - '0' : (c >= 'a' && c <= 'f') ? c - 'a' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 : -1;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Reads up to `max_digits` digits in `base` and returns their value, or -1 when there are none.
|
|
351
|
+
static int32_t read_number(TSLexer *lexer, int32_t base, int max_digits) {
|
|
352
|
+
int32_t value = 0;
|
|
353
|
+
int digits = 0;
|
|
354
|
+
while (digits < max_digits) {
|
|
355
|
+
int32_t digit = hex_value(lexer->lookahead);
|
|
356
|
+
if (digit < 0 || digit >= base) {
|
|
357
|
+
break;
|
|
358
|
+
}
|
|
359
|
+
value = value * base + digit;
|
|
360
|
+
advance(lexer);
|
|
361
|
+
digits++;
|
|
362
|
+
}
|
|
363
|
+
return digits == 0 ? -1 : value;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Decodes the escape after a backslash in `$'...'` as bash does, appending it to `out`.
|
|
367
|
+
static void push_ansi_c_escape(TSLexer *lexer, CodePoints *out) {
|
|
368
|
+
int32_t c = lexer->lookahead;
|
|
369
|
+
int32_t value = -1;
|
|
370
|
+
switch (c) {
|
|
371
|
+
case 'a': value = 7; break;
|
|
372
|
+
case 'b': value = 8; break;
|
|
373
|
+
case 'e':
|
|
374
|
+
case 'E': value = 27; break;
|
|
375
|
+
case 'f': value = 12; break;
|
|
376
|
+
case 'n': value = '\n'; break;
|
|
377
|
+
case 'r': value = '\r'; break;
|
|
378
|
+
case 't': value = '\t'; break;
|
|
379
|
+
case 'v': value = 11; break;
|
|
380
|
+
case '\\':
|
|
381
|
+
case '\'':
|
|
382
|
+
case '"':
|
|
383
|
+
case '?': value = c; break;
|
|
384
|
+
default: break;
|
|
385
|
+
}
|
|
386
|
+
if (value >= 0) {
|
|
387
|
+
advance(lexer);
|
|
388
|
+
array_push(out, value);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
if (c >= '0' && c <= '7') {
|
|
392
|
+
array_push(out, read_number(lexer, 8, 3));
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
int max_digits = c == 'x' ? 2 : c == 'u' ? 4 : c == 'U' ? 8 : 0;
|
|
396
|
+
if (max_digits > 0) {
|
|
397
|
+
advance(lexer);
|
|
398
|
+
value = read_number(lexer, 16, max_digits);
|
|
399
|
+
if (value < 0) {
|
|
400
|
+
array_push(out, '\\');
|
|
401
|
+
array_push(out, c);
|
|
402
|
+
} else {
|
|
403
|
+
array_push(out, value);
|
|
404
|
+
}
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
if (c == 'c') {
|
|
408
|
+
advance(lexer);
|
|
409
|
+
if (!lexer->eof(lexer)) {
|
|
410
|
+
array_push(out, lexer->lookahead & 0x1F);
|
|
411
|
+
advance(lexer);
|
|
412
|
+
}
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
// Bash keeps an unknown escape as written.
|
|
416
|
+
array_push(out, '\\');
|
|
417
|
+
}
|
|
418
|
+
|
|
182
419
|
static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
183
420
|
Heredoc *heredoc = array_back(&scanner->heredocs);
|
|
184
421
|
array_clear(&heredoc->delimiter);
|
|
@@ -189,6 +426,7 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
|
189
426
|
}
|
|
190
427
|
|
|
191
428
|
int32_t quote = 0;
|
|
429
|
+
bool ansi_c = false;
|
|
192
430
|
while (!lexer->eof(lexer)) {
|
|
193
431
|
int32_t c = lexer->lookahead;
|
|
194
432
|
if (quote) {
|
|
@@ -197,36 +435,80 @@ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
|
197
435
|
quote = 0;
|
|
198
436
|
continue;
|
|
199
437
|
}
|
|
438
|
+
if (ansi_c && c == '\\') {
|
|
439
|
+
if (!lexer->eof(lexer)) {
|
|
440
|
+
push_ansi_c_escape(lexer, &heredoc->delimiter);
|
|
441
|
+
}
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
// Inside double quotes a backslash-newline still only continues the line.
|
|
445
|
+
if (quote == '"' && c == '\\' && (lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
|
|
446
|
+
if (lexer->lookahead == '\r') {
|
|
447
|
+
advance(lexer);
|
|
448
|
+
}
|
|
449
|
+
if (lexer->lookahead == '\n') {
|
|
450
|
+
advance(lexer);
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
array_push(&heredoc->delimiter, '\\');
|
|
454
|
+
array_push(&heredoc->delimiter, '\r');
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
200
457
|
if (quote == '"' && c == '\\' &&
|
|
201
458
|
(lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
|
|
202
459
|
lexer->lookahead == '\\')) {
|
|
203
460
|
c = lexer->lookahead;
|
|
204
461
|
advance(lexer);
|
|
205
462
|
}
|
|
206
|
-
array_push(&heredoc->delimiter,
|
|
463
|
+
array_push(&heredoc->delimiter, c);
|
|
207
464
|
continue;
|
|
208
465
|
}
|
|
209
|
-
|
|
466
|
+
// Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
|
|
467
|
+
// the word (`<<EO`true`F`).
|
|
468
|
+
if (iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
210
469
|
break;
|
|
211
470
|
}
|
|
212
471
|
advance(lexer);
|
|
213
|
-
if (c == '\'' ||
|
|
472
|
+
if (c == '$' && (lexer->lookahead == '\'' || lexer->lookahead == '"')) {
|
|
473
|
+
// `$'...'` and `$"..."` are quoting constructs, so their `$` is not part of the word.
|
|
474
|
+
c = lexer->lookahead;
|
|
475
|
+
advance(lexer);
|
|
214
476
|
quote = c;
|
|
477
|
+
ansi_c = c == '\'';
|
|
215
478
|
heredoc->is_raw = true;
|
|
216
|
-
} else if (c == '
|
|
479
|
+
} else if (c == '\'' || c == '"') {
|
|
480
|
+
quote = c;
|
|
481
|
+
ansi_c = false;
|
|
217
482
|
heredoc->is_raw = true;
|
|
483
|
+
} else if (c == '\\') {
|
|
484
|
+
// A backslash-newline only continues the line; any other backslash quotes the delimiter.
|
|
485
|
+
bool carriage_return = lexer->lookahead == '\r';
|
|
486
|
+
if (carriage_return) {
|
|
487
|
+
advance(lexer);
|
|
488
|
+
}
|
|
218
489
|
if (lexer->lookahead == '\n') {
|
|
219
490
|
advance(lexer);
|
|
491
|
+
} else if (carriage_return) {
|
|
492
|
+
heredoc->is_raw = true;
|
|
493
|
+
array_push(&heredoc->delimiter, '\r');
|
|
220
494
|
} else if (!lexer->eof(lexer)) {
|
|
221
|
-
|
|
495
|
+
heredoc->is_raw = true;
|
|
496
|
+
array_push(&heredoc->delimiter, lexer->lookahead);
|
|
222
497
|
advance(lexer);
|
|
223
498
|
}
|
|
224
499
|
} else {
|
|
225
|
-
|
|
500
|
+
// Bash counts a backquote in the delimiter word as quoting, so the body stays literal.
|
|
501
|
+
if (c == '`') {
|
|
502
|
+
heredoc->is_raw = true;
|
|
503
|
+
}
|
|
504
|
+
array_push(&heredoc->delimiter, c);
|
|
226
505
|
}
|
|
227
506
|
}
|
|
228
507
|
|
|
229
|
-
|
|
508
|
+
// A heredoc whose state cannot be serialized in full is rejected, so the parse shows an error
|
|
509
|
+
// rather than comparing against a partial delimiter.
|
|
510
|
+
if ((heredoc->delimiter.size == 0 && !heredoc->is_raw) ||
|
|
511
|
+
serialized_size(scanner) > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
230
512
|
return false;
|
|
231
513
|
}
|
|
232
514
|
lexer->mark_end(lexer);
|
|
@@ -250,46 +532,23 @@ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *val
|
|
|
250
532
|
} else if (!valid_symbols[HEREDOC_ARROW]) {
|
|
251
533
|
return false;
|
|
252
534
|
}
|
|
535
|
+
if (scanner->heredocs.size >= UINT8_MAX ||
|
|
536
|
+
serialized_size(scanner) + HEREDOC_HEADER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
253
539
|
lexer->mark_end(lexer);
|
|
254
|
-
Heredoc heredoc = {
|
|
540
|
+
Heredoc heredoc = {
|
|
541
|
+
.is_raw = false,
|
|
542
|
+
.allows_indent = allows_indent,
|
|
543
|
+
.started = false,
|
|
544
|
+
.depth = current_depth(scanner),
|
|
545
|
+
.delimiter = array_new(),
|
|
546
|
+
};
|
|
255
547
|
array_push(&scanner->heredocs, heredoc);
|
|
256
548
|
lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
|
|
257
549
|
return true;
|
|
258
550
|
}
|
|
259
551
|
|
|
260
|
-
// A name followed by `=`, `+=`, or `[subscript]=` starts an assignment.
|
|
261
|
-
static bool scan_variable_name(TSLexer *lexer) {
|
|
262
|
-
while (is_name_char(lexer->lookahead)) {
|
|
263
|
-
advance(lexer);
|
|
264
|
-
}
|
|
265
|
-
lexer->mark_end(lexer);
|
|
266
|
-
if (lexer->lookahead == '+') {
|
|
267
|
-
advance(lexer);
|
|
268
|
-
return lexer->lookahead == '=';
|
|
269
|
-
}
|
|
270
|
-
if (lexer->lookahead == '[') {
|
|
271
|
-
uint32_t depth = 0;
|
|
272
|
-
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
|
273
|
-
if (lexer->lookahead == '[') {
|
|
274
|
-
depth++;
|
|
275
|
-
} else if (lexer->lookahead == ']' && --depth == 0) {
|
|
276
|
-
advance(lexer);
|
|
277
|
-
break;
|
|
278
|
-
} else if (lexer->lookahead == '\\') {
|
|
279
|
-
advance(lexer);
|
|
280
|
-
}
|
|
281
|
-
advance(lexer);
|
|
282
|
-
}
|
|
283
|
-
if (depth != 0) {
|
|
284
|
-
return false;
|
|
285
|
-
}
|
|
286
|
-
if (lexer->lookahead == '+') {
|
|
287
|
-
advance(lexer);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
return lexer->lookahead == '=';
|
|
291
|
-
}
|
|
292
|
-
|
|
293
552
|
// The right side of `=~` extends to unquoted whitespace outside parentheses.
|
|
294
553
|
static bool scan_regex(TSLexer *lexer) {
|
|
295
554
|
while (is_blank(lexer->lookahead)) {
|
|
@@ -298,10 +557,16 @@ static bool scan_regex(TSLexer *lexer) {
|
|
|
298
557
|
uint32_t depth = 0;
|
|
299
558
|
bool did_advance = false;
|
|
300
559
|
while (!lexer->eof(lexer)) {
|
|
560
|
+
lexer->mark_end(lexer);
|
|
301
561
|
int32_t c = lexer->lookahead;
|
|
302
562
|
if (depth == 0 && iswspace(c)) {
|
|
303
563
|
break;
|
|
304
564
|
}
|
|
565
|
+
// Like bash, treat `|` and parentheses as regex characters, while other metacharacters end
|
|
566
|
+
// the regex even without surrounding blanks (`[[ a =~ a&&b ]]`).
|
|
567
|
+
if (depth == 0 && (c == '&' || c == ';' || c == '<' || c == '>')) {
|
|
568
|
+
break;
|
|
569
|
+
}
|
|
305
570
|
if (c == '\\') {
|
|
306
571
|
advance(lexer);
|
|
307
572
|
} else if (c == '\'' || c == '"') {
|
|
@@ -323,7 +588,9 @@ static bool scan_regex(TSLexer *lexer) {
|
|
|
323
588
|
advance(lexer);
|
|
324
589
|
did_advance = true;
|
|
325
590
|
}
|
|
326
|
-
lexer->
|
|
591
|
+
if (lexer->eof(lexer)) {
|
|
592
|
+
lexer->mark_end(lexer);
|
|
593
|
+
}
|
|
327
594
|
lexer->result_symbol = REGEX;
|
|
328
595
|
return did_advance;
|
|
329
596
|
}
|
|
@@ -352,6 +619,186 @@ static bool scan_string_content(TSLexer *lexer) {
|
|
|
352
619
|
return did_advance;
|
|
353
620
|
}
|
|
354
621
|
|
|
622
|
+
static inline bool is_word_break(int32_t c) {
|
|
623
|
+
return iswspace(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
|
|
627
|
+
|
|
628
|
+
typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
|
|
629
|
+
|
|
630
|
+
// From the `(` of an extglob group, reads to its matching `)`: a group holding quotes or expansions
|
|
631
|
+
// is parsed as `extglob_pattern`, while the word token keeps literal ones.
|
|
632
|
+
// The nesting depth up to which grammar.js's word token lexes a literal group (`extglobGroup(…, 3)`);
|
|
633
|
+
// a deeper group is parsed as `extglob_pattern`.
|
|
634
|
+
#define EXTGLOB_LITERAL_DEPTH 3
|
|
635
|
+
|
|
636
|
+
static GroupKind scan_extglob_group(TSLexer *lexer) {
|
|
637
|
+
uint32_t depth = 0;
|
|
638
|
+
while (!lexer->eof(lexer)) {
|
|
639
|
+
int32_t c = lexer->lookahead;
|
|
640
|
+
if (c == '$' || c == '`' || c == '\'' || c == '"') {
|
|
641
|
+
return GROUP_STRUCTURED;
|
|
642
|
+
}
|
|
643
|
+
advance(lexer);
|
|
644
|
+
if (c == '\\') {
|
|
645
|
+
// The word token's literal groups cannot hold a line continuation.
|
|
646
|
+
if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
647
|
+
return lexer->eof(lexer) ? GROUP_UNSUPPORTED : GROUP_STRUCTURED;
|
|
648
|
+
}
|
|
649
|
+
advance(lexer);
|
|
650
|
+
} else if (c == '(') {
|
|
651
|
+
if (++depth > EXTGLOB_LITERAL_DEPTH) {
|
|
652
|
+
return GROUP_STRUCTURED;
|
|
653
|
+
}
|
|
654
|
+
} else if (c == ')' && --depth == 0) {
|
|
655
|
+
return GROUP_LITERAL;
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
return GROUP_UNSUPPORTED;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// The literal text of a word up to a structured extglob group (`foo` in `foo@($x)`), which the
|
|
662
|
+
// word token's longest match would otherwise run into.
|
|
663
|
+
static bool continue_extglob_prefix(TSLexer *lexer, bool consumed) {
|
|
664
|
+
while (!lexer->eof(lexer)) {
|
|
665
|
+
int32_t c = lexer->lookahead;
|
|
666
|
+
if (is_extglob_operator(c)) {
|
|
667
|
+
lexer->mark_end(lexer);
|
|
668
|
+
advance(lexer);
|
|
669
|
+
if (lexer->lookahead == '(') {
|
|
670
|
+
GroupKind kind = scan_extglob_group(lexer);
|
|
671
|
+
if (kind == GROUP_STRUCTURED) {
|
|
672
|
+
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
673
|
+
return consumed;
|
|
674
|
+
}
|
|
675
|
+
if (kind == GROUP_UNSUPPORTED) {
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
} else if (c == '\\') {
|
|
680
|
+
advance(lexer);
|
|
681
|
+
// A backslash-newline continues the line; the word token handles that.
|
|
682
|
+
if (lexer->eof(lexer) || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
683
|
+
return false;
|
|
684
|
+
}
|
|
685
|
+
advance(lexer);
|
|
686
|
+
} else if (is_word_break(c)) {
|
|
687
|
+
return false;
|
|
688
|
+
} else {
|
|
689
|
+
advance(lexer);
|
|
690
|
+
}
|
|
691
|
+
consumed = true;
|
|
692
|
+
}
|
|
693
|
+
return false;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// A leading `#` starts a comment except in an assignment value (`A=#@($x)`).
|
|
697
|
+
static bool scan_extglob_prefix(TSLexer *lexer, bool in_value) {
|
|
698
|
+
return (lexer->lookahead != '#' || in_value) && continue_extglob_prefix(lexer, false);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// Skips a quoted span from its opening quote through its closing one.
|
|
702
|
+
static void skip_quoted(TSLexer *lexer) {
|
|
703
|
+
int32_t quote = lexer->lookahead;
|
|
704
|
+
advance(lexer);
|
|
705
|
+
while (!lexer->eof(lexer) && lexer->lookahead != quote) {
|
|
706
|
+
if (lexer->lookahead == '\\' && quote != '\'') {
|
|
707
|
+
advance(lexer);
|
|
708
|
+
if (lexer->eof(lexer)) {
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
advance(lexer);
|
|
713
|
+
}
|
|
714
|
+
advance(lexer);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// Skips a parenthesized span such as a command substitution's, quotes included.
|
|
718
|
+
static void skip_parenthesized(TSLexer *lexer) {
|
|
719
|
+
uint32_t depth = 0;
|
|
720
|
+
while (!lexer->eof(lexer)) {
|
|
721
|
+
int32_t c = lexer->lookahead;
|
|
722
|
+
if (c == '\'' || c == '"' || c == '`') {
|
|
723
|
+
skip_quoted(lexer);
|
|
724
|
+
continue;
|
|
725
|
+
}
|
|
726
|
+
if (c == '\\') {
|
|
727
|
+
advance(lexer);
|
|
728
|
+
} else if (c == '(') {
|
|
729
|
+
depth++;
|
|
730
|
+
} else if (c == ')' && --depth == 0) {
|
|
731
|
+
advance(lexer);
|
|
732
|
+
return;
|
|
733
|
+
}
|
|
734
|
+
advance(lexer);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// A name at a position where both an assignment and a word may start: `name=`, `name+=`, and
|
|
739
|
+
// `name[…]=` are assignments, and otherwise the name may begin the literal text before a structured
|
|
740
|
+
// extglob group (`foo@($x)`), in which `+(` is an extglob operator rather than an unfinished `+=`.
|
|
741
|
+
static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbols) {
|
|
742
|
+
while (is_name_char(lexer->lookahead)) {
|
|
743
|
+
advance(lexer);
|
|
744
|
+
}
|
|
745
|
+
lexer->mark_end(lexer);
|
|
746
|
+
lexer->result_symbol = VARIABLE_NAME;
|
|
747
|
+
if (lexer->lookahead == '=') {
|
|
748
|
+
return true;
|
|
749
|
+
}
|
|
750
|
+
if (lexer->lookahead == '+') {
|
|
751
|
+
advance(lexer);
|
|
752
|
+
if (lexer->lookahead == '=') {
|
|
753
|
+
return true;
|
|
754
|
+
}
|
|
755
|
+
if (!valid_symbols[EXTGLOB_PREFIX]) {
|
|
756
|
+
return false;
|
|
757
|
+
}
|
|
758
|
+
if (lexer->lookahead == '(') {
|
|
759
|
+
GroupKind kind = scan_extglob_group(lexer);
|
|
760
|
+
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
761
|
+
return kind == GROUP_STRUCTURED || (kind == GROUP_LITERAL && continue_extglob_prefix(lexer, true));
|
|
762
|
+
}
|
|
763
|
+
return continue_extglob_prefix(lexer, true);
|
|
764
|
+
}
|
|
765
|
+
if (lexer->lookahead == '[') {
|
|
766
|
+
// Like bash, the subscript extends to the matching `]` across lines; without one the name
|
|
767
|
+
// still starts an assignment, so the parse reports the missing `]`.
|
|
768
|
+
uint32_t depth = 0;
|
|
769
|
+
while (!lexer->eof(lexer)) {
|
|
770
|
+
int32_t c = lexer->lookahead;
|
|
771
|
+
if (c == '[') {
|
|
772
|
+
depth++;
|
|
773
|
+
} else if (c == ']' && --depth == 0) {
|
|
774
|
+
advance(lexer);
|
|
775
|
+
break;
|
|
776
|
+
} else if (c == '\\') {
|
|
777
|
+
advance(lexer);
|
|
778
|
+
} else if (c == '\'' || c == '"' || c == '`') {
|
|
779
|
+
// Brackets inside quotes do not count (`A["]"]=x`).
|
|
780
|
+
skip_quoted(lexer);
|
|
781
|
+
continue;
|
|
782
|
+
} else if (c == '$') {
|
|
783
|
+
advance(lexer);
|
|
784
|
+
if (lexer->lookahead == '(') {
|
|
785
|
+
skip_parenthesized(lexer);
|
|
786
|
+
}
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
advance(lexer);
|
|
790
|
+
}
|
|
791
|
+
if (depth == 0 && lexer->lookahead == '+') {
|
|
792
|
+
advance(lexer);
|
|
793
|
+
}
|
|
794
|
+
if (depth != 0 || lexer->lookahead == '=') {
|
|
795
|
+
return true;
|
|
796
|
+
}
|
|
797
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
798
|
+
}
|
|
799
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
800
|
+
}
|
|
801
|
+
|
|
355
802
|
static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
356
803
|
bool error_recovery = valid_symbols[ERROR_RECOVERY];
|
|
357
804
|
// Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
|
|
@@ -359,22 +806,54 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
359
806
|
int32_t first = lexer->lookahead;
|
|
360
807
|
bool at_eof = lexer->eof(lexer);
|
|
361
808
|
|
|
809
|
+
if (scanner->terminator_owed) {
|
|
810
|
+
scanner->terminator_owed = false;
|
|
811
|
+
if (valid_symbols[NEWLINE] && lexer->lookahead == '}') {
|
|
812
|
+
lexer->result_symbol = NEWLINE;
|
|
813
|
+
return true;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// Counted before a heredoc body may start, so a newline right after `$(` is inside it.
|
|
818
|
+
if (!error_recovery &&
|
|
819
|
+
(valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
|
|
820
|
+
valid_symbols[BRACKET_SUBSTITUTION_START])) {
|
|
821
|
+
// Nesting too deep for the serialized state is left to the parse as an error.
|
|
822
|
+
if (serialized_size(scanner) >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
823
|
+
return false;
|
|
824
|
+
}
|
|
825
|
+
lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
|
|
826
|
+
: valid_symbols[BRACE_SUBSTITUTION_START] ? BRACE_SUBSTITUTION_START
|
|
827
|
+
: BRACKET_SUBSTITUTION_START;
|
|
828
|
+
uint8_t closer = lexer->result_symbol == SUBSTITUTION_START ? ')'
|
|
829
|
+
: lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
|
|
830
|
+
: ']';
|
|
831
|
+
array_push(&scanner->closers, closer);
|
|
832
|
+
return true;
|
|
833
|
+
}
|
|
834
|
+
|
|
362
835
|
if (scanner->heredocs.size > 0) {
|
|
363
|
-
|
|
364
|
-
if (
|
|
365
|
-
!(lexer->lookahead == '`' && !
|
|
366
|
-
return scan_heredoc_content(scanner, lexer);
|
|
836
|
+
int32_t active = active_heredoc_index(scanner);
|
|
837
|
+
if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
|
|
838
|
+
!(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw)) {
|
|
839
|
+
return scan_heredoc_content(scanner, lexer, (uint32_t)active);
|
|
367
840
|
}
|
|
368
841
|
// A newline inside a string is part of it, so bodies start only outside strings.
|
|
369
|
-
|
|
370
|
-
|
|
842
|
+
int32_t next = next_heredoc_index(scanner);
|
|
843
|
+
if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && !valid_symbols[STRING_CONTENT]) {
|
|
371
844
|
while (is_blank(lexer->lookahead)) {
|
|
372
845
|
skip(lexer);
|
|
373
846
|
}
|
|
374
847
|
if (lexer->lookahead == '\n') {
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
848
|
+
// The line's heredocs cannot be located once one of them was rejected, so none gets a
|
|
849
|
+
// body; scanning continues so that the change is kept with the token found below.
|
|
850
|
+
if (is_rejected(array_get(&scanner->heredocs, next))) {
|
|
851
|
+
drop_unstarted_heredocs_at_depth(scanner);
|
|
852
|
+
} else {
|
|
853
|
+
array_get(&scanner->heredocs, next)->started = true;
|
|
854
|
+
lexer->result_symbol = HEREDOC_BODY_START;
|
|
855
|
+
return true;
|
|
856
|
+
}
|
|
378
857
|
}
|
|
379
858
|
}
|
|
380
859
|
}
|
|
@@ -393,8 +872,46 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
393
872
|
|
|
394
873
|
if (valid_symbols[CONCAT]) {
|
|
395
874
|
int32_t c = first;
|
|
396
|
-
if (
|
|
397
|
-
|
|
875
|
+
if (c == '\\' && lexer->lookahead == '\\') {
|
|
876
|
+
// A backslash-newline joins lines, so the word continues only if the next line does.
|
|
877
|
+
lexer->mark_end(lexer);
|
|
878
|
+
advance(lexer);
|
|
879
|
+
if (lexer->lookahead == '\r') {
|
|
880
|
+
advance(lexer);
|
|
881
|
+
}
|
|
882
|
+
// Bash drops a backslash at the end of input like a continuation.
|
|
883
|
+
if (lexer->eof(lexer)) {
|
|
884
|
+
lexer->mark_end(lexer);
|
|
885
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
886
|
+
return true;
|
|
887
|
+
}
|
|
888
|
+
if (lexer->lookahead != '\n') {
|
|
889
|
+
lexer->result_symbol = CONCAT;
|
|
890
|
+
return true;
|
|
891
|
+
}
|
|
892
|
+
advance(lexer);
|
|
893
|
+
lexer->mark_end(lexer);
|
|
894
|
+
c = lexer->lookahead;
|
|
895
|
+
// The word continues only if the next line does; otherwise the continuation itself is the
|
|
896
|
+
// token, so that whatever follows it is lexed as it would be without it.
|
|
897
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
898
|
+
if (lexer->eof(lexer) || iswspace(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
899
|
+
return true;
|
|
900
|
+
}
|
|
901
|
+
if (c == '\\') {
|
|
902
|
+
// Another continuation is whitespace too; an escaped character continues the word.
|
|
903
|
+
advance(lexer);
|
|
904
|
+
if (lexer->lookahead == '\r') {
|
|
905
|
+
advance(lexer);
|
|
906
|
+
}
|
|
907
|
+
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
908
|
+
return true;
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
lexer->result_symbol = CONCAT;
|
|
912
|
+
return true;
|
|
913
|
+
}
|
|
914
|
+
if (!at_eof && !iswspace(c) && !is_metacharacter(c) && !(c == '`' && scanner->backtick_depth > 0)) {
|
|
398
915
|
lexer->result_symbol = CONCAT;
|
|
399
916
|
return true;
|
|
400
917
|
}
|
|
@@ -402,6 +919,17 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
402
919
|
|
|
403
920
|
if (valid_symbols[EMPTY_VALUE]) {
|
|
404
921
|
int32_t c = first;
|
|
922
|
+
// A redirection may follow an empty value (`A=>f`), but `<(`/`>(` is a process substitution
|
|
923
|
+
// value; nothing else a value may start with begins with `<` or `>`.
|
|
924
|
+
if ((c == '<' || c == '>') && lexer->lookahead == c) {
|
|
925
|
+
lexer->mark_end(lexer);
|
|
926
|
+
advance(lexer);
|
|
927
|
+
if (lexer->lookahead == '(') {
|
|
928
|
+
return false;
|
|
929
|
+
}
|
|
930
|
+
lexer->result_symbol = EMPTY_VALUE;
|
|
931
|
+
return true;
|
|
932
|
+
}
|
|
405
933
|
if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
|
|
406
934
|
(c == '`' && scanner->backtick_depth > 0)) {
|
|
407
935
|
lexer->result_symbol = EMPTY_VALUE;
|
|
@@ -426,24 +954,49 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
426
954
|
}
|
|
427
955
|
skip(lexer);
|
|
428
956
|
} else if (lexer->lookahead == '\\') {
|
|
429
|
-
|
|
957
|
+
// A continuation is returned as a token, since tree-sitter's lexer cannot always resume
|
|
958
|
+
// after one; a backslash escaping a word's first character can only start an extglob prefix
|
|
959
|
+
// (`\foo@($x)`), and the word token handles it otherwise.
|
|
960
|
+
advance(lexer);
|
|
430
961
|
if (lexer->lookahead == '\r') {
|
|
431
|
-
|
|
962
|
+
advance(lexer);
|
|
432
963
|
}
|
|
433
|
-
if (lexer->lookahead
|
|
434
|
-
|
|
964
|
+
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
965
|
+
advance(lexer);
|
|
966
|
+
lexer->mark_end(lexer);
|
|
967
|
+
lexer->result_symbol = LINE_CONTINUATION;
|
|
968
|
+
return true;
|
|
435
969
|
}
|
|
436
|
-
|
|
970
|
+
if (valid_symbols[EXTGLOB_PREFIX]) {
|
|
971
|
+
advance(lexer);
|
|
972
|
+
return continue_extglob_prefix(lexer, true);
|
|
973
|
+
}
|
|
974
|
+
return false;
|
|
437
975
|
} else {
|
|
438
976
|
break;
|
|
439
977
|
}
|
|
440
978
|
}
|
|
441
979
|
|
|
980
|
+
// Each closer has its own marker, so a `]` or `}` argument inside `$( )` does not end it.
|
|
981
|
+
enum TokenType end = valid_symbols[SUBSTITUTION_END] && lexer->lookahead == ')' ? SUBSTITUTION_END
|
|
982
|
+
: valid_symbols[BRACE_SUBSTITUTION_END] && lexer->lookahead == '}' ? BRACE_SUBSTITUTION_END
|
|
983
|
+
: valid_symbols[BRACKET_SUBSTITUTION_END] && lexer->lookahead == ']' ? BRACKET_SUBSTITUTION_END
|
|
984
|
+
: ERROR_RECOVERY;
|
|
985
|
+
if (end != ERROR_RECOVERY) {
|
|
986
|
+
if (scanner->closers.size > 0) {
|
|
987
|
+
array_pop(&scanner->closers);
|
|
988
|
+
}
|
|
989
|
+
discard_unstarted_heredocs(scanner);
|
|
990
|
+
lexer->result_symbol = end;
|
|
991
|
+
return true;
|
|
992
|
+
}
|
|
993
|
+
|
|
442
994
|
if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
|
|
443
995
|
if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
|
|
444
996
|
advance(lexer);
|
|
445
997
|
lexer->mark_end(lexer);
|
|
446
998
|
scanner->backtick_depth--;
|
|
999
|
+
discard_unstarted_heredocs(scanner);
|
|
447
1000
|
lexer->result_symbol = BACKTICK_CLOSE;
|
|
448
1001
|
return true;
|
|
449
1002
|
}
|
|
@@ -460,18 +1013,27 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
460
1013
|
return scan_heredoc_arrow(scanner, lexer, valid_symbols);
|
|
461
1014
|
}
|
|
462
1015
|
|
|
1016
|
+
// Only one check that consumes input can run, so the prefix scan leaves names before a possible
|
|
1017
|
+
// assignment to their own check; digits are read once as a descriptor or a word prefix.
|
|
463
1018
|
if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
|
|
464
1019
|
while (iswdigit(lexer->lookahead)) {
|
|
465
1020
|
advance(lexer);
|
|
466
1021
|
}
|
|
467
1022
|
lexer->mark_end(lexer);
|
|
468
1023
|
lexer->result_symbol = FILE_DESCRIPTOR;
|
|
469
|
-
|
|
1024
|
+
if (lexer->lookahead == '<' || lexer->lookahead == '>') {
|
|
1025
|
+
return true;
|
|
1026
|
+
}
|
|
1027
|
+
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
if (valid_symbols[EXTGLOB_PREFIX] && !(valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) &&
|
|
1031
|
+
!is_word_break(lexer->lookahead)) {
|
|
1032
|
+
return scan_extglob_prefix(lexer, valid_symbols[EMPTY_VALUE]);
|
|
470
1033
|
}
|
|
471
1034
|
|
|
472
1035
|
if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
|
|
473
|
-
lexer
|
|
474
|
-
return scan_variable_name(lexer);
|
|
1036
|
+
return scan_name_or_extglob_prefix(lexer, valid_symbols);
|
|
475
1037
|
}
|
|
476
1038
|
|
|
477
1039
|
if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
|
|
@@ -488,6 +1050,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
|
488
1050
|
void *tree_sitter_bash_external_scanner_create(void) {
|
|
489
1051
|
Scanner *scanner = ts_calloc(1, sizeof(Scanner));
|
|
490
1052
|
array_init(&scanner->heredocs);
|
|
1053
|
+
array_init(&scanner->closers);
|
|
491
1054
|
return scanner;
|
|
492
1055
|
}
|
|
493
1056
|
|
|
@@ -495,6 +1058,7 @@ void tree_sitter_bash_external_scanner_destroy(void *payload) {
|
|
|
495
1058
|
Scanner *scanner = (Scanner *)payload;
|
|
496
1059
|
scanner_reset(scanner);
|
|
497
1060
|
array_delete(&scanner->heredocs);
|
|
1061
|
+
array_delete(&scanner->closers);
|
|
498
1062
|
ts_free(scanner);
|
|
499
1063
|
}
|
|
500
1064
|
|
|
@@ -506,51 +1070,71 @@ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer
|
|
|
506
1070
|
Scanner *scanner = (Scanner *)payload;
|
|
507
1071
|
unsigned size = 0;
|
|
508
1072
|
buffer[size++] = (char)scanner->backtick_depth;
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
1073
|
+
buffer[size++] = (char)scanner->terminator_owed;
|
|
1074
|
+
uint16_t closer_count = (uint16_t)scanner->closers.size;
|
|
1075
|
+
memcpy(&buffer[size], &closer_count, sizeof(closer_count));
|
|
1076
|
+
size += sizeof(closer_count);
|
|
1077
|
+
if (scanner->closers.size > 0) {
|
|
1078
|
+
memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
|
|
1079
|
+
size += scanner->closers.size;
|
|
1080
|
+
}
|
|
1081
|
+
buffer[size++] = (char)scanner->heredocs.size;
|
|
1082
|
+
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
512
1083
|
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
513
|
-
if (size + 3 + sizeof(uint16_t) + heredoc->delimiter.size > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
514
|
-
break;
|
|
515
|
-
}
|
|
516
1084
|
buffer[size++] = (char)heredoc->is_raw;
|
|
517
1085
|
buffer[size++] = (char)heredoc->allows_indent;
|
|
518
1086
|
buffer[size++] = (char)heredoc->started;
|
|
1087
|
+
memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
|
|
1088
|
+
size += sizeof(heredoc->depth);
|
|
519
1089
|
uint16_t length = (uint16_t)heredoc->delimiter.size;
|
|
520
1090
|
memcpy(&buffer[size], &length, sizeof(length));
|
|
521
1091
|
size += sizeof(length);
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
size += length;
|
|
1092
|
+
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
1093
|
+
size += encode_varint(*array_get(&heredoc->delimiter, j), &buffer[size]);
|
|
525
1094
|
}
|
|
526
|
-
count++;
|
|
527
1095
|
}
|
|
528
|
-
buffer[count_position] = (char)count;
|
|
529
1096
|
return size;
|
|
530
1097
|
}
|
|
531
1098
|
|
|
532
1099
|
void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
|
|
533
1100
|
Scanner *scanner = (Scanner *)payload;
|
|
534
1101
|
scanner_reset(scanner);
|
|
535
|
-
if (length
|
|
1102
|
+
if (length < 2 + sizeof(uint16_t)) {
|
|
536
1103
|
return;
|
|
537
1104
|
}
|
|
538
1105
|
unsigned size = 0;
|
|
539
1106
|
scanner->backtick_depth = (uint8_t)buffer[size++];
|
|
1107
|
+
scanner->terminator_owed = buffer[size++];
|
|
1108
|
+
uint16_t closer_count;
|
|
1109
|
+
memcpy(&closer_count, &buffer[size], sizeof(closer_count));
|
|
1110
|
+
size += sizeof(closer_count);
|
|
1111
|
+
if (size + closer_count >= length) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
array_reserve(&scanner->closers, closer_count);
|
|
1115
|
+
memcpy(scanner->closers.contents, &buffer[size], closer_count);
|
|
1116
|
+
scanner->closers.size = closer_count;
|
|
1117
|
+
size += closer_count;
|
|
540
1118
|
uint8_t count = (uint8_t)buffer[size++];
|
|
541
|
-
for (uint8_t i = 0; i < count; i++) {
|
|
1119
|
+
for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
|
|
542
1120
|
Heredoc heredoc = {.delimiter = array_new()};
|
|
543
1121
|
heredoc.is_raw = buffer[size++];
|
|
544
1122
|
heredoc.allows_indent = buffer[size++];
|
|
545
1123
|
heredoc.started = buffer[size++];
|
|
1124
|
+
memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
|
|
1125
|
+
size += sizeof(heredoc.depth);
|
|
546
1126
|
uint16_t delimiter_length;
|
|
547
1127
|
memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
|
|
548
1128
|
size += sizeof(delimiter_length);
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
1129
|
+
array_reserve(&heredoc.delimiter, delimiter_length);
|
|
1130
|
+
for (uint16_t j = 0; j < delimiter_length; j++) {
|
|
1131
|
+
int32_t c;
|
|
1132
|
+
unsigned consumed = decode_varint(&buffer[size], length - size, &c);
|
|
1133
|
+
if (consumed == 0) {
|
|
1134
|
+
break;
|
|
1135
|
+
}
|
|
1136
|
+
size += consumed;
|
|
1137
|
+
array_push(&heredoc.delimiter, c);
|
|
554
1138
|
}
|
|
555
1139
|
array_push(&scanner->heredocs, heredoc);
|
|
556
1140
|
}
|