@willbooster/tree-sitter-bash 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/package.json +2 -2
- package/tree-sitter-bash.wasm +0 -0
- package/tree-sitter.json +1 -9
- package/src/grammar.json +0 -5000
- package/src/parser.c +0 -263210
- package/src/scanner.c +0 -1659
- package/src/tree_sitter/alloc.h +0 -54
- package/src/tree_sitter/array.h +0 -336
- package/src/tree_sitter/parser.h +0 -286
package/src/scanner.c
DELETED
|
@@ -1,1659 +0,0 @@
|
|
|
1
|
-
#include "tree_sitter/array.h"
|
|
2
|
-
#include "tree_sitter/parser.h"
|
|
3
|
-
|
|
4
|
-
#include <string.h>
|
|
5
|
-
#include <wctype.h>
|
|
6
|
-
|
|
7
|
-
// Must match the order of `externals` in grammar.js.
|
|
8
|
-
enum TokenType {
|
|
9
|
-
HEREDOC_START,
|
|
10
|
-
HEREDOC_BODY_START,
|
|
11
|
-
HEREDOC_CONTENT,
|
|
12
|
-
HEREDOC_END,
|
|
13
|
-
HEREDOC_ARROW,
|
|
14
|
-
HEREDOC_ARROW_DASH,
|
|
15
|
-
CONCAT,
|
|
16
|
-
FILE_DESCRIPTOR,
|
|
17
|
-
VARIABLE_NAME,
|
|
18
|
-
BARE_DOLLAR,
|
|
19
|
-
EMPTY_VALUE,
|
|
20
|
-
REGEX,
|
|
21
|
-
STRING_CONTENT,
|
|
22
|
-
BACKTICK_OPEN,
|
|
23
|
-
BACKTICK_CLOSE,
|
|
24
|
-
NEWLINE,
|
|
25
|
-
SUBSTITUTION_START,
|
|
26
|
-
SUBSTITUTION_END,
|
|
27
|
-
BRACE_SUBSTITUTION_END,
|
|
28
|
-
BRACKET_SUBSTITUTION_END,
|
|
29
|
-
EXTGLOB_PREFIX,
|
|
30
|
-
LINE_CONTINUATION,
|
|
31
|
-
BRACE_SUBSTITUTION_START,
|
|
32
|
-
BRACKET_SUBSTITUTION_START,
|
|
33
|
-
REGEX_START,
|
|
34
|
-
REGEX_CONCAT,
|
|
35
|
-
CLOSER_LINE_RAW_STRING_START,
|
|
36
|
-
CLOSER_LINE_ANSI_C_STRING_START,
|
|
37
|
-
CLOSER_LINE_RAW_STRING_CONTENT,
|
|
38
|
-
CLOSER_LINE_ANSI_C_STRING_CONTENT,
|
|
39
|
-
JOINED_COMMENT,
|
|
40
|
-
ERROR_RECOVERY,
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
typedef Array(int32_t) CodePoints;
|
|
44
|
-
|
|
45
|
-
// A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
|
|
46
|
-
// (not one inside a string) at the substitution depth where the operator appeared, after the bodies
|
|
47
|
-
// of heredocs opened earlier at that depth: bash reads a substitution, including the bodies of the
|
|
48
|
-
// heredocs inside it, before the body of a heredoc whose operator precedes it on the same line.
|
|
49
|
-
typedef struct {
|
|
50
|
-
bool is_raw;
|
|
51
|
-
bool allows_indent;
|
|
52
|
-
bool started;
|
|
53
|
-
// Queued on the same line as a heredoc whose body ended at a line holding the substitution's `)`:
|
|
54
|
-
// bash then reads this body right after that line, wherever the line ends.
|
|
55
|
-
bool deferred;
|
|
56
|
-
uint16_t depth;
|
|
57
|
-
CodePoints delimiter;
|
|
58
|
-
} Heredoc;
|
|
59
|
-
|
|
60
|
-
typedef struct {
|
|
61
|
-
Array(Heredoc) heredocs;
|
|
62
|
-
// Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
|
|
63
|
-
// whether a backquote opens or closes one.
|
|
64
|
-
uint8_t backtick_depth;
|
|
65
|
-
// The closer (`)`, `}`, or `]`) of each open substitution, innermost last.
|
|
66
|
-
Array(uint8_t) closers;
|
|
67
|
-
// The regex group depth when each open substitution started, restored when it ends.
|
|
68
|
-
Array(uint16_t) closer_regex_depths;
|
|
69
|
-
// How many closers were open when the backquotes opened; the ones after them are nested inside.
|
|
70
|
-
uint16_t backtick_closers;
|
|
71
|
-
// A heredoc body ended before the end of its line (at a `${ list; }` closer, or at a delimiter
|
|
72
|
-
// followed by more code); the newline that ended the command's line began the body, so a
|
|
73
|
-
// terminator is still owed before what follows.
|
|
74
|
-
bool terminator_owed;
|
|
75
|
-
// The last token of this scanner was a line continuation that ended a line; nothing concatenates at
|
|
76
|
-
// the start of the next one.
|
|
77
|
-
bool after_line_break;
|
|
78
|
-
// The rest of a line that ended a heredoc body at the substitution's `)` is being read: bash reads it
|
|
79
|
-
// with its line continuations removed, even inside single quotes and comments.
|
|
80
|
-
bool joined_line;
|
|
81
|
-
// A line continuation was split before its newline, where a deferred body started; its newline is still
|
|
82
|
-
// to be read, at the end of a word (`in_word`, of a regex if `in_regex`) or between words.
|
|
83
|
-
bool split_continuation;
|
|
84
|
-
bool split_continuation_in_word;
|
|
85
|
-
bool split_continuation_in_regex;
|
|
86
|
-
// A quoted string on a line that ended a body at the substitution's `)` has just opened; its first text
|
|
87
|
-
// token may be empty, so that it always has one and joining them gives its value.
|
|
88
|
-
bool closer_line_string_opened;
|
|
89
|
-
// The number of unclosed parentheses in the regex being read, which hold blanks and metacharacters.
|
|
90
|
-
uint16_t regex_depth;
|
|
91
|
-
uint16_t backtick_regex_depth;
|
|
92
|
-
} Scanner;
|
|
93
|
-
|
|
94
|
-
static inline uint16_t current_depth(Scanner *scanner) {
|
|
95
|
-
return scanner->backtick_depth + scanner->closers.size;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
|
|
99
|
-
|
|
100
|
-
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
|
|
101
|
-
|
|
102
|
-
static inline bool is_metacharacter(int32_t c) {
|
|
103
|
-
return c == '|' || c == '&' || c == ';' || c == '(' || c == ')' || c == '<' || c == '>';
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// Like bash, only a space and a tab are blanks; every other whitespace character, CR included, is part
|
|
107
|
-
// of a word.
|
|
108
|
-
static inline bool is_blank(int32_t c) { return c == ' ' || c == '\t'; }
|
|
109
|
-
|
|
110
|
-
static inline bool is_separator(int32_t c) { return is_blank(c) || c == '\n'; }
|
|
111
|
-
|
|
112
|
-
static inline bool is_name_start(int32_t c) { return iswalpha(c) || c == '_'; }
|
|
113
|
-
|
|
114
|
-
static inline bool is_name_char(int32_t c) { return iswalnum(c) || c == '_'; }
|
|
115
|
-
|
|
116
|
-
// Whether `$` followed by `c` begins an expansion inside double quotes or a heredoc body.
|
|
117
|
-
static inline bool starts_quoted_expansion(int32_t c) {
|
|
118
|
-
return is_name_start(c) || iswdigit(c) || c == '{' || c == '(' || c == '[' || c == '*' || c == '@' ||
|
|
119
|
-
c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// Code points are serialized as zigzag varints, one byte for code points below 64 and two for the
|
|
123
|
-
// rest of ASCII; every int32 round-trips, including the negative value the lexer reports for invalid
|
|
124
|
-
// UTF-8.
|
|
125
|
-
static unsigned varint_length(int32_t c) {
|
|
126
|
-
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
127
|
-
unsigned length = 1;
|
|
128
|
-
while (value >= 0x80) {
|
|
129
|
-
value >>= 7;
|
|
130
|
-
length++;
|
|
131
|
-
}
|
|
132
|
-
return length;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
static unsigned encode_varint(int32_t c, char *out) {
|
|
136
|
-
uint32_t value = ((uint32_t)c << 1) ^ (uint32_t)(c >> 31);
|
|
137
|
-
unsigned length = 0;
|
|
138
|
-
while (value >= 0x80) {
|
|
139
|
-
out[length++] = (char)(0x80 | (value & 0x7F));
|
|
140
|
-
value >>= 7;
|
|
141
|
-
}
|
|
142
|
-
out[length++] = (char)value;
|
|
143
|
-
return length;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// Returns 0 when the input ends before the varint does.
|
|
147
|
-
static unsigned decode_varint(const char *in, unsigned available, int32_t *c) {
|
|
148
|
-
uint32_t value = 0;
|
|
149
|
-
for (unsigned i = 0; i < available && i < 5; i++) {
|
|
150
|
-
value |= (uint32_t)((unsigned char)in[i] & 0x7F) << (7 * i);
|
|
151
|
-
if (((unsigned char)in[i] & 0x80) == 0) {
|
|
152
|
-
*c = (int32_t)((value >> 1) ^ (0u - (value & 1)));
|
|
153
|
-
return i + 1;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
return 0;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// Serialized layout: backtick depth, the closer count when the backquotes opened (uint16), the owed-terminator,
|
|
160
|
-
// line-break, joined-line, and split-continuation flags, the regex depth and the one saved when the backquotes
|
|
161
|
-
// opened (uint16 each), the closers of the open substitutions (uint16 count, then one byte each, then their
|
|
162
|
-
// saved regex depths as uint16), and heredoc count, then per heredoc its flags, depth (uint16), delimiter
|
|
163
|
-
// length (uint16), and the delimiter as varints.
|
|
164
|
-
#define SCANNER_HEADER_SIZE (2 + 4 * sizeof(uint16_t))
|
|
165
|
-
#define CLOSER_SIZE (1 + sizeof(uint16_t))
|
|
166
|
-
#define HEREDOC_HEADER_SIZE (3 + 2 * sizeof(uint16_t))
|
|
167
|
-
|
|
168
|
-
static unsigned serialized_size(Scanner *scanner) {
|
|
169
|
-
unsigned size = SCANNER_HEADER_SIZE + CLOSER_SIZE * scanner->closers.size + 1;
|
|
170
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
171
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
172
|
-
size += HEREDOC_HEADER_SIZE;
|
|
173
|
-
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
174
|
-
size += varint_length(*array_get(&heredoc->delimiter, j));
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
return size;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
|
|
181
|
-
|
|
182
|
-
static void scanner_reset(Scanner *scanner) {
|
|
183
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
184
|
-
heredoc_delete(array_get(&scanner->heredocs, i));
|
|
185
|
-
}
|
|
186
|
-
array_clear(&scanner->heredocs);
|
|
187
|
-
scanner->backtick_depth = 0;
|
|
188
|
-
array_clear(&scanner->closers);
|
|
189
|
-
array_clear(&scanner->closer_regex_depths);
|
|
190
|
-
scanner->backtick_closers = 0;
|
|
191
|
-
scanner->terminator_owed = false;
|
|
192
|
-
scanner->after_line_break = false;
|
|
193
|
-
scanner->joined_line = false;
|
|
194
|
-
scanner->split_continuation = false;
|
|
195
|
-
scanner->split_continuation_in_word = false;
|
|
196
|
-
scanner->split_continuation_in_regex = false;
|
|
197
|
-
scanner->closer_line_string_opened = false;
|
|
198
|
-
scanner->regex_depth = 0;
|
|
199
|
-
scanner->backtick_regex_depth = 0;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
// The body being read: the most deeply nested started heredoc.
|
|
203
|
-
static int32_t active_heredoc_index(Scanner *scanner) {
|
|
204
|
-
int32_t active = -1;
|
|
205
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
206
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
207
|
-
if (heredoc->started && (active < 0 || heredoc->depth >= array_get(&scanner->heredocs, active)->depth)) {
|
|
208
|
-
active = (int32_t)i;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
return active;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
// The next body to start: the first deferred heredoc, or else the first unstarted heredoc opened at the
|
|
215
|
-
// current substitution depth.
|
|
216
|
-
static int32_t next_heredoc_index(Scanner *scanner) {
|
|
217
|
-
uint16_t depth = current_depth(scanner);
|
|
218
|
-
int32_t next = -1;
|
|
219
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
220
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
221
|
-
if (heredoc->started) {
|
|
222
|
-
continue;
|
|
223
|
-
}
|
|
224
|
-
if (heredoc->deferred) {
|
|
225
|
-
return (int32_t)i;
|
|
226
|
-
}
|
|
227
|
-
if (next < 0 && heredoc->depth == depth) {
|
|
228
|
-
next = (int32_t)i;
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
return next;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
// An entry without a delimiter belongs to an operator whose delimiter was rejected.
|
|
235
|
-
static inline bool is_rejected(Heredoc *heredoc) { return heredoc->delimiter.size == 0 && !heredoc->is_raw; }
|
|
236
|
-
|
|
237
|
-
// Drops the unstarted heredocs at the current depth, which were all queued on the line just ended.
|
|
238
|
-
static void drop_unstarted_heredocs_at_depth(Scanner *scanner) {
|
|
239
|
-
uint16_t depth = current_depth(scanner);
|
|
240
|
-
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
241
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
242
|
-
if (!heredoc->started && !heredoc->deferred && heredoc->depth == depth) {
|
|
243
|
-
heredoc_delete(heredoc);
|
|
244
|
-
array_erase(&scanner->heredocs, i - 1);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
static void remove_heredoc(Scanner *scanner, uint32_t index) {
|
|
250
|
-
heredoc_delete(array_get(&scanner->heredocs, index));
|
|
251
|
-
array_erase(&scanner->heredocs, index);
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// Like bash, drop the heredocs of a substitution that closes before their bodies started.
|
|
255
|
-
static void discard_unstarted_heredocs(Scanner *scanner) {
|
|
256
|
-
uint16_t depth = current_depth(scanner);
|
|
257
|
-
for (uint32_t i = scanner->heredocs.size; i > 0; i--) {
|
|
258
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i - 1);
|
|
259
|
-
if (!heredoc->started && !heredoc->deferred && heredoc->depth > depth) {
|
|
260
|
-
remove_heredoc(scanner, i - 1);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
// Ends a body at a line that holds the substitution's `)`. Like bash, the bodies still queued on the line
|
|
266
|
-
// that began this one are then read right after that line, and the rest of the line is read without its
|
|
267
|
-
// line continuations unless the delimiter was quoted.
|
|
268
|
-
static void end_heredoc_at_closer_line(Scanner *scanner, uint32_t index) {
|
|
269
|
-
Heredoc *ended = array_get(&scanner->heredocs, index);
|
|
270
|
-
scanner->joined_line = !ended->is_raw;
|
|
271
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
272
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
273
|
-
if (!heredoc->started && heredoc->depth == ended->depth) {
|
|
274
|
-
heredoc->deferred = true;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
remove_heredoc(scanner, index);
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
// Whether a deferred body waits for the end of the line that ended a body at the substitution's `)`. That
|
|
281
|
-
// line ends at its first newline even after a backslash or inside quotes, unless it is joined.
|
|
282
|
-
static bool awaits_deferred_body(Scanner *scanner) {
|
|
283
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
284
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
285
|
-
if (heredoc->deferred && !heredoc->started) {
|
|
286
|
-
return true;
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
return false;
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
// Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
|
|
293
|
-
// line (or at the end of input, as bash does with a warning). An unquoted body stops before each
|
|
294
|
-
// expansion so that the parser can read it.
|
|
295
|
-
static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer, uint32_t index) {
|
|
296
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, index);
|
|
297
|
-
// Bash reads a deferred body from the input after its line, never from the text of backquotes that the
|
|
298
|
-
// line opened, so a backquote in it is literal.
|
|
299
|
-
bool in_backquotes = scanner->backtick_depth > 0 && !heredoc->deferred;
|
|
300
|
-
bool did_advance = false;
|
|
301
|
-
bool at_line_start = lexer->get_column(lexer) == 0;
|
|
302
|
-
|
|
303
|
-
for (;;) {
|
|
304
|
-
if (at_line_start) {
|
|
305
|
-
at_line_start = false;
|
|
306
|
-
lexer->mark_end(lexer);
|
|
307
|
-
// Like bash, compare the delimiter with the line after joining backslash-newlines and
|
|
308
|
-
// then stripping the leading tabs of `<<-`.
|
|
309
|
-
bool consumed = false;
|
|
310
|
-
bool escaped = false;
|
|
311
|
-
uint32_t matched = 0;
|
|
312
|
-
// Inside `$( )`, `<( )`, and `>( )`, bash also ends the body at a line that merely starts with
|
|
313
|
-
// the delimiter when a `)` follows anywhere on that line, and reads the rest of the line as
|
|
314
|
-
// code. Such a line ends the content token before it and the end token after the delimiter.
|
|
315
|
-
// Where the backquotes are innermost, bash fixes their extent before it reads the body, so
|
|
316
|
-
// the rule is off.
|
|
317
|
-
uint16_t outside_backquotes = in_backquotes ? scanner->backtick_closers : 0;
|
|
318
|
-
bool in_parenthesized =
|
|
319
|
-
scanner->closers.size > outside_backquotes && *array_back(&scanner->closers) == ')';
|
|
320
|
-
bool prefix_marked = false;
|
|
321
|
-
int32_t escaped_after_prefix = 0;
|
|
322
|
-
// At the end of input the lookahead is 0, which a delimiter holding NUL would otherwise match.
|
|
323
|
-
// Bash compares one line at a time, so a delimiter holding a newline (`<<"E` + newline +
|
|
324
|
-
// `OF"`) never matches.
|
|
325
|
-
for (;;) {
|
|
326
|
-
if (matched == 0 && heredoc->allows_indent && lexer->lookahead == '\t') {
|
|
327
|
-
advance(lexer);
|
|
328
|
-
consumed = true;
|
|
329
|
-
} else if (lexer->lookahead == '\\' && !heredoc->is_raw) {
|
|
330
|
-
advance(lexer);
|
|
331
|
-
consumed = true;
|
|
332
|
-
if (lexer->lookahead != '\n' || lexer->eof(lexer)) {
|
|
333
|
-
// An escape rather than a line continuation: the escaped character is content.
|
|
334
|
-
if (!lexer->eof(lexer)) {
|
|
335
|
-
if (prefix_marked) {
|
|
336
|
-
escaped_after_prefix = lexer->lookahead;
|
|
337
|
-
}
|
|
338
|
-
advance(lexer);
|
|
339
|
-
}
|
|
340
|
-
escaped = true;
|
|
341
|
-
break;
|
|
342
|
-
}
|
|
343
|
-
advance(lexer);
|
|
344
|
-
} else if (matched < heredoc->delimiter.size && !lexer->eof(lexer) && lexer->lookahead != '\n' &&
|
|
345
|
-
lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
|
|
346
|
-
advance(lexer);
|
|
347
|
-
matched++;
|
|
348
|
-
if (matched == heredoc->delimiter.size && in_parenthesized) {
|
|
349
|
-
if (did_advance) {
|
|
350
|
-
lexer->result_symbol = HEREDOC_CONTENT;
|
|
351
|
-
return true;
|
|
352
|
-
}
|
|
353
|
-
lexer->mark_end(lexer);
|
|
354
|
-
prefix_marked = true;
|
|
355
|
-
}
|
|
356
|
-
} else {
|
|
357
|
-
break;
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
// Inside a substitution, bash also ends the body at a delimiter directly followed by the
|
|
361
|
-
// innermost substitution's closer (`)` or `}`) or a closing backquote.
|
|
362
|
-
bool ends_line = lexer->lookahead == '\n' || lexer->eof(lexer) ||
|
|
363
|
-
(scanner->closers.size > 0 && lexer->lookahead == *array_back(&scanner->closers)) ||
|
|
364
|
-
(lexer->lookahead == '`' && in_backquotes);
|
|
365
|
-
if (!escaped && matched == heredoc->delimiter.size && ends_line) {
|
|
366
|
-
if (did_advance) {
|
|
367
|
-
lexer->result_symbol = HEREDOC_CONTENT;
|
|
368
|
-
return true;
|
|
369
|
-
}
|
|
370
|
-
lexer->mark_end(lexer);
|
|
371
|
-
lexer->result_symbol = HEREDOC_END;
|
|
372
|
-
scanner->terminator_owed = lexer->lookahead == '}';
|
|
373
|
-
if (in_parenthesized && lexer->lookahead == ')') {
|
|
374
|
-
end_heredoc_at_closer_line(scanner, index);
|
|
375
|
-
} else {
|
|
376
|
-
remove_heredoc(scanner, index);
|
|
377
|
-
}
|
|
378
|
-
return true;
|
|
379
|
-
}
|
|
380
|
-
if (prefix_marked) {
|
|
381
|
-
// The line is read after joining its backslash-newlines, as for the delimiter; any other
|
|
382
|
-
// escaped character is kept with its backslash, so a newline after it ends the line.
|
|
383
|
-
while (!lexer->eof(lexer) && lexer->lookahead != '\n' && lexer->lookahead != ')') {
|
|
384
|
-
bool backslash = lexer->lookahead == '\\' && !heredoc->is_raw;
|
|
385
|
-
advance(lexer);
|
|
386
|
-
if (backslash && !lexer->eof(lexer) && lexer->lookahead != ')') {
|
|
387
|
-
advance(lexer);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
if (escaped_after_prefix == ')' || lexer->lookahead == ')') {
|
|
391
|
-
lexer->result_symbol = HEREDOC_END;
|
|
392
|
-
scanner->terminator_owed = true;
|
|
393
|
-
end_heredoc_at_closer_line(scanner, index);
|
|
394
|
-
} else {
|
|
395
|
-
lexer->result_symbol = HEREDOC_CONTENT;
|
|
396
|
-
}
|
|
397
|
-
return true;
|
|
398
|
-
}
|
|
399
|
-
// The characters read while trying the delimiter are content; an empty attempt must not
|
|
400
|
-
// count, or a zero-width content token would repeat forever.
|
|
401
|
-
did_advance = did_advance || consumed || matched > 0;
|
|
402
|
-
continue;
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
if (lexer->eof(lexer)) {
|
|
406
|
-
lexer->mark_end(lexer);
|
|
407
|
-
lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
|
|
408
|
-
if (!did_advance) {
|
|
409
|
-
remove_heredoc(scanner, index);
|
|
410
|
-
}
|
|
411
|
-
return true;
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
switch (lexer->lookahead) {
|
|
415
|
-
case '\n':
|
|
416
|
-
advance(lexer);
|
|
417
|
-
did_advance = true;
|
|
418
|
-
at_line_start = true;
|
|
419
|
-
scanner->joined_line = false;
|
|
420
|
-
break;
|
|
421
|
-
case '\\':
|
|
422
|
-
advance(lexer);
|
|
423
|
-
did_advance = true;
|
|
424
|
-
// Inside backquotes a backslash also hides a backquote from a quoted body's end.
|
|
425
|
-
if (!lexer->eof(lexer) &&
|
|
426
|
-
(!heredoc->is_raw || (in_backquotes && lexer->lookahead == '`'))) {
|
|
427
|
-
// A backslash-newline joins lines before the delimiter comparison, while a
|
|
428
|
-
// backslash-CR escapes only the CR, so the LF after it still ends the line.
|
|
429
|
-
advance(lexer);
|
|
430
|
-
}
|
|
431
|
-
break;
|
|
432
|
-
case '`':
|
|
433
|
-
// Bash ends backquotes at the first unescaped backquote, even inside a heredoc body,
|
|
434
|
-
// and ends the body there.
|
|
435
|
-
if (in_backquotes) {
|
|
436
|
-
lexer->mark_end(lexer);
|
|
437
|
-
lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
|
|
438
|
-
if (!did_advance) {
|
|
439
|
-
remove_heredoc(scanner, index);
|
|
440
|
-
}
|
|
441
|
-
return true;
|
|
442
|
-
}
|
|
443
|
-
if (!heredoc->is_raw && scanner->backtick_depth == 0) {
|
|
444
|
-
lexer->mark_end(lexer);
|
|
445
|
-
lexer->result_symbol = HEREDOC_CONTENT;
|
|
446
|
-
return did_advance;
|
|
447
|
-
}
|
|
448
|
-
advance(lexer);
|
|
449
|
-
did_advance = true;
|
|
450
|
-
break;
|
|
451
|
-
case '$':
|
|
452
|
-
if (!heredoc->is_raw) {
|
|
453
|
-
lexer->mark_end(lexer);
|
|
454
|
-
advance(lexer);
|
|
455
|
-
if (starts_quoted_expansion(lexer->lookahead)) {
|
|
456
|
-
if (!did_advance) {
|
|
457
|
-
return false;
|
|
458
|
-
}
|
|
459
|
-
lexer->result_symbol = HEREDOC_CONTENT;
|
|
460
|
-
return true;
|
|
461
|
-
}
|
|
462
|
-
did_advance = true;
|
|
463
|
-
break;
|
|
464
|
-
}
|
|
465
|
-
advance(lexer);
|
|
466
|
-
did_advance = true;
|
|
467
|
-
break;
|
|
468
|
-
default:
|
|
469
|
-
advance(lexer);
|
|
470
|
-
did_advance = true;
|
|
471
|
-
break;
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
// The delimiter word undergoes quote removal; any quoting makes the body literal.
|
|
477
|
-
static inline int32_t hex_value(int32_t c) {
|
|
478
|
-
return iswdigit(c) ? c - '0' : (c >= 'a' && c <= 'f') ? c - 'a' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 : -1;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
// Reads up to `max_digits` digits in `base` and returns their value, or -1 when there are none.
|
|
482
|
-
static int32_t read_number(TSLexer *lexer, int32_t base, int max_digits) {
|
|
483
|
-
int32_t value = 0;
|
|
484
|
-
int digits = 0;
|
|
485
|
-
while (digits < max_digits) {
|
|
486
|
-
int32_t digit = hex_value(lexer->lookahead);
|
|
487
|
-
if (digit < 0 || digit >= base) {
|
|
488
|
-
break;
|
|
489
|
-
}
|
|
490
|
-
value = value * base + digit;
|
|
491
|
-
advance(lexer);
|
|
492
|
-
digits++;
|
|
493
|
-
}
|
|
494
|
-
return digits == 0 ? -1 : value;
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
// Decodes the escape after a backslash in `$'...'` as bash does, appending it to `out`.
|
|
498
|
-
static void push_ansi_c_escape(TSLexer *lexer, CodePoints *out) {
|
|
499
|
-
int32_t c = lexer->lookahead;
|
|
500
|
-
int32_t value = -1;
|
|
501
|
-
switch (c) {
|
|
502
|
-
case 'a': value = 7; break;
|
|
503
|
-
case 'b': value = 8; break;
|
|
504
|
-
case 'e':
|
|
505
|
-
case 'E': value = 27; break;
|
|
506
|
-
case 'f': value = 12; break;
|
|
507
|
-
case 'n': value = '\n'; break;
|
|
508
|
-
case 'r': value = '\r'; break;
|
|
509
|
-
case 't': value = '\t'; break;
|
|
510
|
-
case 'v': value = 11; break;
|
|
511
|
-
case '\\':
|
|
512
|
-
case '\'':
|
|
513
|
-
case '"':
|
|
514
|
-
case '?': value = c; break;
|
|
515
|
-
default: break;
|
|
516
|
-
}
|
|
517
|
-
if (value >= 0) {
|
|
518
|
-
advance(lexer);
|
|
519
|
-
array_push(out, value);
|
|
520
|
-
return;
|
|
521
|
-
}
|
|
522
|
-
if (c >= '0' && c <= '7') {
|
|
523
|
-
array_push(out, read_number(lexer, 8, 3));
|
|
524
|
-
return;
|
|
525
|
-
}
|
|
526
|
-
int max_digits = c == 'x' ? 2 : c == 'u' ? 4 : c == 'U' ? 8 : 0;
|
|
527
|
-
if (max_digits > 0) {
|
|
528
|
-
advance(lexer);
|
|
529
|
-
value = read_number(lexer, 16, max_digits);
|
|
530
|
-
if (value < 0) {
|
|
531
|
-
array_push(out, '\\');
|
|
532
|
-
array_push(out, c);
|
|
533
|
-
} else {
|
|
534
|
-
array_push(out, value);
|
|
535
|
-
}
|
|
536
|
-
return;
|
|
537
|
-
}
|
|
538
|
-
if (c == 'c') {
|
|
539
|
-
advance(lexer);
|
|
540
|
-
if (!lexer->eof(lexer)) {
|
|
541
|
-
array_push(out, lexer->lookahead & 0x1F);
|
|
542
|
-
advance(lexer);
|
|
543
|
-
}
|
|
544
|
-
return;
|
|
545
|
-
}
|
|
546
|
-
// Bash keeps an unknown escape as written.
|
|
547
|
-
array_push(out, '\\');
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
|
|
551
|
-
Heredoc *heredoc = array_back(&scanner->heredocs);
|
|
552
|
-
array_clear(&heredoc->delimiter);
|
|
553
|
-
heredoc->is_raw = false;
|
|
554
|
-
|
|
555
|
-
while (is_blank(lexer->lookahead)) {
|
|
556
|
-
skip(lexer);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
int32_t quote = 0;
|
|
560
|
-
bool ansi_c = false;
|
|
561
|
-
while (!lexer->eof(lexer)) {
|
|
562
|
-
int32_t c = lexer->lookahead;
|
|
563
|
-
if (quote) {
|
|
564
|
-
advance(lexer);
|
|
565
|
-
if (c == quote) {
|
|
566
|
-
quote = 0;
|
|
567
|
-
continue;
|
|
568
|
-
}
|
|
569
|
-
if (scanner->joined_line && c == '\\' && lexer->lookahead == '\n') {
|
|
570
|
-
advance(lexer);
|
|
571
|
-
continue;
|
|
572
|
-
}
|
|
573
|
-
if (scanner->joined_line && quote == '\'' && !ansi_c && c == '\\' && lexer->lookahead != '\'' &&
|
|
574
|
-
!lexer->eof(lexer)) {
|
|
575
|
-
// The character after a backslash is kept as it is, so it cannot begin a line continuation.
|
|
576
|
-
array_push(&heredoc->delimiter, c);
|
|
577
|
-
c = lexer->lookahead;
|
|
578
|
-
advance(lexer);
|
|
579
|
-
} else if (ansi_c && c == '\\') {
|
|
580
|
-
if (!lexer->eof(lexer)) {
|
|
581
|
-
push_ansi_c_escape(lexer, &heredoc->delimiter);
|
|
582
|
-
}
|
|
583
|
-
continue;
|
|
584
|
-
}
|
|
585
|
-
// Inside double quotes a backslash-newline still only continues the line.
|
|
586
|
-
if (quote == '"' && c == '\\' && lexer->lookahead == '\n') {
|
|
587
|
-
advance(lexer);
|
|
588
|
-
continue;
|
|
589
|
-
}
|
|
590
|
-
if (quote == '"' && c == '\\' &&
|
|
591
|
-
(lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
|
|
592
|
-
lexer->lookahead == '\\')) {
|
|
593
|
-
c = lexer->lookahead;
|
|
594
|
-
advance(lexer);
|
|
595
|
-
}
|
|
596
|
-
array_push(&heredoc->delimiter, c);
|
|
597
|
-
continue;
|
|
598
|
-
}
|
|
599
|
-
// Inside backquotes an unquoted backquote closes the substitution; at top level it is part of
|
|
600
|
-
// the word (`<<EO`true`F`).
|
|
601
|
-
if (is_separator(c) || is_metacharacter(c) || (c == '`' && scanner->backtick_depth > 0)) {
|
|
602
|
-
break;
|
|
603
|
-
}
|
|
604
|
-
advance(lexer);
|
|
605
|
-
if (c == '$' && (lexer->lookahead == '\'' || lexer->lookahead == '"')) {
|
|
606
|
-
// `$'...'` and `$"..."` are quoting constructs, so their `$` is not part of the word.
|
|
607
|
-
c = lexer->lookahead;
|
|
608
|
-
advance(lexer);
|
|
609
|
-
quote = c;
|
|
610
|
-
ansi_c = c == '\'';
|
|
611
|
-
heredoc->is_raw = true;
|
|
612
|
-
} else if (c == '\'' || c == '"') {
|
|
613
|
-
quote = c;
|
|
614
|
-
ansi_c = false;
|
|
615
|
-
heredoc->is_raw = true;
|
|
616
|
-
} else if (c == '\\') {
|
|
617
|
-
// A backslash-newline only continues the line; any other backslash quotes the delimiter.
|
|
618
|
-
if (lexer->lookahead == '\n') {
|
|
619
|
-
advance(lexer);
|
|
620
|
-
} else if (!lexer->eof(lexer)) {
|
|
621
|
-
heredoc->is_raw = true;
|
|
622
|
-
array_push(&heredoc->delimiter, lexer->lookahead);
|
|
623
|
-
advance(lexer);
|
|
624
|
-
}
|
|
625
|
-
} else {
|
|
626
|
-
// Bash counts a backquote in the delimiter word as quoting, so the body stays literal.
|
|
627
|
-
if (c == '`') {
|
|
628
|
-
heredoc->is_raw = true;
|
|
629
|
-
}
|
|
630
|
-
array_push(&heredoc->delimiter, c);
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
// A heredoc whose state cannot be serialized in full is rejected, so the parse shows an error
|
|
635
|
-
// rather than comparing against a partial delimiter.
|
|
636
|
-
if ((heredoc->delimiter.size == 0 && !heredoc->is_raw) ||
|
|
637
|
-
serialized_size(scanner) > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
638
|
-
return false;
|
|
639
|
-
}
|
|
640
|
-
lexer->mark_end(lexer);
|
|
641
|
-
lexer->result_symbol = HEREDOC_START;
|
|
642
|
-
return true;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
// Scans `<<` or `<<-` after its first `<` has been read.
|
|
646
|
-
static bool scan_heredoc_arrow_after_first(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
647
|
-
if (lexer->lookahead != '<') {
|
|
648
|
-
return false;
|
|
649
|
-
}
|
|
650
|
-
advance(lexer);
|
|
651
|
-
bool allows_indent = false;
|
|
652
|
-
if (lexer->lookahead == '<') {
|
|
653
|
-
return false;
|
|
654
|
-
}
|
|
655
|
-
if (lexer->lookahead == '-' && valid_symbols[HEREDOC_ARROW_DASH]) {
|
|
656
|
-
advance(lexer);
|
|
657
|
-
allows_indent = true;
|
|
658
|
-
} else if (!valid_symbols[HEREDOC_ARROW]) {
|
|
659
|
-
return false;
|
|
660
|
-
}
|
|
661
|
-
if (scanner->heredocs.size >= UINT8_MAX ||
|
|
662
|
-
serialized_size(scanner) + HEREDOC_HEADER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
663
|
-
return false;
|
|
664
|
-
}
|
|
665
|
-
lexer->mark_end(lexer);
|
|
666
|
-
Heredoc heredoc = {
|
|
667
|
-
.is_raw = false,
|
|
668
|
-
.allows_indent = allows_indent,
|
|
669
|
-
.started = false,
|
|
670
|
-
.depth = current_depth(scanner),
|
|
671
|
-
.delimiter = array_new(),
|
|
672
|
-
};
|
|
673
|
-
array_push(&scanner->heredocs, heredoc);
|
|
674
|
-
lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
|
|
675
|
-
return true;
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
679
|
-
advance(lexer);
|
|
680
|
-
return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
// The literal text of the right side of `=~`, which, like bash, reads `(` and `|` as regex characters
|
|
684
|
-
// and a parenthesized group, blanks and metacharacters included, as part of the word up to its matching
|
|
685
|
-
// `)`. Quotes and expansions are separate parts, so the group depth is kept for the text after them.
|
|
686
|
-
static bool scan_regex(Scanner *scanner, TSLexer *lexer) {
|
|
687
|
-
uint32_t depth = scanner->regex_depth;
|
|
688
|
-
if (depth == 0) {
|
|
689
|
-
while (is_blank(lexer->lookahead)) {
|
|
690
|
-
skip(lexer);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
bool did_advance = false;
|
|
694
|
-
for (;;) {
|
|
695
|
-
lexer->mark_end(lexer);
|
|
696
|
-
if (lexer->eof(lexer)) {
|
|
697
|
-
break;
|
|
698
|
-
}
|
|
699
|
-
int32_t c = lexer->lookahead;
|
|
700
|
-
if (c == '\'' || c == '"' || c == '`' ||
|
|
701
|
-
(depth == 0 && (is_separator(c) || c == '&' || c == ';' || c == ')'))) {
|
|
702
|
-
break;
|
|
703
|
-
}
|
|
704
|
-
// A deferred body starts at the newline that ends the line, even inside a group.
|
|
705
|
-
if (c == '\n' && awaits_deferred_body(scanner)) {
|
|
706
|
-
break;
|
|
707
|
-
}
|
|
708
|
-
advance(lexer);
|
|
709
|
-
if (c == '\\' && lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
710
|
-
// The newline ends the line, after which a deferred body starts; a concatenation splits it.
|
|
711
|
-
break;
|
|
712
|
-
}
|
|
713
|
-
if (c == '$' || c == '<' || c == '>') {
|
|
714
|
-
int32_t next = lexer->lookahead;
|
|
715
|
-
bool starts_part = c == '$' ? starts_quoted_expansion(next) || next == '\'' || next == '"' : next == '(';
|
|
716
|
-
if (starts_part || (c != '$' && depth == 0)) {
|
|
717
|
-
break;
|
|
718
|
-
}
|
|
719
|
-
} else if (c == '\\') {
|
|
720
|
-
if (!lexer->eof(lexer)) {
|
|
721
|
-
advance(lexer);
|
|
722
|
-
}
|
|
723
|
-
} else if (c == '(') {
|
|
724
|
-
depth++;
|
|
725
|
-
} else if (c == ')') {
|
|
726
|
-
depth--;
|
|
727
|
-
} else if (c == '\n') {
|
|
728
|
-
scanner->joined_line = false;
|
|
729
|
-
}
|
|
730
|
-
did_advance = true;
|
|
731
|
-
}
|
|
732
|
-
if (!did_advance || depth > UINT16_MAX) {
|
|
733
|
-
return false;
|
|
734
|
-
}
|
|
735
|
-
scanner->regex_depth = (uint16_t)depth;
|
|
736
|
-
lexer->result_symbol = REGEX;
|
|
737
|
-
return true;
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
static bool scan_string_content(Scanner *scanner, TSLexer *lexer) {
|
|
741
|
-
bool line_breaks = !scanner->joined_line && awaits_deferred_body(scanner);
|
|
742
|
-
bool did_advance = false;
|
|
743
|
-
for (;;) {
|
|
744
|
-
lexer->mark_end(lexer);
|
|
745
|
-
if (lexer->eof(lexer)) {
|
|
746
|
-
break;
|
|
747
|
-
}
|
|
748
|
-
int32_t c = lexer->lookahead;
|
|
749
|
-
if (c == '"' || c == '`') {
|
|
750
|
-
break;
|
|
751
|
-
}
|
|
752
|
-
if (c == '\n' && (scanner->joined_line || line_breaks)) {
|
|
753
|
-
// A deferred body starts right before the newline that ends the line.
|
|
754
|
-
if (did_advance) {
|
|
755
|
-
break;
|
|
756
|
-
}
|
|
757
|
-
scanner->joined_line = false;
|
|
758
|
-
}
|
|
759
|
-
advance(lexer);
|
|
760
|
-
if (c == '$' && starts_quoted_expansion(lexer->lookahead)) {
|
|
761
|
-
break;
|
|
762
|
-
}
|
|
763
|
-
if (c == '\\' && !lexer->eof(lexer) && !(line_breaks && lexer->lookahead == '\n')) {
|
|
764
|
-
advance(lexer);
|
|
765
|
-
}
|
|
766
|
-
did_advance = true;
|
|
767
|
-
}
|
|
768
|
-
lexer->result_symbol = STRING_CONTENT;
|
|
769
|
-
return did_advance;
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
static inline bool is_word_break(int32_t c) {
|
|
773
|
-
return is_separator(c) || is_metacharacter(c) || c == '"' || c == '\'' || c == '`' || c == '$' || c == '\\';
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
// Whether `c` ends a word; `(` and `|` are regex characters in the right side of `=~`.
|
|
777
|
-
static inline bool ends_word(int32_t c, bool regex) {
|
|
778
|
-
return is_separator(c) || (is_metacharacter(c) && !(regex && (c == '(' || c == '|')));
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
static inline bool is_extglob_operator(int32_t c) { return c == '?' || c == '*' || c == '+' || c == '@' || c == '!'; }
|
|
782
|
-
|
|
783
|
-
typedef enum { GROUP_LITERAL, GROUP_STRUCTURED, GROUP_UNSUPPORTED } GroupKind;
|
|
784
|
-
|
|
785
|
-
// From the `(` of an extglob group, reads to its matching `)`: a group holding quotes or expansions
|
|
786
|
-
// is parsed as `extglob_pattern`, while the word token keeps literal ones.
|
|
787
|
-
// The nesting depth up to which grammar.js's word token lexes a literal group (`extglobGroup(…, 3)`);
|
|
788
|
-
// a deeper group is parsed as `extglob_pattern`.
|
|
789
|
-
#define EXTGLOB_LITERAL_DEPTH 3
|
|
790
|
-
|
|
791
|
-
static GroupKind scan_extglob_group(TSLexer *lexer) {
|
|
792
|
-
uint32_t depth = 0;
|
|
793
|
-
while (!lexer->eof(lexer)) {
|
|
794
|
-
int32_t c = lexer->lookahead;
|
|
795
|
-
if (c == '$' || c == '`' || c == '\'' || c == '"') {
|
|
796
|
-
return GROUP_STRUCTURED;
|
|
797
|
-
}
|
|
798
|
-
advance(lexer);
|
|
799
|
-
if (c == '\\') {
|
|
800
|
-
// The word token's literal groups cannot hold a line continuation.
|
|
801
|
-
if (lexer->eof(lexer) || lexer->lookahead == '\n') {
|
|
802
|
-
return lexer->eof(lexer) ? GROUP_UNSUPPORTED : GROUP_STRUCTURED;
|
|
803
|
-
}
|
|
804
|
-
advance(lexer);
|
|
805
|
-
} else if (c == '(') {
|
|
806
|
-
if (++depth > EXTGLOB_LITERAL_DEPTH) {
|
|
807
|
-
return GROUP_STRUCTURED;
|
|
808
|
-
}
|
|
809
|
-
} else if (c == ')' && --depth == 0) {
|
|
810
|
-
return GROUP_LITERAL;
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
return GROUP_UNSUPPORTED;
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
// The literal text of a word up to a structured extglob group (`foo` in `foo@($x)`), which the
|
|
817
|
-
// word token's longest match would otherwise run into.
|
|
818
|
-
static bool continue_extglob_prefix(TSLexer *lexer, bool consumed) {
|
|
819
|
-
while (!lexer->eof(lexer)) {
|
|
820
|
-
int32_t c = lexer->lookahead;
|
|
821
|
-
if (is_extglob_operator(c)) {
|
|
822
|
-
lexer->mark_end(lexer);
|
|
823
|
-
advance(lexer);
|
|
824
|
-
if (lexer->lookahead == '(') {
|
|
825
|
-
GroupKind kind = scan_extglob_group(lexer);
|
|
826
|
-
if (kind == GROUP_STRUCTURED) {
|
|
827
|
-
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
828
|
-
return consumed;
|
|
829
|
-
}
|
|
830
|
-
if (kind == GROUP_UNSUPPORTED) {
|
|
831
|
-
return false;
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
} else if (c == '\\') {
|
|
835
|
-
advance(lexer);
|
|
836
|
-
// A backslash-newline continues the line; the word token handles that.
|
|
837
|
-
if (lexer->eof(lexer) || lexer->lookahead == '\n') {
|
|
838
|
-
return false;
|
|
839
|
-
}
|
|
840
|
-
advance(lexer);
|
|
841
|
-
} else if (is_word_break(c)) {
|
|
842
|
-
return false;
|
|
843
|
-
} else {
|
|
844
|
-
advance(lexer);
|
|
845
|
-
}
|
|
846
|
-
consumed = true;
|
|
847
|
-
}
|
|
848
|
-
return false;
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
// A leading `#` starts a comment except in an assignment value (`A=#@($x)`).
|
|
852
|
-
static bool scan_extglob_prefix(TSLexer *lexer, bool in_value) {
|
|
853
|
-
return (lexer->lookahead != '#' || in_value) && continue_extglob_prefix(lexer, false);
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
// Skips a quoted span from its opening quote through its closing one.
|
|
857
|
-
static void skip_quoted(TSLexer *lexer) {
|
|
858
|
-
int32_t quote = lexer->lookahead;
|
|
859
|
-
advance(lexer);
|
|
860
|
-
while (!lexer->eof(lexer) && lexer->lookahead != quote) {
|
|
861
|
-
if (lexer->lookahead == '\\' && quote != '\'') {
|
|
862
|
-
advance(lexer);
|
|
863
|
-
if (lexer->eof(lexer)) {
|
|
864
|
-
return;
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
advance(lexer);
|
|
868
|
-
}
|
|
869
|
-
advance(lexer);
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
// Skips a parenthesized span such as a command substitution's, quotes included.
|
|
873
|
-
static void skip_parenthesized(TSLexer *lexer) {
|
|
874
|
-
uint32_t depth = 0;
|
|
875
|
-
while (!lexer->eof(lexer)) {
|
|
876
|
-
int32_t c = lexer->lookahead;
|
|
877
|
-
if (c == '\'' || c == '"' || c == '`') {
|
|
878
|
-
skip_quoted(lexer);
|
|
879
|
-
continue;
|
|
880
|
-
}
|
|
881
|
-
if (c == '\\') {
|
|
882
|
-
advance(lexer);
|
|
883
|
-
} else if (c == '(') {
|
|
884
|
-
depth++;
|
|
885
|
-
} else if (c == ')' && --depth == 0) {
|
|
886
|
-
advance(lexer);
|
|
887
|
-
return;
|
|
888
|
-
}
|
|
889
|
-
advance(lexer);
|
|
890
|
-
}
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
// A name at a position where both an assignment and a word may start: `name=`, `name+=`, and
|
|
894
|
-
// `name[…]=` are assignments, and otherwise the name may begin the literal text before a structured
|
|
895
|
-
// extglob group (`foo@($x)`), in which `+(` is an extglob operator rather than an unfinished `+=`.
|
|
896
|
-
static bool scan_name_or_extglob_prefix(TSLexer *lexer, const bool *valid_symbols) {
|
|
897
|
-
while (is_name_char(lexer->lookahead)) {
|
|
898
|
-
advance(lexer);
|
|
899
|
-
}
|
|
900
|
-
lexer->mark_end(lexer);
|
|
901
|
-
lexer->result_symbol = VARIABLE_NAME;
|
|
902
|
-
if (lexer->lookahead == '=') {
|
|
903
|
-
return true;
|
|
904
|
-
}
|
|
905
|
-
if (lexer->lookahead == '+') {
|
|
906
|
-
advance(lexer);
|
|
907
|
-
if (lexer->lookahead == '=') {
|
|
908
|
-
return true;
|
|
909
|
-
}
|
|
910
|
-
if (!valid_symbols[EXTGLOB_PREFIX]) {
|
|
911
|
-
return false;
|
|
912
|
-
}
|
|
913
|
-
if (lexer->lookahead == '(') {
|
|
914
|
-
GroupKind kind = scan_extglob_group(lexer);
|
|
915
|
-
lexer->result_symbol = EXTGLOB_PREFIX;
|
|
916
|
-
return kind == GROUP_STRUCTURED || (kind == GROUP_LITERAL && continue_extglob_prefix(lexer, true));
|
|
917
|
-
}
|
|
918
|
-
return continue_extglob_prefix(lexer, true);
|
|
919
|
-
}
|
|
920
|
-
if (lexer->lookahead == '[') {
|
|
921
|
-
// Like bash, the subscript extends to the matching `]` across lines; without one the name
|
|
922
|
-
// still starts an assignment, so the parse reports the missing `]`.
|
|
923
|
-
uint32_t depth = 0;
|
|
924
|
-
while (!lexer->eof(lexer)) {
|
|
925
|
-
int32_t c = lexer->lookahead;
|
|
926
|
-
if (c == '[') {
|
|
927
|
-
depth++;
|
|
928
|
-
} else if (c == ']' && --depth == 0) {
|
|
929
|
-
advance(lexer);
|
|
930
|
-
break;
|
|
931
|
-
} else if (c == '\\') {
|
|
932
|
-
advance(lexer);
|
|
933
|
-
} else if (c == '\'' || c == '"' || c == '`') {
|
|
934
|
-
// Brackets inside quotes do not count (`A["]"]=x`).
|
|
935
|
-
skip_quoted(lexer);
|
|
936
|
-
continue;
|
|
937
|
-
} else if (c == '$') {
|
|
938
|
-
advance(lexer);
|
|
939
|
-
if (lexer->lookahead == '(') {
|
|
940
|
-
skip_parenthesized(lexer);
|
|
941
|
-
}
|
|
942
|
-
continue;
|
|
943
|
-
}
|
|
944
|
-
advance(lexer);
|
|
945
|
-
}
|
|
946
|
-
if (depth == 0 && lexer->lookahead == '+') {
|
|
947
|
-
advance(lexer);
|
|
948
|
-
}
|
|
949
|
-
if (depth != 0 || lexer->lookahead == '=') {
|
|
950
|
-
return true;
|
|
951
|
-
}
|
|
952
|
-
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
953
|
-
}
|
|
954
|
-
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
// Marks the current position and reports whether a reserved word that ends or continues a compound
|
|
958
|
-
// command (`then`, `do`, `done`, `fi`, `esac`, `else`, `elif`, `}`) starts here.
|
|
959
|
-
static bool at_closing_reserved_word(TSLexer *lexer) {
|
|
960
|
-
static const char *const words[] = {"then", "do", "done", "fi", "esac", "else", "elif", "}"};
|
|
961
|
-
lexer->mark_end(lexer);
|
|
962
|
-
char word[5];
|
|
963
|
-
uint32_t length = 0;
|
|
964
|
-
for (;;) {
|
|
965
|
-
if (lexer->lookahead == '\\') {
|
|
966
|
-
// Bash joins a backslash-newline before it checks where the word ends (`then\` + newline).
|
|
967
|
-
advance(lexer);
|
|
968
|
-
if (lexer->lookahead != '\n') {
|
|
969
|
-
return false;
|
|
970
|
-
}
|
|
971
|
-
advance(lexer);
|
|
972
|
-
} else if (length < sizeof(word) && (iswlower(lexer->lookahead) || lexer->lookahead == '}')) {
|
|
973
|
-
word[length++] = (char)lexer->lookahead;
|
|
974
|
-
advance(lexer);
|
|
975
|
-
} else {
|
|
976
|
-
break;
|
|
977
|
-
}
|
|
978
|
-
}
|
|
979
|
-
if (length == 0 || length == sizeof(word) ||
|
|
980
|
-
!(lexer->eof(lexer) || is_separator(lexer->lookahead) || is_metacharacter(lexer->lookahead))) {
|
|
981
|
-
return false;
|
|
982
|
-
}
|
|
983
|
-
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
|
|
984
|
-
if (strlen(words[i]) == length && strncmp(words[i], word, length) == 0) {
|
|
985
|
-
return true;
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
return false;
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
// After a line continuation inside a word: bash removes every backslash-newline before it splits words, so
|
|
992
|
-
// further continuations right after this one are part of the token (`a\` + `\` + `b` is `ab`), which is a
|
|
993
|
-
// concatenation if the word continues on the next line. Otherwise the continuations themselves are the token,
|
|
994
|
-
// so that whatever follows them is lexed as it would be without them.
|
|
995
|
-
static bool finish_word_continuations(Scanner *scanner, TSLexer *lexer, enum TokenType concat, bool regex) {
|
|
996
|
-
int32_t c;
|
|
997
|
-
for (;;) {
|
|
998
|
-
c = lexer->lookahead;
|
|
999
|
-
if (c != '\\') {
|
|
1000
|
-
break;
|
|
1001
|
-
}
|
|
1002
|
-
advance(lexer);
|
|
1003
|
-
if (lexer->eof(lexer)) {
|
|
1004
|
-
lexer->mark_end(lexer);
|
|
1005
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1006
|
-
return true;
|
|
1007
|
-
}
|
|
1008
|
-
if (lexer->lookahead != '\n') {
|
|
1009
|
-
// An escaped character continues the word.
|
|
1010
|
-
lexer->result_symbol = concat;
|
|
1011
|
-
return true;
|
|
1012
|
-
}
|
|
1013
|
-
advance(lexer);
|
|
1014
|
-
lexer->mark_end(lexer);
|
|
1015
|
-
}
|
|
1016
|
-
if ((c == '<' || c == '>')) {
|
|
1017
|
-
advance(lexer);
|
|
1018
|
-
if (lexer->lookahead == '(') {
|
|
1019
|
-
lexer->result_symbol = concat;
|
|
1020
|
-
return true;
|
|
1021
|
-
}
|
|
1022
|
-
scanner->after_line_break = true;
|
|
1023
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1024
|
-
} else if (lexer->eof(lexer) || (ends_word(c, regex) && !(regex && scanner->regex_depth > 0)) ||
|
|
1025
|
-
(c == '`' && scanner->backtick_depth > 0)) {
|
|
1026
|
-
scanner->after_line_break = true;
|
|
1027
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1028
|
-
} else {
|
|
1029
|
-
lexer->result_symbol = concat;
|
|
1030
|
-
}
|
|
1031
|
-
return true;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
// Ends a line continuation's token at its backslash, since its newline ends the line after which a deferred
|
|
1035
|
-
// body starts; the newline is read after the body.
|
|
1036
|
-
static void split_continuation(Scanner *scanner, TSLexer *lexer, bool in_word, bool in_regex) {
|
|
1037
|
-
lexer->mark_end(lexer);
|
|
1038
|
-
scanner->split_continuation = true;
|
|
1039
|
-
scanner->split_continuation_in_word = in_word;
|
|
1040
|
-
scanner->split_continuation_in_regex = in_regex;
|
|
1041
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// After `$`, a bare dollar sign unless it begins an expansion or a quote.
|
|
1045
|
-
static bool finish_bare_dollar(TSLexer *lexer) {
|
|
1046
|
-
lexer->mark_end(lexer);
|
|
1047
|
-
int32_t c = lexer->lookahead;
|
|
1048
|
-
lexer->result_symbol = BARE_DOLLAR;
|
|
1049
|
-
return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
// The text of a single-quoted string on a line that ended a body at the substitution's `)`. Where the line
|
|
1053
|
-
// is joined, the text is split at its line continuations, which are tokens of their own, and, as when bash
|
|
1054
|
-
// joins the line, the character after any other backslash is kept with it; in a raw string that backslash
|
|
1055
|
-
// is literal, so a quote after it still ends the string. The text also ends before the newline that ends
|
|
1056
|
-
// the line, where a deferred body starts.
|
|
1057
|
-
static bool scan_closer_line_string_content(Scanner *scanner, TSLexer *lexer, bool ansi_c) {
|
|
1058
|
-
bool line_open = scanner->joined_line || awaits_deferred_body(scanner);
|
|
1059
|
-
bool did_advance = scanner->closer_line_string_opened;
|
|
1060
|
-
scanner->closer_line_string_opened = false;
|
|
1061
|
-
for (;;) {
|
|
1062
|
-
lexer->mark_end(lexer);
|
|
1063
|
-
if (lexer->eof(lexer) || lexer->lookahead == '\'') {
|
|
1064
|
-
break;
|
|
1065
|
-
}
|
|
1066
|
-
int32_t c = lexer->lookahead;
|
|
1067
|
-
// A deferred body starts right before the newline that ends the line.
|
|
1068
|
-
if (c == '\n' && line_open && did_advance) {
|
|
1069
|
-
break;
|
|
1070
|
-
}
|
|
1071
|
-
advance(lexer);
|
|
1072
|
-
if (c == '\\' && scanner->joined_line && lexer->lookahead == '\n') {
|
|
1073
|
-
if (did_advance) {
|
|
1074
|
-
break;
|
|
1075
|
-
}
|
|
1076
|
-
advance(lexer);
|
|
1077
|
-
lexer->mark_end(lexer);
|
|
1078
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1079
|
-
return true;
|
|
1080
|
-
}
|
|
1081
|
-
if (c == '\\' && !lexer->eof(lexer) && lexer->lookahead != '\n' && (ansi_c || lexer->lookahead != '\'')) {
|
|
1082
|
-
advance(lexer);
|
|
1083
|
-
} else if (c == '\n') {
|
|
1084
|
-
scanner->joined_line = false;
|
|
1085
|
-
line_open = false;
|
|
1086
|
-
}
|
|
1087
|
-
did_advance = true;
|
|
1088
|
-
}
|
|
1089
|
-
lexer->result_symbol = ansi_c ? CLOSER_LINE_ANSI_C_STRING_CONTENT : CLOSER_LINE_RAW_STRING_CONTENT;
|
|
1090
|
-
return did_advance;
|
|
1091
|
-
}
|
|
1092
|
-
|
|
1093
|
-
// A comment on a joined line, which continues past each line continuation.
|
|
1094
|
-
static bool scan_joined_comment(TSLexer *lexer) {
|
|
1095
|
-
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
|
1096
|
-
bool backslash = lexer->lookahead == '\\';
|
|
1097
|
-
advance(lexer);
|
|
1098
|
-
if (backslash && !lexer->eof(lexer)) {
|
|
1099
|
-
advance(lexer);
|
|
1100
|
-
}
|
|
1101
|
-
}
|
|
1102
|
-
lexer->mark_end(lexer);
|
|
1103
|
-
lexer->result_symbol = JOINED_COMMENT;
|
|
1104
|
-
return true;
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
|
|
1108
|
-
bool error_recovery = valid_symbols[ERROR_RECOVERY];
|
|
1109
|
-
// Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
|
|
1110
|
-
// by the character right after it, even when blanks are skipped below to find a newline.
|
|
1111
|
-
int32_t first = lexer->lookahead;
|
|
1112
|
-
bool at_eof = lexer->eof(lexer);
|
|
1113
|
-
// Kept only if this call returns a token, so the flag lasts until the scanner's next token.
|
|
1114
|
-
bool after_line_break = scanner->after_line_break;
|
|
1115
|
-
scanner->after_line_break = false;
|
|
1116
|
-
|
|
1117
|
-
if (scanner->terminator_owed) {
|
|
1118
|
-
scanner->terminator_owed = false;
|
|
1119
|
-
if (valid_symbols[NEWLINE]) {
|
|
1120
|
-
lexer->result_symbol = NEWLINE;
|
|
1121
|
-
return true;
|
|
1122
|
-
}
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
// Counted before a heredoc body may start, so a newline right after `$(` is inside it.
|
|
1126
|
-
if (!error_recovery &&
|
|
1127
|
-
(valid_symbols[SUBSTITUTION_START] || valid_symbols[BRACE_SUBSTITUTION_START] ||
|
|
1128
|
-
valid_symbols[BRACKET_SUBSTITUTION_START])) {
|
|
1129
|
-
// Nesting too deep for the serialized state is left to the parse as an error.
|
|
1130
|
-
if (serialized_size(scanner) + CLOSER_SIZE > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
|
|
1131
|
-
return false;
|
|
1132
|
-
}
|
|
1133
|
-
lexer->result_symbol = valid_symbols[SUBSTITUTION_START] ? SUBSTITUTION_START
|
|
1134
|
-
: valid_symbols[BRACE_SUBSTITUTION_START] ? BRACE_SUBSTITUTION_START
|
|
1135
|
-
: BRACKET_SUBSTITUTION_START;
|
|
1136
|
-
uint8_t closer = lexer->result_symbol == SUBSTITUTION_START ? ')'
|
|
1137
|
-
: lexer->result_symbol == BRACE_SUBSTITUTION_START ? '}'
|
|
1138
|
-
: ']';
|
|
1139
|
-
array_push(&scanner->closers, closer);
|
|
1140
|
-
array_push(&scanner->closer_regex_depths, scanner->regex_depth);
|
|
1141
|
-
return true;
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
if (!error_recovery && valid_symbols[REGEX_START]) {
|
|
1145
|
-
scanner->regex_depth = 0;
|
|
1146
|
-
lexer->result_symbol = REGEX_START;
|
|
1147
|
-
return true;
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
// Blanks, metacharacters, and newlines inside a regex group are part of the regex.
|
|
1151
|
-
bool in_regex_group = scanner->regex_depth > 0 && (valid_symbols[REGEX] || valid_symbols[REGEX_CONCAT]);
|
|
1152
|
-
|
|
1153
|
-
if (scanner->heredocs.size > 0) {
|
|
1154
|
-
int32_t next = next_heredoc_index(scanner);
|
|
1155
|
-
// Only another deferred body starts at the newline of a split line continuation.
|
|
1156
|
-
bool deferred = next >= 0 && array_get(&scanner->heredocs, next)->deferred;
|
|
1157
|
-
// A deferred body starts at the end of its line even inside the body of an enclosing heredoc, but
|
|
1158
|
-
// the deferred bodies of one line follow one another.
|
|
1159
|
-
int32_t active = active_heredoc_index(scanner);
|
|
1160
|
-
if (active >= 0 && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
|
|
1161
|
-
!(deferred && !array_get(&scanner->heredocs, active)->deferred && lexer->lookahead == '\n' &&
|
|
1162
|
-
valid_symbols[HEREDOC_BODY_START]) &&
|
|
1163
|
-
!(lexer->lookahead == '`' && !array_get(&scanner->heredocs, active)->is_raw &&
|
|
1164
|
-
scanner->backtick_depth == 0)) {
|
|
1165
|
-
return scan_heredoc_content(scanner, lexer, (uint32_t)active);
|
|
1166
|
-
}
|
|
1167
|
-
// A newline inside a string is part of it, so bodies start only outside strings, except for a
|
|
1168
|
-
// deferred body, which bash reads right after its line wherever that line ends.
|
|
1169
|
-
bool in_quotes = valid_symbols[STRING_CONTENT] || valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] ||
|
|
1170
|
-
valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT];
|
|
1171
|
-
if (next >= 0 && valid_symbols[HEREDOC_BODY_START] && (!in_regex_group || deferred) &&
|
|
1172
|
-
(deferred || (!in_quotes && !scanner->split_continuation))) {
|
|
1173
|
-
while (is_blank(lexer->lookahead) && !in_quotes) {
|
|
1174
|
-
skip(lexer);
|
|
1175
|
-
}
|
|
1176
|
-
if (lexer->lookahead == '\n') {
|
|
1177
|
-
// The line's heredocs cannot be located once one of them was rejected, so none gets a
|
|
1178
|
-
// body; scanning continues so that the change is kept with the token found below.
|
|
1179
|
-
if (is_rejected(array_get(&scanner->heredocs, next)) && !deferred) {
|
|
1180
|
-
drop_unstarted_heredocs_at_depth(scanner);
|
|
1181
|
-
} else {
|
|
1182
|
-
array_get(&scanner->heredocs, next)->started = true;
|
|
1183
|
-
scanner->joined_line = false;
|
|
1184
|
-
lexer->result_symbol = HEREDOC_BODY_START;
|
|
1185
|
-
return true;
|
|
1186
|
-
}
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
|
|
1191
|
-
if (error_recovery) {
|
|
1192
|
-
return false;
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
// The right side of `=~` joins its parts with a concatenation of its own, since `(` and `|` continue
|
|
1196
|
-
// it and a group continues it up to its matching `)`.
|
|
1197
|
-
bool regex = valid_symbols[REGEX_CONCAT] && !valid_symbols[CONCAT];
|
|
1198
|
-
enum TokenType concat = regex ? REGEX_CONCAT : CONCAT;
|
|
1199
|
-
bool can_concat = valid_symbols[CONCAT] || regex;
|
|
1200
|
-
|
|
1201
|
-
// The word goes on after the body even where the parse state no longer allows a concatenation: that
|
|
1202
|
-
// state comes from a reparse that reused the words before the body as a complete command, which the
|
|
1203
|
-
// concatenation then breaks down again.
|
|
1204
|
-
if (scanner->split_continuation && lexer->lookahead == '\n') {
|
|
1205
|
-
scanner->split_continuation = false;
|
|
1206
|
-
advance(lexer);
|
|
1207
|
-
lexer->mark_end(lexer);
|
|
1208
|
-
if (scanner->split_continuation_in_word) {
|
|
1209
|
-
bool in_regex = scanner->split_continuation_in_regex;
|
|
1210
|
-
return finish_word_continuations(scanner, lexer, in_regex ? REGEX_CONCAT : CONCAT, in_regex);
|
|
1211
|
-
}
|
|
1212
|
-
scanner->after_line_break = true;
|
|
1213
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1214
|
-
return true;
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
|
-
if (valid_symbols[STRING_CONTENT] && lexer->lookahead != '`') {
|
|
1218
|
-
return scan_string_content(scanner, lexer);
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
if (valid_symbols[CLOSER_LINE_RAW_STRING_CONTENT] || valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]) {
|
|
1222
|
-
return scan_closer_line_string_content(scanner, lexer, valid_symbols[CLOSER_LINE_ANSI_C_STRING_CONTENT]);
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
if (valid_symbols[HEREDOC_START] && scanner->heredocs.size > 0) {
|
|
1226
|
-
return scan_heredoc_start(scanner, lexer);
|
|
1227
|
-
}
|
|
1228
|
-
|
|
1229
|
-
if (regex && scanner->regex_depth > 0 && !at_eof && first != '\\') {
|
|
1230
|
-
lexer->result_symbol = REGEX_CONCAT;
|
|
1231
|
-
return true;
|
|
1232
|
-
}
|
|
1233
|
-
|
|
1234
|
-
// A line continuation after a blank separates words, so nothing concatenates at the start of the
|
|
1235
|
-
// next line. The column costs a scan back to the line start, so it is computed only right after
|
|
1236
|
-
// such a continuation and only where a concatenation could start; a concatenation then clears the
|
|
1237
|
-
// flag, which bounds the scans to one per continuation.
|
|
1238
|
-
|
|
1239
|
-
// Like bash, `a<(cmd)` is one word. `<` and `>` end a word part otherwise, so the concatenation
|
|
1240
|
-
// needs a look at the next character, and a `<<` read along the way is scanned as a heredoc arrow.
|
|
1241
|
-
if (can_concat && (first == '<' || first == '>') &&
|
|
1242
|
-
!(after_line_break && lexer->get_column(lexer) == 0)) {
|
|
1243
|
-
lexer->mark_end(lexer);
|
|
1244
|
-
advance(lexer);
|
|
1245
|
-
if (lexer->lookahead == '(') {
|
|
1246
|
-
lexer->result_symbol = concat;
|
|
1247
|
-
return true;
|
|
1248
|
-
}
|
|
1249
|
-
if (first == '<' && (valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH])) {
|
|
1250
|
-
return scan_heredoc_arrow_after_first(scanner, lexer, valid_symbols);
|
|
1251
|
-
}
|
|
1252
|
-
return false;
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
if (can_concat && !at_eof && (!ends_word(first, regex) || (regex && scanner->regex_depth > 0)) &&
|
|
1256
|
-
!(after_line_break && lexer->get_column(lexer) == 0)) {
|
|
1257
|
-
int32_t c = first;
|
|
1258
|
-
if (c == '\\' && lexer->lookahead == '\\') {
|
|
1259
|
-
// A backslash-newline joins lines, so the word continues only if the next line does.
|
|
1260
|
-
lexer->mark_end(lexer);
|
|
1261
|
-
advance(lexer);
|
|
1262
|
-
// Bash drops a backslash at the end of input like a continuation.
|
|
1263
|
-
if (lexer->eof(lexer)) {
|
|
1264
|
-
lexer->mark_end(lexer);
|
|
1265
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1266
|
-
return true;
|
|
1267
|
-
}
|
|
1268
|
-
if (lexer->lookahead != '\n') {
|
|
1269
|
-
lexer->result_symbol = concat;
|
|
1270
|
-
return true;
|
|
1271
|
-
}
|
|
1272
|
-
if (!scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
1273
|
-
split_continuation(scanner, lexer, true, regex);
|
|
1274
|
-
return true;
|
|
1275
|
-
}
|
|
1276
|
-
advance(lexer);
|
|
1277
|
-
lexer->mark_end(lexer);
|
|
1278
|
-
return finish_word_continuations(scanner, lexer, concat, regex);
|
|
1279
|
-
}
|
|
1280
|
-
if (c != '`' || scanner->backtick_depth == 0) {
|
|
1281
|
-
lexer->result_symbol = concat;
|
|
1282
|
-
return true;
|
|
1283
|
-
}
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
if (valid_symbols[EMPTY_VALUE]) {
|
|
1287
|
-
int32_t c = first;
|
|
1288
|
-
// A redirection may follow an empty value (`A=>f`), but `<(`/`>(` is a process substitution
|
|
1289
|
-
// value; nothing else a value may start with begins with `<` or `>`.
|
|
1290
|
-
if ((c == '<' || c == '>') && lexer->lookahead == c) {
|
|
1291
|
-
lexer->mark_end(lexer);
|
|
1292
|
-
advance(lexer);
|
|
1293
|
-
if (lexer->lookahead == '(') {
|
|
1294
|
-
return false;
|
|
1295
|
-
}
|
|
1296
|
-
lexer->result_symbol = EMPTY_VALUE;
|
|
1297
|
-
return true;
|
|
1298
|
-
}
|
|
1299
|
-
if (at_eof || is_separator(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
|
|
1300
|
-
(c == '`' && scanner->backtick_depth > 0)) {
|
|
1301
|
-
lexer->result_symbol = EMPTY_VALUE;
|
|
1302
|
-
return true;
|
|
1303
|
-
}
|
|
1304
|
-
}
|
|
1305
|
-
|
|
1306
|
-
// The rest of a line that ended a body at the substitution's `)` scans its single-quoted strings itself,
|
|
1307
|
-
// to remove their line continuations where the line is joined, and to end the line inside them. Inside
|
|
1308
|
-
// backquotes, the next backquote ends a quote, as handled below, but a newline still ends the line.
|
|
1309
|
-
if ((scanner->joined_line || awaits_deferred_body(scanner)) && !error_recovery) {
|
|
1310
|
-
while (is_blank(lexer->lookahead) && !in_regex_group) {
|
|
1311
|
-
skip(lexer);
|
|
1312
|
-
}
|
|
1313
|
-
if (lexer->lookahead == '\n' && scanner->joined_line && !in_regex_group) {
|
|
1314
|
-
// The joined line ends here; a token is returned even where the newline is only whitespace,
|
|
1315
|
-
// so that the line's end is kept.
|
|
1316
|
-
advance(lexer);
|
|
1317
|
-
lexer->mark_end(lexer);
|
|
1318
|
-
scanner->joined_line = false;
|
|
1319
|
-
lexer->result_symbol = valid_symbols[NEWLINE] ? NEWLINE : LINE_CONTINUATION;
|
|
1320
|
-
return true;
|
|
1321
|
-
}
|
|
1322
|
-
}
|
|
1323
|
-
if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 && !error_recovery) {
|
|
1324
|
-
if (lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_RAW_STRING_START]) {
|
|
1325
|
-
advance(lexer);
|
|
1326
|
-
lexer->mark_end(lexer);
|
|
1327
|
-
scanner->closer_line_string_opened = true;
|
|
1328
|
-
lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
|
|
1329
|
-
return true;
|
|
1330
|
-
}
|
|
1331
|
-
// A `#` begins a word, and so a comment, after a blank or an operator. Right after a word part, where
|
|
1332
|
-
// another part may follow, it continues the word; a command may also start there, as after `;`, but
|
|
1333
|
-
// a reparse may reuse the statements before it as a complete list, where no part may follow.
|
|
1334
|
-
bool in_word = valid_symbols[BARE_DOLLAR] && !valid_symbols[VARIABLE_NAME];
|
|
1335
|
-
if (lexer->lookahead == '#' && (is_blank(first) || !in_word) && scanner->joined_line &&
|
|
1336
|
-
valid_symbols[JOINED_COMMENT]) {
|
|
1337
|
-
return scan_joined_comment(lexer);
|
|
1338
|
-
}
|
|
1339
|
-
// A regex reads `$` as text unless it begins a part, so it looks for `$'` itself.
|
|
1340
|
-
if (lexer->lookahead == '$' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START] && !valid_symbols[REGEX]) {
|
|
1341
|
-
advance(lexer);
|
|
1342
|
-
if (lexer->lookahead == '\'') {
|
|
1343
|
-
advance(lexer);
|
|
1344
|
-
lexer->mark_end(lexer);
|
|
1345
|
-
scanner->closer_line_string_opened = true;
|
|
1346
|
-
lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
|
|
1347
|
-
return true;
|
|
1348
|
-
}
|
|
1349
|
-
return valid_symbols[BARE_DOLLAR] && finish_bare_dollar(lexer);
|
|
1350
|
-
}
|
|
1351
|
-
}
|
|
1352
|
-
|
|
1353
|
-
if (valid_symbols[REGEX]) {
|
|
1354
|
-
if (scan_regex(scanner, lexer)) {
|
|
1355
|
-
return true;
|
|
1356
|
-
}
|
|
1357
|
-
// Having found no text, the scan stopped before a quote, or after the `$` of `$'`.
|
|
1358
|
-
if ((scanner->joined_line || awaits_deferred_body(scanner)) && scanner->backtick_depth == 0 &&
|
|
1359
|
-
lexer->lookahead == '\'' && valid_symbols[CLOSER_LINE_ANSI_C_STRING_START]) {
|
|
1360
|
-
advance(lexer);
|
|
1361
|
-
lexer->mark_end(lexer);
|
|
1362
|
-
scanner->closer_line_string_opened = true;
|
|
1363
|
-
lexer->result_symbol = CLOSER_LINE_ANSI_C_STRING_START;
|
|
1364
|
-
return true;
|
|
1365
|
-
}
|
|
1366
|
-
// Backquotes are the only other part that the scanner opens.
|
|
1367
|
-
if (lexer->lookahead != '`') {
|
|
1368
|
-
return false;
|
|
1369
|
-
}
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
// A newline ends a statement where a terminator may follow and is whitespace elsewhere.
|
|
1373
|
-
for (;;) {
|
|
1374
|
-
if (is_blank(lexer->lookahead)) {
|
|
1375
|
-
skip(lexer);
|
|
1376
|
-
} else if (lexer->lookahead == '\n') {
|
|
1377
|
-
if (valid_symbols[NEWLINE]) {
|
|
1378
|
-
advance(lexer);
|
|
1379
|
-
lexer->mark_end(lexer);
|
|
1380
|
-
lexer->result_symbol = NEWLINE;
|
|
1381
|
-
return true;
|
|
1382
|
-
}
|
|
1383
|
-
skip(lexer);
|
|
1384
|
-
} else if (lexer->lookahead == '\\') {
|
|
1385
|
-
// A continuation is returned as a token, since tree-sitter's lexer cannot always resume
|
|
1386
|
-
// after one; a backslash escaping a word's first character can only start an extglob prefix
|
|
1387
|
-
// (`\foo@($x)`), and the word token handles it otherwise.
|
|
1388
|
-
advance(lexer);
|
|
1389
|
-
if (lexer->lookahead == '\n' && !scanner->joined_line && awaits_deferred_body(scanner)) {
|
|
1390
|
-
// A reparse may reuse the words before the backslash as a complete command, where no
|
|
1391
|
-
// concatenation is valid, so a backslash right after the previous token is taken to end a
|
|
1392
|
-
// word unless a command or an assignment value may start there.
|
|
1393
|
-
split_continuation(scanner, lexer,
|
|
1394
|
-
first == '\\' && !valid_symbols[VARIABLE_NAME] && !valid_symbols[EMPTY_VALUE], false);
|
|
1395
|
-
return true;
|
|
1396
|
-
}
|
|
1397
|
-
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
|
|
1398
|
-
advance(lexer);
|
|
1399
|
-
lexer->mark_end(lexer);
|
|
1400
|
-
scanner->after_line_break = true;
|
|
1401
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1402
|
-
return true;
|
|
1403
|
-
}
|
|
1404
|
-
if (valid_symbols[EXTGLOB_PREFIX]) {
|
|
1405
|
-
advance(lexer);
|
|
1406
|
-
return continue_extglob_prefix(lexer, true);
|
|
1407
|
-
}
|
|
1408
|
-
return false;
|
|
1409
|
-
} else {
|
|
1410
|
-
break;
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
// Each closer has its own marker, so a `]` or `}` argument inside `$( )` does not end it.
|
|
1415
|
-
enum TokenType end = valid_symbols[SUBSTITUTION_END] && lexer->lookahead == ')' ? SUBSTITUTION_END
|
|
1416
|
-
: valid_symbols[BRACE_SUBSTITUTION_END] && lexer->lookahead == '}' ? BRACE_SUBSTITUTION_END
|
|
1417
|
-
: valid_symbols[BRACKET_SUBSTITUTION_END] && lexer->lookahead == ']' ? BRACKET_SUBSTITUTION_END
|
|
1418
|
-
: ERROR_RECOVERY;
|
|
1419
|
-
if (end != ERROR_RECOVERY) {
|
|
1420
|
-
if (scanner->closers.size > 0) {
|
|
1421
|
-
array_pop(&scanner->closers);
|
|
1422
|
-
scanner->regex_depth = array_pop(&scanner->closer_regex_depths);
|
|
1423
|
-
}
|
|
1424
|
-
discard_unstarted_heredocs(scanner);
|
|
1425
|
-
lexer->result_symbol = end;
|
|
1426
|
-
return true;
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
// Bash ends backquotes at the first unescaped backquote, even inside single quotes (`echo '`),
|
|
1430
|
-
// and then reports the unterminated quote when it runs the substitution. So a quote that the next
|
|
1431
|
-
// backquote cuts short closes the substitution there when it may close, and otherwise is skipped
|
|
1432
|
-
// up to the backquote, which then cannot be parsed; the quote never hides the text after it.
|
|
1433
|
-
if (scanner->backtick_depth > 0 && lexer->lookahead == '\'') {
|
|
1434
|
-
advance(lexer);
|
|
1435
|
-
lexer->mark_end(lexer);
|
|
1436
|
-
// A deferred body starts at the newline that ends the line, so what follows it is read later.
|
|
1437
|
-
bool breaks_line = awaits_deferred_body(scanner) && valid_symbols[CLOSER_LINE_RAW_STRING_START];
|
|
1438
|
-
while (!lexer->eof(lexer) && lexer->lookahead != '\'' && lexer->lookahead != '`' &&
|
|
1439
|
-
!(breaks_line && lexer->lookahead == '\n')) {
|
|
1440
|
-
// A backslash escapes a backquote or a backslash for finding the end of the backquotes; the
|
|
1441
|
-
// quote itself stays literal, so `'\'` still ends at its second quote.
|
|
1442
|
-
bool backslash = lexer->lookahead == '\\';
|
|
1443
|
-
advance(lexer);
|
|
1444
|
-
if (backslash && (lexer->lookahead == '`' || lexer->lookahead == '\\')) {
|
|
1445
|
-
advance(lexer);
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
if (lexer->lookahead != '`') {
|
|
1449
|
-
// A quote that the backquotes do not cut short scans its text itself where a deferred body may
|
|
1450
|
-
// start inside it.
|
|
1451
|
-
if (!lexer->eof(lexer) && breaks_line) {
|
|
1452
|
-
scanner->closer_line_string_opened = true;
|
|
1453
|
-
lexer->result_symbol = CLOSER_LINE_RAW_STRING_START;
|
|
1454
|
-
return true;
|
|
1455
|
-
}
|
|
1456
|
-
return false;
|
|
1457
|
-
}
|
|
1458
|
-
if (valid_symbols[BACKTICK_CLOSE]) {
|
|
1459
|
-
advance(lexer);
|
|
1460
|
-
lexer->mark_end(lexer);
|
|
1461
|
-
scanner->backtick_depth--;
|
|
1462
|
-
scanner->regex_depth = scanner->backtick_regex_depth;
|
|
1463
|
-
discard_unstarted_heredocs(scanner);
|
|
1464
|
-
lexer->result_symbol = BACKTICK_CLOSE;
|
|
1465
|
-
return true;
|
|
1466
|
-
}
|
|
1467
|
-
lexer->mark_end(lexer);
|
|
1468
|
-
lexer->result_symbol = LINE_CONTINUATION;
|
|
1469
|
-
return true;
|
|
1470
|
-
}
|
|
1471
|
-
|
|
1472
|
-
if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
|
|
1473
|
-
if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
|
|
1474
|
-
advance(lexer);
|
|
1475
|
-
lexer->mark_end(lexer);
|
|
1476
|
-
scanner->backtick_depth--;
|
|
1477
|
-
scanner->regex_depth = scanner->backtick_regex_depth;
|
|
1478
|
-
discard_unstarted_heredocs(scanner);
|
|
1479
|
-
lexer->result_symbol = BACKTICK_CLOSE;
|
|
1480
|
-
return true;
|
|
1481
|
-
}
|
|
1482
|
-
if (scanner->backtick_depth == 0 && valid_symbols[BACKTICK_OPEN]) {
|
|
1483
|
-
advance(lexer);
|
|
1484
|
-
lexer->mark_end(lexer);
|
|
1485
|
-
scanner->backtick_depth++;
|
|
1486
|
-
scanner->backtick_closers = (uint16_t)scanner->closers.size;
|
|
1487
|
-
scanner->backtick_regex_depth = scanner->regex_depth;
|
|
1488
|
-
lexer->result_symbol = BACKTICK_OPEN;
|
|
1489
|
-
return true;
|
|
1490
|
-
}
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
if ((valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH]) && lexer->lookahead == '<') {
|
|
1494
|
-
return scan_heredoc_arrow(scanner, lexer, valid_symbols);
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
|
-
// Only one check that consumes input can run, so the prefix scan leaves names before a possible
|
|
1498
|
-
// assignment to their own check; digits are read once as a descriptor or a word prefix.
|
|
1499
|
-
if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
|
|
1500
|
-
while (iswdigit(lexer->lookahead)) {
|
|
1501
|
-
advance(lexer);
|
|
1502
|
-
}
|
|
1503
|
-
lexer->mark_end(lexer);
|
|
1504
|
-
lexer->result_symbol = FILE_DESCRIPTOR;
|
|
1505
|
-
if (lexer->lookahead == '<' || lexer->lookahead == '>') {
|
|
1506
|
-
// Like bash, digits before `<(` or `>(` are a word joined with the process substitution.
|
|
1507
|
-
advance(lexer);
|
|
1508
|
-
return lexer->lookahead != '(';
|
|
1509
|
-
}
|
|
1510
|
-
return valid_symbols[EXTGLOB_PREFIX] && continue_extglob_prefix(lexer, true);
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
if (valid_symbols[EXTGLOB_PREFIX] && !(valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) &&
|
|
1514
|
-
!is_word_break(lexer->lookahead)) {
|
|
1515
|
-
return scan_extglob_prefix(lexer, valid_symbols[EMPTY_VALUE]);
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
|
|
1519
|
-
return scan_name_or_extglob_prefix(lexer, valid_symbols);
|
|
1520
|
-
}
|
|
1521
|
-
|
|
1522
|
-
if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
|
|
1523
|
-
advance(lexer);
|
|
1524
|
-
return finish_bare_dollar(lexer);
|
|
1525
|
-
}
|
|
1526
|
-
|
|
1527
|
-
// Bash reads a reserved word right after a compound command without a separator (`if (x) then`,
|
|
1528
|
-
// `{ (y) }`), so a zero-width terminator is returned before one there. After a word, the reserved
|
|
1529
|
-
// word would be an argument instead; a possible concatenation shows that a word just ended. This
|
|
1530
|
-
// check comes last because it consumes input even when it finds no reserved word.
|
|
1531
|
-
if (valid_symbols[NEWLINE] && !valid_symbols[CONCAT] && at_closing_reserved_word(lexer)) {
|
|
1532
|
-
lexer->result_symbol = NEWLINE;
|
|
1533
|
-
return true;
|
|
1534
|
-
}
|
|
1535
|
-
|
|
1536
|
-
return false;
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
void *tree_sitter_bash_external_scanner_create(void) {
|
|
1540
|
-
Scanner *scanner = ts_calloc(1, sizeof(Scanner));
|
|
1541
|
-
array_init(&scanner->heredocs);
|
|
1542
|
-
array_init(&scanner->closers);
|
|
1543
|
-
array_init(&scanner->closer_regex_depths);
|
|
1544
|
-
return scanner;
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
void tree_sitter_bash_external_scanner_destroy(void *payload) {
|
|
1548
|
-
Scanner *scanner = (Scanner *)payload;
|
|
1549
|
-
scanner_reset(scanner);
|
|
1550
|
-
array_delete(&scanner->heredocs);
|
|
1551
|
-
array_delete(&scanner->closers);
|
|
1552
|
-
array_delete(&scanner->closer_regex_depths);
|
|
1553
|
-
ts_free(scanner);
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
bool tree_sitter_bash_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
|
|
1557
|
-
return scan((Scanner *)payload, lexer, valid_symbols);
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer) {
|
|
1561
|
-
Scanner *scanner = (Scanner *)payload;
|
|
1562
|
-
unsigned size = 0;
|
|
1563
|
-
buffer[size++] = (char)scanner->backtick_depth;
|
|
1564
|
-
memcpy(&buffer[size], &scanner->backtick_closers, sizeof(scanner->backtick_closers));
|
|
1565
|
-
size += sizeof(scanner->backtick_closers);
|
|
1566
|
-
buffer[size++] = (char)(scanner->terminator_owed | scanner->after_line_break << 1 | scanner->joined_line << 2 |
|
|
1567
|
-
scanner->split_continuation << 3 | scanner->split_continuation_in_word << 4 |
|
|
1568
|
-
scanner->split_continuation_in_regex << 5 | scanner->closer_line_string_opened << 6);
|
|
1569
|
-
memcpy(&buffer[size], &scanner->regex_depth, sizeof(scanner->regex_depth));
|
|
1570
|
-
size += sizeof(scanner->regex_depth);
|
|
1571
|
-
memcpy(&buffer[size], &scanner->backtick_regex_depth, sizeof(scanner->backtick_regex_depth));
|
|
1572
|
-
size += sizeof(scanner->backtick_regex_depth);
|
|
1573
|
-
uint16_t closer_count = (uint16_t)scanner->closers.size;
|
|
1574
|
-
memcpy(&buffer[size], &closer_count, sizeof(closer_count));
|
|
1575
|
-
size += sizeof(closer_count);
|
|
1576
|
-
if (scanner->closers.size > 0) {
|
|
1577
|
-
memcpy(&buffer[size], scanner->closers.contents, scanner->closers.size);
|
|
1578
|
-
size += scanner->closers.size;
|
|
1579
|
-
memcpy(&buffer[size], scanner->closer_regex_depths.contents, closer_count * sizeof(uint16_t));
|
|
1580
|
-
size += closer_count * sizeof(uint16_t);
|
|
1581
|
-
}
|
|
1582
|
-
buffer[size++] = (char)scanner->heredocs.size;
|
|
1583
|
-
for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
|
|
1584
|
-
Heredoc *heredoc = array_get(&scanner->heredocs, i);
|
|
1585
|
-
buffer[size++] = (char)heredoc->is_raw;
|
|
1586
|
-
buffer[size++] = (char)heredoc->allows_indent;
|
|
1587
|
-
buffer[size++] = (char)(heredoc->started | heredoc->deferred << 1);
|
|
1588
|
-
memcpy(&buffer[size], &heredoc->depth, sizeof(heredoc->depth));
|
|
1589
|
-
size += sizeof(heredoc->depth);
|
|
1590
|
-
uint16_t length = (uint16_t)heredoc->delimiter.size;
|
|
1591
|
-
memcpy(&buffer[size], &length, sizeof(length));
|
|
1592
|
-
size += sizeof(length);
|
|
1593
|
-
for (uint32_t j = 0; j < heredoc->delimiter.size; j++) {
|
|
1594
|
-
size += encode_varint(*array_get(&heredoc->delimiter, j), &buffer[size]);
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
return size;
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
|
|
1601
|
-
Scanner *scanner = (Scanner *)payload;
|
|
1602
|
-
scanner_reset(scanner);
|
|
1603
|
-
if (length < SCANNER_HEADER_SIZE) {
|
|
1604
|
-
return;
|
|
1605
|
-
}
|
|
1606
|
-
unsigned size = 0;
|
|
1607
|
-
scanner->backtick_depth = (uint8_t)buffer[size++];
|
|
1608
|
-
memcpy(&scanner->backtick_closers, &buffer[size], sizeof(scanner->backtick_closers));
|
|
1609
|
-
size += sizeof(scanner->backtick_closers);
|
|
1610
|
-
scanner->terminator_owed = buffer[size] & 1;
|
|
1611
|
-
scanner->after_line_break = (buffer[size] & 2) != 0;
|
|
1612
|
-
scanner->joined_line = (buffer[size] & 4) != 0;
|
|
1613
|
-
scanner->split_continuation = (buffer[size] & 8) != 0;
|
|
1614
|
-
scanner->split_continuation_in_word = (buffer[size] & 16) != 0;
|
|
1615
|
-
scanner->split_continuation_in_regex = (buffer[size] & 32) != 0;
|
|
1616
|
-
scanner->closer_line_string_opened = (buffer[size++] & 64) != 0;
|
|
1617
|
-
memcpy(&scanner->regex_depth, &buffer[size], sizeof(scanner->regex_depth));
|
|
1618
|
-
size += sizeof(scanner->regex_depth);
|
|
1619
|
-
memcpy(&scanner->backtick_regex_depth, &buffer[size], sizeof(scanner->backtick_regex_depth));
|
|
1620
|
-
size += sizeof(scanner->backtick_regex_depth);
|
|
1621
|
-
uint16_t closer_count;
|
|
1622
|
-
memcpy(&closer_count, &buffer[size], sizeof(closer_count));
|
|
1623
|
-
size += sizeof(closer_count);
|
|
1624
|
-
if (size + CLOSER_SIZE * closer_count >= length) {
|
|
1625
|
-
return;
|
|
1626
|
-
}
|
|
1627
|
-
array_reserve(&scanner->closers, closer_count);
|
|
1628
|
-
memcpy(scanner->closers.contents, &buffer[size], closer_count);
|
|
1629
|
-
scanner->closers.size = closer_count;
|
|
1630
|
-
size += closer_count;
|
|
1631
|
-
array_reserve(&scanner->closer_regex_depths, closer_count);
|
|
1632
|
-
memcpy(scanner->closer_regex_depths.contents, &buffer[size], closer_count * sizeof(uint16_t));
|
|
1633
|
-
scanner->closer_regex_depths.size = closer_count;
|
|
1634
|
-
size += closer_count * sizeof(uint16_t);
|
|
1635
|
-
uint8_t count = (uint8_t)buffer[size++];
|
|
1636
|
-
for (uint8_t i = 0; i < count && size + HEREDOC_HEADER_SIZE <= length; i++) {
|
|
1637
|
-
Heredoc heredoc = {.delimiter = array_new()};
|
|
1638
|
-
heredoc.is_raw = buffer[size++];
|
|
1639
|
-
heredoc.allows_indent = buffer[size++];
|
|
1640
|
-
heredoc.started = buffer[size] & 1;
|
|
1641
|
-
heredoc.deferred = (buffer[size++] & 2) != 0;
|
|
1642
|
-
memcpy(&heredoc.depth, &buffer[size], sizeof(heredoc.depth));
|
|
1643
|
-
size += sizeof(heredoc.depth);
|
|
1644
|
-
uint16_t delimiter_length;
|
|
1645
|
-
memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
|
|
1646
|
-
size += sizeof(delimiter_length);
|
|
1647
|
-
array_reserve(&heredoc.delimiter, delimiter_length);
|
|
1648
|
-
for (uint16_t j = 0; j < delimiter_length; j++) {
|
|
1649
|
-
int32_t c;
|
|
1650
|
-
unsigned consumed = decode_varint(&buffer[size], length - size, &c);
|
|
1651
|
-
if (consumed == 0) {
|
|
1652
|
-
break;
|
|
1653
|
-
}
|
|
1654
|
-
size += consumed;
|
|
1655
|
-
array_push(&heredoc.delimiter, c);
|
|
1656
|
-
}
|
|
1657
|
-
array_push(&scanner->heredocs, heredoc);
|
|
1658
|
-
}
|
|
1659
|
-
}
|