@willbooster/tree-sitter-bash 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scanner.c ADDED
@@ -0,0 +1,557 @@
1
+ #include "tree_sitter/array.h"
2
+ #include "tree_sitter/parser.h"
3
+
4
+ #include <string.h>
5
+ #include <wctype.h>
6
+
7
+ // Must match the order of `externals` in grammar.js.
8
+ enum TokenType {
9
+ HEREDOC_START,
10
+ HEREDOC_BODY_START,
11
+ HEREDOC_CONTENT,
12
+ HEREDOC_END,
13
+ HEREDOC_ARROW,
14
+ HEREDOC_ARROW_DASH,
15
+ CONCAT,
16
+ FILE_DESCRIPTOR,
17
+ VARIABLE_NAME,
18
+ BARE_DOLLAR,
19
+ EMPTY_VALUE,
20
+ REGEX,
21
+ STRING_CONTENT,
22
+ BACKTICK_OPEN,
23
+ BACKTICK_CLOSE,
24
+ NEWLINE,
25
+ ERROR_RECOVERY,
26
+ };
27
+
28
+ typedef Array(char) String;
29
+
30
+ // A heredoc whose `<<` operator has been read. Its body starts at the next newline that ends a line
31
+ // (not one inside a string), after the bodies of heredocs opened earlier.
32
+ typedef struct {
33
+ bool is_raw;
34
+ bool allows_indent;
35
+ bool started;
36
+ String delimiter;
37
+ } Heredoc;
38
+
39
+ typedef struct {
40
+ Array(Heredoc) heredocs;
41
+ // Inside backquotes an unescaped backquote always closes the substitution, so the depth decides
42
+ // whether a backquote opens or closes one.
43
+ uint8_t backtick_depth;
44
+ } Scanner;
45
+
46
+ static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
47
+
48
+ static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
49
+
50
+ static inline bool is_metacharacter(int32_t c) {
51
+ return c == '|' || c == '&' || c == ';' || c == '(' || c == ')' || c == '<' || c == '>';
52
+ }
53
+
54
+ static inline bool is_blank(int32_t c) { return c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'; }
55
+
56
+ static inline bool is_name_start(int32_t c) { return iswalpha(c) || c == '_'; }
57
+
58
+ static inline bool is_name_char(int32_t c) { return iswalnum(c) || c == '_'; }
59
+
60
+ // Whether `$` followed by `c` begins an expansion inside double quotes or a heredoc body.
61
+ static inline bool starts_quoted_expansion(int32_t c) {
62
+ return is_name_start(c) || iswdigit(c) || c == '{' || c == '(' || c == '[' || c == '*' || c == '@' ||
63
+ c == '#' || c == '?' || c == '$' || c == '!' || c == '-';
64
+ }
65
+
66
+ static void heredoc_delete(Heredoc *heredoc) { array_delete(&heredoc->delimiter); }
67
+
68
+ static void scanner_reset(Scanner *scanner) {
69
+ for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
70
+ heredoc_delete(array_get(&scanner->heredocs, i));
71
+ }
72
+ array_clear(&scanner->heredocs);
73
+ scanner->backtick_depth = 0;
74
+ }
75
+
76
+ static void pop_front_heredoc(Scanner *scanner) {
77
+ heredoc_delete(array_get(&scanner->heredocs, 0));
78
+ array_erase(&scanner->heredocs, 0);
79
+ }
80
+
81
+ // Heredoc bodies: the body starts at the newline ending the header line and ends with the delimiter
82
+ // line (or at the end of input, as bash does with a warning). An unquoted body stops before each
83
+ // expansion so that the parser can read it.
84
+ static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer) {
85
+ Heredoc *heredoc = array_get(&scanner->heredocs, 0);
86
+ bool did_advance = false;
87
+ bool at_line_start = lexer->get_column(lexer) == 0;
88
+
89
+ for (;;) {
90
+ if (at_line_start) {
91
+ at_line_start = false;
92
+ lexer->mark_end(lexer);
93
+ bool consumed_tabs = false;
94
+ if (heredoc->allows_indent) {
95
+ while (lexer->lookahead == '\t') {
96
+ advance(lexer);
97
+ consumed_tabs = true;
98
+ }
99
+ }
100
+ uint32_t matched = 0;
101
+ while (matched < heredoc->delimiter.size && lexer->lookahead == *array_get(&heredoc->delimiter, matched)) {
102
+ advance(lexer);
103
+ matched++;
104
+ }
105
+ if (lexer->lookahead == '\r') {
106
+ advance(lexer);
107
+ }
108
+ if (matched == heredoc->delimiter.size && (lexer->lookahead == '\n' || lexer->eof(lexer))) {
109
+ if (did_advance) {
110
+ lexer->result_symbol = HEREDOC_CONTENT;
111
+ return true;
112
+ }
113
+ lexer->mark_end(lexer);
114
+ lexer->result_symbol = HEREDOC_END;
115
+ pop_front_heredoc(scanner);
116
+ return true;
117
+ }
118
+ // The characters read while trying the delimiter are content; an empty attempt must not
119
+ // count, or a zero-width content token would repeat forever.
120
+ did_advance = did_advance || consumed_tabs || matched > 0;
121
+ continue;
122
+ }
123
+
124
+ if (lexer->eof(lexer)) {
125
+ lexer->mark_end(lexer);
126
+ lexer->result_symbol = did_advance ? HEREDOC_CONTENT : HEREDOC_END;
127
+ if (!did_advance) {
128
+ pop_front_heredoc(scanner);
129
+ }
130
+ return true;
131
+ }
132
+
133
+ switch (lexer->lookahead) {
134
+ case '\n':
135
+ advance(lexer);
136
+ did_advance = true;
137
+ at_line_start = true;
138
+ break;
139
+ case '\\':
140
+ advance(lexer);
141
+ did_advance = true;
142
+ if (!heredoc->is_raw && !lexer->eof(lexer)) {
143
+ // A backslash-newline joins lines before the delimiter comparison.
144
+ advance(lexer);
145
+ }
146
+ break;
147
+ case '`':
148
+ if (!heredoc->is_raw) {
149
+ lexer->mark_end(lexer);
150
+ lexer->result_symbol = HEREDOC_CONTENT;
151
+ return did_advance;
152
+ }
153
+ advance(lexer);
154
+ did_advance = true;
155
+ break;
156
+ case '$':
157
+ if (!heredoc->is_raw) {
158
+ lexer->mark_end(lexer);
159
+ advance(lexer);
160
+ if (starts_quoted_expansion(lexer->lookahead)) {
161
+ if (!did_advance) {
162
+ return false;
163
+ }
164
+ lexer->result_symbol = HEREDOC_CONTENT;
165
+ return true;
166
+ }
167
+ did_advance = true;
168
+ break;
169
+ }
170
+ advance(lexer);
171
+ did_advance = true;
172
+ break;
173
+ default:
174
+ advance(lexer);
175
+ did_advance = true;
176
+ break;
177
+ }
178
+ }
179
+ }
180
+
181
+ // The delimiter word undergoes quote removal; any quoting makes the body literal.
182
+ static bool scan_heredoc_start(Scanner *scanner, TSLexer *lexer) {
183
+ Heredoc *heredoc = array_back(&scanner->heredocs);
184
+ array_clear(&heredoc->delimiter);
185
+ heredoc->is_raw = false;
186
+
187
+ while (is_blank(lexer->lookahead)) {
188
+ skip(lexer);
189
+ }
190
+
191
+ int32_t quote = 0;
192
+ while (!lexer->eof(lexer)) {
193
+ int32_t c = lexer->lookahead;
194
+ if (quote) {
195
+ advance(lexer);
196
+ if (c == quote) {
197
+ quote = 0;
198
+ continue;
199
+ }
200
+ if (quote == '"' && c == '\\' &&
201
+ (lexer->lookahead == '$' || lexer->lookahead == '`' || lexer->lookahead == '"' ||
202
+ lexer->lookahead == '\\')) {
203
+ c = lexer->lookahead;
204
+ advance(lexer);
205
+ }
206
+ array_push(&heredoc->delimiter, (char)c);
207
+ continue;
208
+ }
209
+ if (iswspace(c) || is_metacharacter(c)) {
210
+ break;
211
+ }
212
+ advance(lexer);
213
+ if (c == '\'' || c == '"') {
214
+ quote = c;
215
+ heredoc->is_raw = true;
216
+ } else if (c == '\\') {
217
+ heredoc->is_raw = true;
218
+ if (lexer->lookahead == '\n') {
219
+ advance(lexer);
220
+ } else if (!lexer->eof(lexer)) {
221
+ array_push(&heredoc->delimiter, (char)lexer->lookahead);
222
+ advance(lexer);
223
+ }
224
+ } else {
225
+ array_push(&heredoc->delimiter, (char)c);
226
+ }
227
+ }
228
+
229
+ if (heredoc->delimiter.size == 0 && !heredoc->is_raw) {
230
+ return false;
231
+ }
232
+ lexer->mark_end(lexer);
233
+ lexer->result_symbol = HEREDOC_START;
234
+ return true;
235
+ }
236
+
237
+ static bool scan_heredoc_arrow(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
238
+ advance(lexer);
239
+ if (lexer->lookahead != '<') {
240
+ return false;
241
+ }
242
+ advance(lexer);
243
+ bool allows_indent = false;
244
+ if (lexer->lookahead == '<') {
245
+ return false;
246
+ }
247
+ if (lexer->lookahead == '-' && valid_symbols[HEREDOC_ARROW_DASH]) {
248
+ advance(lexer);
249
+ allows_indent = true;
250
+ } else if (!valid_symbols[HEREDOC_ARROW]) {
251
+ return false;
252
+ }
253
+ lexer->mark_end(lexer);
254
+ Heredoc heredoc = {.is_raw = false, .allows_indent = allows_indent, .started = false, .delimiter = array_new()};
255
+ array_push(&scanner->heredocs, heredoc);
256
+ lexer->result_symbol = allows_indent ? HEREDOC_ARROW_DASH : HEREDOC_ARROW;
257
+ return true;
258
+ }
259
+
260
+ // A name followed by `=`, `+=`, or `[subscript]=` starts an assignment.
261
+ static bool scan_variable_name(TSLexer *lexer) {
262
+ while (is_name_char(lexer->lookahead)) {
263
+ advance(lexer);
264
+ }
265
+ lexer->mark_end(lexer);
266
+ if (lexer->lookahead == '+') {
267
+ advance(lexer);
268
+ return lexer->lookahead == '=';
269
+ }
270
+ if (lexer->lookahead == '[') {
271
+ uint32_t depth = 0;
272
+ while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
273
+ if (lexer->lookahead == '[') {
274
+ depth++;
275
+ } else if (lexer->lookahead == ']' && --depth == 0) {
276
+ advance(lexer);
277
+ break;
278
+ } else if (lexer->lookahead == '\\') {
279
+ advance(lexer);
280
+ }
281
+ advance(lexer);
282
+ }
283
+ if (depth != 0) {
284
+ return false;
285
+ }
286
+ if (lexer->lookahead == '+') {
287
+ advance(lexer);
288
+ }
289
+ }
290
+ return lexer->lookahead == '=';
291
+ }
292
+
293
+ // The right side of `=~` extends to unquoted whitespace outside parentheses.
294
+ static bool scan_regex(TSLexer *lexer) {
295
+ while (is_blank(lexer->lookahead)) {
296
+ skip(lexer);
297
+ }
298
+ uint32_t depth = 0;
299
+ bool did_advance = false;
300
+ while (!lexer->eof(lexer)) {
301
+ int32_t c = lexer->lookahead;
302
+ if (depth == 0 && iswspace(c)) {
303
+ break;
304
+ }
305
+ if (c == '\\') {
306
+ advance(lexer);
307
+ } else if (c == '\'' || c == '"') {
308
+ advance(lexer);
309
+ while (!lexer->eof(lexer) && lexer->lookahead != c) {
310
+ if (c == '"' && lexer->lookahead == '\\') {
311
+ advance(lexer);
312
+ }
313
+ advance(lexer);
314
+ }
315
+ } else if (c == '(') {
316
+ depth++;
317
+ } else if (c == ')') {
318
+ if (depth == 0) {
319
+ break;
320
+ }
321
+ depth--;
322
+ }
323
+ advance(lexer);
324
+ did_advance = true;
325
+ }
326
+ lexer->mark_end(lexer);
327
+ lexer->result_symbol = REGEX;
328
+ return did_advance;
329
+ }
330
+
331
+ static bool scan_string_content(TSLexer *lexer) {
332
+ bool did_advance = false;
333
+ for (;;) {
334
+ lexer->mark_end(lexer);
335
+ if (lexer->eof(lexer)) {
336
+ break;
337
+ }
338
+ int32_t c = lexer->lookahead;
339
+ if (c == '"' || c == '`') {
340
+ break;
341
+ }
342
+ advance(lexer);
343
+ if (c == '$' && starts_quoted_expansion(lexer->lookahead)) {
344
+ break;
345
+ }
346
+ if (c == '\\' && !lexer->eof(lexer)) {
347
+ advance(lexer);
348
+ }
349
+ did_advance = true;
350
+ }
351
+ lexer->result_symbol = STRING_CONTENT;
352
+ return did_advance;
353
+ }
354
+
355
+ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
356
+ bool error_recovery = valid_symbols[ERROR_RECOVERY];
357
+ // Tokens that must touch the previous one (concatenation, an empty assignment value) are decided
358
+ // by the character right after it, even when blanks are skipped below to find a newline.
359
+ int32_t first = lexer->lookahead;
360
+ bool at_eof = lexer->eof(lexer);
361
+
362
+ if (scanner->heredocs.size > 0) {
363
+ Heredoc *front = array_get(&scanner->heredocs, 0);
364
+ if (front->started && (valid_symbols[HEREDOC_CONTENT] || valid_symbols[HEREDOC_END]) &&
365
+ !(lexer->lookahead == '`' && !front->is_raw)) {
366
+ return scan_heredoc_content(scanner, lexer);
367
+ }
368
+ // A newline inside a string is part of it, so bodies start only outside strings.
369
+ if (!front->started && (front->delimiter.size > 0 || front->is_raw) && valid_symbols[HEREDOC_BODY_START] &&
370
+ !valid_symbols[STRING_CONTENT]) {
371
+ while (is_blank(lexer->lookahead)) {
372
+ skip(lexer);
373
+ }
374
+ if (lexer->lookahead == '\n') {
375
+ front->started = true;
376
+ lexer->result_symbol = HEREDOC_BODY_START;
377
+ return true;
378
+ }
379
+ }
380
+ }
381
+
382
+ if (error_recovery) {
383
+ return false;
384
+ }
385
+
386
+ if (valid_symbols[STRING_CONTENT] && lexer->lookahead != '`') {
387
+ return scan_string_content(lexer);
388
+ }
389
+
390
+ if (valid_symbols[HEREDOC_START] && scanner->heredocs.size > 0) {
391
+ return scan_heredoc_start(scanner, lexer);
392
+ }
393
+
394
+ if (valid_symbols[CONCAT]) {
395
+ int32_t c = first;
396
+ if (!at_eof && !iswspace(c) && !is_metacharacter(c) &&
397
+ !(c == '`' && scanner->backtick_depth > 0)) {
398
+ lexer->result_symbol = CONCAT;
399
+ return true;
400
+ }
401
+ }
402
+
403
+ if (valid_symbols[EMPTY_VALUE]) {
404
+ int32_t c = first;
405
+ if (at_eof || iswspace(c) || c == ';' || c == '&' || c == '|' || c == ')' ||
406
+ (c == '`' && scanner->backtick_depth > 0)) {
407
+ lexer->result_symbol = EMPTY_VALUE;
408
+ return true;
409
+ }
410
+ }
411
+
412
+ if (valid_symbols[REGEX]) {
413
+ return scan_regex(lexer);
414
+ }
415
+
416
+ // A newline ends a statement where a terminator may follow and is whitespace elsewhere.
417
+ for (;;) {
418
+ if (is_blank(lexer->lookahead)) {
419
+ skip(lexer);
420
+ } else if (lexer->lookahead == '\n') {
421
+ if (valid_symbols[NEWLINE]) {
422
+ advance(lexer);
423
+ lexer->mark_end(lexer);
424
+ lexer->result_symbol = NEWLINE;
425
+ return true;
426
+ }
427
+ skip(lexer);
428
+ } else if (lexer->lookahead == '\\') {
429
+ skip(lexer);
430
+ if (lexer->lookahead == '\r') {
431
+ skip(lexer);
432
+ }
433
+ if (lexer->lookahead != '\n') {
434
+ return false;
435
+ }
436
+ skip(lexer);
437
+ } else {
438
+ break;
439
+ }
440
+ }
441
+
442
+ if ((valid_symbols[BACKTICK_OPEN] || valid_symbols[BACKTICK_CLOSE]) && lexer->lookahead == '`') {
443
+ if (scanner->backtick_depth > 0 && valid_symbols[BACKTICK_CLOSE]) {
444
+ advance(lexer);
445
+ lexer->mark_end(lexer);
446
+ scanner->backtick_depth--;
447
+ lexer->result_symbol = BACKTICK_CLOSE;
448
+ return true;
449
+ }
450
+ if (scanner->backtick_depth == 0 && valid_symbols[BACKTICK_OPEN]) {
451
+ advance(lexer);
452
+ lexer->mark_end(lexer);
453
+ scanner->backtick_depth++;
454
+ lexer->result_symbol = BACKTICK_OPEN;
455
+ return true;
456
+ }
457
+ }
458
+
459
+ if ((valid_symbols[HEREDOC_ARROW] || valid_symbols[HEREDOC_ARROW_DASH]) && lexer->lookahead == '<') {
460
+ return scan_heredoc_arrow(scanner, lexer, valid_symbols);
461
+ }
462
+
463
+ if (valid_symbols[FILE_DESCRIPTOR] && iswdigit(lexer->lookahead)) {
464
+ while (iswdigit(lexer->lookahead)) {
465
+ advance(lexer);
466
+ }
467
+ lexer->mark_end(lexer);
468
+ lexer->result_symbol = FILE_DESCRIPTOR;
469
+ return lexer->lookahead == '<' || lexer->lookahead == '>';
470
+ }
471
+
472
+ if (valid_symbols[VARIABLE_NAME] && is_name_start(lexer->lookahead)) {
473
+ lexer->result_symbol = VARIABLE_NAME;
474
+ return scan_variable_name(lexer);
475
+ }
476
+
477
+ if (valid_symbols[BARE_DOLLAR] && lexer->lookahead == '$') {
478
+ advance(lexer);
479
+ lexer->mark_end(lexer);
480
+ int32_t c = lexer->lookahead;
481
+ lexer->result_symbol = BARE_DOLLAR;
482
+ return lexer->eof(lexer) || !(starts_quoted_expansion(c) || c == '\'' || c == '"');
483
+ }
484
+
485
+ return false;
486
+ }
487
+
488
+ void *tree_sitter_bash_external_scanner_create(void) {
489
+ Scanner *scanner = ts_calloc(1, sizeof(Scanner));
490
+ array_init(&scanner->heredocs);
491
+ return scanner;
492
+ }
493
+
494
+ void tree_sitter_bash_external_scanner_destroy(void *payload) {
495
+ Scanner *scanner = (Scanner *)payload;
496
+ scanner_reset(scanner);
497
+ array_delete(&scanner->heredocs);
498
+ ts_free(scanner);
499
+ }
500
+
501
+ bool tree_sitter_bash_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
502
+ return scan((Scanner *)payload, lexer, valid_symbols);
503
+ }
504
+
505
+ unsigned tree_sitter_bash_external_scanner_serialize(void *payload, char *buffer) {
506
+ Scanner *scanner = (Scanner *)payload;
507
+ unsigned size = 0;
508
+ buffer[size++] = (char)scanner->backtick_depth;
509
+ unsigned count_position = size++;
510
+ uint8_t count = 0;
511
+ for (uint32_t i = 0; i < scanner->heredocs.size && count < UINT8_MAX; i++) {
512
+ Heredoc *heredoc = array_get(&scanner->heredocs, i);
513
+ if (size + 3 + sizeof(uint16_t) + heredoc->delimiter.size > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
514
+ break;
515
+ }
516
+ buffer[size++] = (char)heredoc->is_raw;
517
+ buffer[size++] = (char)heredoc->allows_indent;
518
+ buffer[size++] = (char)heredoc->started;
519
+ uint16_t length = (uint16_t)heredoc->delimiter.size;
520
+ memcpy(&buffer[size], &length, sizeof(length));
521
+ size += sizeof(length);
522
+ if (length > 0) {
523
+ memcpy(&buffer[size], heredoc->delimiter.contents, length);
524
+ size += length;
525
+ }
526
+ count++;
527
+ }
528
+ buffer[count_position] = (char)count;
529
+ return size;
530
+ }
531
+
532
+ void tree_sitter_bash_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
533
+ Scanner *scanner = (Scanner *)payload;
534
+ scanner_reset(scanner);
535
+ if (length == 0) {
536
+ return;
537
+ }
538
+ unsigned size = 0;
539
+ scanner->backtick_depth = (uint8_t)buffer[size++];
540
+ uint8_t count = (uint8_t)buffer[size++];
541
+ for (uint8_t i = 0; i < count; i++) {
542
+ Heredoc heredoc = {.delimiter = array_new()};
543
+ heredoc.is_raw = buffer[size++];
544
+ heredoc.allows_indent = buffer[size++];
545
+ heredoc.started = buffer[size++];
546
+ uint16_t delimiter_length;
547
+ memcpy(&delimiter_length, &buffer[size], sizeof(delimiter_length));
548
+ size += sizeof(delimiter_length);
549
+ if (delimiter_length > 0) {
550
+ array_reserve(&heredoc.delimiter, delimiter_length);
551
+ memcpy(heredoc.delimiter.contents, &buffer[size], delimiter_length);
552
+ heredoc.delimiter.size = delimiter_length;
553
+ size += delimiter_length;
554
+ }
555
+ array_push(&scanner->heredocs, heredoc);
556
+ }
557
+ }
@@ -0,0 +1,54 @@
1
+ #ifndef TREE_SITTER_ALLOC_H_
2
+ #define TREE_SITTER_ALLOC_H_
3
+
4
+ #ifdef __cplusplus
5
+ extern "C" {
6
+ #endif
7
+
8
+ #include <stdbool.h>
9
+ #include <stdio.h>
10
+ #include <stdlib.h>
11
+
12
+ // Allow clients to override allocation functions
13
+ #ifdef TREE_SITTER_REUSE_ALLOCATOR
14
+
15
+ extern void *(*ts_current_malloc)(size_t size);
16
+ extern void *(*ts_current_calloc)(size_t count, size_t size);
17
+ extern void *(*ts_current_realloc)(void *ptr, size_t size);
18
+ extern void (*ts_current_free)(void *ptr);
19
+
20
+ #ifndef ts_malloc
21
+ #define ts_malloc ts_current_malloc
22
+ #endif
23
+ #ifndef ts_calloc
24
+ #define ts_calloc ts_current_calloc
25
+ #endif
26
+ #ifndef ts_realloc
27
+ #define ts_realloc ts_current_realloc
28
+ #endif
29
+ #ifndef ts_free
30
+ #define ts_free ts_current_free
31
+ #endif
32
+
33
+ #else
34
+
35
+ #ifndef ts_malloc
36
+ #define ts_malloc malloc
37
+ #endif
38
+ #ifndef ts_calloc
39
+ #define ts_calloc calloc
40
+ #endif
41
+ #ifndef ts_realloc
42
+ #define ts_realloc realloc
43
+ #endif
44
+ #ifndef ts_free
45
+ #define ts_free free
46
+ #endif
47
+
48
+ #endif
49
+
50
+ #ifdef __cplusplus
51
+ }
52
+ #endif
53
+
54
+ #endif // TREE_SITTER_ALLOC_H_