@willbooster/tree-sitter-bash 1.1.2 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -6,20 +6,13 @@
6
6
  [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
7
7
  [![wbfy](https://img.shields.io/badge/wbfy-20.20.0-1e90ff.svg)](https://github.com/WillBooster/shared/tree/main/packages/wbfy)
8
8
 
9
- Bash grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter), rewritten from
10
- [tree-sitter/tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash). The syntax trees differ from the
11
- original grammar's.
9
+ Bash grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter), forked from
10
+ [tree-sitter/tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash). We are grateful to its authors and
11
+ contributors. This is not an official release of that project.
12
12
 
13
- - Heredoc bodies are parsed wherever bash reads them: from the newline that ends the line holding the `<<` operator,
14
- in operator order. Any statement may follow the operator on that line (`cat <<EOF; echo done`), and several heredocs
15
- may share a line. A `heredoc_body` node appears as an extra next to the command that opened it. A heredoc whose
16
- delimiter does not fit the parser's 1 KiB scanner state is an `ERROR` rather than a guessed body.
17
- - Redirections stay inside the command they belong to, in source order (`npm > out run x`), and a command may
18
- consist of assignments and redirections only.
19
- - `$((…))` is always an arithmetic expansion, including inside heredoc bodies; escaped `\$` and `` \` `` are literal.
20
- - A newline is a statement terminator only where one may end a statement; elsewhere it is whitespace.
21
- - Like bash, only a space, a tab, and a newline separate words. Any other whitespace is part of a word, including
22
- the CR of a CRLF line ending, so a CRLF script parses as bash runs it (`echo a` + CRLF passes `a` + CR).
13
+ This fork aims to parse scripts exactly as bash runs them, so that tools can tell code from data in untrusted
14
+ scripts. It fixes parsing bugs and raises conformance with bash, checked by differential tests against bash itself.
15
+ As a result, its syntax trees differ from the original grammar's; review your queries when migrating.
23
16
 
24
17
  ## Usage
25
18
 
@@ -44,7 +37,10 @@ script/parse-examples
44
37
 
45
38
  `bun run test` runs:
46
39
 
47
- - the corpus in `test/corpus`;
40
+ - the corpus in `test/corpus`, with the native build and with the Wasm build (the first run downloads the WASI SDK);
41
+ - an incremental-parsing check (`test/unit/incremental.test.ts`): `tree-sitter fuzz` edits each corpus case at random,
42
+ reparses it, undoes the edits, and reparses again. `TREE_SITTER_SEED`, `TREE_SITTER_ITERATIONS`, and
43
+ `TREE_SITTER_EDITS` run other or more edits;
48
44
  - the Node.js binding test;
49
45
  - a check that real-world scripts cloned into `examples/` fail to parse exactly as listed in
50
46
  `script/known-failures.txt`. The first run clones them, which takes a few minutes. The example repositories are
@@ -59,6 +55,9 @@ script/parse-examples
59
55
  - a performance check (`test/unit/performance.test.ts`) that a 240 KB line parses in linear time, since consumers
60
56
  parse untrusted scripts.
61
57
 
58
+ CI also runs these tests on every platform that gets a prebuild, and fuzzes the parser with libFuzzer and sanitizers
59
+ (`.github/workflows/robustness.yml`).
60
+
62
61
  ### References
63
62
 
64
63
  - [Bash man page](http://man7.org/linux/man-pages/man1/bash.1.html#SHELL_GRAMMAR)
package/grammar.js CHANGED
@@ -19,6 +19,7 @@ const EXTGLOB_GROUP = extglobGroup('[?*+@]', 3);
19
19
  const EXTGLOB_NEGATION = extglobGroup('!', 3);
20
20
 
21
21
  // Text inside `${…}` and subscripts stops before `<(`/`>(`, so a process substitution there is parsed.
22
+ const EXPANSION_TEXT_BREAKS = ['}', '$', '`', '"', '\'', '\\\\', '<', '>'];
22
23
  const SUBSCRIPT_TEXT = /([^\[\]$`"'\\<>]|[<>]+[^(\[\]$`"'\\<>]|\\.)+/;
23
24
 
24
25
  const PREC = {
@@ -77,6 +78,13 @@ module.exports = grammar({
77
78
  $._line_continuation,
78
79
  $._brace_substitution_start,
79
80
  $._bracket_substitution_start,
81
+ $._regex_start,
82
+ $._regex_concat,
83
+ $._closer_line_raw_string_start,
84
+ $._closer_line_ansi_c_string_start,
85
+ $._closer_line_raw_string_content,
86
+ $._closer_line_ansi_c_string_content,
87
+ $._joined_comment,
80
88
  $.__error_recovery,
81
89
  ],
82
90
 
@@ -349,12 +357,19 @@ module.exports = grammar({
349
357
  field('right', $._argument),
350
358
  )),
351
359
 
360
+ // Like bash, the right side is one word whose parenthesized groups may hold blanks and
361
+ // metacharacters; its quotes and expansions are parsed as in any other word.
352
362
  regex_test: $ => prec(PREC.COMPARE, seq(
353
363
  field('left', $._word),
354
364
  field('operator', '=~'),
355
- field('right', $.regex),
365
+ $._regex_start,
366
+ field('right', choice($._regex_part, alias($._regex_concatenation, $.concatenation))),
356
367
  )),
357
368
 
369
+ _regex_concatenation: $ => prec.right(seq($._regex_part, repeat1(seq($._regex_concat, $._regex_part)))),
370
+
371
+ _regex_part: $ => choice($.regex, $._quoted_or_expansion),
372
+
358
373
  _test_negation: $ => prec(PREC.UNARY, seq('!', $._test_expression)),
359
374
 
360
375
  _test_logical: $ => choice(
@@ -550,9 +565,23 @@ module.exports = grammar({
550
565
  '"',
551
566
  ),
552
567
 
553
- raw_string: _ => /'[^']*'/,
568
+ // Where a heredoc inside `$( )` ends on a line that holds the `)`, bash reads the rest of that line
569
+ // with its line continuations removed, even inside single quotes, and reads the bodies still queued
570
+ // before that line right after it, even where it ends inside quotes; the scanner then splits the
571
+ // quoted text around the continuations and the bodies.
572
+ raw_string: $ => choice(
573
+ /'[^']*'/,
574
+ seq($._closer_line_raw_string_start, repeat(alias($._closer_line_raw_string_content, $.string_content)), '\''),
575
+ ),
554
576
 
555
- ansi_c_string: _ => /\$'([^'\\]|\\(.|\n))*'/,
577
+ ansi_c_string: $ => choice(
578
+ /\$'([^'\\]|\\(.|\n))*'/,
579
+ seq(
580
+ $._closer_line_ansi_c_string_start,
581
+ repeat(alias($._closer_line_ansi_c_string_content, $.string_content)),
582
+ '\'',
583
+ ),
584
+ ),
556
585
 
557
586
  translated_string: $ => seq('$"', repeat(choice(
558
587
  $.string_content,
@@ -582,10 +611,6 @@ module.exports = grammar({
582
611
  alias(token.immediate(/[0-9]+|[*@?$_-]/), $.special_variable_name),
583
612
  alias(token.immediate(choice('#', '!')), $.special_variable_name),
584
613
  alias($._expansion_subscript, $.subscript),
585
- // A blank directly after `${` opens a function substitution, so these only take effect
586
- // after a prefix (`${# y}`), which bash parses and rejects only when expanding.
587
- alias(/[A-Za-z_][A-Za-z0-9_]*/, $.variable_name),
588
- alias(/[0-9]+|[*@?$_-]/, $.special_variable_name),
589
614
  )),
590
615
  optional($._expansion_operation),
591
616
  ),
@@ -594,6 +619,7 @@ module.exports = grammar({
594
619
  // `$name` expansion is left out, since its `$` would tie with the special parameter in `${$}`.
595
620
  repeat(field('argument', choice(
596
621
  alias($._expansion_text, $.word),
622
+ alias($._blank_expansion_text, $.word),
597
623
  alias($._angle_text, $.word),
598
624
  $.string,
599
625
  $.raw_string,
@@ -621,6 +647,7 @@ module.exports = grammar({
621
647
  )),
622
648
  repeat(field('argument', choice(
623
649
  alias($._expansion_text, $.word),
650
+ alias($._blank_expansion_text, $.word),
624
651
  alias($._angle_text, $.word),
625
652
  $._quoted_or_expansion,
626
653
  alias($._bare_dollar, $.word),
@@ -630,12 +657,21 @@ module.exports = grammar({
630
657
  // Text without an operator (`${x y}`) is rejected by bash only when expanding.
631
658
  repeat1(field('argument', choice(
632
659
  alias($._expansion_text, $.word),
660
+ alias($._blank_expansion_text, $.word),
633
661
  alias($._angle_text, $.word),
634
662
  $._quoted_or_expansion,
635
663
  ))),
636
664
  ),
637
665
 
638
- _expansion_text: _ => token(prec(-1, /([^}$`"'\\<>]|[<>]+[^(}$`"'\\<>]|\\(.|\n))+/)),
666
+ _expansion_text: _ => token(prec(-1, seq(
667
+ choice(noneOf(...EXPANSION_TEXT_BREAKS, ' ', '\\t', '\\n'), ...expansionTextUnits().slice(1)),
668
+ repeat(choice(...expansionTextUnits())),
669
+ ))),
670
+
671
+ // Text led by a blank outranks `}`: otherwise the lexer, having consumed the blank as the start of
672
+ // text, extends it into a `}` token that swallows the blank (`${x:- }`), which a reparse after an
673
+ // edit does not reproduce, and the blank, which bash expands, would be lost.
674
+ _blank_expansion_text: _ => token(prec(1, seq(/[ \t\n]/, repeat(choice(...expansionTextUnits()))))),
639
675
 
640
676
  // A `<` or `>` that the text above cannot end with, e.g. before `}`. A single character, so that a
641
677
  // run before `(` leaves its last `<`/`>` to open a process substitution (`${x:-a<<(b)}`).
@@ -746,7 +782,8 @@ module.exports = grammar({
746
782
 
747
783
  _arithmetic_parenthesized: $ => seq('(', $._arithmetic_expression, ')'),
748
784
 
749
- comment: _ => token(prec(-10, /#.*/)),
785
+ // A comment on such a line also continues past each line continuation.
786
+ comment: $ => choice(token(prec(-10, /#.*/)), $._joined_comment),
750
787
  },
751
788
  });
752
789
 
@@ -760,6 +797,11 @@ function noneOf(...characters) {
760
797
  return new RegExp(`[^${characters.join('')}]`);
761
798
  }
762
799
 
800
+ // The units of text inside `${…}`: a character, a run of `<`/`>` not before `(`, and an escape.
801
+ function expansionTextUnits() {
802
+ return [noneOf(...EXPANSION_TEXT_BREAKS), seq(/[<>]+/, noneOf('(', ...EXPANSION_TEXT_BREAKS)), /\\(.|\n)/];
803
+ }
804
+
763
805
  /**
764
806
  * A regular expression matching a literal extglob group opened by `prefix`, whose body may contain
765
807
  * groups nested up to `depth` levels.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@willbooster/tree-sitter-bash",
3
- "version": "1.1.2",
3
+ "version": "1.1.4",
4
4
  "description": "Bash grammar for tree-sitter",
5
5
  "keywords": [
6
6
  "incremental",
package/src/grammar.json CHANGED
@@ -1602,17 +1602,77 @@
1602
1602
  "value": "=~"
1603
1603
  }
1604
1604
  },
1605
+ {
1606
+ "type": "SYMBOL",
1607
+ "name": "_regex_start"
1608
+ },
1605
1609
  {
1606
1610
  "type": "FIELD",
1607
1611
  "name": "right",
1608
1612
  "content": {
1609
- "type": "SYMBOL",
1610
- "name": "regex"
1613
+ "type": "CHOICE",
1614
+ "members": [
1615
+ {
1616
+ "type": "SYMBOL",
1617
+ "name": "_regex_part"
1618
+ },
1619
+ {
1620
+ "type": "ALIAS",
1621
+ "content": {
1622
+ "type": "SYMBOL",
1623
+ "name": "_regex_concatenation"
1624
+ },
1625
+ "named": true,
1626
+ "value": "concatenation"
1627
+ }
1628
+ ]
1611
1629
  }
1612
1630
  }
1613
1631
  ]
1614
1632
  }
1615
1633
  },
1634
+ "_regex_concatenation": {
1635
+ "type": "PREC_RIGHT",
1636
+ "value": 0,
1637
+ "content": {
1638
+ "type": "SEQ",
1639
+ "members": [
1640
+ {
1641
+ "type": "SYMBOL",
1642
+ "name": "_regex_part"
1643
+ },
1644
+ {
1645
+ "type": "REPEAT1",
1646
+ "content": {
1647
+ "type": "SEQ",
1648
+ "members": [
1649
+ {
1650
+ "type": "SYMBOL",
1651
+ "name": "_regex_concat"
1652
+ },
1653
+ {
1654
+ "type": "SYMBOL",
1655
+ "name": "_regex_part"
1656
+ }
1657
+ ]
1658
+ }
1659
+ }
1660
+ ]
1661
+ }
1662
+ },
1663
+ "_regex_part": {
1664
+ "type": "CHOICE",
1665
+ "members": [
1666
+ {
1667
+ "type": "SYMBOL",
1668
+ "name": "regex"
1669
+ },
1670
+ {
1671
+ "type": "SYMBOL",
1672
+ "name": "_quoted_or_expansion"
1673
+ }
1674
+ ]
1675
+ },
1616
1676
  "_test_negation": {
1617
1677
  "type": "PREC",
1618
1678
  "value": 14,
@@ -2881,12 +2941,72 @@
2881
2941
  ]
2882
2942
  },
2883
2943
  "raw_string": {
2884
- "type": "PATTERN",
2885
- "value": "'[^']*'"
2944
+ "type": "CHOICE",
2945
+ "members": [
2946
+ {
2947
+ "type": "PATTERN",
2948
+ "value": "'[^']*'"
2949
+ },
2950
+ {
2951
+ "type": "SEQ",
2952
+ "members": [
2953
+ {
2954
+ "type": "SYMBOL",
2955
+ "name": "_closer_line_raw_string_start"
2956
+ },
2957
+ {
2958
+ "type": "REPEAT",
2959
+ "content": {
2960
+ "type": "ALIAS",
2961
+ "content": {
2962
+ "type": "SYMBOL",
2963
+ "name": "_closer_line_raw_string_content"
2964
+ },
2965
+ "named": true,
2966
+ "value": "string_content"
2967
+ }
2968
+ },
2969
+ {
2970
+ "type": "STRING",
2971
+ "value": "'"
2972
+ }
2973
+ ]
2974
+ }
2975
+ ]
2886
2976
  },
2887
2977
  "ansi_c_string": {
2888
- "type": "PATTERN",
2889
- "value": "\\$'([^'\\\\]|\\\\(.|\\n))*'"
2978
+ "type": "CHOICE",
2979
+ "members": [
2980
+ {
2981
+ "type": "PATTERN",
2982
+ "value": "\\$'([^'\\\\]|\\\\(.|\\n))*'"
2983
+ },
2984
+ {
2985
+ "type": "SEQ",
2986
+ "members": [
2987
+ {
2988
+ "type": "SYMBOL",
2989
+ "name": "_closer_line_ansi_c_string_start"
2990
+ },
2991
+ {
2992
+ "type": "REPEAT",
2993
+ "content": {
2994
+ "type": "ALIAS",
2995
+ "content": {
2996
+ "type": "SYMBOL",
2997
+ "name": "_closer_line_ansi_c_string_content"
2998
+ },
2999
+ "named": true,
3000
+ "value": "string_content"
3001
+ }
3002
+ },
3003
+ {
3004
+ "type": "STRING",
3005
+ "value": "'"
3006
+ }
3007
+ ]
3008
+ }
3009
+ ]
2890
3010
  },
2891
3011
  "translated_string": {
2892
3012
  "type": "SEQ",
@@ -3072,24 +3192,6 @@
3072
3192
  },
3073
3193
  "named": true,
3074
3194
  "value": "subscript"
3075
- },
3076
- {
3077
- "type": "ALIAS",
3078
- "content": {
3079
- "type": "PATTERN",
3080
- "value": "[A-Za-z_][A-Za-z0-9_]*"
3081
- },
3082
- "named": true,
3083
- "value": "variable_name"
3084
- },
3085
- {
3086
- "type": "ALIAS",
3087
- "content": {
3088
- "type": "PATTERN",
3089
- "value": "[0-9]+|[*@?$_-]"
3090
- },
3091
- "named": true,
3092
- "value": "special_variable_name"
3093
3195
  }
3094
3196
  ]
3095
3197
  }
@@ -3150,6 +3252,15 @@
3150
3252
  "named": true,
3151
3253
  "value": "word"
3152
3254
  },
3255
+ {
3256
+ "type": "ALIAS",
3257
+ "content": {
3258
+ "type": "SYMBOL",
3259
+ "name": "_blank_expansion_text"
3260
+ },
3261
+ "named": true,
3262
+ "value": "word"
3263
+ },
3153
3264
  {
3154
3265
  "type": "ALIAS",
3155
3266
  "content": {
@@ -3375,6 +3486,15 @@
3375
3486
  "named": true,
3376
3487
  "value": "word"
3377
3488
  },
3489
+ {
3490
+ "type": "ALIAS",
3491
+ "content": {
3492
+ "type": "SYMBOL",
3493
+ "name": "_blank_expansion_text"
3494
+ },
3495
+ "named": true,
3496
+ "value": "word"
3497
+ },
3378
3498
  {
3379
3499
  "type": "ALIAS",
3380
3500
  "content": {
@@ -3445,6 +3565,15 @@
3445
3565
  "named": true,
3446
3566
  "value": "word"
3447
3567
  },
3568
+ {
3569
+ "type": "ALIAS",
3570
+ "content": {
3571
+ "type": "SYMBOL",
3572
+ "name": "_blank_expansion_text"
3573
+ },
3574
+ "named": true,
3575
+ "value": "word"
3576
+ },
3448
3577
  {
3449
3578
  "type": "ALIAS",
3450
3579
  "content": {
@@ -3470,8 +3599,109 @@
3470
3599
  "type": "PREC",
3471
3600
  "value": -1,
3472
3601
  "content": {
3473
- "type": "PATTERN",
3474
- "value": "([^}$`\"'\\\\<>]|[<>]+[^(}$`\"'\\\\<>]|\\\\(.|\\n))+"
3602
+ "type": "SEQ",
3603
+ "members": [
3604
+ {
3605
+ "type": "CHOICE",
3606
+ "members": [
3607
+ {
3608
+ "type": "PATTERN",
3609
+ "value": "[^}$`\"'\\\\<> \\t\\n]"
3610
+ },
3611
+ {
3612
+ "type": "SEQ",
3613
+ "members": [
3614
+ {
3615
+ "type": "PATTERN",
3616
+ "value": "[<>]+"
3617
+ },
3618
+ {
3619
+ "type": "PATTERN",
3620
+ "value": "[^(}$`\"'\\\\<>]"
3621
+ }
3622
+ ]
3623
+ },
3624
+ {
3625
+ "type": "PATTERN",
3626
+ "value": "\\\\(.|\\n)"
3627
+ }
3628
+ ]
3629
+ },
3630
+ {
3631
+ "type": "REPEAT",
3632
+ "content": {
3633
+ "type": "CHOICE",
3634
+ "members": [
3635
+ {
3636
+ "type": "PATTERN",
3637
+ "value": "[^}$`\"'\\\\<>]"
3638
+ },
3639
+ {
3640
+ "type": "SEQ",
3641
+ "members": [
3642
+ {
3643
+ "type": "PATTERN",
3644
+ "value": "[<>]+"
3645
+ },
3646
+ {
3647
+ "type": "PATTERN",
3648
+ "value": "[^(}$`\"'\\\\<>]"
3649
+ }
3650
+ ]
3651
+ },
3652
+ {
3653
+ "type": "PATTERN",
3654
+ "value": "\\\\(.|\\n)"
3655
+ }
3656
+ ]
3657
+ }
3658
+ }
3659
+ ]
3660
+ }
3661
+ }
3662
+ },
3663
+ "_blank_expansion_text": {
3664
+ "type": "TOKEN",
3665
+ "content": {
3666
+ "type": "PREC",
3667
+ "value": 1,
3668
+ "content": {
3669
+ "type": "SEQ",
3670
+ "members": [
3671
+ {
3672
+ "type": "PATTERN",
3673
+ "value": "[ \\t\\n]"
3674
+ },
3675
+ {
3676
+ "type": "REPEAT",
3677
+ "content": {
3678
+ "type": "CHOICE",
3679
+ "members": [
3680
+ {
3681
+ "type": "PATTERN",
3682
+ "value": "[^}$`\"'\\\\<>]"
3683
+ },
3684
+ {
3685
+ "type": "SEQ",
3686
+ "members": [
3687
+ {
3688
+ "type": "PATTERN",
3689
+ "value": "[<>]+"
3690
+ },
3691
+ {
3692
+ "type": "PATTERN",
3693
+ "value": "[^(}$`\"'\\\\<>]"
3694
+ }
3695
+ ]
3696
+ },
3697
+ {
3698
+ "type": "PATTERN",
3699
+ "value": "\\\\(.|\\n)"
3700
+ }
3701
+ ]
3702
+ }
3703
+ }
3704
+ ]
3475
3705
  }
3476
3706
  }
3477
3707
  },
@@ -4579,15 +4809,24 @@
4579
4809
  ]
4580
4810
  },
4581
4811
  "comment": {
4582
- "type": "TOKEN",
4583
- "content": {
4584
- "type": "PREC",
4585
- "value": -10,
4586
- "content": {
4587
- "type": "PATTERN",
4588
- "value": "#.*"
4812
+ "type": "CHOICE",
4813
+ "members": [
4814
+ {
4815
+ "type": "TOKEN",
4816
+ "content": {
4817
+ "type": "PREC",
4818
+ "value": -10,
4819
+ "content": {
4820
+ "type": "PATTERN",
4821
+ "value": "#.*"
4822
+ }
4823
+ }
4824
+ },
4825
+ {
4826
+ "type": "SYMBOL",
4827
+ "name": "_joined_comment"
4589
4828
  }
4590
- }
4829
+ ]
4591
4830
  }
4592
4831
  },
4593
4832
  "extras": [
@@ -4716,6 +4955,34 @@
4716
4955
  "type": "SYMBOL",
4717
4956
  "name": "_bracket_substitution_start"
4718
4957
  },
4958
+ {
4959
+ "type": "SYMBOL",
4960
+ "name": "_regex_start"
4961
+ },
4962
+ {
4963
+ "type": "SYMBOL",
4964
+ "name": "_regex_concat"
4965
+ },
4966
+ {
4967
+ "type": "SYMBOL",
4968
+ "name": "_closer_line_raw_string_start"
4969
+ },
4970
+ {
4971
+ "type": "SYMBOL",
4972
+ "name": "_closer_line_ansi_c_string_start"
4973
+ },
4974
+ {
4975
+ "type": "SYMBOL",
4976
+ "name": "_closer_line_raw_string_content"
4977
+ },
4978
+ {
4979
+ "type": "SYMBOL",
4980
+ "name": "_closer_line_ansi_c_string_content"
4981
+ },
4982
+ {
4983
+ "type": "SYMBOL",
4984
+ "name": "_joined_comment"
4985
+ },
4719
4986
  {
4720
4987
  "type": "SYMBOL",
4721
4988
  "name": "__error_recovery"