@willbooster/tree-sitter-rust 1.10.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -18,7 +18,7 @@ This fork fixes parsing bugs and raises conformance with [The Rust Reference](ht
18
18
 
19
19
  The npm package ships `tree-sitter-rust.wasm` for
20
20
  [@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter), which runs in Node.js, Bun,
21
- browsers, and Cloudflare Workers.
21
+ browsers, and Cloudflare Workers. The compact ABI 16 parser requires runtime 1.3.0 or later.
22
22
 
23
23
  In Node.js and Bun, load the grammar from its path:
24
24
 
@@ -61,13 +61,12 @@ The package also ships the queries in `queries/` and the node types in `src/node
61
61
 
62
62
  In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-rust) and on
63
63
  [willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter), the runtime this package is tested and
64
- fuzzed with (the grammar also loads in the upstream `tree-sitter` crate 0.27, whose error recovery never ends on some
65
- malformed input):
64
+ fuzzed with. The compact ABI 16 parser requires runtime 1.3.0 or later:
66
65
 
67
66
  ```toml
68
67
  [dependencies]
69
- tree-sitter = { package = "willbooster-tree-sitter", version = "1" }
70
- tree-sitter-rust = { package = "willbooster-tree-sitter-rust", version = "1" }
68
+ tree-sitter = { package = "willbooster-tree-sitter", version = "1.3.0" }
69
+ tree-sitter-rust = { package = "willbooster-tree-sitter-rust", version = "2" }
71
70
  ```
72
71
 
73
72
  ```rust
@@ -103,6 +102,13 @@ not the ones this package ships with. Its first run downloads that CLI from the
103
102
  with `cargo` (whose build runs the CMake that `mise.toml` pins) when the download fails or the release has no binary that runs here. Run other CLI
104
103
  commands through it as well (e.g. `script/tree-sitter parse file.rs`).
105
104
 
105
+ `bun run generate` records a fresh ABI 16 generation profile from the applicable `test/corpus` cases and Git-tracked
106
+ files in `examples/`, then generates compact parser tables. The parser also embeds metadata from `tree-sitter.json`. After changing a grammar,
107
+ `tree-sitter.json`, a corpus case, or a tracked example,
108
+ regenerate and commit `src/`. Stage added or removed examples with `git add -A examples` before generation so the profile uses the intended file list.
109
+ Profiles in `.tmp/generation-profiles/` are temporary and must not be committed. `bun run build-wasm`, `bun run build/ci`,
110
+ and the release build regenerate the parsers before compiling them.
111
+
106
112
  `bun run test` runs:
107
113
 
108
114
  - the corpus in `test/corpus`, with the native build and with the Wasm build (the first run downloads the WASI SDK);
package/grammar.js CHANGED
@@ -34,6 +34,11 @@ const numericSuffixWithoutExponent = new RegExp(String.raw`[\p{XID_Start}--[eE]]
34
34
  const zeroNumericSuffix = new RegExp(String.raw`[\p{XID_Start}--[eEbox]][_\p{XID_Continue}]*`, 'v');
35
35
  const hexadecimalSuffix = new RegExp(String.raw`[\p{XID_Start}--[a-fA-F]][_\p{XID_Continue}]*`, 'v');
36
36
 
37
+ const nonAsciiOrRawMacroName = new RegExp(
38
+ String.raw`(?:[\p{XID_Start}--[\x00-\x7f]][_\p{XID_Continue}]*|[_a-zA-Z][_\p{XID_Continue}]*[\p{XID_Continue}--[\x00-\x7f]][_\p{XID_Continue}]*|r#[_\p{XID_Start}][_\p{XID_Continue}]*)`,
39
+ 'v'
40
+ );
41
+
37
42
  const extendedFloatTypes = ['f16', 'f128'];
38
43
 
39
44
  const numericTypes = [
@@ -117,7 +122,7 @@ module.exports = grammar({
117
122
  $.string_content,
118
123
  $.string_close,
119
124
  $._raw_string_literal_start,
120
- $.raw_string_literal_content,
125
+ $._raw_string_literal_content,
121
126
  $._raw_string_literal_end,
122
127
  $._trailing_dot_float_literal,
123
128
  $._outer_block_doc_comment_marker,
@@ -128,6 +133,7 @@ module.exports = grammar({
128
133
  $._error_sentinel,
129
134
  $._foreign_declaration_context,
130
135
  $._foreign_declaration_start,
136
+ $._macro_metavariable_name,
131
137
  ],
132
138
 
133
139
  supertypes: ($) => [$._declaration_statement, $._expression, $._type, $._literal, $._literal_pattern, $._pattern],
@@ -143,8 +149,9 @@ module.exports = grammar({
143
149
  ],
144
150
 
145
151
  conflicts: ($) => [
152
+ [$._token_pattern, $.token_repetition_pattern],
153
+ [$.token_tree, $.token_repetition],
146
154
  [$.field_initializer_list, $.block],
147
- [$._declaration_statement, $.shorthand_field_initializer, $.field_initializer],
148
155
  [$._expression_except_range, $.shorthand_field_initializer],
149
156
  [$._declaration_statement, $.attributed_expression],
150
157
  [$._expression_except_range, $.struct_expression],
@@ -210,7 +217,7 @@ module.exports = grammar({
210
217
  const rules = seq(repeat(seq($.macro_rule, ';')), optional($.macro_rule));
211
218
 
212
219
  return seq(
213
- 'macro_rules!',
220
+ choice('macro_rules!', seq(alias('macro_rules', 'macro_rules!'), '!')),
214
221
  field('name', choice($.identifier, $._reserved_identifier)),
215
222
  choice(seq('(', rules, ')', ';'), seq('[', rules, ']', ';'), seq('{', rules, '}'))
216
223
  );
@@ -232,7 +239,8 @@ module.exports = grammar({
232
239
  $.token_tree_pattern,
233
240
  $.token_repetition_pattern,
234
241
  $.token_binding_pattern,
235
- $.metavariable,
242
+ $._macro_metavariable,
243
+ '$',
236
244
  $._non_special_token
237
245
  ),
238
246
 
@@ -243,10 +251,27 @@ module.exports = grammar({
243
251
  seq('{', repeat($._token_pattern), '}')
244
252
  ),
245
253
 
246
- token_binding_pattern: ($) => prec(1, seq(field('name', $.metavariable), ':', field('type', $.fragment_specifier))),
254
+ token_binding_pattern: ($) =>
255
+ prec(1, seq(field('name', $._macro_metavariable), ':', field('type', $.fragment_specifier))),
247
256
 
248
257
  token_repetition_pattern: ($) =>
249
- seq('$', '(', repeat($._token_pattern), ')', optional(/[^+*?]+/), choice('+', '*', '?')),
258
+ prec.dynamic(
259
+ 2,
260
+ seq(
261
+ '$',
262
+ '(',
263
+ repeat($._token_pattern),
264
+ ')',
265
+ optional(
266
+ macroRepetitionSeparator(
267
+ $._raw_string_literal_start,
268
+ $._raw_string_literal_content,
269
+ $._raw_string_literal_end
270
+ )
271
+ ),
272
+ choice('+', '*', '?')
273
+ )
274
+ ),
250
275
 
251
276
  fragment_specifier: () =>
252
277
  choice(
@@ -267,12 +292,29 @@ module.exports = grammar({
267
292
  'vis'
268
293
  ),
269
294
 
270
- _tokens: ($) => choice($.token_tree, $.token_repetition, $.metavariable, $._non_special_token),
295
+ _tokens: ($) => choice($.token_tree, $.token_repetition, $._macro_metavariable, '$', $._non_special_token),
271
296
 
272
297
  token_tree: ($) =>
273
298
  choice(seq('(', repeat($._tokens), ')'), seq('[', repeat($._tokens), ']'), seq('{', repeat($._tokens), '}')),
274
299
 
275
- token_repetition: ($) => seq('$', '(', repeat($._tokens), ')', optional(/[^+*?]+/), choice('+', '*', '?')),
300
+ token_repetition: ($) =>
301
+ prec.dynamic(
302
+ 2,
303
+ seq(
304
+ '$',
305
+ '(',
306
+ repeat($._tokens),
307
+ ')',
308
+ optional(
309
+ macroRepetitionSeparator(
310
+ $._raw_string_literal_start,
311
+ $._raw_string_literal_content,
312
+ $._raw_string_literal_end
313
+ )
314
+ ),
315
+ choice('+', '*', '?')
316
+ )
317
+ ),
276
318
 
277
319
  _non_special_token: ($) =>
278
320
  choice(
@@ -1451,7 +1493,7 @@ module.exports = grammar({
1451
1493
  raw_string_literal: ($) =>
1452
1494
  seq(
1453
1495
  $._raw_string_literal_start,
1454
- alias($.raw_string_literal_content, $.string_content),
1496
+ alias($._raw_string_literal_content, $.string_content),
1455
1497
  $._raw_string_literal_end
1456
1498
  ),
1457
1499
 
@@ -1528,19 +1570,36 @@ module.exports = grammar({
1528
1570
  // oxlint-disable-next-line no-useless-escape -- tree-sitter's regex parser rejects an unescaped `[` in a character class.
1529
1571
  shebang: () => /#![\r\f\t\v ]*([^\[\n].*)?\n/,
1530
1572
 
1531
- _reserved_identifier: ($) => alias(choice(...reservedIdentifiers, 'auto'), $.identifier),
1573
+ _reserved_identifier: ($) => alias(choice(...reservedIdentifiers, 'auto', 'macro_rules'), $.identifier),
1532
1574
 
1533
- _type_identifier: ($) => alias(choice($.identifier, 'auto'), $.type_identifier),
1575
+ _type_identifier: ($) => alias(choice($.identifier, 'auto', 'macro_rules'), $.type_identifier),
1534
1576
  _field_identifier: ($) => alias($.identifier, $.field_identifier),
1535
1577
 
1536
1578
  self: () => 'self',
1537
1579
  super: () => 'super',
1538
1580
  crate: () => 'crate',
1539
1581
 
1540
- metavariable: () => /\$[a-zA-Z_]\w*/,
1582
+ _macro_metavariable: ($) => choice($.metavariable, alias($._spaced_metavariable, $.metavariable)),
1583
+ _spaced_metavariable: ($) =>
1584
+ prec.dynamic(1, seq('$', choice($._macro_metavariable_name, token(prec(1, nonAsciiOrRawMacroName))))),
1585
+ metavariable: () => /\$(r#)?[_\p{XID_Start}][_\p{XID_Continue}]*/u,
1541
1586
  },
1542
1587
  });
1543
1588
 
1589
+ /**
1590
+ * @param {RuleOrLiteral} start
1591
+ * @param {RuleOrLiteral} content
1592
+ * @param {RuleOrLiteral} end
1593
+ */
1594
+ function macroRepetitionSeparator(start, content, end) {
1595
+ return choice(
1596
+ /[^\s/+*?$"][^/+*?$"]*|\/(?:[^/*+*?$"][^/+*?$"]*)?|\$/,
1597
+ /[bc]?"(?:[^"\\]|\\[\s\S])*"/,
1598
+ /b?'(?:\\(?:[^xu]|u[0-9a-fA-F]{4}|u\{[0-9a-fA-F]+\}|x[0-9a-fA-F]{2})|[^\\'])'/,
1599
+ seq(start, content, end)
1600
+ );
1601
+ }
1602
+
1544
1603
  /**
1545
1604
  * @param {RuleOrLiteral} sep
1546
1605
  * @param {RuleOrLiteral} rule
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@willbooster/tree-sitter-rust",
3
- "version": "1.10.2",
3
+ "version": "2.0.0",
4
4
  "description": "Rust grammar for tree-sitter",
5
5
  "keywords": [
6
6
  "incremental",
@@ -22,10 +22,11 @@
22
22
  "*.wasm"
23
23
  ],
24
24
  "scripts": {
25
- "build-wasm": "script/tree-sitter build --wasm",
26
- "build/ci": "script/tree-sitter generate && script/tree-sitter build --wasm",
25
+ "build-wasm": "script/generate --build-wasm",
26
+ "build/ci": "bun run build-wasm",
27
27
  "cleanup": "bun wb lint --fix --format",
28
28
  "format": "bun wb lint --format",
29
+ "generate": "script/generate",
29
30
  "lint": "bun wb lint",
30
31
  "lint-fix": "bun wb lint --fix",
31
32
  "prepare": "(bun lefthook install || true) && (test ! -f .lefthook/normalize-bun-lockfile.sh || bash .lefthook/normalize-bun-lockfile.sh)",
@@ -46,7 +47,7 @@
46
47
  "@willbooster/oxfmt-config": "1.2.2",
47
48
  "@willbooster/oxlint-config": "1.4.8",
48
49
  "@willbooster/wb": "22.13.2",
49
- "@willbooster/web-tree-sitter": "1.0.17",
50
+ "@willbooster/web-tree-sitter": "1.4.0",
50
51
  "conventional-changelog-conventionalcommits": "10.4.0",
51
52
  "lefthook": "2.1.14",
52
53
  "oxfmt": "0.70.0",
@@ -62,7 +63,7 @@
62
63
  "wrangler": "4.136.2"
63
64
  },
64
65
  "peerDependencies": {
65
- "@willbooster/web-tree-sitter": "^1.0.0"
66
+ "@willbooster/web-tree-sitter": "^1.3.0"
66
67
  },
67
68
  "peerDependenciesMeta": {
68
69
  "@willbooster/web-tree-sitter": {
@@ -3096,6 +3096,11 @@
3096
3096
  ]
3097
3097
  }
3098
3098
  },
3099
+ {
3100
+ "type": "metavariable",
3101
+ "named": true,
3102
+ "fields": {}
3103
+ },
3099
3104
  {
3100
3105
  "type": "mod_item",
3101
3106
  "named": true,
@@ -5631,10 +5636,6 @@
5631
5636
  "type": "meta",
5632
5637
  "named": false
5633
5638
  },
5634
- {
5635
- "type": "metavariable",
5636
- "named": true
5637
- },
5638
5639
  {
5639
5640
  "type": "mod",
5640
5641
  "named": false
Binary file
package/tree-sitter.json CHANGED
@@ -22,7 +22,7 @@
22
22
  }
23
23
  ],
24
24
  "metadata": {
25
- "version": "1.10.2",
25
+ "version": "2.0.0",
26
26
  "license": "MIT",
27
27
  "description": "Rust grammar for tree-sitter",
28
28
  "authors": [