@willbooster/tree-sitter-c 1.5.1 → 2.0.0

package/README.md

@@ -59,6 +59,10 @@ parser.setLanguage(await Language.load(c));
 
 The package also ships `grammar.js`, the scanner sources `src/scanner.c`, `src/pragma.h` and `src/identifier.h`, the queries in `queries/`,
 and the node types in `src/node-types.json` for grammars extending C (such as C++).
+Derived grammars must handle every inherited external token in their scanner, including `_preproc_function_name`.
+Port both the `_preproc_function_name` dispatch and the `_preproc_lparen` splice-skipping loop from `src/scanner.c`,
+using `scan_function_macro_name` from `src/pragma.h`. The macro-name token ends before the splices used to determine
+adjacency, so the parameter scanner must skip them again to reach `(`.
 
 In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-c) and on
 [willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter), the runtime this package is tested and
@@ -68,7 +72,7 @@ malformed input):
 ```toml
 [dependencies]
 tree-sitter = { package = "willbooster-tree-sitter", version = "1" }
-tree-sitter-c = { package = "willbooster-tree-sitter-c", version = "1" }
+tree-sitter-c = { package = "willbooster-tree-sitter-c", version = "2" }
 ```
 
 ```rust

package/grammar.js

@@ -36,7 +36,7 @@ const PREC = {
 };
 
 const LINE_COMMENT = seq('//', /(\\+(.|\r?\n)|[^\\\n])*/);
-const PRAGMA_SPACING = repeat(choice(/\s/, /\\\r?\n/));
+const PRAGMA_SPACING = repeat(choice(/\s/, /\\(?:\r\n?|\n\r?)/));
 const PREPROC_ARGUMENT = /\S([^/\n]|\/[^*]|\\\r?\n)*/;
 const VA_ARG_KEYWORDS = choice('va_arg', '__builtin_va_arg');
 
@@ -71,6 +71,7 @@ module.exports = Object.assign(
       [$.type_definition, $.sized_type_specifier],
       [$.type_definition, $._sized_bit_int_specifier],
       [$.attributed_statement],
+      [$._single_attributed_statement],
       [$._declaration_modifiers, $.attributed_statement],
       [$.enum_specifier],
       [$.type_specifier, $._old_style_parameter_list],
@@ -91,9 +92,10 @@ module.exports = Object.assign(
       sym('_preproc_newline'),
       sym('_preproc_lparen'),
       sym('_preproc_directive_arg'),
+      sym('_preproc_function_name'),
     ],
 
-    extras: ($) => [$.pragma_operator, /\s|\\\r?\n/, $.comment],
+    extras: ($) => [$.pragma_operator, /\s|\\(?:\r\n?|\n\r?)/, $.comment],
 
     inline: ($) => [
       $._non_identifier_type_specifier,
@@ -168,7 +170,7 @@ module.exports = Object.assign(
             PRAGMA_SPACING,
             optional(choice('L', 'u8', 'u', 'U')),
             '"',
-            repeat(choice(/[^\\"\n]/, seq('\\', choice(/./, /\r?\n/)))),
+            repeat(choice(/[^\\"\r\n]/, seq('\\', choice(/[^\r\n]/, /\r\n?|\n\r?/)))),
             '"',
             PRAGMA_SPACING,
             ')'
@@ -202,7 +204,7 @@ module.exports = Object.assign(
       preproc_function_def: ($) =>
         seq(
           preprocessor('define'),
-          field('name', $.identifier),
+          field('name', alias(sym('_preproc_function_name'), $.identifier)),
           field('parameters', $.preproc_params),
           field('value', optional($.preproc_arg)),
           sym('_preproc_newline')
@@ -218,6 +220,7 @@ module.exports = Object.assign(
         ),
 
       ...preprocIf('', () => sym('_block_item')),
+      ...preprocIf('_in_single_case', () => sym('_single_case_body'), 0, true),
       ...preprocIf('_in_field_declaration_list', () => sym('_field_declaration_list_item')),
       ...preprocIf('_in_enumerator_list', () => seq(sym('enumerator'), ',')),
       ...preprocIf('_in_enumerator_list_no_comma', () => sym('enumerator'), -1),
@@ -901,7 +904,65 @@ module.exports = Object.assign(
       else_clause: ($) => seq('else', $.statement),
 
       switch_statement: ($) =>
-        seq('switch', field('condition', $.parenthesized_expression), field('body', $.compound_statement)),
+        seq('switch', field('condition', $.parenthesized_expression), field('body', $._single_statement)),
+
+      _single_statement: ($) =>
+        choice(
+          $.compound_statement,
+          $.expression_statement,
+          $.switch_statement,
+          $.return_statement,
+          $.break_statement,
+          $.continue_statement,
+          $.goto_statement,
+          $.seh_try_statement,
+          $.seh_leave_statement,
+          alias($._single_case_statement, $.case_statement),
+          alias($._single_labeled_statement, $.labeled_statement),
+          alias($._single_if_statement, $.if_statement),
+          alias($._single_while_statement, $.while_statement),
+          alias($._single_do_statement, $.do_statement),
+          alias($._single_for_statement, $.for_statement),
+          alias($._single_attributed_statement, $.attributed_statement)
+        ),
+
+      _single_case_statement: ($) =>
+        prec.right(
+          seq(
+            choice(
+              seq('case', field('value', $.expression), optional(seq('...', field('end_value', $.expression)))),
+              'default'
+            ),
+            ':',
+            optional($._single_case_body)
+          )
+        ),
+      _single_case_body: ($) =>
+        choice(
+          $._single_statement,
+          $.declaration,
+          $.type_definition,
+          alias(sym('preproc_if_in_single_case'), sym('preproc_if')),
+          alias(sym('preproc_ifdef_in_single_case'), sym('preproc_ifdef'))
+        ),
+      _single_labeled_statement: ($) =>
+        seq(field('label', $._statement_identifier), ':', choice($._single_statement, $.declaration)),
+      _single_if_statement: ($) =>
+        prec.right(
+          seq(
+            'if',
+            field('condition', $.parenthesized_expression),
+            field('consequence', $._single_statement),
+            optional(field('alternative', alias($._single_else_clause, $.else_clause)))
+          )
+        ),
+      _single_else_clause: ($) => seq('else', $._single_statement),
+      _single_while_statement: ($) =>
+        seq('while', field('condition', $.parenthesized_expression), field('body', $._single_statement)),
+      _single_do_statement: ($) =>
+        seq('do', field('body', $._single_statement), 'while', field('condition', $.parenthesized_expression), ';'),
+      _single_for_statement: ($) => seq('for', '(', $._for_statement_body, ')', field('body', $._single_statement)),
+      _single_attributed_statement: ($) => seq(repeat1($.attribute_declaration), $._single_statement),
 
       case_statement: ($) =>
         prec.right(
@@ -1370,7 +1431,6 @@ module.exports = Object.assign(
 
       identifier: () =>
         /(\p{XID_Start}|\$|_|\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8})(\p{XID_Continue}|\$|\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8})*/u,
-
       _type_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('type_identifier')),
       _field_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('field_identifier')),
       _statement_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('statement_identifier')),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@willbooster/tree-sitter-c",
-  "version": "1.5.1",
+  "version": "2.0.0",
   "description": "C grammar for tree-sitter",
   "keywords": [
     "incremental",

package/src/node-types.json

@@ -1031,6 +1031,10 @@
           "type": "break_statement",
           "named": true
         },
+        {
+          "type": "case_statement",
+          "named": true
+        },
         {
           "type": "compound_statement",
           "named": true
@@ -1067,6 +1071,14 @@
           "type": "labeled_statement",
           "named": true
         },
+        {
+          "type": "preproc_if",
+          "named": true
+        },
+        {
+          "type": "preproc_ifdef",
+          "named": true
+        },
         {
           "type": "return_statement",
           "named": true
@@ -3880,9 +3892,69 @@
         "multiple": false,
         "required": true,
         "types": [
+          {
+            "type": "attributed_statement",
+            "named": true
+          },
+          {
+            "type": "break_statement",
+            "named": true
+          },
+          {
+            "type": "case_statement",
+            "named": true
+          },
           {
             "type": "compound_statement",
             "named": true
+          },
+          {
+            "type": "continue_statement",
+            "named": true
+          },
+          {
+            "type": "do_statement",
+            "named": true
+          },
+          {
+            "type": "expression_statement",
+            "named": true
+          },
+          {
+            "type": "for_statement",
+            "named": true
+          },
+          {
+            "type": "goto_statement",
+            "named": true
+          },
+          {
+            "type": "if_statement",
+            "named": true
+          },
+          {
+            "type": "labeled_statement",
+            "named": true
+          },
+          {
+            "type": "return_statement",
+            "named": true
+          },
+          {
+            "type": "seh_leave_statement",
+            "named": true
+          },
+          {
+            "type": "seh_try_statement",
+            "named": true
+          },
+          {
+            "type": "switch_statement",
+            "named": true
+          },
+          {
+            "type": "while_statement",
+            "named": true
           }
         ]
       },

package/src/pragma.h

@@ -4,7 +4,10 @@
 #include "tree_sitter/parser.h"
 #include "identifier.h"
 
+static const char pragma_word[] = "_Pragma";
+
 static bool scan_pragma_spacing(TSLexer *lexer);
+static bool scan_pragma_suffix(TSLexer *lexer);
 static bool pragma_space(int32_t c);
 static bool scan_pragma_word(TSLexer *lexer);
 static bool scan_preproc_newline(TSLexer *lexer, bool skip);
@@ -18,6 +21,10 @@ static bool scan_pragma(TSLexer *lexer) {
         lexer->advance(lexer, true);
     }
     if (!scan_pragma_word(lexer)) return false;
+    return scan_pragma_suffix(lexer);
+}
+
+static bool scan_pragma_suffix(TSLexer *lexer) {
     if (!scan_pragma_spacing(lexer) || lexer->lookahead != '(') return false;
     lexer->advance(lexer, false);
     if (!scan_pragma_spacing(lexer)) return false;
@@ -33,10 +40,7 @@ static bool scan_pragma(TSLexer *lexer) {
         if (lexer->lookahead == '\\') {
             lexer->advance(lexer, false);
             if (lexer->eof(lexer)) return false;
-            if (lexer->lookahead == '\r') {
-                lexer->advance(lexer, false);
-                if (lexer->lookahead != '\n') return false;
-            }
+            if (scan_preproc_newline(lexer, false)) continue;
         }
         lexer->advance(lexer, false);
     }
@@ -53,9 +57,7 @@ static bool scan_pragma_spacing(TSLexer *lexer) {
             lexer->advance(lexer, false);
         } else if (lexer->lookahead == '\\') {
             lexer->advance(lexer, false);
-            if (lexer->lookahead == '\r') lexer->advance(lexer, false);
-            if (lexer->lookahead != '\n') return false;
-            lexer->advance(lexer, false);
+            if (!scan_preproc_newline(lexer, false)) return false;
         } else if (lexer->lookahead == '/') {
             lexer->advance(lexer, false);
             if (lexer->lookahead == '/') {
@@ -298,7 +300,7 @@ static bool preproc_word(int32_t c, bool continuation) {
 }
 
 static bool scan_pragma_word(TSLexer *lexer) {
-    const char *word = "_Pragma";
+    const char *word = pragma_word;
     for (; *word; word++) {
         if (lexer->lookahead != *word) return false;
         lexer->advance(lexer, false);
@@ -331,4 +333,49 @@ static bool pragma_space(int32_t c) {
            c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000;
 }
 
+static bool scan_function_macro_name(TSLexer *lexer, bool allow_pragma, TSSymbol pragma_symbol) {
+    bool has_name = false;
+    bool is_pragma = true;
+    unsigned pragma_length = 0;
+    for (;;) {
+        while (!has_name && pragma_space(lexer->lookahead)) lexer->advance(lexer, true);
+        int32_t c = lexer->lookahead;
+        if (c == '\\') {
+            lexer->advance(lexer, false);
+            if (scan_preproc_newline(lexer, !has_name)) {
+                if (!has_name) continue;
+                if (!scan_preproc_splices(lexer)) return false;
+                break;
+            }
+            is_pragma = false;
+            unsigned digits = lexer->lookahead == 'u' ? 4 : lexer->lookahead == 'U' ? 8 : 0;
+            if (!digits) return false;
+            lexer->advance(lexer, false);
+            for (unsigned i = 0; i < digits; i++) {
+                c = lexer->lookahead;
+                if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) return false;
+                lexer->advance(lexer, false);
+            }
+        } else {
+            bool identifier = has_name
+                ? set_contains(preproc_identifier_continue, sizeof(preproc_identifier_continue) / sizeof(TSCharacterRange), c)
+                : set_contains(preproc_identifier_start, sizeof(preproc_identifier_start) / sizeof(TSCharacterRange), c);
+            if (!identifier) break;
+            if (is_pragma) {
+                if (pragma_length < sizeof(pragma_word) - 1 && c == pragma_word[pragma_length]) pragma_length++;
+                else is_pragma = false;
+            }
+            lexer->advance(lexer, false);
+        }
+        has_name = true;
+        lexer->mark_end(lexer);
+    }
+    bool adjacent = has_name && lexer->lookahead == '(';
+    if (allow_pragma && is_pragma && pragma_length == sizeof(pragma_word) - 1 && scan_pragma_suffix(lexer)) {
+        lexer->result_symbol = pragma_symbol;
+        return true;
+    }
+    return adjacent;
+}
+
 #endif

package/src/scanner.c

@@ -1,6 +1,6 @@
 #include "pragma.h"
 
-enum TokenType { PRAGMA_OPERATOR, PREPROC_ARG, PREPROC_NEWLINE, PREPROC_LPAREN, PREPROC_DIRECTIVE_ARG };
+enum TokenType { PRAGMA_OPERATOR, PREPROC_ARG, PREPROC_NEWLINE, PREPROC_LPAREN, PREPROC_DIRECTIVE_ARG, PREPROC_FUNCTION_NAME };
 
 void *tree_sitter_c_external_scanner_create(void) {
     return NULL;
@@ -24,13 +24,23 @@ void tree_sitter_c_external_scanner_deserialize(void *payload, const char *buffe
 
 bool tree_sitter_c_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
     (void)payload;
+    if (valid_symbols[PREPROC_FUNCTION_NAME] && !valid_symbols[PREPROC_LPAREN]) {
+        lexer->result_symbol = PREPROC_FUNCTION_NAME;
+        return scan_function_macro_name(lexer, valid_symbols[PRAGMA_OPERATOR], PRAGMA_OPERATOR);
+    }
     bool directive_text = !valid_symbols[PREPROC_ARG] && valid_symbols[PREPROC_DIRECTIVE_ARG];
     TSSymbol argument_symbol = directive_text ? PREPROC_DIRECTIVE_ARG : PREPROC_ARG;
-    if (valid_symbols[PREPROC_LPAREN] && lexer->lookahead == '(') {
-        lexer->advance(lexer, false);
-        lexer->mark_end(lexer);
-        lexer->result_symbol = PREPROC_LPAREN;
-        return true;
+    if (valid_symbols[PREPROC_LPAREN]) {
+        while (lexer->lookahead == '\\') {
+            lexer->advance(lexer, true);
+            if (!scan_preproc_newline(lexer, true)) return false;
+        }
+        if (lexer->lookahead == '(') {
+            lexer->advance(lexer, false);
+            lexer->mark_end(lexer);
+            lexer->result_symbol = PREPROC_LPAREN;
+            return true;
+        }
     }
     if (valid_symbols[PREPROC_NEWLINE]) {
         for (;;) {

package/tree-sitter.json

@@ -16,7 +16,7 @@
     }
   ],
   "metadata": {
-    "version": "1.5.1",
+    "version": "2.0.0",
     "license": "MIT",
     "description": "C grammar for tree-sitter",
     "authors": [