@willbooster/tree-sitter-c 1.5.2 → 2.0.0

package/README.md

@@ -59,6 +59,10 @@ parser.setLanguage(await Language.load(c));
 
 The package also ships `grammar.js`, the scanner sources `src/scanner.c`, `src/pragma.h` and `src/identifier.h`, the queries in `queries/`,
 and the node types in `src/node-types.json` for grammars extending C (such as C++).
+Derived grammars must handle every inherited external token in their scanner, including `_preproc_function_name`.
+Port both the `_preproc_function_name` dispatch and the `_preproc_lparen` splice-skipping loop from `src/scanner.c`,
+using `scan_function_macro_name` from `src/pragma.h`. The macro-name token ends before the splices used to determine
+adjacency, so the parameter scanner must skip them again to reach `(`.
 
 In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-c) and on
 [willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter), the runtime this package is tested and
@@ -68,7 +72,7 @@ malformed input):
 ```toml
 [dependencies]
 tree-sitter = { package = "willbooster-tree-sitter", version = "1" }
-tree-sitter-c = { package = "willbooster-tree-sitter-c", version = "1" }
+tree-sitter-c = { package = "willbooster-tree-sitter-c", version = "2" }
 ```
 
 ```rust

package/grammar.js

@@ -36,7 +36,7 @@ const PREC = {
 };
 
 const LINE_COMMENT = seq('//', /(\\+(.|\r?\n)|[^\\\n])*/);
-const PRAGMA_SPACING = repeat(choice(/\s/, /\\\r?\n/));
+const PRAGMA_SPACING = repeat(choice(/\s/, /\\(?:\r\n?|\n\r?)/));
 const PREPROC_ARGUMENT = /\S([^/\n]|\/[^*]|\\\r?\n)*/;
 const VA_ARG_KEYWORDS = choice('va_arg', '__builtin_va_arg');
 
@@ -92,9 +92,10 @@ module.exports = Object.assign(
       sym('_preproc_newline'),
       sym('_preproc_lparen'),
       sym('_preproc_directive_arg'),
+      sym('_preproc_function_name'),
     ],
 
-    extras: ($) => [$.pragma_operator, /\s|\\\r?\n/, $.comment],
+    extras: ($) => [$.pragma_operator, /\s|\\(?:\r\n?|\n\r?)/, $.comment],
 
     inline: ($) => [
       $._non_identifier_type_specifier,
@@ -169,7 +170,7 @@ module.exports = Object.assign(
             PRAGMA_SPACING,
             optional(choice('L', 'u8', 'u', 'U')),
             '"',
-            repeat(choice(/[^\\"\n]/, seq('\\', choice(/./, /\r?\n/)))),
+            repeat(choice(/[^\\"\r\n]/, seq('\\', choice(/[^\r\n]/, /\r\n?|\n\r?/)))),
             '"',
             PRAGMA_SPACING,
             ')'
@@ -203,7 +204,7 @@ module.exports = Object.assign(
       preproc_function_def: ($) =>
         seq(
           preprocessor('define'),
-          field('name', $.identifier),
+          field('name', alias(sym('_preproc_function_name'), $.identifier)),
           field('parameters', $.preproc_params),
           field('value', optional($.preproc_arg)),
           sym('_preproc_newline')
@@ -1430,7 +1431,6 @@ module.exports = Object.assign(
 
       identifier: () =>
         /(\p{XID_Start}|\$|_|\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8})(\p{XID_Continue}|\$|\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8})*/u,
-
       _type_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('type_identifier')),
       _field_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('field_identifier')),
       _statement_identifier: ($) => alias(choice($.identifier, VA_ARG_KEYWORDS), sym('statement_identifier')),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@willbooster/tree-sitter-c",
-  "version": "1.5.2",
+  "version": "2.0.0",
   "description": "C grammar for tree-sitter",
   "keywords": [
     "incremental",

package/src/pragma.h

@@ -4,7 +4,10 @@
 #include "tree_sitter/parser.h"
 #include "identifier.h"
 
+static const char pragma_word[] = "_Pragma";
+
 static bool scan_pragma_spacing(TSLexer *lexer);
+static bool scan_pragma_suffix(TSLexer *lexer);
 static bool pragma_space(int32_t c);
 static bool scan_pragma_word(TSLexer *lexer);
 static bool scan_preproc_newline(TSLexer *lexer, bool skip);
@@ -18,6 +21,10 @@ static bool scan_pragma(TSLexer *lexer) {
         lexer->advance(lexer, true);
     }
     if (!scan_pragma_word(lexer)) return false;
+    return scan_pragma_suffix(lexer);
+}
+
+static bool scan_pragma_suffix(TSLexer *lexer) {
     if (!scan_pragma_spacing(lexer) || lexer->lookahead != '(') return false;
     lexer->advance(lexer, false);
     if (!scan_pragma_spacing(lexer)) return false;
@@ -33,10 +40,7 @@ static bool scan_pragma(TSLexer *lexer) {
         if (lexer->lookahead == '\\') {
             lexer->advance(lexer, false);
             if (lexer->eof(lexer)) return false;
-            if (lexer->lookahead == '\r') {
-                lexer->advance(lexer, false);
-                if (lexer->lookahead != '\n') return false;
-            }
+            if (scan_preproc_newline(lexer, false)) continue;
         }
         lexer->advance(lexer, false);
     }
@@ -53,9 +57,7 @@ static bool scan_pragma_spacing(TSLexer *lexer) {
             lexer->advance(lexer, false);
         } else if (lexer->lookahead == '\\') {
             lexer->advance(lexer, false);
-            if (lexer->lookahead == '\r') lexer->advance(lexer, false);
-            if (lexer->lookahead != '\n') return false;
-            lexer->advance(lexer, false);
+            if (!scan_preproc_newline(lexer, false)) return false;
         } else if (lexer->lookahead == '/') {
             lexer->advance(lexer, false);
             if (lexer->lookahead == '/') {
@@ -298,7 +300,7 @@ static bool preproc_word(int32_t c, bool continuation) {
 }
 
 static bool scan_pragma_word(TSLexer *lexer) {
-    const char *word = "_Pragma";
+    const char *word = pragma_word;
     for (; *word; word++) {
         if (lexer->lookahead != *word) return false;
         lexer->advance(lexer, false);
@@ -331,4 +333,49 @@ static bool pragma_space(int32_t c) {
            c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000;
 }
 
+static bool scan_function_macro_name(TSLexer *lexer, bool allow_pragma, TSSymbol pragma_symbol) {
+    bool has_name = false;
+    bool is_pragma = true;
+    unsigned pragma_length = 0;
+    for (;;) {
+        while (!has_name && pragma_space(lexer->lookahead)) lexer->advance(lexer, true);
+        int32_t c = lexer->lookahead;
+        if (c == '\\') {
+            lexer->advance(lexer, false);
+            if (scan_preproc_newline(lexer, !has_name)) {
+                if (!has_name) continue;
+                if (!scan_preproc_splices(lexer)) return false;
+                break;
+            }
+            is_pragma = false;
+            unsigned digits = lexer->lookahead == 'u' ? 4 : lexer->lookahead == 'U' ? 8 : 0;
+            if (!digits) return false;
+            lexer->advance(lexer, false);
+            for (unsigned i = 0; i < digits; i++) {
+                c = lexer->lookahead;
+                if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) return false;
+                lexer->advance(lexer, false);
+            }
+        } else {
+            bool identifier = has_name
+                ? set_contains(preproc_identifier_continue, sizeof(preproc_identifier_continue) / sizeof(TSCharacterRange), c)
+                : set_contains(preproc_identifier_start, sizeof(preproc_identifier_start) / sizeof(TSCharacterRange), c);
+            if (!identifier) break;
+            if (is_pragma) {
+                if (pragma_length < sizeof(pragma_word) - 1 && c == pragma_word[pragma_length]) pragma_length++;
+                else is_pragma = false;
+            }
+            lexer->advance(lexer, false);
+        }
+        has_name = true;
+        lexer->mark_end(lexer);
+    }
+    bool adjacent = has_name && lexer->lookahead == '(';
+    if (allow_pragma && is_pragma && pragma_length == sizeof(pragma_word) - 1 && scan_pragma_suffix(lexer)) {
+        lexer->result_symbol = pragma_symbol;
+        return true;
+    }
+    return adjacent;
+}
+
 #endif

package/src/scanner.c

@@ -1,6 +1,6 @@
 #include "pragma.h"
 
-enum TokenType { PRAGMA_OPERATOR, PREPROC_ARG, PREPROC_NEWLINE, PREPROC_LPAREN, PREPROC_DIRECTIVE_ARG };
+enum TokenType { PRAGMA_OPERATOR, PREPROC_ARG, PREPROC_NEWLINE, PREPROC_LPAREN, PREPROC_DIRECTIVE_ARG, PREPROC_FUNCTION_NAME };
 
 void *tree_sitter_c_external_scanner_create(void) {
     return NULL;
@@ -24,13 +24,23 @@ void tree_sitter_c_external_scanner_deserialize(void *payload, const char *buffe
 
 bool tree_sitter_c_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
     (void)payload;
+    if (valid_symbols[PREPROC_FUNCTION_NAME] && !valid_symbols[PREPROC_LPAREN]) {
+        lexer->result_symbol = PREPROC_FUNCTION_NAME;
+        return scan_function_macro_name(lexer, valid_symbols[PRAGMA_OPERATOR], PRAGMA_OPERATOR);
+    }
     bool directive_text = !valid_symbols[PREPROC_ARG] && valid_symbols[PREPROC_DIRECTIVE_ARG];
     TSSymbol argument_symbol = directive_text ? PREPROC_DIRECTIVE_ARG : PREPROC_ARG;
-    if (valid_symbols[PREPROC_LPAREN] && lexer->lookahead == '(') {
-        lexer->advance(lexer, false);
-        lexer->mark_end(lexer);
-        lexer->result_symbol = PREPROC_LPAREN;
-        return true;
+    if (valid_symbols[PREPROC_LPAREN]) {
+        while (lexer->lookahead == '\\') {
+            lexer->advance(lexer, true);
+            if (!scan_preproc_newline(lexer, true)) return false;
+        }
+        if (lexer->lookahead == '(') {
+            lexer->advance(lexer, false);
+            lexer->mark_end(lexer);
+            lexer->result_symbol = PREPROC_LPAREN;
+            return true;
+        }
     }
     if (valid_symbols[PREPROC_NEWLINE]) {
         for (;;) {

package/tree-sitter.json

@@ -16,7 +16,7 @@
     }
   ],
   "metadata": {
-    "version": "1.5.2",
+    "version": "2.0.0",
     "license": "MIT",
     "description": "C grammar for tree-sitter",
     "authors": [