@marko/tree-sitter 0.3.0 → 0.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@marko/tree-sitter",
-  "version": "0.3.0",
+  "version": "0.4.1",
   "description": "Tree-sitter grammar for the Marko templating language",
   "keywords": [
     "incremental",
@@ -34,7 +34,7 @@
     "tree-sitter.json"
   ],
   "dependencies": {
-    "node-addon-api": "^8.3.0",
+    "node-addon-api": "^8.9.2",
     "node-gyp-build": "^4.8.4"
   },
   "devDependencies": {
@@ -42,15 +42,15 @@
     "@changesets/cli": "^2.31.0",
     "@types/degit": "^2.8.6",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^20.11.16",
+    "@types/node": "^22.20.4",
     "degit": "^2.8.4",
-    "htmljs-parser": "^5.14.0",
-    "mocha": "^10.2.0",
-    "node-gyp": "^11.0.0",
+    "htmljs-parser": "^5.20.0",
+    "mocha": "^12.0.2",
+    "node-gyp": "^13.0.2",
     "prebuildify": "^6.0.1",
-    "tree-sitter": "^0.25.0",
+    "tree-sitter": "^0.25.1",
     "tree-sitter-cli": "^0.26.9",
-    "tsx": "^4.19.2",
+    "tsx": "^4.23.15",
     "web-tree-sitter": "^0.26.9"
   },
   "peerDependencies": {
@@ -81,6 +81,6 @@
     "prebuild": "prebuildify --napi --strip",
     "build:wasm": "tree-sitter build --wasm -o tree-sitter-marko.wasm",
     "test:wasm": "tsx tools/check-wasm.mts",
-    "version": "changeset version && node tools/sync-tree-sitter-version.mjs && pnpm install --lockfile-only"
+    "version": "changeset version && node tools/sync-tree-sitter-version.mjs && tree-sitter generate && pnpm install --lockfile-only"
   }
 }

package/queries/highlights.scm

@@ -13,6 +13,7 @@
 (shorthand_class) @property
 
 (attr_name) @attribute
+(attr_method_async) @keyword
 
 ; Binding positions: patterns get the TS injection; types are captured
 ; flatly because a bare type is not a valid TS program (matching the

package/src/parser.c

@@ -7965,8 +7965,8 @@ TS_PUBLIC const TSLanguage *tree_sitter_marko(void) {
     .max_reserved_word_set_size = 0,
     .metadata = {
       .major_version = 0,
-      .minor_version = 3,
-      .patch_version = 0,
+      .minor_version = 4,
+      .patch_version = 1,
     },
   };
   return &language;

package/src/scanner.c

@@ -555,16 +555,57 @@ static bool should_terminate(EStream *es, ExprState *e, int32_t code,
   return false;
 }
 
+// Only JavaScript has `delete` and `void` operators; in a type `void` is the
+// `void` type. `return`, `throw` and `yield` are left out: JavaScript allows
+// no line break after them, so they never continue an expression.
 static const char *const UNARY_KEYWORDS[] = {
-    "async", "await", "class", "function", "new", "typeof", "void", NULL,
+    "async", "await", "class", "function", "new", "typeof", "delete", "void",
+    NULL,
 };
 static const char *const TS_UNARY_KEYWORDS[] = {
-    "async",  "await",   "class", "function", "new",   "typeof", "void",
-    "asserts", "infer",  "is",    "keyof",    "readonly", "unique", NULL,
+    "async",   "await", "class", "function", "new",      "typeof",
+    "asserts", "infer", "is",    "keyof",    "readonly", "unique", NULL,
 };
 static const char *const BINARY_KEYWORDS[] = {
     "as", "extends", "instanceof", "in", "satisfies", NULL,
 };
+static const char *const RELATIONAL_KEYWORDS[] = {"in", "instanceof", NULL};
+
+static bool es_peek_matches(EStream *es, uint32_t k, const char *str) {
+  for (uint32_t j = 0; str[j]; j++) {
+    if (es_peek(es, k + j) != (int32_t)str[j]) return false;
+  }
+  return true;
+}
+
+// prepareScriptlet's type look-ahead from offset `k`: `declare`, `interface`
+// or `type` (after spaces) before a name, or `type` before `{`/`*`, starts a
+// type; `type = 1` and `type in x` stay JavaScript.
+static bool looks_like_type_statement(EStream *es, uint32_t k) {
+  static const char *const TYPE_KEYWORDS[] = {"declare", "interface", "type",
+                                              NULL};
+  while (is_indent_code(es_peek(es, k))) k++;
+  for (int i = 0; TYPE_KEYWORDS[i]; i++) {
+    if (!es_peek_matches(es, k, TYPE_KEYWORDS[i])) continue;
+
+    uint32_t n = k + (uint32_t)strlen(TYPE_KEYWORDS[i]);
+    if (!is_indent_code(es_peek(es, n))) return false;
+    while (is_indent_code(es_peek(es, n))) n++;
+
+    int32_t c = es_peek(es, n);
+    if (c == '{' || c == '*') return strcmp(TYPE_KEYWORDS[i], "type") == 0;
+    if (!is_word_code(c) || (c >= '0' && c <= '9')) return false;
+    for (int b = 0; BINARY_KEYWORDS[b]; b++) {
+      if (es_peek_matches(es, n, BINARY_KEYWORDS[b]) &&
+          !is_word_code(
+              es_peek(es, n + (uint32_t)strlen(BINARY_KEYWORDS[b])))) {
+        return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
 
 static int64_t look_behind_for(EStream *es, int64_t pos, const char *str) {
   int64_t len = (int64_t)strlen(str);
@@ -586,6 +627,19 @@ static int64_t look_behind_while_ws(EStream *es, int64_t pos) {
   return 0;
 }
 
+// Returns where a whole keyword ending at `pos` starts, or -1.
+static int64_t look_behind_for_keyword(EStream *es, int64_t pos,
+                                       const char *const *keywords) {
+  for (int i = 0; keywords[i]; i++) {
+    int64_t keyword_pos = look_behind_for(es, pos, keywords[i]);
+    if (keyword_pos != -1) {
+      int32_t before = keyword_pos > 0 ? es_at(es, keyword_pos - 1) : -1;
+      return is_word_or_period_code(before) ? -1 : keyword_pos;
+    }
+  }
+  return -1;
+}
+
 // lookBehindForOperator: `pos` is a buf index just past the candidate.
 static int64_t look_behind_for_operator(EStream *es, ExprState *e,
                                         int64_t pos) {
@@ -593,12 +647,33 @@ static int64_t look_behind_for_operator(EStream *es, ExprState *e,
   int32_t code = es_at(es, cur_pos);
 
   switch (code) {
-    case '&': case '*': case '^': case ':': case '=': case '!':
+    case '&': case '*': case '^': case ':': case '=':
     case '<': case '%': case '|': case '?': case '~':
       // The operator may sit just before the token start (cur_pos == -1, eg
       // resuming a tag var type after its ":"); callers only test != -1.
       return cur_pos < 0 ? 0 : cur_pos;
 
+    case '!': {
+      // After an operand, `!` is a TypeScript non-null assertion (postfix),
+      // as is each `!` of a run after one (`x!!`); after a keyword operator
+      // (`typeof!a`, `a in!b`) it is the prefix `!`.
+      int64_t operand_end = cur_pos - 1;
+      while (es_at(es, operand_end) == '!') operand_end--;
+      int32_t operand = es_at(es, operand_end);
+      switch (operand) {
+        case ')': case ']': case '"': case '\'': case '`':
+          return -1;
+        default:
+          return is_word_code(operand) &&
+                         look_behind_for_operator(es, e, operand_end + 1) ==
+                             -1 &&
+                         look_behind_for_keyword(es, operand_end,
+                                                 RELATIONAL_KEYWORDS) == -1
+                     ? -1
+                     : (cur_pos < 0 ? 0 : cur_pos);
+      }
+    }
+
     case '>':
       if (es_at(es, cur_pos - 1) == '=') return cur_pos - 1;
       return e->in_type ? -1 : (cur_pos < 0 ? 0 : cur_pos);
@@ -629,16 +704,8 @@ static int64_t look_behind_for_operator(EStream *es, ExprState *e,
 
     default: {
       if (code < 'a' || code > 'z') return -1;
-      const char *const *keywords =
-          e->in_type ? TS_UNARY_KEYWORDS : UNARY_KEYWORDS;
-      for (int i = 0; keywords[i]; i++) {
-        int64_t keyword_pos = look_behind_for(es, cur_pos, keywords[i]);
-        if (keyword_pos != -1) {
-          int32_t before = keyword_pos > 0 ? es_at(es, keyword_pos - 1) : -1;
-          return is_word_or_period_code(before) ? -1 : keyword_pos;
-        }
-      }
-      return -1;
+      return look_behind_for_keyword(
+          es, cur_pos, e->in_type ? TS_UNARY_KEYWORDS : UNARY_KEYWORDS);
     }
   }
 }
@@ -743,9 +810,15 @@ static bool check_for_operators(EStream *es, ExprState *e, bool eol) {
     }
 
     int32_t next_c = es_peek(es, k);
-    // In HTML mode a "</" is a close tag rather than a less-than operator.
-    if (!is_concise(s) && next_c == '<' && es_peek(es, k + 1) == '/') {
-      return false;
+    // A "</" close tag (html mode) or a "<!--" html comment is markup rather
+    // than a less-than operator, which look_ahead_for_operator would otherwise
+    // continue across.
+    if (next_c == '<') {
+      if (!is_concise(s) && es_peek(es, k + 1) == '/') return false;
+      if (es_peek(es, k + 1) == '!' && es_peek(es, k + 2) == '-' &&
+          es_peek(es, k + 3) == '-') {
+        return false;
+      }
     }
 
     if (next_c >= 0 && !should_terminate(es, e, next_c, k)) {
@@ -1643,7 +1716,16 @@ static bool scan_scriptlet_body(Scanner *s, TSLexer *lexer, const bool *valid,
     memset(&cfg, 0, sizeof(cfg));
     cfg.operators = true;
     cfg.terminated_by_eol = true;
-    if (!scan_expr_token(s, lexer, cfg, NULL)) return false;
+    EStream es;
+    es_init(&es, s, lexer, true);
+    mark(s, lexer);  // empty expression: zero-width token
+    if (looks_like_type_statement(&es, 0)) {
+      cfg.in_type = true;
+      cfg.force_type = true;
+    }
+    bool ok = scan_expr_es(&es, cfg, NULL);
+    es_free(&es);
+    if (!ok) return false;
     *result = SCRIPTLET_EXPR;
     return true;
   }
@@ -2501,6 +2583,19 @@ static bool scan_frame_content(Scanner *s, TSLexer *lexer, const bool *valid,
 // Concise mode content (CONCISE_HTML_CONTENT)
 // --------------------------------------------------------------------------
 
+// In concise mode only whitespace may follow a comment block on its line; the
+// comment event still fires before the error.
+static void check_block_comment_tail(Scanner *s, EStream *es) {
+  for (uint32_t k = 0;; k++) {
+    int32_t c = es_peek(es, k);
+    if (c < 0 || is_line(c)) return;
+    if (!is_ws(c)) {
+      s->fail_next = 1;  // INVALID_CHARACTER after this token
+      return;
+    }
+  }
+}
+
 static bool scan_concise(Scanner *s, TSLexer *lexer, const bool *valid,
                          TSSymbol *result) {
   // Trivia: newlines and indentation.
@@ -2574,6 +2669,7 @@ static bool scan_concise(Scanner *s, TSLexer *lexer, const bool *valid,
     } else if (n1 == '*') {
       *result = BLOCK_COMMENT;
       ok = es_consume_block_comment(&es) && valid[BLOCK_COMMENT];
+      if (ok) check_block_comment_tail(s, &es);
     }
     es_free(&es);
     return ok;
@@ -2609,6 +2705,14 @@ static bool scan_concise(Scanner *s, TSLexer *lexer, const bool *valid,
     }
   }
 
+  if (c == ';') {
+    // An empty line, as after a tag ended by ";": the ";" is trivia and only
+    // comments may follow it.
+    skipc(s, lexer);
+    s->semicolon_line = 1;
+    return scan_concise(s, lexer, valid, result);
+  }
+
   if (c == '<') {
     // Begin a mixed-mode html block; HTML_CONTENT handles the "<".
     s->begin_mixed_mode = 1;
@@ -2633,12 +2737,17 @@ static bool scan_concise(Scanner *s, TSLexer *lexer, const bool *valid,
   }
 
   if (c == '-') {
-    if (es_peek(&es, 1) == '-') {
+    int32_t n1 = es_peek(&es, 1);
+    if (n1 == '-') {
       ok = valid[BLOCK_OPEN] && scan_block_open(s, &es, result);
+      es_free(&es);
+      return ok;
     }
-    // else: INVALID_LINE_START: single hyphen
-    es_free(&es);
-    return ok;
+    if ((t && t->type == TAG_TEXT) || !is_word_code(n1)) {
+      es_free(&es);
+      return false;  // INVALID_LINE_START: single hyphen
+    }
+    // A tag name starting with "-" (eg "-if"): the open tag below reads it.
   }
 
   if (c == '/') {
@@ -2649,20 +2758,7 @@ static bool scan_concise(Scanner *s, TSLexer *lexer, const bool *valid,
       ok = valid[LINE_COMMENT];
     } else if (n1 == '*') {
       ok = es_consume_block_comment(&es);
-      if (ok) {
-        // In concise mode, only whitespace may follow a comment block; the
-        // comment event still fires before the error.
-        uint32_t k = 0;
-        while (true) {
-          int32_t c2 = es_peek(&es, k);
-          if (c2 < 0 || is_line(c2)) break;
-          if (!is_ws(c2)) {
-            s->fail_next = 1;  // INVALID_CHARACTER after this token
-            break;
-          }
-          k++;
-        }
-      }
+      if (ok) check_block_comment_tail(s, &es);
       *result = BLOCK_COMMENT;
       ok = ok && valid[BLOCK_COMMENT];
     }
@@ -2703,6 +2799,14 @@ static bool is_tag_name_terminator(Scanner *s, int32_t c, int32_t next) {
   }
 }
 
+// Nothing of the tag name is read yet: OPEN_TAG still skips "," and
+// whitespace, and TAG_NAME starts at the next char.
+static bool tag_name_pending(Scanner *s) {
+  TagFrame *t = top_tag(s);
+  return t && !s->tag_name_done && !s->in_shorthand && t->name_len == 0 &&
+         !(t->flags & TF_NAME_INTERP);
+}
+
 // Finish the current tag name segment (called when a terminator is seen).
 static void finalize_tag_name(Scanner *s, int32_t stop_char) {
   if (!s->tag_name_done) {
@@ -2835,23 +2939,10 @@ static bool scan_statement_tail(Scanner *s, TSLexer *lexer, const bool *valid,
   cfg.terminated_by_eol = true;
   cfg.consume_indented = true;
 
-  // lookAheadFor("declare " | "interface " | "type ") from pos + 1.
-  static const char *const PREFIXES[] = {"declare ", "interface ", "type ",
-                                         NULL};
-  for (int i = 0; PREFIXES[i]; i++) {
-    const char *p = PREFIXES[i];
-    bool match_p = true;
-    for (uint32_t j = 0; p[j]; j++) {
-      if (es_peek(es, 1 + j) != (int32_t)p[j]) {
-        match_p = false;
-        break;
-      }
-    }
-    if (match_p) {
-      cfg.in_type = true;
-      cfg.force_type = true;
-      break;
-    }
+  // prepareStatement: the code starts one past the tag name.
+  if (looks_like_type_statement(es, 1)) {
+    cfg.in_type = true;
+    cfg.force_type = true;
   }
 
   if (!scan_expr_es(es, cfg, NULL)) return false;
@@ -2872,6 +2963,14 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
     return scan_statement_tail(s, lexer, valid, es);
   }
 
+  // ---- a ";" read before the (empty) name, eg ",;", ended the tag there
+  if (s->semicolon_line) {
+    if (!valid[CONCISE_OPEN_TAG_END]) return false;
+    open_tag_ended(s, false);
+    lexer->result_symbol = CONCISE_OPEN_TAG_END;  // zero-width
+    return true;
+  }
+
   // ---- grammar-position expressions (no trivia before them)
   if (valid[ARGS_EXPR] && !valid[ATTR_NAME]) {
     bool empty = false;
@@ -3105,6 +3204,22 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
 
   // ---- trivia loop (only when no lookahead is pending)
   while (es_drained(es)) {
+    if (concise && tag_name_pending(s) &&
+        (at_eof(lexer) || is_line(lexer->lookahead) ||
+         lexer->lookahead == ';')) {
+      // The open tag ends before a name (eg a line of just "," or ",;"): it
+      // is named empty where it ends, which is after a ";".
+      if (!valid[TAG_NAME_EMPTY]) return false;
+      if (!at_eof(lexer) && lexer->lookahead == ';') {
+        skipc(s, lexer);
+        s->semicolon_line = 1;  // the open tag ends on the next scan
+      }
+      finalize_tag_name(s, -1);
+      mark(s, lexer);
+      lexer->result_symbol = TAG_NAME_EMPTY;
+      return true;
+    }
+
     if (at_eof(lexer)) {
       if (!s->tag_name_done || s->in_shorthand) finalize_tag_name(s, -1);
       s->statement_pending = 0;  // an empty statement expression: no events
@@ -3119,7 +3234,7 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
 
     int32_t cc = lexer->lookahead;
 
-    if ((!s->tag_name_done || s->in_shorthand) &&
+    if ((!s->tag_name_done || s->in_shorthand) && !tag_name_pending(s) &&
         (is_ws(cc) || cc == ',')) {
       // TAG_NAME terminates at whitespace/comma before any trivia skipping.
       finalize_tag_name(s, cc);
@@ -3163,6 +3278,29 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
       continue;
     }
 
+    if (cc == '/' && tag_name_pending(s)) {
+      // OPEN_TAG reads comments before TAG_NAME (eg ", /* c */ div"), so the
+      // name starts after them. An open tag comment is a hidden token anyway,
+      // so skip it as trivia; any other "/" ends an empty name here.
+      mark(s, lexer);
+      int32_t n1 = es_peek(es, 1);
+      if (n1 != '/' && n1 != '*') break;
+      es_next(es);      // "/": skipping the rest drops it from the token
+      skipc(s, lexer);  // "/" or "*"
+      if (n1 == '/') {
+        while (!at_eof(lexer) && !is_line(lexer->lookahead)) skipc(s, lexer);
+      } else {
+        int32_t prev = 0;
+        while (!at_eof(lexer) && !(prev == '*' && lexer->lookahead == '/')) {
+          prev = lexer->lookahead;
+          skipc(s, lexer);
+        }
+        if (at_eof(lexer)) return false;  // MALFORMED_COMMENT
+        skipc(s, lexer);
+      }
+      continue;
+    }
+
     break;
   }
 
@@ -3179,6 +3317,18 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
 
   // ---- tag name section (TAG_NAME handles its chars before OPEN_TAG does)
   if (!s->tag_name_done || s->in_shorthand) {
+    if (concise && c == '-' && tag_name_pending(s)) {
+      // OPEN_TAG reads "-" before TAG_NAME: a name only starts with "-" and a
+      // word char (eg "-if"), and "--" ends the tag before a name.
+      int32_t n1 = es_peek(es, 1);
+      if (n1 == '-') {
+        if (!valid[TAG_NAME_EMPTY]) return false;
+        finalize_tag_name(s, c);
+        lexer->result_symbol = TAG_NAME_EMPTY;  // zero-width at the "--"
+        return true;
+      }
+      if (!is_word_code(n1)) return false;  // MALFORMED_OPEN_TAG
+    }
     TSSymbol name_sym = 0;
     int r = scan_tag_name_token(s, valid, es, &name_sym);
     if (r == 0) return false;

package/tree-sitter.json

@@ -16,7 +16,7 @@
     }
   ],
   "metadata": {
-    "version": "0.3.0",
+    "version": "0.4.1",
     "license": "MIT",
     "description": "Tree-sitter grammar for the Marko templating language",
     "links": {