@marko/tree-sitter 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@marko/tree-sitter",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Tree-sitter grammar for the Marko templating language",
   "keywords": [
     "incremental",
@@ -38,6 +38,8 @@
     "node-gyp-build": "^4.8.4"
   },
   "devDependencies": {
+    "@changesets/changelog-github": "^0.7.0",
+    "@changesets/cli": "^2.31.0",
     "@types/degit": "^2.8.6",
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.11.16",
@@ -61,12 +63,16 @@
   },
   "scripts": {
     "build": "tree-sitter generate && node-gyp rebuild",
+    "change": "changeset add",
     "generate": "tree-sitter generate",
     "install": "node-gyp-build",
+    "release:publish": "changeset publish",
+    "release:tag": "changeset tag",
     "test": "mocha \"./__tests__/*.test.mts\"",
     "prebuild": "prebuildify --napi --strip",
     "build:wasm": "tree-sitter build --wasm -o tree-sitter-marko.wasm",
-    "test:wasm": "tsx tools/check-wasm.mts"
+    "test:wasm": "tsx tools/check-wasm.mts",
+    "version": "changeset version && node tools/sync-tree-sitter-version.mjs && npm install --package-lock-only"
   },
   "tree-sitter": [
     {

package/src/parser.c

@@ -7864,7 +7864,7 @@ TS_PUBLIC const TSLanguage *tree_sitter_marko(void) {
     .max_reserved_word_set_size = 0,
     .metadata = {
       .major_version = 0,
-      .minor_version = 1,
+      .minor_version = 2,
       .patch_version = 0,
     },
   };

package/src/scanner.c

@@ -623,6 +623,7 @@ static int64_t look_behind_for_operator(EStream *es, ExprState *e,
     }
 
     default: {
+      if (code < 'a' || code > 'z') return -1;
       const char *const *keywords =
           e->in_type ? TS_UNARY_KEYWORDS : UNARY_KEYWORDS;
       for (int i = 0; keywords[i]; i++) {
@@ -659,6 +660,7 @@ static int64_t look_ahead_for_operator(EStream *es, ExprState *e, uint32_t k) {
     }
 
     default: {
+      if (c < 'a' || c > 'z') return -1;
       for (int i = 0; BINARY_KEYWORDS[i]; i++) {
         const char *kw = BINARY_KEYWORDS[i];
         size_t len = strlen(kw);
@@ -880,6 +882,11 @@ static bool scan_expr_inner(EStream *es, ExprState *e) {
       continue;
     }
 
+    if (is_word_code(code)) {
+      es_next(es);
+      continue;
+    }
+
     if (!e->group_len) {
       if (e->cfg.terminated_by_whitespace && is_ws(code)) {
         if (!check_for_operators(es, e, false)) return true;
@@ -1017,6 +1024,12 @@ static bool scan_expr_inner(EStream *es, ExprState *e) {
         es_next(es);
         break;
       case '{':
+        if (e->in_type && !e->force_type) {
+          int64_t prev_pos = look_behind_while_ws(es, (int64_t)s->buf_len - 1);
+          if (look_behind_for_operator(es, e, prev_pos) == -1) {
+            e->in_type = false;
+          }
+        }
         if (e->group_len >= sizeof(e->group)) return false;
         e->group[e->group_len++] = '}';
         es_next(es);
@@ -1235,14 +1248,53 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
 static bool scan_frame_content(Scanner *s, TSLexer *lexer, const bool *valid,
                                TSSymbol *result);
 
+static inline void skip_or_peek(Scanner *s, TSLexer *lexer, bool consume) {
+  if (consume) skipc(s, lexer);
+  else lexer->advance(lexer, false);
+}
+
+// Skip whitespace and `//`/`/* */` comments, returning true if a `,` follows.
+// `consume` skips them as trivia, otherwise it is raw lookahead beyond the
+// committed token end. Mirrors the concise-mode eol scan in OPEN_TAG.
+static bool scan_line_attr_trivia(Scanner *s, TSLexer *lexer, bool consume) {
+  for (;;) {
+    if (at_eof(lexer)) return false;
+    int32_t c = lexer->lookahead;
+    if (is_ws(c)) {
+      skip_or_peek(s, lexer, consume);
+    } else if (c == '/') {
+      skip_or_peek(s, lexer, consume);
+      if (lexer->lookahead == '/') {
+        // line comment
+        skip_or_peek(s, lexer, consume);
+        while (!at_eof(lexer) && !is_line(lexer->lookahead))
+          skip_or_peek(s, lexer, consume);
+      } else if (lexer->lookahead == '*') {
+        // block comment
+        int32_t prev = 0;
+        skip_or_peek(s, lexer, consume);
+        while (!at_eof(lexer) && !(prev == '*' && lexer->lookahead == '/')) {
+          prev = lexer->lookahead;
+          skip_or_peek(s, lexer, consume);
+        }
+        if (at_eof(lexer)) return false;
+        skip_or_peek(s, lexer, consume);
+      } else {
+        return false;
+      }
+    } else {
+      break;
+    }
+  }
+  return lexer->lookahead == ',';
+}
+
 // After most tokens inside a concise open tag, look ahead to see whether a
-// `,` follows (over any whitespace, including newlines): if so the open tag
-// continues on the next line (consumeWhitespaceIfBefore(",")).
+// `,` follows (over whitespace and comments): if so the open tag continues on
+// the next line with another line attribute.
 static void concise_tag_epilogue(Scanner *s, TSLexer *lexer) {
   if (!s->tag_open || !is_concise(s) || s->in_attr_group) return;
-  // raw lookahead beyond the committed token end
-  while (!at_eof(lexer) && is_ws(lexer->lookahead)) lexer->advance(lexer, false);
-  s->tag_comma_continue = !at_eof(lexer) && lexer->lookahead == ',';
+  s->tag_comma_continue = scan_line_attr_trivia(s, lexer, false);
 }
 
 // --------------------------------------------------------------------------
@@ -2939,9 +2991,8 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
         continue;
       }
       if (s->tag_comma_continue) {
-        // consumeWhitespaceIfBefore(","): skip to and through the comma.
-        while (!at_eof(lexer) && is_ws(lexer->lookahead)) skipc(s, lexer);
-        if (!at_eof(lexer) && lexer->lookahead == ',') skipc(s, lexer);
+        // comma continues the open tag with another line attribute
+        if (scan_line_attr_trivia(s, lexer, true)) skipc(s, lexer);
         while (!at_eof(lexer) && is_ws(lexer->lookahead)) skipc(s, lexer);
         s->tag_comma_continue = 0;
         continue;

package/tree-sitter.json

@@ -7,14 +7,16 @@
       "title": "Marko",
       "scope": "source.marko",
       "path": ".",
-      "file-types": ["marko"],
+      "file-types": [
+        "marko"
+      ],
       "injection-regex": "^marko$",
       "highlights": "queries/highlights.scm",
       "injections": "queries/injections.scm"
     }
   ],
   "metadata": {
-    "version": "0.1.0",
+    "version": "0.2.0",
     "license": "MIT",
     "description": "Tree-sitter grammar for the Marko templating language",
     "links": {