@marko/tree-sitter 0.1.0 → 0.3.0

package/src/scanner.c

@@ -79,6 +79,7 @@ enum TokenType {
   TYPES_CLOSE,              // ">"
   ATTR_GROUP_OPEN,          // "["
   ATTR_GROUP_CLOSE,         // "]"
+  ATTR_METHOD_ASYNC,        // "async" before a shorthand method
   ATTR_NAME,
   ATTR_EQ,                  // "="
   ATTR_BOUND_EQ,            // ":="
@@ -499,7 +500,11 @@ static bool should_terminate(EStream *es, ExprState *e, int32_t code,
           return next == '>';
         case '>':
           // `=>` continues unless it is the very start of the expression.
-          return at_start || prev != '=';
+          if (at_start) return true;
+          if (prev == '=') return false;
+          // A whitespace preceded ">=" is always a comparison: a closed tag
+          // would have put the "=" in its body content.
+          return !(is_ws(prev) && next == '=');
         default:
           return false;
       }
@@ -623,6 +628,7 @@ static int64_t look_behind_for_operator(EStream *es, ExprState *e,
     }
 
     default: {
+      if (code < 'a' || code > 'z') return -1;
       const char *const *keywords =
           e->in_type ? TS_UNARY_KEYWORDS : UNARY_KEYWORDS;
       for (int i = 0; keywords[i]; i++) {
@@ -659,6 +665,7 @@ static int64_t look_ahead_for_operator(EStream *es, ExprState *e, uint32_t k) {
     }
 
     default: {
+      if (c < 'a' || c > 'z') return -1;
       for (int i = 0; BINARY_KEYWORDS[i]; i++) {
         const char *kw = BINARY_KEYWORDS[i];
         size_t len = strlen(kw);
@@ -736,6 +743,11 @@ static bool check_for_operators(EStream *es, ExprState *e, bool eol) {
     }
 
     int32_t next_c = es_peek(es, k);
+    // In HTML mode a "</" is a close tag rather than a less-than operator.
+    if (!is_concise(s) && next_c == '<' && es_peek(es, k + 1) == '/') {
+      return false;
+    }
+
     if (next_c >= 0 && !should_terminate(es, e, next_c, k)) {
       int64_t consume = look_ahead_for_operator(es, e, k);
       if (consume != -1) {
@@ -880,6 +892,11 @@ static bool scan_expr_inner(EStream *es, ExprState *e) {
       continue;
     }
 
+    if (is_word_code(code)) {
+      es_next(es);
+      continue;
+    }
+
     if (!e->group_len) {
       if (e->cfg.terminated_by_whitespace && is_ws(code)) {
         if (!check_for_operators(es, e, false)) return true;
@@ -1017,6 +1034,12 @@ static bool scan_expr_inner(EStream *es, ExprState *e) {
         es_next(es);
         break;
       case '{':
+        if (e->in_type && !e->force_type) {
+          int64_t prev_pos = look_behind_while_ws(es, (int64_t)s->buf_len - 1);
+          if (look_behind_for_operator(es, e, prev_pos) == -1) {
+            e->in_type = false;
+          }
+        }
         if (e->group_len >= sizeof(e->group)) return false;
         e->group[e->group_len++] = '}';
         es_next(es);
@@ -1235,14 +1258,53 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
 static bool scan_frame_content(Scanner *s, TSLexer *lexer, const bool *valid,
                                TSSymbol *result);
 
+static inline void skip_or_peek(Scanner *s, TSLexer *lexer, bool consume) {
+  if (consume) skipc(s, lexer);
+  else lexer->advance(lexer, false);
+}
+
+// Skip whitespace and `//`/`/* */` comments, returning true if a `,` follows.
+// `consume` skips them as trivia, otherwise it is raw lookahead beyond the
+// committed token end. Mirrors the concise-mode eol scan in OPEN_TAG.
+static bool scan_line_attr_trivia(Scanner *s, TSLexer *lexer, bool consume) {
+  for (;;) {
+    if (at_eof(lexer)) return false;
+    int32_t c = lexer->lookahead;
+    if (is_ws(c)) {
+      skip_or_peek(s, lexer, consume);
+    } else if (c == '/') {
+      skip_or_peek(s, lexer, consume);
+      if (lexer->lookahead == '/') {
+        // line comment
+        skip_or_peek(s, lexer, consume);
+        while (!at_eof(lexer) && !is_line(lexer->lookahead))
+          skip_or_peek(s, lexer, consume);
+      } else if (lexer->lookahead == '*') {
+        // block comment
+        int32_t prev = 0;
+        skip_or_peek(s, lexer, consume);
+        while (!at_eof(lexer) && !(prev == '*' && lexer->lookahead == '/')) {
+          prev = lexer->lookahead;
+          skip_or_peek(s, lexer, consume);
+        }
+        if (at_eof(lexer)) return false;
+        skip_or_peek(s, lexer, consume);
+      } else {
+        return false;
+      }
+    } else {
+      break;
+    }
+  }
+  return lexer->lookahead == ',';
+}
+
 // After most tokens inside a concise open tag, look ahead to see whether a
-// `,` follows (over any whitespace, including newlines): if so the open tag
-// continues on the next line (consumeWhitespaceIfBefore(",")).
+// `,` follows (over whitespace and comments): if so the open tag continues on
+// the next line with another line attribute.
 static void concise_tag_epilogue(Scanner *s, TSLexer *lexer) {
   if (!s->tag_open || !is_concise(s) || s->in_attr_group) return;
-  // raw lookahead beyond the committed token end
-  while (!at_eof(lexer) && is_ws(lexer->lookahead)) lexer->advance(lexer, false);
-  s->tag_comma_continue = !at_eof(lexer) && lexer->lookahead == ',';
+  s->tag_comma_continue = scan_line_attr_trivia(s, lexer, false);
 }
 
 // --------------------------------------------------------------------------
@@ -1283,6 +1345,144 @@ static ExprCfg cfg_type_expr(void) {
   return cfg;
 }
 
+static inline bool is_attr_name_char(int32_t c) {
+  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+         (c >= '0' && c <= '9') || c == '_' || c == '$' || c == '-';
+}
+
+// Whitespace the async keyword may be separated from its method by. A newline
+// ends a concise mode attribute, so only indentation counts there.
+static inline bool skip_async_space(EStream *es, Scanner *s, uint32_t k) {
+  int32_t c = es_peek(es, k);
+  if (c < 0) return false;
+  return is_concise(s) ? is_indent_code(c) : is_ws(c);
+}
+
+// A token cannot be taken back, so the whole signature is checked up front:
+// the parser defers instead, until the args close and a "{" follows.
+// The attribute name is exactly "async": a longer name ("asyncfoo") or one
+// carrying a modifier ("async:mod") is an ordinary attribute name.
+static bool is_async_keyword(EStream *es) {
+  static const char kw[] = "async";
+  for (uint32_t i = 0; i < 5; i++) {
+    if (es_peek(es, i) != kw[i]) return false;
+  }
+  int32_t c = es_peek(es, 5);
+  return c != ':' && !is_attr_name_char(c);
+}
+
+// Skips a balanced group, stepping over the literals and comments whose
+// contents would otherwise be counted as delimiters.
+static bool skip_async_group(EStream *es, uint32_t *pk, int32_t open,
+                             int32_t close) {
+  uint32_t k = *pk;
+  uint32_t depth = 0;
+  int32_t prev = 0;
+
+  do {
+    int32_t c = es_peek(es, k);
+    if (c < 0) return false;
+
+    if (c == '"' || c == '\'' || c == '`') {
+      int32_t quote = c;
+      uint32_t tmpl_depth = 0;
+      k++;
+      for (;;) {
+        c = es_peek(es, k);
+        if (c < 0) return false;
+        k++;
+        if (c == '\\') {
+          if (es_peek(es, k) < 0) return false;
+          k++;
+        } else if (quote == '`' && c == '$' && es_peek(es, k) == '{') {
+          tmpl_depth++;
+          k++;
+        } else if (tmpl_depth && c == '}') {
+          tmpl_depth--;
+        } else if (c == quote && !tmpl_depth) {
+          break;
+        }
+      }
+      prev = quote;
+      continue;
+    }
+
+    if (c == '/') {
+      int32_t next = es_peek(es, k + 1);
+      if (!can_follow_division(prev)) {  // a regular expression literal
+        bool in_class = false;
+        k++;
+        for (;;) {
+          c = es_peek(es, k);
+          if (c < 0 || c == '\n') return false;
+          k++;
+          if (c == '\\') {
+            if (es_peek(es, k) < 0) return false;
+            k++;
+          } else if (c == '[') in_class = true;
+          else if (c == ']') in_class = false;
+          else if (c == '/' && !in_class) break;
+        }
+        prev = '/';
+        continue;
+      }
+      if (next == '/') {
+        while ((c = es_peek(es, k)) >= 0 && c != '\n') k++;
+        continue;
+      }
+      if (next == '*') {
+        k += 2;
+        while ((c = es_peek(es, k)) >= 0 &&
+               !(c == '*' && es_peek(es, k + 1) == '/')) {
+          k++;
+        }
+        if (c < 0) return false;
+        k += 2;
+        continue;
+      }
+    }
+
+    if (c == open) depth++;
+    else if (c == close) depth--;
+    if (!is_ws(c)) prev = c;
+    k++;
+  } while (depth);
+
+  *pk = k;
+  return true;
+}
+
+static bool is_async_method_ahead(EStream *es, Scanner *s) {
+  uint32_t k = 0;
+
+  // In concise mode a newline ends the attribute, so the method cannot
+  // continue onto the next line.
+  bool had_space = false;
+  while (skip_async_space(es, s, k)) {
+    k++;
+    had_space = true;
+  }
+
+  // An optional method name, which must be separated from the keyword.
+  if (is_attr_name_char(es_peek(es, k))) {
+    if (!had_space) return false;
+    while (is_attr_name_char(es_peek(es, k))) k++;
+    while (skip_async_space(es, s, k)) k++;
+  }
+
+  // Optional type params, then the params, then the body opener.
+  if (es_peek(es, k) == '<') {
+    if (!skip_async_group(es, &k, '<', '>')) return false;
+    while (skip_async_space(es, s, k)) k++;
+  }
+
+  if (es_peek(es, k) != '(') return false;
+  if (!skip_async_group(es, &k, '(', ')')) return false;
+
+  while (skip_async_space(es, s, k)) k++;
+  return es_peek(es, k) == '{';
+}
+
 static ExprCfg cfg_attr_name(Scanner *s) {
   ExprCfg cfg;
   memset(&cfg, 0, sizeof(cfg));
@@ -2939,9 +3139,8 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
         continue;
       }
       if (s->tag_comma_continue) {
-        // consumeWhitespaceIfBefore(","): skip to and through the comma.
-        while (!at_eof(lexer) && is_ws(lexer->lookahead)) skipc(s, lexer);
-        if (!at_eof(lexer) && lexer->lookahead == ',') skipc(s, lexer);
+        // comma continues the open tag with another line attribute
+        if (scan_line_attr_trivia(s, lexer, true)) skipc(s, lexer);
         while (!at_eof(lexer) && is_ws(lexer->lookahead)) skipc(s, lexer);
         s->tag_comma_continue = 0;
         continue;
@@ -3222,6 +3421,18 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
     s->tag_has_attrs = 1;
   }
 
+  // Both readings span the same text, so only the symbol depends on the
+  // lookahead; marking first is what keeps the extent right while peeking.
+  if (c == 'a' && is_async_keyword(es)) {
+    for (int i = 0; i < 5; i++) es_next_mark(es);  // "async"
+    bool method = is_async_method_ahead(es, s);
+    if (!valid[method ? ATTR_METHOD_ASYNC : ATTR_NAME]) return false;
+    s->attr_stage = ATTR_NAME_STAGE;
+    lexer->result_symbol = method ? ATTR_METHOD_ASYNC : ATTR_NAME;
+    concise_tag_epilogue(s, lexer);
+    return true;
+  }
+
   if (c == '<') return false;  // INVALID_ATTRIBUTE_NAME
   {
     bool empty = false;
@@ -3499,7 +3710,8 @@ static const char *const TOKEN_NAMES[] = {
     "PARAM_TYPE", "PARAM_EQ", "PARAM_DEFAULT", "PARAM_COMMA",
     "PARAMS_CLOSE", "TYPE_ARGS_OPEN",
     "TYPE_PARAMS_OPEN", "TYPE_EXPR", "TYPES_CLOSE", "ATTR_GROUP_OPEN",
-    "ATTR_GROUP_CLOSE", "ATTR_NAME", "ATTR_EQ", "ATTR_BOUND_EQ",
+    "ATTR_GROUP_CLOSE", "ATTR_METHOD_ASYNC", "ATTR_NAME", "ATTR_EQ",
+    "ATTR_BOUND_EQ",
     "ATTR_SPREAD_START", "ATTR_VALUE_EXPR", "METHOD_BODY_OPEN",
     "METHOD_BODY_EXPR", "METHOD_BODY_CLOSE", "OPEN_TAG_END",
     "OPEN_TAG_END_SELF", "CONCISE_OPEN_TAG_END", "CLOSE_TAG_START",

package/tree-sitter.json

@@ -7,14 +7,16 @@
       "title": "Marko",
       "scope": "source.marko",
       "path": ".",
-      "file-types": ["marko"],
+      "file-types": [
+        "marko"
+      ],
       "injection-regex": "^marko$",
       "highlights": "queries/highlights.scm",
       "injections": "queries/injections.scm"
     }
   ],
   "metadata": {
-    "version": "0.1.0",
+    "version": "0.3.0",
     "license": "MIT",
     "description": "Tree-sitter grammar for the Marko templating language",
     "links": {