@marko/tree-sitter 0.2.0 → 0.3.0

package/src/scanner.c

@@ -79,6 +79,7 @@ enum TokenType {
   TYPES_CLOSE,              // ">"
   ATTR_GROUP_OPEN,          // "["
   ATTR_GROUP_CLOSE,         // "]"
+  ATTR_METHOD_ASYNC,        // "async" before a shorthand method
   ATTR_NAME,
   ATTR_EQ,                  // "="
   ATTR_BOUND_EQ,            // ":="
@@ -499,7 +500,11 @@ static bool should_terminate(EStream *es, ExprState *e, int32_t code,
           return next == '>';
         case '>':
           // `=>` continues unless it is the very start of the expression.
-          return at_start || prev != '=';
+          if (at_start) return true;
+          if (prev == '=') return false;
+          // A whitespace preceded ">=" is always a comparison: a closed tag
+          // would have put the "=" in its body content.
+          return !(is_ws(prev) && next == '=');
         default:
           return false;
       }
@@ -738,6 +743,11 @@ static bool check_for_operators(EStream *es, ExprState *e, bool eol) {
     }
 
     int32_t next_c = es_peek(es, k);
+    // In HTML mode a "</" is a close tag rather than a less-than operator.
+    if (!is_concise(s) && next_c == '<' && es_peek(es, k + 1) == '/') {
+      return false;
+    }
+
     if (next_c >= 0 && !should_terminate(es, e, next_c, k)) {
       int64_t consume = look_ahead_for_operator(es, e, k);
       if (consume != -1) {
@@ -1335,6 +1345,144 @@ static ExprCfg cfg_type_expr(void) {
   return cfg;
 }
 
+static inline bool is_attr_name_char(int32_t c) {
+  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+         (c >= '0' && c <= '9') || c == '_' || c == '$' || c == '-';
+}
+
+// Whitespace the async keyword may be separated from its method by. A newline
+// ends a concise mode attribute, so only indentation counts there.
+static inline bool skip_async_space(EStream *es, Scanner *s, uint32_t k) {
+  int32_t c = es_peek(es, k);
+  if (c < 0) return false;
+  return is_concise(s) ? is_indent_code(c) : is_ws(c);
+}
+
+// A token cannot be taken back, so the whole signature is checked up front:
+// the parser defers instead, until the args close and a "{" follows.
+// The attribute name is exactly "async": a longer name ("asyncfoo") or one
+// carrying a modifier ("async:mod") is an ordinary attribute name.
+static bool is_async_keyword(EStream *es) {
+  static const char kw[] = "async";
+  for (uint32_t i = 0; i < 5; i++) {
+    if (es_peek(es, i) != kw[i]) return false;
+  }
+  int32_t c = es_peek(es, 5);
+  return c != ':' && !is_attr_name_char(c);
+}
+
+// Skips a balanced group, stepping over the literals and comments whose
+// contents would otherwise be counted as delimiters.
+static bool skip_async_group(EStream *es, uint32_t *pk, int32_t open,
+                             int32_t close) {
+  uint32_t k = *pk;
+  uint32_t depth = 0;
+  int32_t prev = 0;
+
+  do {
+    int32_t c = es_peek(es, k);
+    if (c < 0) return false;
+
+    if (c == '"' || c == '\'' || c == '`') {
+      int32_t quote = c;
+      uint32_t tmpl_depth = 0;
+      k++;
+      for (;;) {
+        c = es_peek(es, k);
+        if (c < 0) return false;
+        k++;
+        if (c == '\\') {
+          if (es_peek(es, k) < 0) return false;
+          k++;
+        } else if (quote == '`' && c == '$' && es_peek(es, k) == '{') {
+          tmpl_depth++;
+          k++;
+        } else if (tmpl_depth && c == '}') {
+          tmpl_depth--;
+        } else if (c == quote && !tmpl_depth) {
+          break;
+        }
+      }
+      prev = quote;
+      continue;
+    }
+
+    if (c == '/') {
+      int32_t next = es_peek(es, k + 1);
+      if (!can_follow_division(prev)) {  // a regular expression literal
+        bool in_class = false;
+        k++;
+        for (;;) {
+          c = es_peek(es, k);
+          if (c < 0 || c == '\n') return false;
+          k++;
+          if (c == '\\') {
+            if (es_peek(es, k) < 0) return false;
+            k++;
+          } else if (c == '[') in_class = true;
+          else if (c == ']') in_class = false;
+          else if (c == '/' && !in_class) break;
+        }
+        prev = '/';
+        continue;
+      }
+      if (next == '/') {
+        while ((c = es_peek(es, k)) >= 0 && c != '\n') k++;
+        continue;
+      }
+      if (next == '*') {
+        k += 2;
+        while ((c = es_peek(es, k)) >= 0 &&
+               !(c == '*' && es_peek(es, k + 1) == '/')) {
+          k++;
+        }
+        if (c < 0) return false;
+        k += 2;
+        continue;
+      }
+    }
+
+    if (c == open) depth++;
+    else if (c == close) depth--;
+    if (!is_ws(c)) prev = c;
+    k++;
+  } while (depth);
+
+  *pk = k;
+  return true;
+}
+
+static bool is_async_method_ahead(EStream *es, Scanner *s) {
+  uint32_t k = 0;
+
+  // In concise mode a newline ends the attribute, so the method cannot
+  // continue onto the next line.
+  bool had_space = false;
+  while (skip_async_space(es, s, k)) {
+    k++;
+    had_space = true;
+  }
+
+  // An optional method name, which must be separated from the keyword.
+  if (is_attr_name_char(es_peek(es, k))) {
+    if (!had_space) return false;
+    while (is_attr_name_char(es_peek(es, k))) k++;
+    while (skip_async_space(es, s, k)) k++;
+  }
+
+  // Optional type params, then the params, then the body opener.
+  if (es_peek(es, k) == '<') {
+    if (!skip_async_group(es, &k, '<', '>')) return false;
+    while (skip_async_space(es, s, k)) k++;
+  }
+
+  if (es_peek(es, k) != '(') return false;
+  if (!skip_async_group(es, &k, '(', ')')) return false;
+
+  while (skip_async_space(es, s, k)) k++;
+  return es_peek(es, k) == '{';
+}
+
 static ExprCfg cfg_attr_name(Scanner *s) {
   ExprCfg cfg;
   memset(&cfg, 0, sizeof(cfg));
@@ -3273,6 +3421,18 @@ static bool scan_open_tag_es(Scanner *s, TSLexer *lexer, const bool *valid,
     s->tag_has_attrs = 1;
   }
 
+  // Both readings span the same text, so only the symbol depends on the
+  // lookahead; marking first is what keeps the extent right while peeking.
+  if (c == 'a' && is_async_keyword(es)) {
+    for (int i = 0; i < 5; i++) es_next_mark(es);  // "async"
+    bool method = is_async_method_ahead(es, s);
+    if (!valid[method ? ATTR_METHOD_ASYNC : ATTR_NAME]) return false;
+    s->attr_stage = ATTR_NAME_STAGE;
+    lexer->result_symbol = method ? ATTR_METHOD_ASYNC : ATTR_NAME;
+    concise_tag_epilogue(s, lexer);
+    return true;
+  }
+
   if (c == '<') return false;  // INVALID_ATTRIBUTE_NAME
   {
     bool empty = false;
@@ -3550,7 +3710,8 @@ static const char *const TOKEN_NAMES[] = {
     "PARAM_TYPE", "PARAM_EQ", "PARAM_DEFAULT", "PARAM_COMMA",
     "PARAMS_CLOSE", "TYPE_ARGS_OPEN",
     "TYPE_PARAMS_OPEN", "TYPE_EXPR", "TYPES_CLOSE", "ATTR_GROUP_OPEN",
-    "ATTR_GROUP_CLOSE", "ATTR_NAME", "ATTR_EQ", "ATTR_BOUND_EQ",
+    "ATTR_GROUP_CLOSE", "ATTR_METHOD_ASYNC", "ATTR_NAME", "ATTR_EQ",
+    "ATTR_BOUND_EQ",
     "ATTR_SPREAD_START", "ATTR_VALUE_EXPR", "METHOD_BODY_OPEN",
     "METHOD_BODY_EXPR", "METHOD_BODY_CLOSE", "OPEN_TAG_END",
     "OPEN_TAG_END_SELF", "CONCISE_OPEN_TAG_END", "CLOSE_TAG_START",

package/tree-sitter.json

@@ -16,7 +16,7 @@
     }
   ],
   "metadata": {
-    "version": "0.2.0",
+    "version": "0.3.0",
     "license": "MIT",
     "description": "Tree-sitter grammar for the Marko templating language",
     "links": {