@konomanoasa/tree-sitter-gitattributes 0.5.0 → 0.6.0

package/src/scanner.c

@@ -9,6 +9,7 @@ enum Token {
   RULE_START,
   MACRO_START,
   END_OF_FILE,
+  LINE_ENDING,
   LAYOUT,
   SEPARATOR,
   ATTRIBUTE_START,
@@ -37,6 +38,7 @@ enum Token {
   PATH_SEPARATOR,
   ESCAPE,
   QUOTED_ESCAPE,
+  ESCAPE_PREFIX,
   SET_OPEN,
   SET_NEGATION,
   SET_TEXT,
@@ -49,6 +51,23 @@ enum Token {
   CLASS_OPEN,
   CLASS_NAME,
   CLASS_CLOSE,
+  MISSING_SET_CLOSE,
+  INCOMPLETE_SET_CLOSE,
+  INVALID_ENCODING,
+  NEGATIVE_PATTERN,
+  ENDED_ESCAPE,
+  INCOMPLETE_ESCAPE,
+  INVALID_QUOTED_ESCAPE,
+  ENDED_QUOTED_ESCAPE,
+  INCOMPLETE_QUOTED_ESCAPE,
+  MISSING_QUOTE_CLOSE,
+  INCOMPLETE_QUOTE_CLOSE,
+  MISSING_ATTRIBUTE_SEPARATOR,
+  MISSING_ATTRIBUTE_NAME,
+  INCOMPLETE_ATTRIBUTE_NAME,
+  INVALID_NAME_CHARACTER,
+  INCOMPLETE_NAME_CHARACTER,
+  INVALID_NAME_ESCAPE,
   ERROR_SENTINEL
 };
 
@@ -69,10 +88,10 @@ typedef struct {
 typedef struct {
   uint32_t position, leading, line_end, content_end;
   uint32_t token_end, name_start, name_end;
-  uint32_t next_end, set_end, set_close_end;
+  uint32_t next_end, prefix_end, set_end, set_close_end;
   Class class;
   uint32_t lower_end, range_operator_end, upper_end;
-  uint8_t kind, next, lower_kind, upper_kind;
+  uint8_t kind, next, prefix_kind, lower_kind, upper_kind;
   uint8_t quoted, component_start, after_separator;
 } Scanner;
 
@@ -90,6 +109,16 @@ static bool whitespace(int32_t c) {
   return c == ' ' || c == '\t' || c == '\r';
 }
 
+static bool alphanumeric(int32_t c) {
+  return (c >= 'a' && c <= 'z') ||
+    (c >= 'A' && c <= 'Z') ||
+    (c >= '0' && c <= '9');
+}
+
+static bool name_character(int32_t c, bool first) {
+  return alphanumeric(c) || c == '_' || c == '.' || (!first && c == '-');
+}
+
 static void advance(Scanner *s, TSLexer *lexer) {
   lexer->advance(lexer, false);
   s->position++;
@@ -101,6 +130,20 @@ static bool emit(TSLexer *lexer, const bool *valid, enum Token token) {
   return true;
 }
 
+static enum Token
+absent(TSLexer *lexer, enum Token missing, enum Token incomplete) {
+  return lexer->eof(lexer) ? incomplete : missing;
+}
+
+// Include the next decoded character so edits invalidate the run.
+static bool invalid_run(Scanner *s, TSLexer *lexer, const bool *valid) {
+  do advance(s, lexer);
+  while (!lexer->eof(lexer) && lexer->lookahead == DECODE_ERROR);
+  lexer->mark_end(lexer);
+  if (!lexer->eof(lexer)) lexer->advance(lexer, false);
+  return emit(lexer, valid, INVALID_ENCODING);
+}
+
 static bool consume(
   Scanner *s,
   TSLexer *lexer,
@@ -108,16 +151,8 @@ static bool consume(
   enum Token token,
   uint32_t end
 ) {
-  if (!valid[token]) return false;
-  while (s->position < end) {
-    if (
-      lexer->eof(lexer) ||
-      lexer->lookahead == '\n' ||
-      lexer->lookahead == DECODE_ERROR
-    )
-      return false;
-    advance(s, lexer);
-  }
+  if (token == INVALID_ENCODING) return invalid_run(s, lexer, valid);
+  while (s->position < end) advance(s, lexer);
   lexer->mark_end(lexer);
   return emit(lexer, valid, token);
 }
@@ -134,13 +169,8 @@ static bool text_run(
   enum Token token,
   uint32_t end
 ) {
-  if (lexer->lookahead == DECODE_ERROR) return false;
-  while (
-    s->position < end &&
-    !lexer->eof(lexer) &&
-    lexer->lookahead != '\n' &&
-    lexer->lookahead != DECODE_ERROR
-  )
+  if (lexer->lookahead == DECODE_ERROR) return invalid_run(s, lexer, valid);
+  while (s->position < end && lexer->lookahead != DECODE_ERROR)
     advance(s, lexer);
   lexer->mark_end(lexer);
   return emit(lexer, valid, token);
@@ -178,9 +208,12 @@ static int32_t simple_escape(int32_t c) {
 static void quoted_escape(Cursor *r) {
   const int32_t c = r->lexer->lookahead;
   r->kind = QUOTED_ESCAPE;
-  if (r->raw >= r->limit || c == DECODE_ERROR) {
+  if (r->raw >= r->limit) {
+    r->c = INVALID_UNIT;
+    r->kind = absent(r->lexer, ENDED_QUOTED_ESCAPE, INCOMPLETE_QUOTED_ESCAPE);
+  } else if (c == DECODE_ERROR) {
     r->c = INVALID_UNIT;
-    r->kind = ERROR_SENTINEL;
+    r->kind = ESCAPE_PREFIX;
   } else if (c >= '0' && c <= '3') {
     int32_t value = 0;
     unsigned digits = 0;
@@ -194,12 +227,14 @@ static void quoted_escape(Cursor *r) {
     if (digits == 3) r->c = value;
     else {
       r->c = INVALID_UNIT;
-      r->kind = ERROR_SENTINEL;
+      r->kind = r->raw == r->limit
+        ? absent(r->lexer, ENDED_QUOTED_ESCAPE, INCOMPLETE_QUOTED_ESCAPE)
+        : ENDED_QUOTED_ESCAPE;
     }
   } else {
     raw_step(r);
     r->c = simple_escape(c);
-    if (r->c == INVALID_UNIT) r->kind = ERROR_SENTINEL;
+    if (r->c == INVALID_UNIT) r->kind = INVALID_QUOTED_ESCAPE;
   }
 }
 
@@ -212,7 +247,7 @@ static void step(Cursor *r) {
   raw_step(r);
   r->c = c;
   if (c == DECODE_ERROR) {
-    r->kind = ERROR_SENTINEL;
+    r->kind = INVALID_ENCODING;
     while (r->raw < r->limit && r->lexer->lookahead == DECODE_ERROR)
       raw_step(r);
   } else if (r->s->quoted && r->start < r->s->content_end && c == '\\')
@@ -369,7 +404,9 @@ static Item item(Cursor *r) {
   if (r->kind) a.kind = r->kind;
   step(r);
   if (c == '\\') {
-    if (r->c < 0) a.kind = ERROR_SENTINEL;
+    if (r->c == CURSOR_END)
+      a.kind = absent(r->lexer, ENDED_ESCAPE, INCOMPLETE_ESCAPE);
+    else if (r->c < 0) a.kind = ESCAPE_PREFIX;
     else {
       a.kind = ESCAPE;
       a.end = r->end;
@@ -391,18 +428,36 @@ static bool literal_boundary(const Cursor *r) {
   return c == '\\' || c == '*' || c == '?' || c == '/' || c == '[';
 }
 
+static void hold_prefix(Scanner *s, Cursor *r, uint32_t prefix_end) {
+  s->next = ESCAPE_PREFIX;
+  s->prefix_end = prefix_end;
+  s->prefix_kind = r->kind;
+  if (r->kind == ESCAPE_PREFIX) {
+    s->prefix_end = r->end;
+    s->prefix_kind = INVALID_ENCODING;
+    step(r);
+  }
+  s->next_end = r->end;
+}
+
 // Consume lookahead units fully so edits inside them invalidate the decision.
 static bool prepare_glob(Scanner *s, TSLexer *lexer, const bool *valid) {
   Cursor r = cursor(s, lexer, s->content_end);
   bool separator = r.c == '/';
   const uint32_t start = s->position;
   s->next_end = r.end;
-  if (r.c < 0 || (r.c == '!' && start == content_start(s))) return false;
+  if (r.c < 0) {
+    if (r.kind == ESCAPE_PREFIX) hold_prefix(s, &r, r.end);
+    else s->next = r.kind;
+  } else if (r.c == '!' && start == content_start(s))
+    s->next = NEGATIVE_PATTERN;
   else if (r.c == '?') s->next = SINGLE_CHARACTER;
   else if (r.c == '/') s->next = PATH_SEPARATOR;
   else if (r.c == '\\') {
     step(&r);
-    if (r.c < 0) return false;
+    if (r.c == CURSOR_END)
+      s->next = absent(lexer, ENDED_ESCAPE, INCOMPLETE_ESCAPE);
+    else if (r.c < 0) hold_prefix(s, &r, s->next_end);
     else {
       s->next = ESCAPE;
       s->next_end = r.end;
@@ -438,7 +493,7 @@ static bool prepare_glob(Scanner *s, TSLexer *lexer, const bool *valid) {
     s->component_start = false;
     return emit(lexer, valid, LITERAL);
   }
-  s->component_start = separator;
+  s->component_start = separator || s->next == NEGATIVE_PATTERN;
   if (separator) s->after_separator = true;
   return emit(lexer, valid, GLOB_START);
 }
@@ -485,6 +540,7 @@ static bool prepare_set(Scanner *s, TSLexer *lexer, const bool *valid) {
       if (s->next_end == start) {
         s->next = current.kind;
         s->next_end = current.end;
+        if (current.kind == ESCAPE_PREFIX) hold_prefix(s, &r, current.end);
       }
       break;
     }
@@ -519,7 +575,7 @@ static bool class_piece(Scanner *s, TSLexer *lexer, const bool *valid) {
     if (!next_class(&r, false, &s->class)) s->class.start = NONE;
     return emit(lexer, valid, COMPOUND_CLOSE);
   }
-  if (lexer->lookahead == DECODE_ERROR) return false;
+  if (lexer->lookahead == DECODE_ERROR) return invalid_run(s, lexer, valid);
   Cursor r = cursor(s, lexer, s->class.closing_start);
   const enum Token token = r.kind ? (enum Token)r.kind : CLASS_NAME;
   uint32_t end = r.end;
@@ -550,20 +606,26 @@ static bool range_piece(Scanner *s, TSLexer *lexer, const bool *valid) {
   );
 }
 
-static enum Token name_token(const Cursor *r) {
-  if (r->c < 0) return ERROR_SENTINEL;
+static enum Token name_token(const Cursor *r, bool first) {
+  if (r->c < 0) return (enum Token)r->kind;
+  if (!r->kind && r->c == '\r' && r->lexer->eof(r->lexer))
+    return INCOMPLETE_NAME_CHARACTER;
+  if (!name_character(r->c, first))
+    return r->kind == QUOTED_ESCAPE ? INVALID_NAME_ESCAPE
+                                    : INVALID_NAME_CHARACTER;
   return r->kind == QUOTED_ESCAPE ? QUOTED_ESCAPE : NAME_TEXT;
 }
 
 static bool name_piece(Scanner *s, TSLexer *lexer, const bool *valid) {
-  if (lexer->lookahead == DECODE_ERROR) return false;
+  if (lexer->lookahead == DECODE_ERROR) return invalid_run(s, lexer, valid);
   Cursor r = cursor(s, lexer, s->name_end);
-  const enum Token token = name_token(&r);
+  if (r.kind == ESCAPE_PREFIX) s->next = INVALID_ENCODING;
+  const enum Token token = name_token(&r, s->position == s->name_start);
   uint32_t end = r.end;
   lexer->mark_end(lexer);
-  if (token == NAME_TEXT) {
+  if (token == NAME_TEXT || token == INVALID_NAME_CHARACTER) {
     step(&r);
-    while (name_token(&r) == token) {
+    while (name_token(&r, false) == token) {
       end = r.end;
       lexer->mark_end(lexer);
       step(&r);
@@ -587,6 +649,21 @@ static bool prepare_attribute(Scanner *s, TSLexer *lexer, const bool *valid) {
   return emit(lexer, valid, ATTRIBUTE_START);
 }
 
+static bool end_line(Scanner *s, TSLexer *lexer, const bool *valid) {
+  enum Token token = END_OF_FILE;
+  if (!lexer->eof(lexer)) {
+    if (!valid[LINE_ENDING]) return false;
+    if (lexer->lookahead == '\r') advance(s, lexer);
+    if (lexer->lookahead != '\n') return false;
+    advance(s, lexer);
+    token = LINE_ENDING;
+  }
+  lexer->mark_end(lexer);
+  if (!emit(lexer, valid, token)) return false;
+  *s = (Scanner){0};
+  return true;
+}
+
 static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
   if (valid[ERROR_SENTINEL]) return false;
   if (valid[LINE_START]) return start_line(s, lexer, valid);
@@ -599,11 +676,19 @@ static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
     return start_content(s, lexer, valid);
   lexer->mark_end(lexer);
   if (
-    valid[SET_CLOSE] &&
-    s->position == s->set_end &&
-    s->set_close_end > s->set_end
-  )
-    return consume(s, lexer, valid, SET_CLOSE, s->set_close_end);
+    (valid[SET_CLOSE] ||
+      valid[MISSING_SET_CLOSE] ||
+      valid[INCOMPLETE_SET_CLOSE]) &&
+    s->position == s->set_end
+  ) {
+    if (s->set_close_end > s->set_end)
+      return consume(s, lexer, valid, SET_CLOSE, s->set_close_end);
+    return emit(
+      lexer,
+      valid,
+      absent(lexer, MISSING_SET_CLOSE, INCOMPLETE_SET_CLOSE)
+    );
+  }
   if (valid[ATTRIBUTE_START]) return prepare_attribute(s, lexer, valid);
   if (valid[ATTRIBUTE_END] && s->position == s->token_end)
     return emit(lexer, valid, ATTRIBUTE_END);
@@ -616,6 +701,15 @@ static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
     return emit(lexer, valid, VALUE_START);
   if (valid[VALUE_END] && s->position == s->token_end)
     return emit(lexer, valid, VALUE_END);
+  if (
+    (valid[MISSING_ATTRIBUTE_NAME] || valid[INCOMPLETE_ATTRIBUTE_NAME]) &&
+    s->position == s->name_end
+  )
+    return emit(
+      lexer,
+      valid,
+      absent(lexer, MISSING_ATTRIBUTE_NAME, INCOMPLETE_ATTRIBUTE_NAME)
+    );
   if (
     (valid[UNSET_MARKER] || valid[UNSPECIFIED_MARKER]) &&
     s->position < s->name_start
@@ -632,16 +726,22 @@ static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
   if (valid[ASSIGNMENT_OPERATOR] && lexer->lookahead == '=')
     return take(s, lexer, valid, ASSIGNMENT_OPERATOR);
   if (
-    valid[QUOTE_CLOSE] &&
+    (valid[QUOTE_CLOSE] ||
+      valid[MISSING_QUOTE_CLOSE] ||
+      valid[INCOMPLETE_QUOTE_CLOSE]) &&
     s->quoted &&
-    s->position == s->content_end &&
-    s->content_end < s->line_end
-  )
-    return take(s, lexer, valid, QUOTE_CLOSE);
+    s->position == s->content_end
+  ) {
+    if (s->content_end < s->line_end) return take(s, lexer, valid, QUOTE_CLOSE);
+    return emit(
+      lexer,
+      valid,
+      absent(lexer, MISSING_QUOTE_CLOSE, INCOMPLETE_QUOTE_CLOSE)
+    );
+  }
   if (valid[PATTERN_END] && s->position >= s->content_end)
     return emit(lexer, valid, PATTERN_END);
-  if (s->position >= s->line_end)
-    return lexer->eof(lexer) && emit(lexer, valid, END_OF_FILE);
+  if (s->position >= s->line_end) return end_line(s, lexer, valid);
   if (s->kind == COMMENT_START) {
     if (s->position == s->leading) return take(s, lexer, valid, COMMENT_MARKER);
     return text_run(s, lexer, valid, COMMENT_TEXT, s->line_end);
@@ -651,17 +751,15 @@ static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
   if (valid[MACRO_MARKER])
     return consume(s, lexer, valid, MACRO_MARKER, s->name_start);
   if (valid[SEPARATOR] || valid[LAYOUT]) {
-    if (!whitespace(lexer->lookahead)) return false;
+    if (!whitespace(lexer->lookahead))
+      return emit(lexer, valid, MISSING_ATTRIBUTE_SEPARATOR);
     while (s->position < s->line_end && whitespace(lexer->lookahead))
       advance(s, lexer);
     lexer->mark_end(lexer);
     return emit(lexer, valid, s->position < s->line_end ? SEPARATOR : LAYOUT);
   }
-  if (valid[GLOB_START] && s->position < s->content_end)
-    return prepare_glob(s, lexer, valid);
-  if (
-    (valid[SET_ITEM_START] || valid[SET_NEGATION]) && s->position < s->set_end
-  )
+  if (valid[GLOB_START]) return prepare_glob(s, lexer, valid);
+  if (valid[SET_ITEM_START] || valid[SET_NEGATION])
     return prepare_set(s, lexer, valid);
   if (s->class.start < s->position && s->position < s->class.end)
     return class_piece(s, lexer, valid);
@@ -673,6 +771,10 @@ static bool scan(Scanner *s, TSLexer *lexer, const bool *valid) {
   const enum Token token = (enum Token)s->next;
   if (token == CLASS_OPEN)
     return class_delimiter(s, lexer, valid, COMPOUND_OPEN);
+  if (token == ESCAPE_PREFIX) {
+    s->next = s->prefix_kind;
+    return consume(s, lexer, valid, token, s->prefix_end);
+  }
   return consume(s, lexer, valid, token, s->next_end);
 }
 

package/tree-sitter.json

@@ -12,7 +12,7 @@
     }
   ],
   "metadata": {
-    "version": "0.5.0",
+    "version": "0.6.0",
     "license": "MIT",
     "description": "Tree-sitter grammar for gitattributes.",
     "links": {