tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/runtime/stack.h

@@ -12,116 +12,119 @@ extern "C" {
 
 typedef struct Stack Stack;
 
-typedef unsigned int StackVersion;
-
+typedef unsigned StackVersion;
 #define STACK_VERSION_NONE ((StackVersion)-1)
 
 typedef struct {
   TreeArray trees;
   StackVersion version;
 } StackSlice;
-
 typedef Array(StackSlice) StackSliceArray;
 
 typedef struct {
-  bool stopped_at_error;
-  StackSliceArray slices;
-} StackPopResult;
-
-enum {
-  StackIterateNone,
-  StackIterateStop = 1 << 0,
-  StackIteratePop = 1 << 1,
-};
-
-typedef unsigned int StackIterateAction;
-
-typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state,
-                                                   TreeArray *trees,
-                                                   uint32_t tree_count,
-                                                   bool is_done,
-                                                   bool is_pending);
-
-/*
- *  Create a parse stack.
- */
-Stack *ts_stack_new();
-
-/*
- *  Release any resources reserved by a parse stack.
- */
+  Length position;
+  unsigned depth;
+  TSStateId state;
+} StackSummaryEntry;
+typedef Array(StackSummaryEntry) StackSummary;
+
+// Create a stack.
+Stack *ts_stack_new(TreePool *);
+
+// Release the memory reserved for a given stack.
 void ts_stack_delete(Stack *);
 
-/*
- *  Get the stack's current number of versions.
- */
+// Get the stack's current number of versions.
 uint32_t ts_stack_version_count(const Stack *);
 
-/*
- *  Get the state at the top of the given version of the stack. If the stack is
- *  empty, this returns the initial state (0).
- */
-TSStateId ts_stack_top_state(const Stack *, StackVersion);
+// Get the state at the top of the given version of the stack. If the stack is
+// empty, this returns the initial state, 0.
+TSStateId ts_stack_state(const Stack *, StackVersion);
 
-unsigned ts_stack_push_count(const Stack *, StackVersion);
+// Get the last external token associated with a given version of the stack.
+Tree *ts_stack_last_external_token(const Stack *, StackVersion);
 
-void ts_stack_decrease_push_count(Stack *, StackVersion, unsigned);
+// Set the last external token associated with a given version of the stack.
+void ts_stack_set_last_external_token(Stack *, StackVersion, Tree *);
 
-const TSExternalTokenState *ts_stack_external_token_state(const Stack *, StackVersion);
+// Get the position of the given version of the stack within the document.
+Length ts_stack_position(const Stack *, StackVersion);
 
-void ts_stack_set_external_token_state(Stack *, StackVersion, const TSExternalTokenState *);
+// Push a tree and state onto the given version of the stack.
+//
+// This transfers ownership of the tree to the Stack. Callers that
+// need to retain ownership of the tree for their own purposes should
+// first retain the tree.
+void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId);
 
-/*
- *  Get the position at the top of the given version of the stack. If the stack
- *  is empty, this returns zero.
- */
-Length ts_stack_top_position(const Stack *, StackVersion);
+// Pop the given number of entries from the given version of the stack. This
+// operation can increase the number of stack versions by revealing multiple
+// versions which had previously been merged. It returns an array that
+// specifies the index of each revealed version and the trees that were
+// removed from that version.
+StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
 
-/*
- *  Push a tree and state onto the given head of the stack. This could cause
- *  the version to merge with an existing version.
- */
-bool ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId);
+// Remove an error at the top of the given version of the stack.
+TreeArray ts_stack_pop_error(Stack *, StackVersion);
 
-/*
- *  Pop the given number of entries from the given version of the stack. This
- *  operation can increase the number of stack versions by revealing multiple
- *  versions which had previously been merged. It returns a struct that
- *  indicates the index of each revealed version and the trees removed from that
- *  version.
- */
-StackPopResult ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
+// Remove any pending trees from the top of the given version of the stack.
+StackSliceArray ts_stack_pop_pending(Stack *, StackVersion);
 
-StackPopResult ts_stack_iterate(Stack *, StackVersion, StackIterateCallback,
-                                void *);
+// Remove any all trees from the given version of the stack.
+StackSliceArray ts_stack_pop_all(Stack *, StackVersion);
 
-StackPopResult ts_stack_pop_pending(Stack *, StackVersion);
+// Get the maximum number of tree nodes reachable from this version of the stack
+// since the last error was detected.
+unsigned ts_stack_node_count_since_error(const Stack *, StackVersion);
 
-StackPopResult ts_stack_pop_all(Stack *, StackVersion);
+int ts_stack_dynamic_precedence(Stack *, StackVersion);
 
-ErrorStatus ts_stack_error_status(const Stack *, StackVersion);
+// Compute a summary of all the parse states near the top of the given
+// version of the stack and store the summary for later retrieval.
+void ts_stack_record_summary(Stack *, StackVersion, unsigned max_depth);
 
+// Retrieve a summary of all the parse states near the top of the
+// given version of the stack.
+StackSummary *ts_stack_get_summary(Stack *, StackVersion);
+
+// Get the total cost of all errors on the given version of the stack.
+unsigned ts_stack_error_cost(const Stack *, StackVersion version);
+
+// Determine whether the given two stack versions can be merged.
 bool ts_stack_merge(Stack *, StackVersion, StackVersion);
 
+// Merge the given two stack versions if possible, returning true
+// if they were successfully merged and false otherwise.
+bool ts_stack_can_merge(Stack *, StackVersion, StackVersion);
+
+TSSymbol ts_stack_resume(Stack *, StackVersion);
+
+void ts_stack_pause(Stack *, StackVersion, TSSymbol);
+
 void ts_stack_halt(Stack *, StackVersion);
 
-bool ts_stack_is_halted(Stack *, StackVersion);
+bool ts_stack_is_active(const Stack *, StackVersion);
+
+bool ts_stack_is_paused(const Stack *, StackVersion);
+
+bool ts_stack_is_halted(const Stack *, StackVersion);
 
 void ts_stack_renumber_version(Stack *, StackVersion, StackVersion);
 
+void ts_stack_swap_versions(Stack *, StackVersion, StackVersion);
+
 StackVersion ts_stack_copy_version(Stack *, StackVersion);
 
-/*
- *  Remove the given version from the stack.
- */
+// Remove the given version from the stack.
 void ts_stack_remove_version(Stack *, StackVersion);
 
-/*
- *  Remove all entries from the stack.
- */
 void ts_stack_clear(Stack *);
 
-bool ts_stack_print_dot_graph(Stack *, const char **, FILE *);
+bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *);
+
+typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t);
+
+void ts_stack_iterate(Stack *, StackVersion, StackIterateCallback, void *);
 
 #ifdef __cplusplus
 }

data/ext/tree-sitter/tree-sitter/src/runtime/string_input.c

@@ -8,7 +8,7 @@ typedef struct {
   uint32_t length;
 } TSStringInput;
 
-const char *ts_string_input_read(void *payload, uint32_t *bytes_read) {
+static const char *ts_string_input__read(void *payload, uint32_t *bytes_read) {
   TSStringInput *input = (TSStringInput *)payload;
   if (input->position >= input->length) {
     *bytes_read = 0;
@@ -20,7 +20,7 @@ const char *ts_string_input_read(void *payload, uint32_t *bytes_read) {
   return input->string + previous_position;
 }
 
-int ts_string_input_seek(void *payload, uint32_t character, uint32_t byte) {
+static int ts_string_input__seek(void *payload, uint32_t byte, TSPoint _) {
   TSStringInput *input = (TSStringInput *)payload;
   input->position = byte;
   return (byte < input->length);
@@ -32,19 +32,13 @@ TSInput ts_string_input_make(const char *string) {
 
 TSInput ts_string_input_make_with_length(const char *string, uint32_t length) {
   TSStringInput *input = ts_malloc(sizeof(TSStringInput));
-  if (!input)
-    goto error;
-
   input->string = string;
   input->position = 0;
   input->length = length;
   return (TSInput){
     .payload = input,
-    .read = ts_string_input_read,
-    .seek = ts_string_input_seek,
+    .read = ts_string_input__read,
+    .seek = ts_string_input__seek,
     .encoding = TSInputEncodingUTF8,
   };
-
-error:
-  return (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 };
 }

data/ext/tree-sitter/tree-sitter/src/runtime/tree.c

@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <ctype.h>
 #include <limits.h>
 #include <stdbool.h>
 #include <string.h>
@@ -6,30 +7,45 @@
 #include "runtime/alloc.h"
 #include "runtime/tree.h"
 #include "runtime/length.h"
+#include "runtime/language.h"
 #include "runtime/error_costs.h"
 
 TSStateId TS_TREE_STATE_NONE = USHRT_MAX;
 
-Tree *ts_tree_make_leaf(TSSymbol sym, Length padding, Length size,
-                          TSSymbolMetadata metadata) {
-  Tree *result = ts_malloc(sizeof(Tree));
-  *result = (Tree){
-    .ref_count = 1,
-    .symbol = sym,
-    .size = size,
-    .child_count = 0,
-    .visible_child_count = 0,
-    .named_child_count = 0,
-    .children = NULL,
-    .padding = padding,
-    .visible = metadata.visible,
-    .named = metadata.named,
-    .has_changes = false,
-    .first_leaf.symbol = sym,
-  };
-  return result;
+// ExternalTokenState
+
+void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) {
+  self->length = length;
+  if (length > sizeof(self->short_data)) {
+    self->long_data = ts_malloc(length);
+    memcpy(self->long_data, content, length);
+  } else {
+    memcpy(self->short_data, content, length);
+  }
+}
+
+void ts_external_token_state_delete(TSExternalTokenState *self) {
+  if (self->length > sizeof(self->short_data)) {
+    ts_free(self->long_data);
+  }
 }
 
+const char *ts_external_token_state_data(const TSExternalTokenState *self) {
+  if (self->length > sizeof(self->short_data)) {
+    return self->long_data;
+  } else {
+    return self->short_data;
+  }
+}
+
+bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalTokenState *b) {
+  return a == b ||
+    (a->length == b->length &&
+     memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0);
+}
+
+// TreeArray
+
 bool ts_tree_array_copy(TreeArray self, TreeArray *dest) {
   Tree **contents = NULL;
   if (self.capacity > 0) {
@@ -45,43 +61,11 @@ bool ts_tree_array_copy(TreeArray self, TreeArray *dest) {
   return true;
 }
 
-void ts_tree_array_delete(TreeArray *self) {
-  for (uint32_t i = 0; i < self->size; i++)
-    ts_tree_release(self->contents[i]);
-  array_delete(self);
-}
-
-uint32_t ts_tree_array_essential_count(const TreeArray *self) {
-  uint32_t result = 0;
+void ts_tree_array_delete(TreePool *pool, TreeArray *self) {
   for (uint32_t i = 0; i < self->size; i++) {
-    Tree *tree = self->contents[i];
-    if (!tree->extra && tree->symbol != ts_builtin_sym_error)
-      result++;
-  }
-  return result;
-}
-
-TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) {
-  TreeArray result = array_new();
-  if (self->size == 0 || remove_count == 0) return result;
-
-  uint32_t count = 0;
-  uint32_t split_index = self->size - 1;
-  for (; split_index + 1 > 0; split_index--) {
-    Tree *tree = self->contents[split_index];
-    if (!tree->extra) {
-      count++;
-      if (count == remove_count) break;
-    }
-  }
-
-  array_grow(&result, self->size - split_index);
-  for (uint32_t i = split_index; i < self->size; i++) {
-    array_push(&result, self->contents[i]);
+    ts_tree_release(pool, self->contents[i]);
   }
-
-  self->size = split_index;
-  return result;
+  array_delete(self);
 }
 
 TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) {
@@ -95,162 +79,331 @@ TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) {
   }
 
   self->size = i + 1;
-  array_reverse(&result);
+  ts_tree_array_reverse(&result);
   return result;
 }
 
-Tree *ts_tree_make_error(Length size, Length padding, char lookahead_char) {
-  Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size,
-                                     (TSSymbolMetadata){
-                                       .visible = true, .named = true,
-                                     });
+void ts_tree_array_reverse(TreeArray *self) {
+  for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) {
+    size_t reverse_index = self->size - 1 - i;
+    Tree *swap = self->contents[i];
+    self->contents[i] = self->contents[reverse_index];
+    self->contents[reverse_index] = swap;
+  }
+}
+
+// TreePool
+
+static const uint32_t MAX_TREE_POOL_SIZE = 1024;
+
+void ts_tree_pool_init(TreePool *self) {
+  array_init(&self->free_trees);
+  array_init(&self->tree_stack);
+}
+
+void ts_tree_pool_delete(TreePool *self) {
+  if (self->free_trees.contents) {
+    for (unsigned i = 0; i < self->free_trees.size; i++) {
+      ts_free(self->free_trees.contents[i]);
+    }
+    array_delete(&self->free_trees);
+  }
+  if (self->tree_stack.contents) array_delete(&self->tree_stack);
+}
+
+Tree *ts_tree_pool_allocate(TreePool *self) {
+  if (self->free_trees.size > 0) {
+    return array_pop(&self->free_trees);
+  } else {
+    return ts_malloc(sizeof(Tree));
+  }
+}
+
+void ts_tree_pool_free(TreePool *self, Tree *tree) {
+  if (self->free_trees.size < MAX_TREE_POOL_SIZE) {
+    array_push(&self->free_trees, tree);
+  } else {
+    ts_free(tree);
+  }
+}
+
+// Tree
+
+Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length size, const TSLanguage *language) {
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
+  Tree *result = ts_tree_pool_allocate(pool);
+  *result = (Tree){
+    .ref_count = 1,
+    .symbol = symbol,
+    .size = size,
+    .visible_child_count = 0,
+    .named_child_count = 0,
+    .alias_sequence_id = 0,
+    .padding = padding,
+    .visible = metadata.visible,
+    .named = metadata.named,
+    .node_count = 1,
+    .has_changes = false,
+    .first_leaf = {
+      .symbol = symbol,
+      .lex_mode = {0, 0},
+    },
+    .has_external_tokens = false,
+  };
+  return result;
+}
+
+Tree *ts_tree_make_error(TreePool *pool, Length size, Length padding, int32_t lookahead_char,
+                         const TSLanguage *language) {
+  Tree *result = ts_tree_make_leaf(pool, ts_builtin_sym_error, padding, size, language);
   result->fragile_left = true;
   result->fragile_right = true;
   result->lookahead_char = lookahead_char;
   return result;
 }
 
-Tree *ts_tree_make_copy(Tree *self) {
-  Tree *result = ts_malloc(sizeof(Tree));
+Tree *ts_tree_make_copy(TreePool *pool, Tree *self) {
+  Tree *result = ts_tree_pool_allocate(pool);
   *result = *self;
   result->ref_count = 1;
   return result;
 }
 
-void ts_tree_assign_parents(Tree *self, TreePath *path) {
-  array_clear(path);
-  array_push(path, ((TreePathEntry){self, length_zero(), 0}));
-  while (path->size > 0) {
-    Tree *tree = array_pop(path).tree;
+static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language) {
+  Tree *tree = self;
+  for (unsigned i = 0; i < count; i++) {
+    if (tree->ref_count > 1 || tree->children.size != 2) break;
+
+    Tree *child = tree->children.contents[0];
+    if (
+      child->ref_count > 1 ||
+      child->children.size != 2 ||
+      child->symbol != tree->symbol
+    ) break;
+
+    Tree *grandchild = child->children.contents[0];
+    if (
+      grandchild->ref_count > 1 ||
+      grandchild->children.size != 2 ||
+      grandchild->symbol != tree->symbol
+    ) break;
+
+    tree->children.contents[0] = grandchild;
+    grandchild->context.parent = tree;
+    grandchild->context.index = -1;
+
+    child->children.contents[0] = grandchild->children.contents[1];
+    child->children.contents[0]->context.parent = child;
+    child->children.contents[0]->context.index = -1;
+
+    grandchild->children.contents[1] = child;
+    grandchild->children.contents[1]->context.parent = grandchild;
+    grandchild->children.contents[1]->context.index = -1;
+
+    tree = grandchild;
+  }
+
+  while (tree != self) {
+    tree = tree->context.parent;
+    Tree *child = tree->children.contents[0];
+    Tree *grandchild = child->children.contents[1];
+    ts_tree_set_children(grandchild, &grandchild->children, language);
+    ts_tree_set_children(child, &child->children, language);
+    ts_tree_set_children(tree, &tree->children, language);
+  }
+}
+
+void ts_tree__balance(Tree *self, const TSLanguage *language) {
+  if (self->children.contents[0]->repeat_depth > self->children.contents[1]->repeat_depth) {
+    unsigned n = self->children.contents[0]->repeat_depth - self->children.contents[1]->repeat_depth;
+    for (unsigned i = n / 2; i > 0; i /= 2) {
+      ts_tree__compress(self, i, language);
+      n -= i;
+    }
+  }
+}
+
+void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *language) {
+  self->context.parent = NULL;
+  array_clear(&pool->tree_stack);
+  array_push(&pool->tree_stack, self);
+  while (pool->tree_stack.size > 0) {
+    Tree *tree = array_pop(&pool->tree_stack);
+
+    if (tree->repeat_depth > 0) {
+      ts_tree__balance(tree, language);
+    }
+
     Length offset = length_zero();
-    for (uint32_t i = 0; i < tree->child_count; i++) {
-      Tree *child = tree->children[i];
-      if (child->context.parent != tree || child->context.index != i) {
+    const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id);
+    uint32_t non_extra_index = 0;
+    bool earlier_child_was_changed = false;
+    for (uint32_t i = 0; i < tree->children.size; i++) {
+      Tree *child = tree->children.contents[i];
+      if (earlier_child_was_changed || child->context.parent != tree || child->context.index != i) {
+        earlier_child_was_changed = true;
         child->context.parent = tree;
         child->context.index = i;
         child->context.offset = offset;
-        array_push(path, ((TreePathEntry){child, length_zero(), 0}));
+        if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) {
+          TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]);
+          child->context.alias_symbol = alias_sequence[non_extra_index];
+          child->context.alias_is_named = metadata.named;
+        } else {
+          child->context.alias_symbol = 0;
+          child->context.alias_is_named = false;
+        }
+        array_push(&pool->tree_stack, child);
       }
       offset = length_add(offset, ts_tree_total_size(child));
+      if (!child->extra) non_extra_index++;
     }
   }
 }
 
+void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *language) {
+  if (self->children.size > 0 && children->contents != self->children.contents) {
+    array_delete(&self->children);
+  }
 
-void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
-  if (self->child_count > 0)
-    ts_free(self->children);
-
-  self->children = children;
-  self->child_count = child_count;
+  self->children = *children;
   self->named_child_count = 0;
   self->visible_child_count = 0;
   self->error_cost = 0;
+  self->repeat_depth = 0;
+  self->node_count = 1;
   self->has_external_tokens = false;
-  self->has_external_token_state = false;
+  self->dynamic_precedence = 0;
+
+  uint32_t non_extra_index = 0;
+  const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id);
 
-  for (uint32_t i = 0; i < child_count; i++) {
-    Tree *child = children[i];
+  for (uint32_t i = 0; i < self->children.size; i++) {
+    Tree *child = self->children.contents[i];
 
     if (i == 0) {
       self->padding = child->padding;
       self->size = child->size;
+      self->bytes_scanned = child->bytes_scanned;
     } else {
+      uint32_t bytes_scanned = ts_tree_total_bytes(self) + child->bytes_scanned;
+      if (bytes_scanned > self->bytes_scanned) self->bytes_scanned = bytes_scanned;
       self->size = length_add(self->size, ts_tree_total_size(child));
     }
 
-    self->error_cost += child->error_cost;
+    if (child->symbol != ts_builtin_sym_error_repeat) {
+      self->error_cost += child->error_cost;
+    }
+    self->dynamic_precedence += child->dynamic_precedence;
+    self->node_count += child->node_count;
 
-    if (child->visible) {
+    if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) {
       self->visible_child_count++;
-      if (child->named)
+      if (ts_language_symbol_metadata(language, alias_sequence[non_extra_index]).named) {
         self->named_child_count++;
-    } else if (child->child_count > 0) {
+      }
+    } else if (child->visible) {
+      self->visible_child_count++;
+      if (child->named) self->named_child_count++;
+    } else if (child->children.size > 0) {
       self->visible_child_count += child->visible_child_count;
       self->named_child_count += child->named_child_count;
     }
 
     if (child->has_external_tokens) self->has_external_tokens = true;
-    if (child->has_external_token_state) self->has_external_token_state = true;
 
     if (child->symbol == ts_builtin_sym_error) {
       self->fragile_left = self->fragile_right = true;
       self->parse_state = TS_TREE_STATE_NONE;
     }
+
+    if (!child->extra) non_extra_index++;
   }
 
-  if (self->symbol == ts_builtin_sym_error) {
-    self->error_cost += ERROR_COST_PER_SKIPPED_CHAR * self->size.chars +
+  if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) {
+    self->error_cost += ERROR_COST_PER_RECOVERY +
+                        ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes +
                         ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row;
-    for (uint32_t i = 0; i < child_count; i++)
-      if (!self->children[i]->extra)
+    for (uint32_t i = 0; i < self->children.size; i++) {
+      Tree *child = self->children.contents[i];
+      if (child->extra) continue;
+      if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue;
+      if (child->visible) {
         self->error_cost += ERROR_COST_PER_SKIPPED_TREE;
+      } else {
+        self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count;
+      }
+    }
   }
 
-  if (child_count > 0) {
-    self->first_leaf = children[0]->first_leaf;
-    if (children[0]->fragile_left)
-      self->fragile_left = true;
-    if (children[child_count - 1]->fragile_right)
-      self->fragile_right = true;
+  if (self->children.size > 0) {
+    Tree *first_child = self->children.contents[0];
+    Tree *last_child = self->children.contents[self->children.size - 1];
+    self->first_leaf = first_child->first_leaf;
+    if (first_child->fragile_left) self->fragile_left = true;
+    if (last_child->fragile_right) self->fragile_right = true;
+    if (
+      self->children.size == 2 &&
+      !self->visible && !self->named &&
+      first_child->symbol == self->symbol &&
+      last_child->symbol == self->symbol
+    ) {
+      if (first_child->repeat_depth > last_child->repeat_depth) {
+        self->repeat_depth = first_child->repeat_depth + 1;
+      } else {
+        self->repeat_depth = last_child->repeat_depth + 1;
+      }
+    }
   }
 }
 
-Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count,
-                          Tree **children, TSSymbolMetadata metadata) {
-  Tree *result =
-    ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata);
-  ts_tree_set_children(result, child_count, children);
+Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children,
+                        unsigned alias_sequence_id, const TSLanguage *language) {
+  Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language);
+  result->alias_sequence_id = alias_sequence_id;
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
+    result->fragile_left = true;
+    result->fragile_right = true;
+  }
+  ts_tree_set_children(result, children, language);
   return result;
 }
 
-Tree *ts_tree_make_error_node(TreeArray *children) {
-  for (uint32_t i = 0; i < children->size; i++) {
-    Tree *child = children->contents[i];
-    if (child->symbol == ts_builtin_sym_error && child->child_count > 0) {
-      array_splice(children, i, 1, child->child_count, child->children);
-      i += child->child_count - 1;
-      for (uint32_t j = 0; j < child->child_count; j++)
-        ts_tree_retain(child->children[j]);
-      ts_tree_release(child);
-    }
-  }
-
-  Tree *result = ts_tree_make_node(
-    ts_builtin_sym_error, children->size, children->contents,
-    (TSSymbolMetadata){.extra = false, .visible = true, .named = true });
+Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) {
+  return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language);
+}
 
-  result->fragile_left = true;
-  result->fragile_right = true;
+Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) {
+  Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language);
+  result->is_missing = true;
+  result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
   return result;
 }
 
 void ts_tree_retain(Tree *self) {
   assert(self->ref_count > 0);
   self->ref_count++;
+  assert(self->ref_count != 0);
 }
 
-void ts_tree_release(Tree *self) {
-  if (!self)
-    return;
-
-recur:
-  assert(self->ref_count > 0);
-  self->ref_count--;
-
-  if (self->ref_count == 0) {
-    if (self->child_count > 0) {
-      for (uint32_t i = 0; i < self->child_count - 1; i++)
-        ts_tree_release(self->children[i]);
-      Tree *last_child = self->children[self->child_count - 1];
-      ts_free(self->children);
-      ts_free(self);
-
-      self = last_child;
-      goto recur;
+void ts_tree_release(TreePool *pool, Tree *self) {
+  array_clear(&pool->tree_stack);
+  array_push(&pool->tree_stack, self);
+  while (pool->tree_stack.size > 0) {
+    Tree *tree = array_pop(&pool->tree_stack);
+    assert(tree->ref_count > 0);
+    tree->ref_count--;
+    if (tree->ref_count == 0) {
+      if (tree->children.size > 0) {
+        for (uint32_t i = 0; i < tree->children.size; i++) {
+          array_push(&pool->tree_stack, tree->children.contents[i]);
+        }
+        array_delete(&tree->children);
+      } else if (tree->has_external_tokens) {
+        ts_external_token_state_delete(&tree->external_token_state);
+      }
+      ts_tree_pool_free(pool, tree);
     }
-
-    ts_free(self);
   }
 }
 
@@ -275,43 +428,25 @@ uint32_t ts_tree_end_column(const Tree *self) {
 
 bool ts_tree_eq(const Tree *self, const Tree *other) {
   if (self) {
-    if (!other)
-      return false;
+    if (!other) return false;
   } else {
     return !other;
   }
 
-  if (self->symbol != other->symbol)
-    return false;
-  if (self->visible != other->visible)
-    return false;
-  if (self->named != other->named)
-    return false;
-  if (self->symbol == ts_builtin_sym_error)
-    return self->lookahead_char == other->lookahead_char;
-  if (self->child_count != other->child_count)
-    return false;
-  if (self->visible_child_count != other->visible_child_count)
-    return false;
-  if (self->named_child_count != other->named_child_count)
-    return false;
-  for (uint32_t i = 0; i < self->child_count; i++)
-    if (!ts_tree_eq(self->children[i], other->children[i]))
-      return false;
-  return true;
-}
-
-bool ts_tree_tokens_eq(const Tree *self, const Tree *other) {
-  if (self->child_count > 0 || other->child_count > 0) return false;
   if (self->symbol != other->symbol) return false;
+  if (self->visible != other->visible) return false;
+  if (self->named != other->named) return false;
   if (self->padding.bytes != other->padding.bytes) return false;
   if (self->size.bytes != other->size.bytes) return false;
-  if (self->extra != other->extra) return false;
-  if (self->has_external_token_state) {
-    if (!other->has_external_token_state) return false;
-    if (!ts_external_token_state_eq(&self->external_token_state, &other->external_token_state)) return false;
-  } else {
-    if (other->has_external_token_state) return false;
+  if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
+  if (self->children.size != other->children.size) return false;
+  if (self->visible_child_count != other->visible_child_count) return false;
+  if (self->named_child_count != other->named_child_count) return false;
+
+  for (uint32_t i = 0; i < self->children.size; i++) {
+    if (!ts_tree_eq(self->children.contents[i], other->children.contents[i])) {
+      return false;
+    }
   }
   return true;
 }
@@ -321,13 +456,13 @@ int ts_tree_compare(const Tree *left, const Tree *right) {
     return -1;
   if (right->symbol < left->symbol)
     return 1;
-  if (left->child_count < right->child_count)
+  if (left->children.size < right->children.size)
     return -1;
-  if (right->child_count < left->child_count)
+  if (right->children.size < left->children.size)
     return 1;
-  for (uint32_t i = 0; i < left->child_count; i++) {
-    Tree *left_child = left->children[i];
-    Tree *right_child = right->children[i];
+  for (uint32_t i = 0; i < left->children.size; i++) {
+    Tree *left_child = left->children.contents[i];
+    Tree *right_child = right->children.contents[i];
     switch (ts_tree_compare(left_child, right_child)) {
       case -1:
         return -1;
@@ -340,10 +475,28 @@ int ts_tree_compare(const Tree *left, const Tree *right) {
   return 0;
 }
 
-static inline long min(long a, long b) {
+static inline long min_byte(long a, long b) {
   return a <= b ? a : b;
 }
 
+bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) {
+  if (edit_byte_offset >= self->bytes_scanned) return false;
+  self->has_changes = true;
+  if (self->children.size > 0) {
+    uint32_t child_start_byte = 0;
+    for (uint32_t i = 0; i < self->children.size; i++) {
+      Tree *child = self->children.contents[i];
+      if (child_start_byte > edit_byte_offset) break;
+      ts_tree_invalidate_lookahead(child, edit_byte_offset - child_start_byte);
+      child_start_byte += ts_tree_total_bytes(child);
+    }
+  }
+  return true;
+}
+
+static inline TSPoint ts_tree_total_extent(const Tree *self) {
+  return point_add(self->padding.extent, self->size.extent);
+}
 
 void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
   uint32_t old_end_byte = edit->start_byte + edit->bytes_removed;
@@ -356,14 +509,12 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
   self->has_changes = true;
 
   if (edit->start_byte < self->padding.bytes) {
-    length_set_unknown_chars(&self->padding);
     if (self->padding.bytes >= old_end_byte) {
       uint32_t trailing_padding_bytes = self->padding.bytes - old_end_byte;
       TSPoint trailing_padding_extent = point_sub(self->padding.extent, old_end_point);
       self->padding.bytes = new_end_byte + trailing_padding_bytes;
       self->padding.extent = point_add(new_end_point, trailing_padding_extent);
     } else {
-      length_set_unknown_chars(&self->size);
       uint32_t removed_content_bytes = old_end_byte - self->padding.bytes;
       TSPoint removed_content_extent = point_sub(old_end_point, self->padding.extent);
       self->size.bytes = self->size.bytes - removed_content_bytes;
@@ -372,11 +523,9 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
       self->padding.extent = new_end_point;
     }
   } else if (edit->start_byte == self->padding.bytes && edit->bytes_removed == 0) {
-    length_set_unknown_chars(&self->padding);
     self->padding.bytes = self->padding.bytes + edit->bytes_added;
     self->padding.extent = point_add(self->padding.extent, edit->extent_added);
   } else {
-    length_set_unknown_chars(&self->size);
     uint32_t trailing_content_bytes = ts_tree_total_bytes(self) - old_end_byte;
     TSPoint trailing_content_extent = point_sub(ts_tree_total_extent(self), old_end_point);
     self->size.bytes = new_end_byte + trailing_content_bytes - self->padding.bytes;
@@ -387,37 +536,35 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
   long remaining_bytes_to_delete = 0;
   TSPoint remaining_extent_to_delete = {0, 0};
   Length child_left, child_right = length_zero();
-  for (uint32_t i = 0; i < self->child_count; i++) {
-    Tree *child = self->children[i];
+  for (uint32_t i = 0; i < self->children.size; i++) {
+    Tree *child = self->children.contents[i];
     child_left = child_right;
+    child_right = length_add(child_left, ts_tree_total_size(child));
 
-    if (!found_first_child) {
-      child_right = length_add(child_left, ts_tree_total_size(child));
-      if (child_right.bytes >= edit->start_byte) {
-        found_first_child = true;
-        TSInputEdit child_edit = {
-          .start_byte = edit->start_byte - child_left.bytes,
-          .bytes_added = edit->bytes_added,
-          .bytes_removed = edit->bytes_removed,
-          .start_point = point_sub(edit->start_point, child_left.extent),
-          .extent_added = edit->extent_added,
-          .extent_removed = edit->extent_removed,
-        };
-
-        if (old_end_byte > child_right.bytes) {
-          child_edit.bytes_removed = child_right.bytes - edit->start_byte;
-          child_edit.extent_removed = point_sub(child_right.extent, edit->start_point);
-          remaining_bytes_to_delete = old_end_byte - child_right.bytes;
-          remaining_extent_to_delete = point_sub(old_end_point, child_right.extent);
-        }
+    if (!found_first_child && child_right.bytes >= edit->start_byte) {
+      found_first_child = true;
+      TSInputEdit child_edit = {
+        .start_byte = edit->start_byte - child_left.bytes,
+        .bytes_added = edit->bytes_added,
+        .bytes_removed = edit->bytes_removed,
+        .start_point = point_sub(edit->start_point, child_left.extent),
+        .extent_added = edit->extent_added,
+        .extent_removed = edit->extent_removed,
+      };
 
-        ts_tree_edit(child, &child_edit);
+      if (old_end_byte > child_right.bytes) {
+        child_edit.bytes_removed = child_right.bytes - edit->start_byte;
+        child_edit.extent_removed = point_sub(child_right.extent, edit->start_point);
+        remaining_bytes_to_delete = old_end_byte - child_right.bytes;
+        remaining_extent_to_delete = point_sub(old_end_point, child_right.extent);
       }
+
+      ts_tree_edit(child, &child_edit);
     } else if (remaining_bytes_to_delete > 0) {
       TSInputEdit child_edit = {
         .start_byte = 0,
         .bytes_added = 0,
-        .bytes_removed = min(remaining_bytes_to_delete, ts_tree_total_bytes(child)),
+        .bytes_removed = min_byte(remaining_bytes_to_delete, ts_tree_total_bytes(child)),
         .start_point = {0, 0},
         .extent_added = {0, 0},
         .extent_removed = point_min(remaining_extent_to_delete, ts_tree_total_size(child).extent),
@@ -425,6 +572,8 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
       remaining_bytes_to_delete -= child_edit.bytes_removed;
       remaining_extent_to_delete = point_sub(remaining_extent_to_delete, child_edit.extent_removed);
       ts_tree_edit(child, &child_edit);
+    } else {
+      ts_tree_invalidate_lookahead(child, edit->start_byte - child_left.bytes);
     }
 
     child_right = length_add(child_left, ts_tree_total_size(child));
@@ -432,79 +581,81 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
   }
 }
 
-const TSExternalTokenState *ts_tree_last_external_token_state(const Tree *tree) {
-  while (tree->child_count > 0) {
-    for (uint32_t i = tree->child_count - 1; i + 1 > 0; i--) {
-      Tree *child = tree->children[i];
-      if (child->has_external_token_state) {
+Tree *ts_tree_last_external_token(Tree *tree) {
+  if (!tree->has_external_tokens) return NULL;
+  while (tree->children.size > 0) {
+    for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) {
+      Tree *child = tree->children.contents[i];
+      if (child->has_external_tokens) {
         tree = child;
         break;
-      } else if (child->has_external_tokens) {
-        return NULL;
       }
     }
   }
-  return &tree->external_token_state;
+  return tree;
 }
 
 static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) {
   if (c == 0)
     return snprintf(s, n, "EOF");
+  if (c == -1)
+    return snprintf(s, n, "INVALID");
   else if (c == '\n')
     return snprintf(s, n, "'\\n'");
   else if (c == '\t')
     return snprintf(s, n, "'\\t'");
   else if (c == '\r')
     return snprintf(s, n, "'\\r'");
-  else if (c < 128)
+  else if (0 < c && c < 128 && isprint(c))
     return snprintf(s, n, "'%c'", c);
   else
     return snprintf(s, n, "%d", c);
 }
 
-static size_t ts_tree__write_to_string(const Tree *self,
-                                       const TSLanguage *language, char *string,
-                                       size_t limit, bool is_root,
+static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *language,
+                                       char *string, size_t limit, bool is_root,
                                        bool include_all) {
-  if (!self)
-    return snprintf(string, limit, "(NULL)");
+  if (!self) return snprintf(string, limit, "(NULL)");
 
   char *cursor = string;
   char **writer = (limit > 0) ? &cursor : &string;
-  bool visible = include_all || is_root || (self->visible && self->named);
-
-  if (visible && !is_root)
+  bool visible =
+    include_all ||
+    is_root ||
+    self->is_missing ||
+    (self->visible && self->named) ||
+    self->context.alias_is_named;
+
+  if (visible && !is_root) {
     cursor += snprintf(*writer, limit, " ");
+  }
 
   if (visible) {
-    if (self->symbol == ts_builtin_sym_error && self->child_count == 0 &&
-        self->size.chars > 0) {
+    if (self->symbol == ts_builtin_sym_error && self->children.size == 0 && self->size.bytes > 0) {
       cursor += snprintf(*writer, limit, "(UNEXPECTED ");
-      cursor +=
-        ts_tree__write_char_to_string(*writer, limit, self->lookahead_char);
+      cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char);
+    } else if (self->is_missing) {
+      cursor += snprintf(*writer, limit, "(MISSING");
     } else {
-      cursor += snprintf(*writer, limit, "(%s",
-                         ts_language_symbol_name(language, self->symbol));
+      TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
+      const char *symbol_name = ts_language_symbol_name(language, symbol);
+      cursor += snprintf(*writer, limit, "(%s", symbol_name);
     }
   }
 
-  for (uint32_t i = 0; i < self->child_count; i++) {
-    Tree *child = self->children[i];
-    cursor += ts_tree__write_to_string(child, language, *writer, limit, false,
-                                       include_all);
+  for (uint32_t i = 0; i < self->children.size; i++) {
+    Tree *child = self->children.contents[i];
+    cursor += ts_tree__write_to_string(child, language, *writer, limit, false, include_all);
   }
 
-  if (visible)
-    cursor += snprintf(*writer, limit, ")");
+  if (visible) cursor += snprintf(*writer, limit, ")");
 
   return cursor - string;
 }
 
-char *ts_tree_string(const Tree *self, const TSLanguage *language,
-                     bool include_all) {
-  static char SCRATCH[1];
-  size_t size =
-    ts_tree__write_to_string(self, language, SCRATCH, 0, true, include_all) + 1;
+char *ts_tree_string(const Tree *self, const TSLanguage *language, bool include_all) {
+  char scratch_string[1];
+  size_t size = ts_tree__write_to_string(self, language, scratch_string, 0, true, include_all) + 1;
   char *result = ts_malloc(size * sizeof(char));
   ts_tree__write_to_string(self, language, result, size, true, include_all);
   return result;
@@ -512,40 +663,38 @@ char *ts_tree_string(const Tree *self, const TSLanguage *language,
 
 void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset,
                               const TSLanguage *language, FILE *f) {
-  fprintf(f, "tree_%p [label=\"%s\"", self,
-          ts_language_symbol_name(language, self->symbol));
+  TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
+  fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol));
 
-  if (self->child_count == 0)
+  if (self->children.size == 0)
     fprintf(f, ", shape=plaintext");
   if (self->extra)
     fprintf(f, ", fontcolor=gray");
 
-  fprintf(f, ", tooltip=\"range:%u - %u\nstate:%d\nerror-cost:%u\"]\n",
-          byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state,
-          self->error_cost);
-  for (uint32_t i = 0; i < self->child_count; i++) {
-    const Tree *child = self->children[i];
+  fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n",
+          self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state,
+          self->error_cost, self->repeat_depth);
+  for (uint32_t i = 0; i < self->children.size; i++) {
+    const Tree *child = self->children.contents[i];
     ts_tree__print_dot_graph(child, byte_offset, language, f);
     fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i);
     byte_offset += ts_tree_total_bytes(child);
   }
 }
 
-void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language,
-                             FILE *f) {
+void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) {
   fprintf(f, "digraph tree {\n");
   fprintf(f, "edge [arrowhead=none]\n");
   ts_tree__print_dot_graph(self, 0, language, f);
   fprintf(f, "}\n");
 }
 
-bool ts_external_token_state_eq(const TSExternalTokenState *self,
-                                const TSExternalTokenState *other) {
-  if (self == other) {
-    return true;
-  } else if (!self || !other) {
-    return false;
-  } else {
-    return memcmp(self, other, sizeof(TSExternalTokenState)) == 0;
-  }
+static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}};
+
+bool ts_tree_external_token_state_eq(const Tree *self, const Tree *other) {
+  const TSExternalTokenState *state1 = &empty_state;
+  const TSExternalTokenState *state2 = &empty_state;
+  if (self && self->has_external_tokens) state1 = &self->external_token_state;
+  if (other && other->has_external_tokens) state2 = &other->external_token_state;
+  return ts_external_token_state_eq(state1, state2);
 }