tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/runtime/parser.h

@@ -10,28 +10,33 @@ extern "C" {
 #include "runtime/lexer.h"
 #include "runtime/reusable_node.h"
 #include "runtime/reduce_action.h"
+#include "runtime/tree.h"
+
+typedef struct {
+  Tree *token;
+  Tree *last_external_token;
+  uint32_t byte_index;
+} TokenCache;
 
 typedef struct {
   Lexer lexer;
   Stack *stack;
+  TreePool tree_pool;
   const TSLanguage *language;
   ReduceActionSet reduce_actions;
   Tree *finished_tree;
-  bool is_split;
-  bool print_debugging_graphs;
   Tree scratch_tree;
-  Tree *cached_token;
-  uint32_t cached_token_byte_index;
+  TokenCache token_cache;
   ReusableNode reusable_node;
-  TreePath tree_path1;
-  TreePath tree_path2;
   void *external_scanner_payload;
-  Tree *last_external_token;
+  bool in_ambiguity;
+  bool print_debugging_graphs;
+  unsigned accept_count;
 } Parser;
 
 bool parser_init(Parser *);
 void parser_destroy(Parser *);
-Tree *parser_parse(Parser *, TSInput, Tree *);
+Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error);
 void parser_set_language(Parser *, const TSLanguage *);
 
 #ifdef __cplusplus

data/ext/tree-sitter/tree-sitter/src/runtime/point.h

@@ -3,18 +3,23 @@
 
 #include "tree_sitter/runtime.h"
 
+static inline TSPoint point__new(unsigned row, unsigned column) {
+  TSPoint result = {row, column};
+  return result;
+}
+
 static inline TSPoint point_add(TSPoint a, TSPoint b) {
   if (b.row > 0)
-    return (TSPoint){a.row + b.row, b.column};
+    return point__new(a.row + b.row, b.column);
   else
-    return (TSPoint){a.row, a.column + b.column};
+    return point__new(a.row, a.column + b.column);
 }
 
 static inline TSPoint point_sub(TSPoint a, TSPoint b) {
   if (a.row > b.row)
-    return (TSPoint){a.row - b.row, a.column};
+    return point__new(a.row - b.row, a.column);
   else
-    return (TSPoint){0, a.column - b.column};
+    return point__new(0, a.column - b.column);
 }
 
 static inline bool point_lte(TSPoint a, TSPoint b) {

data/ext/tree-sitter/tree-sitter/src/runtime/reduce_action.h

@@ -11,6 +11,8 @@ extern "C" {
 typedef struct {
   uint32_t count;
   TSSymbol symbol;
+  int dynamic_precedence;
+  unsigned short alias_sequence_id;
 } ReduceAction;
 
 typedef Array(ReduceAction) ReduceActionSet;

data/ext/tree-sitter/tree-sitter/src/runtime/reusable_node.h

@@ -3,48 +3,44 @@
 typedef struct {
   Tree *tree;
   uint32_t byte_index;
-  bool has_preceding_external_token;
-  const TSExternalTokenState *preceding_external_token_state;
+  Tree *last_external_token;
 } ReusableNode;
 
 static inline ReusableNode reusable_node_new(Tree *tree) {
-  return (ReusableNode){
-    .tree = tree,
-    .byte_index = 0,
-    .has_preceding_external_token = false,
-    .preceding_external_token_state = NULL,
-  };
+  ReusableNode result = {tree, 0, NULL};
+  return result;
 }
 
 static inline void reusable_node_pop(ReusableNode *self) {
   self->byte_index += ts_tree_total_bytes(self->tree);
   if (self->tree->has_external_tokens) {
-    self->has_preceding_external_token = true;
-    self->preceding_external_token_state = ts_tree_last_external_token_state(self->tree);
+    self->last_external_token = ts_tree_last_external_token(self->tree);
   }
 
   while (self->tree) {
     Tree *parent = self->tree->context.parent;
     uint32_t next_index = self->tree->context.index + 1;
-    if (parent && parent->child_count > next_index) {
-      self->tree = parent->children[next_index];
+    if (parent && parent->children.size > next_index) {
+      self->tree = parent->children.contents[next_index];
       return;
     }
     self->tree = parent;
   }
 }
 
-static inline void reusable_node_pop_leaf(ReusableNode *self) {
-  while (self->tree->child_count > 0)
-    self->tree = self->tree->children[0];
-  reusable_node_pop(self);
+static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) {
+  ReusableNode result = *self;
+  while (result.tree->children.size > 0)
+    result.tree = result.tree->children.contents[0];
+  reusable_node_pop(&result);
+  return result;
 }
 
 static inline bool reusable_node_breakdown(ReusableNode *self) {
-  if (self->tree->child_count == 0) {
+  if (self->tree->children.size == 0) {
     return false;
   } else {
-    self->tree = self->tree->children[0];
+    self->tree = self->tree->children.contents[0];
     return true;
   }
 }

data/ext/tree-sitter/tree-sitter/src/runtime/stack.c

@@ -1,4 +1,5 @@
 #include "runtime/alloc.h"
+#include "runtime/language.h"
 #include "runtime/tree.h"
 #include "runtime/array.h"
 #include "runtime/stack.h"
@@ -10,14 +11,17 @@
 #define MAX_NODE_POOL_SIZE 50
 #define MAX_ITERATOR_COUNT 64
 
-#define INLINE static inline __attribute__((always_inline))
+#ifdef _WIN32
+#define inline __forceinline
+#else
+#define inline static inline __attribute__((always_inline))
+#endif
 
 typedef struct StackNode StackNode;
 
 typedef struct {
   StackNode *node;
   Tree *tree;
-  unsigned push_count;
   bool is_pending;
 } StackLink;
 
@@ -26,32 +30,39 @@ struct StackNode {
   Length position;
   StackLink links[MAX_LINK_COUNT];
   short unsigned int link_count;
-  short unsigned int ref_count;
+  uint32_t ref_count;
   unsigned error_cost;
-  unsigned error_count;
+  unsigned node_count;
+  int dynamic_precedence;
 };
 
 typedef struct {
+  StackNode *node;
   TreeArray trees;
   uint32_t tree_count;
-  StackNode *node;
   bool is_pending;
-  unsigned push_count;
 } Iterator;
 
 typedef struct {
-  uint32_t goal_tree_count;
-  bool found_error;
-  bool found_valid_path;
-} StackPopSession;
+  void *payload;
+  StackIterateCallback callback;
+} StackIterateSession;
 
 typedef Array(StackNode *) StackNodeArray;
 
+typedef enum {
+  StackStatusActive,
+  StackStatusPaused,
+  StackStatusHalted,
+} StackStatus;
+
 typedef struct {
   StackNode *node;
-  bool is_halted;
-  unsigned push_count;
-  const TSExternalTokenState *external_token_state;
+  Tree *last_external_token;
+  StackSummary *summary;
+  unsigned node_count_at_last_error;
+  TSSymbol lookahead_when_paused;
+  StackStatus status;
 } StackHead;
 
 struct Stack {
@@ -60,190 +71,221 @@ struct Stack {
   Array(Iterator) iterators;
   StackNodeArray node_pool;
   StackNode *base_node;
+  TreePool *tree_pool;
+};
+
+typedef unsigned StackAction;
+enum {
+  StackActionNone,
+  StackActionStop = 1,
+  StackActionPop = 2,
 };
 
+typedef StackAction (*StackCallback)(void *, const Iterator *);
+
 static void stack_node_retain(StackNode *self) {
   if (!self)
     return;
-  assert(self->ref_count != 0);
+  assert(self->ref_count > 0);
   self->ref_count++;
+  assert(self->ref_count != 0);
 }
 
-static void stack_node_release(StackNode *self, StackNodeArray *pool) {
-  if (!self)
-    return;
+static void stack_node_release(StackNode *self, StackNodeArray *pool, TreePool *tree_pool) {
+recur:
   assert(self->ref_count != 0);
   self->ref_count--;
-  if (self->ref_count == 0) {
-    for (int i = 0; i < self->link_count; i++) {
-      if (self->links[i].tree)
-        ts_tree_release(self->links[i].tree);
-      stack_node_release(self->links[i].node, pool);
-    }
+  if (self->ref_count > 0) return;
 
-    if (pool->size < MAX_NODE_POOL_SIZE) {
-      array_push(pool, self);
-    } else {
-      ts_free(self);
+  StackNode *first_predecessor = NULL;
+  if (self->link_count > 0) {
+    for (unsigned i = self->link_count - 1; i > 0; i--) {
+      if (self->links[i].tree) ts_tree_release(tree_pool, self->links[i].tree);
+      stack_node_release(self->links[i].node, pool, tree_pool);
     }
+    if (self->links[0].tree) ts_tree_release(tree_pool, self->links[0].tree);
+    first_predecessor = self->links[0].node;
   }
-}
 
-static StackNode *stack_node_new(StackNode *next, Tree *tree, bool is_pending,
-                                 TSStateId state, Length position,
-                                 StackNodeArray *pool) {
-  StackNode *node;
-  if (pool->size > 0)
-    node = array_pop(pool);
-  else if (!(node = ts_malloc(sizeof(StackNode))))
-    return NULL;
-
-  *node = (StackNode){
-    .ref_count = 1,
-    .link_count = 0,
-    .links = {},
-    .state = state,
-    .position = position,
-  };
+  if (pool->size < MAX_NODE_POOL_SIZE) {
+    array_push(pool, self);
+  } else {
+    ts_free(self);
+  }
+
+  if (first_predecessor) {
+    self = first_predecessor;
+    goto recur;
+  }
+}
 
-  if (next) {
-    stack_node_retain(next);
+static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_pending,
+                                 TSStateId state, StackNodeArray *pool) {
+  StackNode *node = pool->size > 0 ?
+    array_pop(pool) :
+    ts_malloc(sizeof(StackNode));
+  *node = (StackNode){.ref_count = 1, .link_count = 0, .state = state};
 
+  if (previous_node) {
     node->link_count = 1;
     node->links[0] = (StackLink){
-      .node = next, .tree = tree, .is_pending = is_pending, .push_count = 0,
+      .node = previous_node,
+      .tree = tree,
+      .is_pending = is_pending,
     };
 
-    node->error_count = next->error_count;
-    node->error_cost = next->error_cost;
+    node->position = previous_node->position;
+    node->error_cost = previous_node->error_cost;
+    node->dynamic_precedence = previous_node->dynamic_precedence;
+    node->node_count = previous_node->node_count;
 
     if (tree) {
-      ts_tree_retain(tree);
       node->error_cost += tree->error_cost;
-
-      if (state == ERROR_STATE) {
-        if (!tree->extra) {
-          node->error_cost += ERROR_COST_PER_SKIPPED_TREE +
-                              ERROR_COST_PER_SKIPPED_CHAR *
-                                (tree->padding.chars + tree->size.chars) +
-                              ERROR_COST_PER_SKIPPED_LINE *
-                                (tree->padding.extent.row + tree->size.extent.row);
-        }
-      }
-    } else {
-      node->error_count++;
+      node->position = length_add(node->position, ts_tree_total_size(tree));
+      node->dynamic_precedence += tree->dynamic_precedence;
+      if (!tree->extra) node->node_count += tree->node_count;
     }
   } else {
-    node->error_count = 0;
+    node->position = length_zero();
     node->error_cost = 0;
   }
 
   return node;
 }
 
+static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) {
+  return
+    left == right ||
+    (left &&
+     right &&
+     left->symbol == right->symbol &&
+     ((left->error_cost > 0 && right->error_cost > 0) ||
+      (left->children.size == 0 && right->children.size == 0 &&
+       left->padding.bytes == right->padding.bytes &&
+       left->size.bytes == right->size.bytes &&
+       left->extra == right->extra &&
+       ts_tree_external_token_state_eq(left, right))));
+}
+
 static void stack_node_add_link(StackNode *self, StackLink link) {
+  if (link.node == self) return;
+
   for (int i = 0; i < self->link_count; i++) {
     StackLink existing_link = self->links[i];
-    if (existing_link.tree == link.tree ||
-        (existing_link.tree && link.tree &&
-         ts_tree_tokens_eq(existing_link.tree, link.tree))) {
-      if (existing_link.node == link.node)
-        return;
-      if (existing_link.node->state == link.node->state) {
-        for (int j = 0; j < link.node->link_count; j++)
+    if (stack__tree_is_equivalent(existing_link.tree, link.tree)) {
+      if (existing_link.node == link.node) return;
+      if (existing_link.node->state == link.node->state &&
+          existing_link.node->position.bytes == link.node->position.bytes) {
+        for (int j = 0; j < link.node->link_count; j++) {
           stack_node_add_link(existing_link.node, link.node->links[j]);
+        }
         return;
       }
     }
   }
 
-  if (self->link_count < MAX_LINK_COUNT) {
-    stack_node_retain(link.node);
-    if (link.tree)
-      ts_tree_retain(link.tree);
+  if (self->link_count == MAX_LINK_COUNT) return;
 
-    self->links[self->link_count++] = (StackLink){
-      .node = link.node,
-      .tree = link.tree,
-      .is_pending = link.is_pending,
-      .push_count = link.push_count,
-    };
+  stack_node_retain(link.node);
+  if (link.tree) ts_tree_retain(link.tree);
+  self->links[self->link_count++] = link;
+
+  unsigned node_count = link.node->node_count;
+  if (link.tree) node_count += link.tree->node_count;
+  if (node_count > self->node_count) self->node_count = node_count;
+}
+
+static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) {
+  if (self->node) {
+    if (self->last_external_token) {
+      ts_tree_release(tree_pool, self->last_external_token);
+    }
+    if (self->summary) {
+      array_delete(self->summary);
+      ts_free(self->summary);
+    }
+    stack_node_release(self->node, pool, tree_pool);
   }
 }
 
-static StackVersion ts_stack__add_version(Stack *self, StackNode *node,
-                                          unsigned push_count,
-                                          const TSExternalTokenState *external_token_state) {
+static StackVersion ts_stack__add_version(Stack *self, StackVersion original_version,
+                                          StackNode *node) {
   StackHead head = {
     .node = node,
-    .is_halted = false,
-    .push_count = push_count,
-    .external_token_state = external_token_state,
+    .node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error,
+    .last_external_token = self->heads.contents[original_version].last_external_token,
+    .status = StackStatusActive,
+    .lookahead_when_paused = 0,
   };
   array_push(&self->heads, head);
   stack_node_retain(node);
+  if (head.last_external_token) ts_tree_retain(head.last_external_token);
   return (StackVersion)(self->heads.size - 1);
 }
 
-static void ts_stack__add_slice(Stack *self, StackNode *node, TreeArray *trees,
-                                unsigned push_count,
-                                const TSExternalTokenState *external_token_state) {
+static void ts_stack__add_slice(Stack *self, StackVersion original_version,
+                                StackNode *node, TreeArray *trees) {
   for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
     StackVersion version = self->slices.contents[i].version;
     if (self->heads.contents[version].node == node) {
-      StackSlice slice = { *trees, version };
+      StackSlice slice = {*trees, version};
       array_insert(&self->slices, i + 1, slice);
       return;
     }
   }
 
-  StackVersion version = ts_stack__add_version(self, node, push_count, external_token_state);
+  StackVersion version = ts_stack__add_version(self, original_version, node);
   StackSlice slice = { *trees, version };
   array_push(&self->slices, slice);
 }
 
-INLINE StackPopResult stack__iter(Stack *self, StackVersion version,
-                                  StackIterateCallback callback, void *payload) {
+inline StackSliceArray stack__iter(Stack *self, StackVersion version,
+                                   StackCallback callback, void *payload,
+                                   int goal_tree_count) {
   array_clear(&self->slices);
   array_clear(&self->iterators);
 
   StackHead *head = array_get(&self->heads, version);
-  unsigned push_count = head->push_count;
-  const TSExternalTokenState *external_token_state = head->external_token_state;
   Iterator iterator = {
     .node = head->node,
     .trees = array_new(),
     .tree_count = 0,
     .is_pending = true,
-    .push_count = 0,
   };
+
+  bool include_trees = false;
+  if (goal_tree_count >= 0) {
+    include_trees = true;
+    array_reserve(&iterator.trees, goal_tree_count);
+  }
+
   array_push(&self->iterators, iterator);
 
   while (self->iterators.size > 0) {
     for (uint32_t i = 0, size = self->iterators.size; i < size; i++) {
       Iterator *iterator = &self->iterators.contents[i];
       StackNode *node = iterator->node;
-      bool is_done = node == self->base_node;
-
-      StackIterateAction action =
-        callback(payload, node->state, &iterator->trees, iterator->tree_count,
-                 is_done, iterator->is_pending);
 
-      bool should_pop = action & StackIteratePop;
-      bool should_stop = action & StackIterateStop || node->link_count == 0;
+      StackAction action = callback(payload, iterator);
+      bool should_pop = action & StackActionPop;
+      bool should_stop = action & StackActionStop || node->link_count == 0;
 
       if (should_pop) {
         TreeArray trees = iterator->trees;
         if (!should_stop)
           ts_tree_array_copy(trees, &trees);
-        array_reverse(&trees);
-        ts_stack__add_slice(self, node, &trees, push_count + iterator->push_count,
-                            external_token_state);
+        ts_tree_array_reverse(&trees);
+        ts_stack__add_slice(
+          self,
+          version,
+          node,
+          &trees
+        );
       }
 
       if (should_stop) {
         if (!should_pop)
-          ts_tree_array_delete(&iterator->trees);
+          ts_tree_array_delete(self->tree_pool, &iterator->trees);
         array_erase(&self->iterators, i);
         i--, size--;
         continue;
@@ -258,52 +300,51 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version,
         } else {
           if (self->iterators.size >= MAX_ITERATOR_COUNT) continue;
           link = node->links[j];
-          array_push(&self->iterators, self->iterators.contents[i]);
+          Iterator current_iterator = self->iterators.contents[i];
+          array_push(&self->iterators, current_iterator);
           next_iterator = array_back(&self->iterators);
           ts_tree_array_copy(next_iterator->trees, &next_iterator->trees);
         }
 
         next_iterator->node = link.node;
-        next_iterator->push_count += link.push_count;
         if (link.tree) {
+          if (include_trees) {
+            array_push(&next_iterator->trees, link.tree);
+            ts_tree_retain(link.tree);
+          }
+
           if (!link.tree->extra) {
             next_iterator->tree_count++;
-            if (!link.is_pending)
+            if (!link.is_pending) {
               next_iterator->is_pending = false;
+            }
           }
-          array_push(&next_iterator->trees, link.tree);
-          ts_tree_retain(link.tree);
         } else {
+          next_iterator->tree_count++;
           next_iterator->is_pending = false;
         }
       }
     }
   }
 
-  return (StackPopResult){ false, self->slices };
+  return self->slices;
 }
 
-Stack *ts_stack_new() {
+Stack *ts_stack_new(TreePool *tree_pool) {
   Stack *self = ts_calloc(1, sizeof(Stack));
 
   array_init(&self->heads);
   array_init(&self->slices);
   array_init(&self->iterators);
   array_init(&self->node_pool);
-  array_grow(&self->heads, 4);
-  array_grow(&self->slices, 4);
-  array_grow(&self->iterators, 4);
-  array_grow(&self->node_pool, MAX_NODE_POOL_SIZE);
+  array_reserve(&self->heads, 4);
+  array_reserve(&self->slices, 4);
+  array_reserve(&self->iterators, 4);
+  array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE);
 
-  self->base_node =
-    stack_node_new(NULL, NULL, false, 1, length_zero(), &self->node_pool);
-  stack_node_retain(self->base_node);
-  array_push(&self->heads, ((StackHead){
-    self->base_node,
-    false,
-    0,
-    NULL
-  }));
+  self->tree_pool = tree_pool;
+  self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool);
+  ts_stack_clear(self);
 
   return self;
 }
@@ -313,9 +354,10 @@ void ts_stack_delete(Stack *self) {
     array_delete(&self->slices);
   if (self->iterators.contents)
     array_delete(&self->iterators);
-  stack_node_release(self->base_node, &self->node_pool);
-  for (uint32_t i = 0; i < self->heads.size; i++)
-    stack_node_release(self->heads.contents[i].node, &self->node_pool);
+  stack_node_release(self->base_node, &self->node_pool, self->tree_pool);
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool);
+  }
   array_clear(&self->heads);
   if (self->node_pool.contents) {
     for (uint32_t i = 0; i < self->node_pool.size; i++)
@@ -330,214 +372,289 @@ uint32_t ts_stack_version_count(const Stack *self) {
   return self->heads.size;
 }
 
-TSStateId ts_stack_top_state(const Stack *self, StackVersion version) {
+TSStateId ts_stack_state(const Stack *self, StackVersion version) {
   return array_get(&self->heads, version)->node->state;
 }
 
-Length ts_stack_top_position(const Stack *self, StackVersion version) {
+Length ts_stack_position(const Stack *self, StackVersion version) {
   return array_get(&self->heads, version)->node->position;
 }
 
-unsigned ts_stack_push_count(const Stack *self, StackVersion version) {
-  return array_get(&self->heads, version)->push_count;
-}
-
-void ts_stack_decrease_push_count(Stack *self, StackVersion version,
-                                  unsigned decrement) {
-  array_get(&self->heads, version)->push_count -= decrement;
+Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->last_external_token;
 }
 
-const TSExternalTokenState *ts_stack_external_token_state(const Stack *self, StackVersion version) {
-  return array_get(&self->heads, version)->external_token_state;
-}
-
-void ts_stack_set_external_token_state(Stack *self, StackVersion version, const TSExternalTokenState *state) {
-  array_get(&self->heads, version)->external_token_state = state;
+void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *token) {
+  StackHead *head = array_get(&self->heads, version);
+  if (token) ts_tree_retain(token);
+  if (head->last_external_token) ts_tree_release(self->tree_pool, head->last_external_token);
+  head->last_external_token = token;
 }
 
-ErrorStatus ts_stack_error_status(const Stack *self, StackVersion version) {
+unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
   StackHead *head = array_get(&self->heads, version);
-  return (ErrorStatus){
-    .cost = head->node->error_cost,
-    .count = head->node->error_count,
-    .push_count = head->push_count,
-  };
+  unsigned result = head->node->error_cost;
+  if (
+    head->status == StackStatusPaused ||
+    (head->node->state == ERROR_STATE && !head->node->links[0].tree)) {
+    result += ERROR_COST_PER_RECOVERY;
+  }
+  return result;
 }
 
-unsigned ts_stack_error_count(const Stack *self, StackVersion version) {
-  StackNode *node = array_get(&self->heads, version)->node;
-  return node->error_count;
+unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) {
+  StackHead *head = array_get(&self->heads, version);
+  if (head->node->node_count < head->node_count_at_last_error) {
+    head->node_count_at_last_error = head->node->node_count;
+  }
+  return head->node->node_count - head->node_count_at_last_error;
 }
 
-bool ts_stack_push(Stack *self, StackVersion version, Tree *tree,
-                   bool is_pending, TSStateId state) {
+void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) {
   StackHead *head = array_get(&self->heads, version);
-  StackNode *node = head->node;
-  Length position = node->position;
-  if (tree)
-    position = length_add(position, ts_tree_total_size(tree));
-  StackNode *new_node =
-    stack_node_new(node, tree, is_pending, state, position, &self->node_pool);
-  if (!new_node)
-    return false;
-  stack_node_release(node, &self->node_pool);
+  StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool);
+  if (!tree) head->node_count_at_last_error = new_node->node_count;
   head->node = new_node;
-  if (state == ERROR_STATE) {
-    new_node->links[0].push_count = head->push_count;
-    head->push_count = 0;
-  } else
-    head->push_count++;
-  return true;
 }
 
-StackPopResult ts_stack_iterate(Stack *self, StackVersion version,
-                                StackIterateCallback callback, void *payload) {
-  return stack__iter(self, version, callback, payload);
+inline StackAction iterate_callback(void *payload, const Iterator *iterator) {
+  StackIterateSession *session = payload;
+  session->callback(
+    session->payload,
+    iterator->node->state,
+    iterator->tree_count
+  );
+  return StackActionNone;
 }
 
-INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state,
-                                             TreeArray *trees, uint32_t tree_count,
-                                             bool is_done, bool is_pending) {
-  StackPopSession *pop_session = (StackPopSession *)payload;
+void ts_stack_iterate(Stack *self, StackVersion version,
+                      StackIterateCallback callback, void *payload) {
+  StackIterateSession session = {payload, callback};
+  stack__iter(self, version, iterate_callback, &session, -1);
+}
 
-  if (tree_count == pop_session->goal_tree_count) {
-    pop_session->found_valid_path = true;
-    return StackIteratePop | StackIterateStop;
+inline StackAction pop_count_callback(void *payload, const Iterator *iterator) {
+  unsigned *goal_tree_count = payload;
+  if (iterator->tree_count == *goal_tree_count) {
+    return StackActionPop | StackActionStop;
+  } else {
+    return StackActionNone;
   }
+}
+
+StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) {
+  return stack__iter(self, version, pop_count_callback, &count, count);
+}
 
-  if (state == ERROR_STATE) {
-    if (pop_session->found_valid_path || pop_session->found_error) {
-      return StackIterateStop;
+inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) {
+  if (iterator->tree_count >= 1) {
+    if (iterator->is_pending) {
+      return StackActionPop | StackActionStop;
     } else {
-      pop_session->found_error = true;
-      return StackIteratePop | StackIterateStop;
+      return StackActionStop;
     }
+  } else {
+    return StackActionNone;
   }
-  return StackIterateNone;
 }
 
-StackPopResult ts_stack_pop_count(Stack *self, StackVersion version,
-                                  uint32_t count) {
-  StackPopSession session = {
-    .goal_tree_count = count, .found_error = false, .found_valid_path = false,
-  };
-  StackPopResult pop = stack__iter(self, version, pop_count_callback, &session);
-  if (session.found_error) {
-    if (session.found_valid_path) {
-      StackSlice error_slice = pop.slices.contents[0];
-      ts_tree_array_delete(&error_slice.trees);
-      array_erase(&pop.slices, 0);
-      if (array_front(&pop.slices)->version != error_slice.version) {
-        ts_stack_remove_version(self, error_slice.version);
-        for (StackVersion i = 0; i < pop.slices.size; i++)
-          pop.slices.contents[i].version--;
-      }
-    } else {
-      pop.stopped_at_error = true;
-    }
+StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
+  StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0);
+  if (pop.size > 0) {
+    ts_stack_renumber_version(self, pop.contents[0].version, version);
+    pop.contents[0].version = version;
   }
   return pop;
 }
 
-INLINE StackIterateAction pop_pending_callback(void *payload, TSStateId state,
-                                               TreeArray *trees,
-                                               uint32_t tree_count, bool is_done,
-                                               bool is_pending) {
-  if (tree_count >= 1) {
-    if (is_pending) {
-      return StackIteratePop | StackIterateStop;
+inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
+  if (iterator->trees.size > 0) {
+    bool *found_error = payload;
+    if (!*found_error && iterator->trees.contents[0]->symbol == ts_builtin_sym_error) {
+      *found_error = true;
+      return StackActionPop | StackActionStop;
     } else {
-      return StackIterateStop;
+      return StackActionStop;
     }
   } else {
-    return StackIterateNone;
+    return StackActionNone;
   }
 }
 
-StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) {
-  StackPopResult pop = stack__iter(self, version, pop_pending_callback, NULL);
-  if (pop.slices.size > 0) {
-    ts_stack_renumber_version(self, pop.slices.contents[0].version, version);
-    pop.slices.contents[0].version = version;
+TreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
+  StackNode *node = array_get(&self->heads, version)->node;
+  for (unsigned i = 0; i < node->link_count; i++) {
+    if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) {
+      bool found_error = false;
+      StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1);
+      if (pop.size > 0) {
+        assert(pop.size == 1);
+        ts_stack_renumber_version(self, pop.contents[0].version, version);
+        return pop.contents[0].trees;
+      }
+      break;
+    }
   }
-  return pop;
+  return (TreeArray){.size = 0};
+}
+
+inline StackAction pop_all_callback(void *payload, const Iterator *iterator) {
+  return iterator->node->link_count == 0 ? StackActionPop : StackActionNone;
 }
 
-INLINE StackIterateAction pop_all_callback(void *payload, TSStateId state,
-                                           TreeArray *trees, uint32_t tree_count,
-                                           bool is_done, bool is_pending) {
-  return is_done ? (StackIteratePop | StackIterateStop) : StackIterateNone;
+StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) {
+  return stack__iter(self, version, pop_all_callback, NULL, 0);
 }
 
-StackPopResult ts_stack_pop_all(Stack *self, StackVersion version) {
-  return stack__iter(self, version, pop_all_callback, NULL);
+typedef struct {
+  StackSummary *summary;
+  unsigned max_depth;
+} SummarizeStackSession;
+
+inline StackAction summarize_stack_callback(void *payload, const Iterator *iterator) {
+  SummarizeStackSession *session = payload;
+  TSStateId state = iterator->node->state;
+  unsigned depth = iterator->tree_count;
+  if (depth > session->max_depth) return StackActionStop;
+  for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
+    StackSummaryEntry entry = session->summary->contents[i];
+    if (entry.depth < depth) break;
+    if (entry.depth == depth && entry.state == state) return StackActionNone;
+  }
+  array_push(session->summary, ((StackSummaryEntry){
+    .position = iterator->node->position,
+    .depth = depth,
+    .state = state,
+  }));
+  return StackActionNone;
+}
+
+void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth) {
+  SummarizeStackSession session = {
+    .summary = ts_malloc(sizeof(StackSummary)),
+    .max_depth = max_depth
+  };
+  array_init(session.summary);
+  stack__iter(self, version, summarize_stack_callback, &session, -1);
+  self->heads.contents[version].summary = session.summary;
+}
+
+StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->summary;
+}
+
+int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->node->dynamic_precedence;
 }
 
 void ts_stack_remove_version(Stack *self, StackVersion version) {
-  StackNode *node = array_get(&self->heads, version)->node;
-  stack_node_release(node, &self->node_pool);
+  stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->tree_pool);
   array_erase(&self->heads, version);
 }
 
 void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
   assert(v2 < v1);
   assert((uint32_t)v1 < self->heads.size);
-  stack_node_release(self->heads.contents[v2].node, &self->node_pool);
-  self->heads.contents[v2] = self->heads.contents[v1];
+  StackHead *source_head = &self->heads.contents[v1];
+  StackHead *target_head = &self->heads.contents[v2];
+  if (target_head->summary && !source_head->summary) {
+    source_head->summary = target_head->summary;
+    target_head->summary = NULL;
+  }
+  stack_head_delete(target_head, &self->node_pool, self->tree_pool);
+  *target_head = *source_head;
   array_erase(&self->heads, v1);
 }
 
+void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) {
+  StackHead temporary_head = self->heads.contents[v1];
+  self->heads.contents[v1] = self->heads.contents[v2];
+  self->heads.contents[v2] = temporary_head;
+}
+
 StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
   assert(version < self->heads.size);
   array_push(&self->heads, self->heads.contents[version]);
-  stack_node_retain(array_back(&self->heads)->node);
+  StackHead *head = array_back(&self->heads);
+  stack_node_retain(head->node);
+  if (head->last_external_token) ts_tree_retain(head->last_external_token);
+  head->summary = NULL;
   return self->heads.size - 1;
 }
 
-bool ts_stack_merge(Stack *self, StackVersion version, StackVersion new_version) {
-  StackHead *head = &self->heads.contents[version];
-  StackHead *new_head = &self->heads.contents[new_version];
-  StackNode *node = head->node;
-  StackNode *new_node = new_head->node;
-
-  if (new_node->state == node->state &&
-      new_node->position.chars == node->position.chars &&
-      new_node->error_count == node->error_count &&
-      new_node->error_cost == node->error_cost &&
-      new_head->external_token_state == head->external_token_state) {
-    for (uint32_t j = 0; j < new_node->link_count; j++)
-      stack_node_add_link(node, new_node->links[j]);
-    if (new_head->push_count > head->push_count)
-      head->push_count = new_head->push_count;
-    ts_stack_remove_version(self, new_version);
-    return true;
-  } else {
-    return false;
+bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
+  if (!ts_stack_can_merge(self, version1, version2)) return false;
+  StackHead *head1 = &self->heads.contents[version1];
+  StackHead *head2 = &self->heads.contents[version2];
+  for (uint32_t i = 0; i < head2->node->link_count; i++) {
+    stack_node_add_link(head1->node, head2->node->links[i]);
+  }
+  if (head1->node->state == ERROR_STATE) {
+    head1->node_count_at_last_error = head1->node->node_count;
   }
+  ts_stack_remove_version(self, version2);
+  return true;
+}
+
+bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) {
+  StackHead *head1 = &self->heads.contents[version1];
+  StackHead *head2 = &self->heads.contents[version2];
+  return
+    head1->status == StackStatusActive &&
+    head2->status == StackStatusActive &&
+    head1->node->state == head2->node->state &&
+    head1->node->position.bytes == head2->node->position.bytes &&
+    head1->node->error_cost == head2->node->error_cost &&
+    ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token);
 }
 
 void ts_stack_halt(Stack *self, StackVersion version) {
-  array_get(&self->heads, version)->is_halted = true;
+  array_get(&self->heads, version)->status = StackStatusHalted;
+}
+
+void ts_stack_pause(Stack *self, StackVersion version, TSSymbol lookahead) {
+  StackHead *head = array_get(&self->heads, version);
+  head->status = StackStatusPaused;
+  head->lookahead_when_paused = lookahead;
+  head->node_count_at_last_error = head->node->node_count;
 }
 
-bool ts_stack_is_halted(Stack *self, StackVersion version) {
-  return array_get(&self->heads, version)->is_halted;
+bool ts_stack_is_active(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusActive;
+}
+
+bool ts_stack_is_halted(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusHalted;
+}
+
+bool ts_stack_is_paused(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusPaused;
+}
+
+TSSymbol ts_stack_resume(Stack *self, StackVersion version) {
+  StackHead *head = array_get(&self->heads, version);
+  assert(head->status == StackStatusPaused);
+  TSSymbol result = head->lookahead_when_paused;
+  head->status = StackStatusActive;
+  head->lookahead_when_paused = 0;
+  return result;
 }
 
 void ts_stack_clear(Stack *self) {
   stack_node_retain(self->base_node);
-  for (uint32_t i = 0; i < self->heads.size; i++)
-    stack_node_release(self->heads.contents[i].node, &self->node_pool);
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool);
+  }
   array_clear(&self->heads);
   array_push(&self->heads, ((StackHead){
-    self->base_node,
-    false,
-    0,
-    NULL
+    .node = self->base_node,
+    .last_external_token = NULL,
+    .status = StackStatusActive,
+    .lookahead_when_paused = 0,
   }));
 }
 
-bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
+bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) {
   bool was_recording_allocations = ts_toggle_allocation_recording(false);
   if (!f)
     f = stderr;
@@ -546,28 +663,31 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
   fprintf(f, "rankdir=\"RL\";\n");
   fprintf(f, "edge [arrowhead=none]\n");
 
-  Array(StackNode *)visited_nodes = array_new();
+  Array(StackNode *) visited_nodes = array_new();
 
   array_clear(&self->iterators);
   for (uint32_t i = 0; i < self->heads.size; i++) {
     StackHead *head = &self->heads.contents[i];
-    if (head->is_halted)
-      continue;
+    if (head->status == StackStatusHalted) continue;
+
     fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
-    fprintf(
-      f,
-      "node_head_%u -> node_%p [label=%u, fontcolor=blue, weight=10000, "
-      "labeltooltip=\"push_count: %u",
-      i, head->node, i, head->push_count);
-
-    if (head->external_token_state) {
-      const TSExternalTokenState *s = head->external_token_state;
-      fprintf(f,
-        "\nexternal_token_state: "
-        "%2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X %2X",
-        (*s)[0], (*s)[1], (*s)[2], (*s)[3], (*s)[4], (*s)[5], (*s)[6], (*s)[7],
-        (*s)[8], (*s)[9], (*s)[10], (*s)[11], (*s)[12], (*s)[13], (*s)[14], (*s)[15]
-      );
+    fprintf(f, "node_head_%u -> node_%p [", i, head->node);
+
+    if (head->status == StackStatusPaused) {
+      fprintf(f, "color=red ");
+    }
+    fprintf(f,
+      "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u",
+      i,
+      ts_stack_node_count_since_error(self, i),
+      ts_stack_error_cost(self, i)
+    );
+
+    if (head->last_external_token) {
+      TSExternalTokenState *state = &head->last_external_token->external_token_state;
+      const char *data = ts_external_token_state_data(state);
+      fprintf(f, "\nexternal_token_state:");
+      for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
     }
 
     fprintf(f, "\"]\n");
@@ -579,8 +699,8 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
     all_iterators_done = true;
 
     for (uint32_t i = 0; i < self->iterators.size; i++) {
-      Iterator *iterator = &self->iterators.contents[i];
-      StackNode *node = iterator->node;
+      Iterator iterator = self->iterators.contents[i];
+      StackNode *node = iterator.node;
 
       for (uint32_t j = 0; j < visited_nodes.size; j++) {
         if (visited_nodes.contents[j] == node) {
@@ -589,8 +709,7 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
         }
       }
 
-      if (!node)
-        continue;
+      if (!node) continue;
       all_iterators_done = false;
 
       fprintf(f, "node_%p [", node);
@@ -602,10 +721,15 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
       else
         fprintf(f, "label=\"%d\"", node->state);
 
-      fprintf(f,
-              " tooltip=\"position: %u,%u\nerror_count: %u\nerror_cost: %u\"];\n",
-              node->position.extent.row, node->position.extent.column, node->error_count,
-              node->error_cost);
+      fprintf(
+        f,
+        " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
+        node->position.extent.row,
+        node->position.extent.column,
+        node->node_count,
+        node->error_cost,
+        node->dynamic_precedence
+      );
 
       for (int j = 0; j < node->link_count; j++) {
         StackLink link = node->links[j];
@@ -616,34 +740,32 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
           fprintf(f, "fontcolor=gray ");
 
         if (!link.tree) {
-          fprintf(f, "color=red, tooltip=\"push_count: %u\"", link.push_count);
-        } else if (link.tree->symbol == ts_builtin_sym_error) {
-          fprintf(f, "label=\"ERROR\"");
+          fprintf(f, "color=red");
         } else {
           fprintf(f, "label=\"");
-          if (!link.tree->named)
-            fprintf(f, "'");
-          const char *name = symbol_names[link.tree->symbol];
+          if (link.tree->visible && !link.tree->named) fprintf(f, "'");
+          const char *name = ts_language_symbol_name(language, link.tree->symbol);
           for (const char *c = name; *c; c++) {
-            if (*c == '\"' || *c == '\\')
-              fprintf(f, "\\");
+            if (*c == '\"' || *c == '\\') fprintf(f, "\\");
             fprintf(f, "%c", *c);
           }
-          if (!link.tree->named)
-            fprintf(f, "'");
-          fprintf(f, "\" labeltooltip=\"error_cost: %u\"",
-                  link.tree->error_cost);
+          if (link.tree->visible && !link.tree->named) fprintf(f, "'");
+          fprintf(f, "\"");
+          fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
+                  link.tree->error_cost,
+                  link.tree->dynamic_precedence);
         }
 
         fprintf(f, "];\n");
 
+        Iterator *next_iterator;
         if (j == 0) {
-          iterator->node = link.node;
+          next_iterator = &self->iterators.contents[i];
         } else {
-          array_push(&self->iterators, *iterator);
-          Iterator *next_iterator = array_back(&self->iterators);
-          next_iterator->node = link.node;
+          array_push(&self->iterators, iterator);
+          next_iterator = array_back(&self->iterators);
         }
+        next_iterator->node = link.node;
       }
 
       array_push(&visited_nodes, node);