tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/{spec/runtime/stack_spec.cc → test/runtime/stack_test.cc}

@@ -1,4 +1,4 @@
-#include "spec_helper.h"
+#include "test_helper.h"
 #include "helpers/tree_helpers.h"
 #include "helpers/point_helpers.h"
 #include "helpers/record_alloc.h"
@@ -20,10 +20,10 @@ enum {
 };
 
 Length operator*(const Length &length, uint32_t factor) {
-  return {length.bytes * factor, length.chars * factor, {0, length.extent.column * factor}};
+  return {length.bytes * factor, {0, length.extent.column * factor}};
 }
 
-void free_slice_array(StackSliceArray *slices) {
+void free_slice_array(TreePool *pool, StackSliceArray *slices) {
   for (size_t i = 0; i < slices->size; i++) {
     StackSlice slice = slices->contents[i];
 
@@ -38,7 +38,7 @@ void free_slice_array(StackSliceArray *slices) {
 
     if (!matches_prior_trees) {
       for (size_t j = 0; j < slice.trees.size; j++)
-        ts_tree_release(slice.trees.contents[j]);
+        ts_tree_release(pool, slice.trees.contents[j]);
       array_delete(&slice.trees);
     }
   }
@@ -54,12 +54,12 @@ vector<StackEntry> get_stack_entries(Stack *stack, StackVersion version) {
   ts_stack_iterate(
     stack,
     version,
-    [](void *payload, TSStateId state, TreeArray *trees, uint32_t tree_count, bool is_done, bool is_pending) -> StackIterateAction {
+    [](void *payload, TSStateId state, uint32_t tree_count) {
       auto entries = static_cast<vector<StackEntry> *>(payload);
       StackEntry entry = {state, tree_count};
-      if (find(entries->begin(), entries->end(), entry) == entries->end())
+      if (find(entries->begin(), entries->end(), entry) == entries->end()) {
         entries->push_back(entry);
-      return StackIterateNone;
+      }
     }, &result);
   return result;
 }
@@ -70,48 +70,60 @@ describe("Stack", [&]() {
   Stack *stack;
   const size_t tree_count = 11;
   Tree *trees[tree_count];
-  Length tree_len = {2, 3, {0, 3}};
+  Length tree_len = {3, {0, 3}};
+  TreePool pool;
 
   before_each([&]() {
     record_alloc::start();
 
-    stack = ts_stack_new();
+    ts_tree_pool_init(&pool);
+    stack = ts_stack_new(&pool);
 
-    for (size_t i = 0; i < tree_count; i++)
-      trees[i] = ts_tree_make_leaf(i, length_zero(), tree_len, {
-        true, true, false, true,
-      });
+    TSLanguage dummy_language;
+    TSSymbolMetadata symbol_metadata[50] = {};
+    dummy_language.symbol_metadata = symbol_metadata;
+
+    for (size_t i = 0; i < tree_count; i++) {
+      trees[i] = ts_tree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language);
+    }
   });
 
   after_each([&]() {
     ts_stack_delete(stack);
-    for (size_t i = 0; i < tree_count; i++)
-      ts_tree_release(trees[i]);
+    for (size_t i = 0; i < tree_count; i++) {
+      ts_tree_release(&pool, trees[i]);
+    }
+    ts_tree_pool_delete(&pool);
 
     record_alloc::stop();
     AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
   });
 
+  auto push = [&](StackVersion version, Tree *tree, TSStateId state) {
+    ts_tree_retain(tree);
+    ts_stack_push(stack, version, tree, false, state);
+  };
+
   describe("push(version, tree, is_pending, state)", [&]() {
     it("adds entries to the given version of the stack", [&]() {
       AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
-      AssertThat(ts_stack_top_state(stack, 0), Equals(1));
-      AssertThat(ts_stack_top_position(stack, 0), Equals(length_zero()));
+      AssertThat(ts_stack_state(stack, 0), Equals(1));
+      AssertThat(ts_stack_position(stack, 0), Equals(length_zero()));
 
       // . <──0── A*
-      ts_stack_push(stack, 0, trees[0], false, stateA);
-      AssertThat(ts_stack_top_state(stack, 0), Equals(stateA));
-      AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len));
+      push(0, trees[0], stateA);
+      AssertThat(ts_stack_state(stack, 0), Equals(stateA));
+      AssertThat(ts_stack_position(stack, 0), Equals(tree_len));
 
       // . <──0── A <──1── B*
-      ts_stack_push(stack, 0, trees[1], false, stateB);
-      AssertThat(ts_stack_top_state(stack, 0), Equals(stateB));
-      AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2));
+      push(0, trees[1], stateB);
+      AssertThat(ts_stack_state(stack, 0), Equals(stateB));
+      AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 2));
 
       // . <──0── A <──1── B <──2── C*
-      ts_stack_push(stack, 0, trees[2], false, stateC);
-      AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
-      AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 3));
+      push(0, trees[2], stateC);
+      AssertThat(ts_stack_state(stack, 0), Equals(stateC));
+      AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 3));
 
       AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
         {stateC, 0},
@@ -120,31 +132,25 @@ describe("Stack", [&]() {
         {1, 3},
       })));
     });
-
-    it("increments the version's push count", [&]() {
-      AssertThat(ts_stack_push_count(stack, 0), Equals<unsigned>(0));
-      ts_stack_push(stack, 0, trees[0], false, stateA);
-      AssertThat(ts_stack_push_count(stack, 0), Equals<unsigned>(1));
-    });
   });
 
   describe("merge()", [&]() {
     before_each([&]() {
-      // . <──0── A <──1── B*
+      // . <──0── A <─*
       //          ↑
-      //          └───2─── C*
-      ts_stack_push(stack, 0, trees[0], false, stateA);
+      //          └───*
+      push(0, trees[0], stateA);
       ts_stack_copy_version(stack, 0);
-      ts_stack_push(stack, 0, trees[1], false, stateB);
-      ts_stack_push(stack, 1, trees[2], false, stateC);
     });
 
     it("combines versions that have the same top states and positions", [&]() {
       // . <──0── A <──1── B <──3── D*
       //          ↑
       //          └───2─── C <──4── D*
-      ts_stack_push(stack, 0, trees[3], false, stateD);
-      ts_stack_push(stack, 1, trees[4], false, stateD);
+      push(0, trees[1], stateB);
+      push(1, trees[2], stateC);
+      push(0, trees[3], stateD);
+      push(1, trees[4], stateD);
 
       // . <──0── A <──1── B <──3── D*
       //          ↑                 |
@@ -161,6 +167,12 @@ describe("Stack", [&]() {
     });
 
     it("does not combine versions that have different states", [&]() {
+      // . <──0── A <──1── B*
+      //          ↑
+      //          └───2─── C*
+      push(0, trees[1], stateB);
+      push(1, trees[2], stateC);
+
       AssertThat(ts_stack_merge(stack, 0, 1), IsFalse());
       AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
     });
@@ -170,8 +182,10 @@ describe("Stack", [&]() {
       //          ↑
       //          └───2─── C <──4── D*
       trees[3]->size = tree_len * 3;
-      ts_stack_push(stack, 0, trees[3], false, stateD);
-      ts_stack_push(stack, 1, trees[4], false, stateD);
+      push(0, trees[1], stateB);
+      push(1, trees[2], stateC);
+      push(0, trees[3], stateD);
+      push(1, trees[4], stateD);
 
       AssertThat(ts_stack_merge(stack, 0, 1), IsFalse());
       AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
@@ -182,10 +196,12 @@ describe("Stack", [&]() {
         // . <──0── A <──1── B <──3── D <──5── E*
         //          ↑
         //          └───2─── C <──4── D <──5── E*
-        ts_stack_push(stack, 0, trees[3], false, stateD);
-        ts_stack_push(stack, 0, trees[5], false, stateE);
-        ts_stack_push(stack, 1, trees[4], false, stateD);
-        ts_stack_push(stack, 1, trees[5], false, stateE);
+        push(0, trees[1], stateB);
+        push(1, trees[2], stateC);
+        push(0, trees[3], stateD);
+        push(1, trees[4], stateD);
+        push(0, trees[5], stateE);
+        push(1, trees[5], stateE);
 
         // . <──0── A <──1── B <──3── D <──5── E*
         //          ↑                 |
@@ -202,31 +218,53 @@ describe("Stack", [&]() {
         })));
       });
     });
+
+    describe("when one of the versions contains an extra (e.g. ERROR) tree of size zero", [&]() {
+      it("does not create a loop in the stack", [&]() {
+        // . <──0── A <────1──── B*
+        //          ↑
+        //          └2─ A <──1── B*
+        trees[2]->extra = true;
+        trees[2]->size = tree_len * 0;
+
+        push(0, trees[1], stateB);
+        push(1, trees[2], stateA);
+        push(1, trees[1], stateB);
+
+        // . <──0── A <──1── B*
+        AssertThat(ts_stack_merge(stack, 0, 1), IsTrue());
+        AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
+        AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
+          {stateB, 0},
+          {stateA, 1},
+          {1, 2},
+        })));
+      });
+    });
   });
 
   describe("pop_count(version, count)", [&]() {
     before_each([&]() {
       // . <──0── A <──1── B <──2── C*
-      ts_stack_push(stack, 0, trees[0], false, stateA);
-      ts_stack_push(stack, 0, trees[1], false, stateB);
-      ts_stack_push(stack, 0, trees[2], false, stateC);
+      push(0, trees[0], stateA);
+      push(0, trees[1], stateB);
+      push(0, trees[2], stateC);
     });
 
     it("creates a new version with the given number of entries removed", [&]() {
       // . <──0── A <──1── B <──2── C*
       //          ↑
       //          └─*
-      StackPopResult pop = ts_stack_pop_count(stack, 0, 2);
-      AssertThat(pop.stopped_at_error, Equals(false));
-      AssertThat(pop.slices.size, Equals<size_t>(1));
+      StackSliceArray pop = ts_stack_pop_count(stack, 0, 2);
+      AssertThat(pop.size, Equals<size_t>(1));
       AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
 
-      StackSlice slice = pop.slices.contents[0];
+      StackSlice slice = pop.contents[0];
       AssertThat(slice.version, Equals<StackVersion>(1));
       AssertThat(slice.trees, Equals(vector<Tree *>({ trees[1], trees[2] })));
-      AssertThat(ts_stack_top_state(stack, 1), Equals(stateA));
+      AssertThat(ts_stack_state(stack, 1), Equals(stateA));
 
-      free_slice_array(&pop.slices);
+      free_slice_array(&pool,&pop);
     });
 
     it("does not count 'extra' trees toward the given count", [&]() {
@@ -235,49 +273,14 @@ describe("Stack", [&]() {
       // . <──0── A <──1── B <──2── C*
       // ↑
       // └─*
-      StackPopResult pop = ts_stack_pop_count(stack, 0, 2);
-      AssertThat(pop.stopped_at_error, Equals(false));
-      AssertThat(pop.slices.size, Equals<size_t>(1));
+      StackSliceArray pop = ts_stack_pop_count(stack, 0, 2);
+      AssertThat(pop.size, Equals<size_t>(1));
 
-      StackSlice slice = pop.slices.contents[0];
+      StackSlice slice = pop.contents[0];
       AssertThat(slice.trees, Equals(vector<Tree *>({ trees[0], trees[1], trees[2] })));
-      AssertThat(ts_stack_top_state(stack, 1), Equals(1));
-
-      free_slice_array(&pop.slices);
-    });
-
-    it("stops popping entries early if it reaches an error tree", [&]() {
-      // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D*
-      ts_stack_push(stack, 0, trees[3], false, ERROR_STATE);
-      ts_stack_push(stack, 0, trees[4], false, stateD);
-
-      // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D*
-      //                                       ↑
-      //                                       └─*
-      StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
-      AssertThat(pop.stopped_at_error, Equals(true));
+      AssertThat(ts_stack_state(stack, 1), Equals(1));
 
-      AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
-      AssertThat(ts_stack_top_state(stack, 1), Equals(ERROR_STATE));
-
-      AssertThat(pop.slices.size, Equals<size_t>(1));
-      StackSlice slice = pop.slices.contents[0];
-      AssertThat(slice.version, Equals<StackVersion>(1));
-      AssertThat(slice.trees, Equals(vector<Tree *>({ trees[4] })));
-
-      free_slice_array(&pop.slices);
-    });
-
-    it("preserves the push count of the popped version", [&]() {
-      // . <──0── A <──1── B <──2── C*
-      //          ↑
-      //          └─*
-      StackPopResult pop = ts_stack_pop_count(stack, 0, 2);
-
-      AssertThat(ts_stack_push_count(stack, 0), Equals<unsigned>(3));
-      AssertThat(ts_stack_push_count(stack, 1), Equals<unsigned>(3));
-
-      free_slice_array(&pop.slices);
+      free_slice_array(&pool,&pop);
     });
 
     describe("when the version has been merged", [&]() {
@@ -285,14 +288,14 @@ describe("Stack", [&]() {
         // . <──0── A <──1── B <──2── C <──3── D <──10── I*
         //          ↑                          |
         //          └───4─── E <──5── F <──6───┘
-        ts_stack_push(stack, 0, trees[3], false, stateD);
-        StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
-        free_slice_array(&pop.slices);
-        ts_stack_push(stack, 1, trees[4], false, stateE);
-        ts_stack_push(stack, 1, trees[5], false, stateF);
-        ts_stack_push(stack, 1, trees[6], false, stateD);
+        push(0, trees[3], stateD);
+        StackSliceArray pop = ts_stack_pop_count(stack, 0, 3);
+        free_slice_array(&pool,&pop);
+        push(1, trees[4], stateE);
+        push(1, trees[5], stateF);
+        push(1, trees[6], stateD);
         ts_stack_merge(stack, 0, 1);
-        ts_stack_push(stack, 0, trees[10], false, stateI);
+        push(0, trees[10], stateI);
 
         AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
         AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
@@ -314,14 +317,14 @@ describe("Stack", [&]() {
           //          |        └*
           //          |
           //          └───4─── E*
-          StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
-          AssertThat(pop.slices.size, Equals<size_t>(2));
+          StackSliceArray pop = ts_stack_pop_count(stack, 0, 3);
+          AssertThat(pop.size, Equals<size_t>(2));
 
-          StackSlice slice1 = pop.slices.contents[0];
+          StackSlice slice1 = pop.contents[0];
           AssertThat(slice1.version, Equals<StackVersion>(1));
           AssertThat(slice1.trees, Equals(vector<Tree *>({ trees[2], trees[3], trees[10] })));
 
-          StackSlice slice2 = pop.slices.contents[1];
+          StackSlice slice2 = pop.contents[1];
           AssertThat(slice2.version, Equals<StackVersion>(2));
           AssertThat(slice2.trees, Equals(vector<Tree *>({ trees[5], trees[6], trees[10] })));
 
@@ -347,7 +350,7 @@ describe("Stack", [&]() {
             {1, 2},
           })));
 
-          free_slice_array(&pop.slices);
+          free_slice_array(&pool,&pop);
         });
       });
 
@@ -358,18 +361,18 @@ describe("Stack", [&]() {
           //          └───5─── F <──6── G <──7───┘
           //                                     |
           //                                     └*
-          StackPopResult pop = ts_stack_pop_count(stack, 0, 1);
-          AssertThat(pop.slices.size, Equals<size_t>(1));
+          StackSliceArray pop = ts_stack_pop_count(stack, 0, 1);
+          AssertThat(pop.size, Equals<size_t>(1));
 
-          StackSlice slice1 = pop.slices.contents[0];
+          StackSlice slice1 = pop.contents[0];
           AssertThat(slice1.version, Equals<StackVersion>(1));
           AssertThat(slice1.trees, Equals(vector<Tree *>({ trees[10] })));
 
           AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
-          AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
-          AssertThat(ts_stack_top_state(stack, 1), Equals(stateD));
+          AssertThat(ts_stack_state(stack, 0), Equals(stateI));
+          AssertThat(ts_stack_state(stack, 1), Equals(stateD));
 
-          free_slice_array(&pop.slices);
+          free_slice_array(&pool,&pop);
         });
       });
 
@@ -380,22 +383,22 @@ describe("Stack", [&]() {
           //          ├───4─── E <──5── F <──6───┘
           //          |
           //          └*
-          StackPopResult pop = ts_stack_pop_count(stack, 0, 4);
-          AssertThat(pop.slices.size, Equals<size_t>(2));
+          StackSliceArray pop = ts_stack_pop_count(stack, 0, 4);
+          AssertThat(pop.size, Equals<size_t>(2));
 
-          StackSlice slice1 = pop.slices.contents[0];
+          StackSlice slice1 = pop.contents[0];
           AssertThat(slice1.version, Equals<StackVersion>(1));
           AssertThat(slice1.trees, Equals(vector<Tree *>({ trees[1], trees[2], trees[3], trees[10] })));
 
-          StackSlice slice2 = pop.slices.contents[1];
+          StackSlice slice2 = pop.contents[1];
           AssertThat(slice2.version, Equals<StackVersion>(1));
-          AssertThat(slice2.trees, Equals(vector<Tree *>({ trees[4], trees[5], trees[6], trees[10] })))
+          AssertThat(slice2.trees, Equals(vector<Tree *>({ trees[4], trees[5], trees[6], trees[10] })));
 
           AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
-          AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
-          AssertThat(ts_stack_top_state(stack, 1), Equals(stateA));
+          AssertThat(ts_stack_state(stack, 0), Equals(stateI));
+          AssertThat(ts_stack_state(stack, 1), Equals(stateA));
 
-          free_slice_array(&pop.slices);
+          free_slice_array(&pool,&pop);
         });
       });
 
@@ -406,12 +409,12 @@ describe("Stack", [&]() {
           //          ├───4─── E <──5── F <──6───┘
           //          |                          |
           //          └───7─── G <──8── H <──9───┘
-          StackPopResult pop = ts_stack_pop_count(stack, 0, 4);
-          free_slice_array(&pop.slices);
-          ts_stack_push(stack, 1, trees[7], false, stateG);
-          ts_stack_push(stack, 1, trees[8], false, stateH);
-          ts_stack_push(stack, 1, trees[9], false, stateD);
-          ts_stack_push(stack, 1, trees[10], false, stateI);
+          StackSliceArray pop = ts_stack_pop_count(stack, 0, 4);
+          free_slice_array(&pool,&pop);
+          push(1, trees[7], stateG);
+          push(1, trees[8], stateH);
+          push(1, trees[9], stateD);
+          push(1, trees[10], stateI);
           ts_stack_merge(stack, 0, 1);
 
           AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
@@ -436,27 +439,27 @@ describe("Stack", [&]() {
           //          |
           //          └───7─── G <──8── H*
           pop = ts_stack_pop_count(stack, 0, 2);
-          AssertThat(pop.slices.size, Equals<size_t>(3));
+          AssertThat(pop.size, Equals<size_t>(3));
 
-          StackSlice slice1 = pop.slices.contents[0];
+          StackSlice slice1 = pop.contents[0];
           AssertThat(slice1.version, Equals<StackVersion>(1));
-          AssertThat(slice1.trees, Equals(vector<Tree *>({ trees[3], trees[10] })))
+          AssertThat(slice1.trees, Equals(vector<Tree *>({ trees[3], trees[10] })));
 
-          StackSlice slice2 = pop.slices.contents[1];
+          StackSlice slice2 = pop.contents[1];
           AssertThat(slice2.version, Equals<StackVersion>(2));
-          AssertThat(slice2.trees, Equals(vector<Tree *>({ trees[6], trees[10] })))
+          AssertThat(slice2.trees, Equals(vector<Tree *>({ trees[6], trees[10] })));
 
-          StackSlice slice3 = pop.slices.contents[2];
+          StackSlice slice3 = pop.contents[2];
           AssertThat(slice3.version, Equals<StackVersion>(3));
-          AssertThat(slice3.trees, Equals(vector<Tree *>({ trees[9], trees[10] })))
+          AssertThat(slice3.trees, Equals(vector<Tree *>({ trees[9], trees[10] })));
 
           AssertThat(ts_stack_version_count(stack), Equals<size_t>(4));
-          AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
-          AssertThat(ts_stack_top_state(stack, 1), Equals(stateC));
-          AssertThat(ts_stack_top_state(stack, 2), Equals(stateF));
-          AssertThat(ts_stack_top_state(stack, 3), Equals(stateH));
+          AssertThat(ts_stack_state(stack, 0), Equals(stateI));
+          AssertThat(ts_stack_state(stack, 1), Equals(stateC));
+          AssertThat(ts_stack_state(stack, 2), Equals(stateF));
+          AssertThat(ts_stack_state(stack, 3), Equals(stateH));
 
-          free_slice_array(&pop.slices);
+          free_slice_array(&pool,&pop);
         });
       });
     });
@@ -464,53 +467,52 @@ describe("Stack", [&]() {
 
   describe("pop_pending(version)", [&]() {
     before_each([&]() {
-      ts_stack_push(stack, 0, trees[0], false, stateA);
+      push(0, trees[0], stateA);
     });
 
     it("removes the top node from the stack if it was pushed in pending mode", [&]() {
       ts_stack_push(stack, 0, trees[1], true, stateB);
+      ts_tree_retain(trees[1]);
 
-      StackPopResult pop = ts_stack_pop_pending(stack, 0);
-      AssertThat(pop.stopped_at_error, Equals(false));
-      AssertThat(pop.slices.size, Equals<size_t>(1));
+      StackSliceArray pop = ts_stack_pop_pending(stack, 0);
+      AssertThat(pop.size, Equals<size_t>(1));
 
       AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
         {stateA, 0},
         {1, 1},
       })));
 
-      free_slice_array(&pop.slices);
+      free_slice_array(&pool,&pop);
     });
 
     it("skips entries whose trees are extra", [&]() {
       ts_stack_push(stack, 0, trees[1], true, stateB);
+      ts_tree_retain(trees[1]);
 
       trees[2]->extra = true;
       trees[3]->extra = true;
 
-      ts_stack_push(stack, 0, trees[2], false, stateB);
-      ts_stack_push(stack, 0, trees[3], false, stateB);
+      push(0, trees[2], stateB);
+      push(0, trees[3], stateB);
 
-      StackPopResult pop = ts_stack_pop_pending(stack, 0);
-      AssertThat(pop.stopped_at_error, Equals(false));
-      AssertThat(pop.slices.size, Equals<size_t>(1));
+      StackSliceArray pop = ts_stack_pop_pending(stack, 0);
+      AssertThat(pop.size, Equals<size_t>(1));
 
-      AssertThat(pop.slices.contents[0].trees, Equals(vector<Tree *>({ trees[1], trees[2], trees[3] })));
+      AssertThat(pop.contents[0].trees, Equals(vector<Tree *>({ trees[1], trees[2], trees[3] })));
 
       AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
         {stateA, 0},
         {1, 1},
       })));
 
-      free_slice_array(&pop.slices);
+      free_slice_array(&pool,&pop);
     });
 
     it("does nothing if the top node was not pushed in pending mode", [&]() {
-      ts_stack_push(stack, 0, trees[1], false, stateB);
+      push(0, trees[1], stateB);
 
-      StackPopResult pop = ts_stack_pop_pending(stack, 0);
-      AssertThat(pop.stopped_at_error, Equals(false));
-      AssertThat(pop.slices.size, Equals<size_t>(0));
+      StackSliceArray pop = ts_stack_pop_pending(stack, 0);
+      AssertThat(pop.size, Equals<size_t>(0));
 
       AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
         {stateB, 0},
@@ -518,33 +520,68 @@ describe("Stack", [&]() {
         {1, 2},
       })));
 
-      free_slice_array(&pop.slices);
+      free_slice_array(&pool,&pop);
     });
   });
 
   describe("setting external token state", [&]() {
-    TSExternalTokenState external_token_state1, external_token_state2;
+    before_each([&]() {
+      trees[1]->has_external_tokens = true;
+      trees[2]->has_external_tokens = true;
+      ts_external_token_state_init(&trees[1]->external_token_state, NULL, 0);
+      ts_external_token_state_init(&trees[2]->external_token_state, NULL, 0);
+    });
 
     it("allows the state to be retrieved", [&]() {
-      AssertThat(ts_stack_external_token_state(stack, 0), Equals(nullptr));
+      AssertThat(ts_stack_last_external_token(stack, 0), Equals<Tree *>(nullptr));
 
-      ts_stack_set_external_token_state(stack, 0, &external_token_state1);
-      AssertThat(ts_stack_external_token_state(stack, 0), Equals(&external_token_state1));
+      ts_stack_set_last_external_token(stack, 0, trees[1]);
+      AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1]));
 
       ts_stack_copy_version(stack, 0);
-      AssertThat(ts_stack_external_token_state(stack, 0), Equals(&external_token_state1));
+      AssertThat(ts_stack_last_external_token(stack, 1), Equals(trees[1]));
+
+      ts_stack_set_last_external_token(stack, 0, trees[2]);
+      AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[2]));
     });
 
     it("does not merge stack versions with different external token states", [&]() {
+      ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2);
+      ts_external_token_state_init(&trees[2]->external_token_state, "ABCD", 2);
+
       ts_stack_copy_version(stack, 0);
-      ts_stack_push(stack, 0, trees[0], false, 5);
-      ts_stack_push(stack, 1, trees[0], false, 5);
+      push(0, trees[0], 5);
+      push(1, trees[0], 5);
 
-      ts_stack_set_external_token_state(stack, 0, &external_token_state1);
-      ts_stack_set_external_token_state(stack, 0, &external_token_state2);
+      ts_stack_set_last_external_token(stack, 0, trees[1]);
+      ts_stack_set_last_external_token(stack, 1, trees[2]);
 
       AssertThat(ts_stack_merge(stack, 0, 1), IsFalse());
     });
+
+    it("merges stack versions with identical external token states", [&]() {
+      ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2);
+      ts_external_token_state_init(&trees[2]->external_token_state, "abcd", 2);
+
+      ts_stack_copy_version(stack, 0);
+      push(0, trees[0], 5);
+      push(1, trees[0], 5);
+
+      ts_stack_set_last_external_token(stack, 0, trees[1]);
+      ts_stack_set_last_external_token(stack, 1, trees[2]);
+
+      AssertThat(ts_stack_merge(stack, 0, 1), IsTrue());
+    });
+
+    it("does not distinguish between an *empty* external token state and *no* external token state", [&]() {
+      ts_stack_copy_version(stack, 0);
+      push(0, trees[0], 5);
+      push(1, trees[0], 5);
+
+      ts_stack_set_last_external_token(stack, 0, trees[1]);
+
+      AssertThat(ts_stack_merge(stack, 0, 1), IsTrue());
+    });
   });
 });