tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/runtime/document.h

@@ -3,12 +3,15 @@
 
 #include "runtime/parser.h"
 #include "runtime/tree.h"
+#include "runtime/get_changed_ranges.h"
 #include <stdbool.h>
 
 struct TSDocument {
   Parser parser;
   TSInput input;
   Tree *tree;
+  TreePath tree_path1;
+  TreePath tree_path2;
   size_t parse_count;
   bool valid;
   bool owns_input;

data/ext/tree-sitter/tree-sitter/src/runtime/error_costs.h

@@ -1,25 +1,11 @@
 #ifndef RUNTIME_ERROR_COSTS_H_
 #define RUNTIME_ERROR_COSTS_H_
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 #define ERROR_STATE 0
-#define ERROR_COST_PER_SKIPPED_TREE 10
-#define ERROR_COST_PER_SKIPPED_LINE 3
-#define ERROR_COST_PER_SKIPPED_CHAR 0
-
-typedef struct {
-  unsigned count;
-  unsigned cost;
-  unsigned push_count;
-} ErrorStatus;
-
-int error_status_compare(ErrorStatus a, ErrorStatus b);
-
-#ifdef __cplusplus
-}
-#endif
+#define ERROR_COST_PER_RECOVERY 500
+#define ERROR_COST_PER_MISSING_TREE 110
+#define ERROR_COST_PER_SKIPPED_TREE 100
+#define ERROR_COST_PER_SKIPPED_LINE 30
+#define ERROR_COST_PER_SKIPPED_CHAR 1
 
 #endif

data/ext/tree-sitter/tree-sitter/src/runtime/get_changed_ranges.c

@@ -0,0 +1,355 @@
+#include "runtime/get_changed_ranges.h"
+#include "runtime/tree.h"
+#include "runtime/language.h"
+#include "runtime/error_costs.h"
+#include <assert.h>
+
+// #define DEBUG_GET_CHANGED_RANGES
+
+typedef Array(TSRange) RangeArray;
+
+static void range_array_add(RangeArray *results, TSPoint start, TSPoint end) {
+  if (results->size > 0) {
+    TSRange *last_range = array_back(results);
+    if (point_lte(start, last_range->end)) {
+      last_range->end = end;
+      return;
+    }
+  }
+
+  if (point_lt(start, end)) {
+    TSRange range = { start, end };
+    array_push(results, range);
+  }
+}
+
+typedef struct {
+  TreePath path;
+  const TSLanguage *language;
+  unsigned visible_depth;
+  bool in_padding;
+} Iterator;
+
+static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *language) {
+  array_clear(path);
+  array_push(path, ((TreePathEntry){
+    .tree = tree,
+    .position = length_zero(),
+    .child_index = 0,
+    .structural_child_index = 0,
+  }));
+  return (Iterator) {
+    .path = *path,
+    .language = language,
+    .visible_depth = 1,
+    .in_padding = false,
+  };
+}
+
+static bool iterator_done(Iterator *self) {
+  return self->path.size == 0;
+}
+
+Length iterator_start_position(Iterator *self) {
+  TreePathEntry entry = *array_back(&self->path);
+  if (self->in_padding) {
+    return entry.position;
+  } else {
+    return length_add(entry.position, entry.tree->padding);
+  }
+}
+
+Length iterator_end_position(Iterator *self) {
+  TreePathEntry entry = *array_back(&self->path);
+  Length result = length_add(entry.position, entry.tree->padding);
+  if (self->in_padding) {
+    return result;
+  } else {
+    return length_add(result, entry.tree->size);
+  }
+}
+
+static bool iterator_tree_is_visible(const Iterator *self) {
+  TreePathEntry entry = *array_back(&self->path);
+  if (entry.tree->visible) return true;
+  if (self->path.size > 1) {
+    Tree *parent = self->path.contents[self->path.size - 2].tree;
+    const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id);
+    return alias_sequence && alias_sequence[entry.structural_child_index] != 0;
+  }
+  return false;
+}
+
+static void iterator_get_visible_state(const Iterator *self, Tree **tree,
+                                       TSSymbol *alias_symbol, uint32_t *start_byte) {
+  uint32_t i = self->path.size - 1;
+
+  if (self->in_padding) {
+    if (i == 0) return;
+    i--;
+  }
+
+  for (; i + 1 > 0; i--) {
+    TreePathEntry entry = self->path.contents[i];
+
+    if (i > 0) {
+      Tree *parent = self->path.contents[i - 1].tree;
+      const TSSymbol *alias_sequence = ts_language_alias_sequence(
+        self->language,
+        parent->alias_sequence_id
+      );
+      if (alias_sequence) {
+        *alias_symbol = alias_sequence[entry.structural_child_index];
+      }
+    }
+
+    if (entry.tree->visible || *alias_symbol) {
+      *tree = entry.tree;
+      *start_byte = entry.position.bytes;
+      break;
+    }
+  }
+}
+
+static void iterator_ascend(Iterator *self) {
+  if (iterator_done(self)) return;
+  if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--;
+  if (array_back(&self->path)->child_index > 0) self->in_padding = false;
+  self->path.size--;
+}
+
+static bool iterator_descend(Iterator *self, uint32_t goal_position) {
+  if (self->in_padding) return false;
+
+  bool did_descend;
+  do {
+    did_descend = false;
+    TreePathEntry entry = *array_back(&self->path);
+    Length position = entry.position;
+    uint32_t structural_child_index = 0;
+    for (uint32_t i = 0; i < entry.tree->children.size; i++) {
+      Tree *child = entry.tree->children.contents[i];
+      Length child_left = length_add(position, child->padding);
+      Length child_right = length_add(child_left, child->size);
+
+      if (child_right.bytes > goal_position) {
+        array_push(&self->path, ((TreePathEntry){
+          .tree = child,
+          .position = position,
+          .child_index = i,
+          .structural_child_index = structural_child_index,
+        }));
+
+        if (iterator_tree_is_visible(self)) {
+          if (child_left.bytes > goal_position) {
+            self->in_padding = true;
+          } else {
+            self->visible_depth++;
+          }
+          return true;
+        }
+
+        did_descend = true;
+        break;
+      }
+
+      position = child_right;
+      if (!child->extra) structural_child_index++;
+    }
+  } while (did_descend);
+
+  return false;
+}
+
+static void iterator_advance(Iterator *self) {
+  if (self->in_padding) {
+    self->in_padding = false;
+    if (iterator_tree_is_visible(self)) {
+      self->visible_depth++;
+    } else {
+      iterator_descend(self, 0);
+    }
+    return;
+  }
+
+  for (;;) {
+    if (iterator_tree_is_visible(self)) self->visible_depth--;
+    TreePathEntry entry = array_pop(&self->path);
+    if (iterator_done(self)) return;
+
+    Tree *parent = array_back(&self->path)->tree;
+    uint32_t child_index = entry.child_index + 1;
+    if (parent->children.size > child_index) {
+      Length position = length_add(entry.position, ts_tree_total_size(entry.tree));
+      uint32_t structural_child_index = entry.structural_child_index;
+      if (!entry.tree->extra) structural_child_index++;
+      Tree *next_child = parent->children.contents[child_index];
+
+      array_push(&self->path, ((TreePathEntry){
+        .tree = next_child,
+        .position = position,
+        .child_index = child_index,
+        .structural_child_index = structural_child_index,
+      }));
+
+      if (iterator_tree_is_visible(self)) {
+        if (next_child->padding.bytes > 0) {
+          self->in_padding = true;
+        } else {
+          self->visible_depth++;
+        }
+      } else {
+        iterator_descend(self, 0);
+      }
+      break;
+    }
+  }
+}
+
+typedef enum {
+  IteratorDiffers,
+  IteratorMayDiffer,
+  IteratorMatches,
+} IteratorComparison;
+
+IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) {
+  Tree *old_tree = NULL, *new_tree = NULL;
+  uint32_t old_start = 0, new_start = 0;
+  TSSymbol old_alias_symbol = 0, new_alias_symbol = 0;
+  iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
+  iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
+
+  if (!old_tree && !new_tree) return IteratorMatches;
+  if (!old_tree || !new_tree) return IteratorDiffers;
+
+  if (old_alias_symbol == new_alias_symbol) {
+    if (old_start == new_start) {
+      if (!old_tree->has_changes &&
+          old_tree->symbol == new_tree->symbol &&
+          old_tree->symbol != ts_builtin_sym_error &&
+          old_tree->size.bytes == new_tree->size.bytes &&
+          old_tree->parse_state != TS_TREE_STATE_NONE &&
+          new_tree->parse_state != TS_TREE_STATE_NONE &&
+          (old_tree->parse_state == ERROR_STATE) ==
+          (new_tree->parse_state == ERROR_STATE)) {
+        return IteratorMatches;
+      }
+    }
+
+    if (old_tree->symbol == new_tree->symbol) {
+      return IteratorMayDiffer;
+    }
+  }
+
+  return IteratorDiffers;
+}
+
+#ifdef DEBUG_GET_CHANGED_RANGES
+static inline void iterator_print_state(Iterator *self) {
+  TreePathEntry entry = *array_back(&self->path);
+  TSPoint start = iterator_start_position(self).extent;
+  TSPoint end = iterator_end_position(self).extent;
+  const char *name = ts_language_symbol_name(self->language, entry.tree->symbol);
+  printf(
+    "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
+    name, self->in_padding ? "(p)" : "   ",
+    self->visible_depth,
+    start.row, start.column,
+    end.row, end.column
+  );
+}
+#endif
+
+unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree,
+                                    TreePath *path1, TreePath *path2,
+                                    const TSLanguage *language, TSRange **ranges) {
+  RangeArray results = array_new();
+
+  Iterator old_iter = iterator_new(path1, old_tree, language);
+  Iterator new_iter = iterator_new(path2, new_tree, language);
+
+  Length position = iterator_start_position(&old_iter);
+  Length next_position = iterator_start_position(&new_iter);
+  if (position.bytes < next_position.bytes) {
+    range_array_add(&results, position.extent, next_position.extent);
+    position = next_position;
+  } else if (position.bytes > next_position.bytes) {
+    range_array_add(&results, next_position.extent, position.extent);
+    next_position = position;
+  }
+
+  do {
+    #ifdef DEBUG_GET_CHANGED_RANGES
+    printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column);
+    iterator_print_state(&old_iter);
+    printf("\tvs\t");
+    iterator_print_state(&new_iter);
+    puts("");
+    #endif
+
+    bool is_changed = false;
+    switch (iterator_compare(&old_iter, &new_iter)) {
+      case IteratorMatches:
+        next_position = iterator_end_position(&old_iter);
+        break;
+
+      case IteratorMayDiffer:
+        if (iterator_descend(&old_iter, position.bytes)) {
+          if (!iterator_descend(&new_iter, position.bytes)) {
+            is_changed = true;
+            next_position = iterator_end_position(&old_iter);
+          }
+        } else if (iterator_descend(&new_iter, position.bytes)) {
+          is_changed = true;
+          next_position = iterator_end_position(&new_iter);
+        } else {
+          next_position = length_min(
+            iterator_end_position(&old_iter),
+            iterator_end_position(&new_iter)
+          );
+        }
+        break;
+
+      case IteratorDiffers:
+        is_changed = true;
+        next_position = length_min(
+          iterator_end_position(&old_iter),
+          iterator_end_position(&new_iter)
+        );
+        break;
+    }
+
+    while (
+      !iterator_done(&old_iter) &&
+      iterator_end_position(&old_iter).bytes <= next_position.bytes
+    ) iterator_advance(&old_iter);
+
+    while (
+      !iterator_done(&new_iter) &&
+      iterator_end_position(&new_iter).bytes <= next_position.bytes
+    ) iterator_advance(&new_iter);
+
+    while (old_iter.visible_depth > new_iter.visible_depth) iterator_ascend(&old_iter);
+
+    while (new_iter.visible_depth > old_iter.visible_depth) iterator_ascend(&new_iter);
+
+    if (is_changed) {
+      #ifdef DEBUG_GET_CHANGED_RANGES
+      printf(
+        "  change: [[%u, %u] - [%u, %u]]\n",
+        position.extent.row, position.extent.column,
+        next_position.extent.row, next_position.extent.column
+      );
+      #endif
+
+      range_array_add(&results, position.extent, next_position.extent);
+    }
+
+    position = next_position;
+  } while (!iterator_done(&old_iter) && !iterator_done(&new_iter));
+
+  *path1 = old_iter.path;
+  *path2 = new_iter.path;
+  *ranges = results.contents;
+  return results.size;
+}

data/ext/tree-sitter/tree-sitter/src/runtime/get_changed_ranges.h

@@ -0,0 +1,20 @@
+#ifndef RUNTIME_GET_CHANGED_RANGES_H_
+#define RUNTIME_GET_CHANGED_RANGES_H_
+
+#include "runtime/tree.h"
+
+typedef struct {
+  Tree *tree;
+  Length position;
+  uint32_t child_index;
+  uint32_t structural_child_index;
+} TreePathEntry;
+
+typedef Array(TreePathEntry) TreePath;
+
+unsigned ts_tree_get_changed_ranges(
+  Tree *old_tree, Tree *new_tree, TreePath *path1, TreePath *path2,
+  const TSLanguage *language, TSRange **ranges
+);
+
+#endif  // RUNTIME_GET_CHANGED_RANGES_H_

data/ext/tree-sitter/tree-sitter/src/runtime/language.c

@@ -2,55 +2,57 @@
 #include "runtime/tree.h"
 #include "runtime/error_costs.h"
 
-static const TSParseAction SHIFT_ERROR = {
-  .type = TSParseActionTypeShift, .params = {.to_state = ERROR_STATE}
-};
-
 void ts_language_table_entry(const TSLanguage *self, TSStateId state,
                              TSSymbol symbol, TableEntry *result) {
-  uint32_t action_index;
-  if (symbol == ts_builtin_sym_error) {
-    if (state == ERROR_STATE) {
-      result->action_count = 1;
-      result->is_reusable = false;
-      result->depends_on_lookahead = false;
-      result->actions = &SHIFT_ERROR;
-      return;
-    }
-    action_index = 0;
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
+    result->action_count = 0;
+    result->is_reusable = false;
+    result->actions = NULL;
   } else {
     assert(symbol < self->token_count);
-    action_index = self->parse_table[state * self->symbol_count + symbol];
+    uint32_t action_index = self->parse_table[state * self->symbol_count + symbol];
+    const TSParseActionEntry *entry = &self->parse_actions[action_index];
+    result->action_count = entry->count;
+    result->is_reusable = entry->reusable;
+    result->actions = (const TSParseAction *)(entry + 1);
   }
-
-  const TSParseActionEntry *entry = &self->parse_actions[action_index];
-  result->action_count = entry->count;
-  result->is_reusable = entry->reusable;
-  result->depends_on_lookahead = entry->depends_on_lookahead;
-  result->actions = (const TSParseAction *)(entry + 1);
 }
 
 uint32_t ts_language_symbol_count(const TSLanguage *language) {
-  return language->symbol_count;
+  return language->symbol_count + language->alias_count;
 }
 
 uint32_t ts_language_version(const TSLanguage *language) {
   return language->version;
 }
 
-TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language,
-                                             TSSymbol symbol) {
-  if (symbol == ts_builtin_sym_error)
-    return (TSSymbolMetadata){
-      .visible = true, .named = true, .extra = false, .structural = true,
-    };
-  else
+TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) {
+  if (symbol == ts_builtin_sym_error)  {
+    return (TSSymbolMetadata){.visible = true, .named = true};
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return (TSSymbolMetadata){.visible = false, .named = false};
+  } else {
     return language->symbol_metadata[symbol];
+  }
 }
 
 const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) {
-  if (symbol == ts_builtin_sym_error)
+  if (symbol == ts_builtin_sym_error) {
     return "ERROR";
-  else
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return "_ERROR";
+  } else {
     return language->symbol_names[symbol];
+  }
+}
+
+TSSymbolType ts_language_symbol_type(const TSLanguage *language, TSSymbol symbol) {
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
+  if (metadata.named) {
+    return TSSymbolTypeRegular;
+  } else if (metadata.visible) {
+    return TSSymbolTypeAnonymous;
+  } else {
+    return TSSymbolTypeAuxiliary;
+  }
 }