tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/syntax_grammar.cc

@@ -1,34 +1,36 @@
 #include "compiler/syntax_grammar.h"
-#include <vector>
-#include <string>
-#include <utility>
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/built_in_symbols.h"
 
 namespace tree_sitter {
 
-using std::string;
-using std::to_string;
-using std::pair;
-using std::vector;
-using std::set;
+bool ProductionStep::operator==(const ProductionStep &other) const {
+  return symbol == other.symbol &&
+    precedence == other.precedence &&
+    associativity == other.associativity &&
+    alias == other.alias;
+}
 
-SyntaxVariable::SyntaxVariable(const string &name, VariableType type,
-                               const vector<Production> &productions)
-    : name(name), productions(productions), type(type) {}
+bool ProductionStep::operator!=(const ProductionStep &other) const {
+  return !operator==(other);
+}
 
-ProductionStep::ProductionStep(const rules::Symbol &symbol, int precedence,
-                               rules::Associativity associativity)
-    : symbol(symbol), precedence(precedence), associativity(associativity) {}
+bool ProductionStep::operator<(const ProductionStep &other) const {
+ if (symbol < other.symbol) return true;
+ if (other.symbol < symbol) return false;
+ if (precedence < other.precedence) return true;
+ if (other.precedence < precedence) return false;
+ if (associativity < other.associativity) return true;
+ if (other.associativity < associativity) return false;
+ return alias < other.alias;
+}
 
-bool ExternalToken::operator==(const ExternalToken &other) const {
-  return name == other.name && type == other.type &&
-    corresponding_internal_token == other.corresponding_internal_token;
+bool Production::operator==(const Production &other) const {
+  return steps == other.steps && dynamic_precedence == other.dynamic_precedence;
 }
 
-bool ProductionStep::operator==(const ProductionStep &other) const {
-  return symbol == other.symbol && precedence == other.precedence &&
-         associativity == other.associativity;
+bool ExternalToken::operator==(const ExternalToken &other) const {
+  return name == other.name &&
+    type == other.type &&
+    corresponding_internal_token == other.corresponding_internal_token;
 }
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/syntax_grammar.h

@@ -4,47 +4,62 @@
 #include <vector>
 #include <string>
 #include <set>
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/variable.h"
+#include "compiler/rule.h"
+#include "compiler/grammar.h"
 
 namespace tree_sitter {
 
-struct ExternalToken {
-  std::string name;
-  VariableType type;
-  rules::Symbol corresponding_internal_token;
-
-  bool operator==(const ExternalToken &) const;
-};
-
 struct ProductionStep {
-  ProductionStep(const rules::Symbol &, int, rules::Associativity);
-  bool operator==(const ProductionStep &) const;
-
   rules::Symbol symbol;
   int precedence;
   rules::Associativity associativity;
+  rules::Alias alias;
+
+  bool operator==(const ProductionStep &) const;
+  bool operator!=(const ProductionStep &) const;
+  bool operator<(const ProductionStep &) const;
 };
 
-typedef std::vector<ProductionStep> Production;
+struct Production {
+  std::vector<ProductionStep> steps;
+  int dynamic_precedence;
 
-struct SyntaxVariable {
-  SyntaxVariable(const std::string &, VariableType,
-                 const std::vector<Production> &);
+  inline Production() : dynamic_precedence(0) {}
 
+  inline Production(std::vector<ProductionStep> &&steps, int dynamic_precedence = 0) :
+    steps(move(steps)), dynamic_precedence(dynamic_precedence) {}
+
+  bool operator==(const Production &) const;
+  inline ProductionStep &back() { return steps.back(); }
+  inline const ProductionStep &back() const { return steps.back(); }
+  inline bool empty() const { return steps.empty(); }
+  inline size_t size() const { return steps.size(); }
+  inline const ProductionStep &operator[](int i) const { return steps[i]; }
+  inline const ProductionStep &at(int i) const { return steps[i]; }
+  inline std::vector<ProductionStep>::const_iterator begin() const { return steps.begin(); }
+  inline std::vector<ProductionStep>::const_iterator end() const { return steps.end(); }
+};
+
+struct SyntaxVariable {
   std::string name;
-  std::vector<Production> productions;
   VariableType type;
+  std::vector<Production> productions;
 };
 
-typedef std::set<rules::Symbol> ConflictSet;
+struct ExternalToken {
+  std::string name;
+  VariableType type;
+  rules::Symbol corresponding_internal_token;
+
+  bool operator==(const ExternalToken &) const;
+};
 
 struct SyntaxGrammar {
   std::vector<SyntaxVariable> variables;
   std::set<rules::Symbol> extra_tokens;
-  std::set<ConflictSet> expected_conflicts;
+  std::set<std::set<rules::Symbol>> expected_conflicts;
   std::vector<ExternalToken> external_tokens;
+  std::set<rules::Symbol> variables_to_inline;
 };
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/util/make_visitor.h

@@ -0,0 +1,31 @@
+#ifndef COMPILER_UTIL_MAKE_VISITOR_H_
+#define COMPILER_UTIL_MAKE_VISITOR_H_
+
+namespace tree_sitter {
+namespace util {
+
+template <typename... Fns>
+struct visitor;
+
+template <typename Fn>
+struct visitor<Fn> : Fn {
+  using Fn::operator();
+  visitor(Fn fn) : Fn(fn) {}
+};
+
+template <typename Fn, typename... Fns>
+struct visitor<Fn, Fns...> : Fn, visitor<Fns...> {
+  using Fn::operator();
+  using visitor<Fns...>::operator();
+  visitor(Fn fn, Fns... fns) : Fn(fn), visitor<Fns...>(fns...) {}
+};
+
+template <typename... Fns>
+visitor<Fns...> make_visitor(Fns... fns) {
+  return visitor<Fns...>(fns...);
+}
+
+}  // namespace util
+}  // namespace tree_sitter
+
+#endif  // COMPILER_UTIL_MAKE_VISITOR_H_

data/ext/tree-sitter/tree-sitter/src/runtime/alloc.h

@@ -30,7 +30,7 @@ static inline void *ts_realloc(void *buffer, size_t size) {
 }
 
 static inline void ts_free(void *buffer) {
-  return ts_record_free(buffer);
+  ts_record_free(buffer);
 }
 
 static inline bool ts_toggle_allocation_recording(bool value) {

data/ext/tree-sitter/tree-sitter/src/runtime/array.h

@@ -12,18 +12,18 @@ extern "C" {
 #include <stdbool.h>
 #include "runtime/alloc.h"
 
-#define Array(T)     \
-  struct {           \
-    T *contents;     \
+#define Array(T)       \
+  struct {             \
     uint32_t size;     \
     uint32_t capacity; \
+    T *contents;       \
   }
 
 #define array_init(self) \
   ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
 
 #define array_new() \
-  { NULL, 0, 0 }
+  { 0, 0, NULL }
 
 #define array_get(self, index) \
   (assert((uint32_t)index < (self)->size), &(self)->contents[index])
@@ -34,33 +34,30 @@ extern "C" {
 
 #define array_clear(self) ((self)->size = 0)
 
-#define array_grow(self, new_capacity) \
-  array__grow((VoidArray *)(self), array__elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity) \
+  array__reserve((VoidArray *)(self), array__elem_size(self), new_capacity)
 
 #define array_erase(self, index) \
   array__erase((VoidArray *)(self), array__elem_size(self), index)
 
 #define array_delete(self) array__delete((VoidArray *)self)
 
-#define array_push(self, element)                                    \
-  (array_grow((self), (self)->size + 1), \
+#define array_push(self, element)                            \
+  (array__grow((VoidArray *)(self), array__elem_size(self)), \
    (self)->contents[(self)->size++] = (element))
 
 #define array_push_all(self, other)                                       \
-  array_splice((self), (self)->size, 0, (other)->size, (other)->contents)
+  array_splice((self), (self)->size, 0, (other))
 
-#define array_splice(self, index, old_count, new_count, new_elements)          \
+#define array_splice(self, index, old_count, new_array)          \
   array__splice((VoidArray *)(self), array__elem_size(self), index, old_count, \
-                new_count, (new_elements))
+                (new_array)->size, (new_array)->contents)
 
 #define array_insert(self, index, element) \
-  array_splice(self, index, 0, 1, &(element))
+  array__splice((VoidArray *)(self), array__elem_size(self), index, 0, 1, &element)
 
 #define array_pop(self) ((self)->contents[--(self)->size])
 
-#define array_reverse(self) \
-  array__reverse((VoidArray *)(self), array__elem_size(self))
-
 // Private
 
 typedef Array(void) VoidArray;
@@ -83,21 +80,25 @@ static inline void array__erase(VoidArray *self, size_t element_size,
   self->size--;
 }
 
-static inline void array__grow(VoidArray *self, size_t element_size,
-                               uint32_t new_capacity) {
+static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t new_capacity) {
   if (new_capacity > self->capacity) {
-    if (new_capacity < 2 * self->capacity)
-      new_capacity = 2 * self->capacity;
-    if (new_capacity < 8)
-      new_capacity = 8;
-    if (self->contents)
+    if (self->contents) {
       self->contents = ts_realloc(self->contents, new_capacity * element_size);
-    else
+    } else {
       self->contents = ts_calloc(new_capacity, element_size);
+    }
     self->capacity = new_capacity;
   }
 }
 
+static inline void array__grow(VoidArray *self, size_t element_size) {
+  if (self->size == self->capacity) {
+    size_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    array__reserve(self, element_size, new_capacity);
+  }
+}
+
 static inline void array__splice(VoidArray *self, size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, void *elements) {
@@ -106,7 +107,7 @@ static inline void array__splice(VoidArray *self, size_t element_size,
   uint32_t new_end = index + new_count;
   assert(old_end <= self->size);
 
-  array__grow(self, element_size, new_size);
+  array__reserve(self, element_size, new_size);
 
   char *contents = (char *)self->contents;
   if (self->size > old_end)
@@ -118,18 +119,6 @@ static inline void array__splice(VoidArray *self, size_t element_size,
   self->size += new_count - old_count;
 }
 
-static inline void array__reverse(VoidArray *self, size_t element_size) {
-  char swap[element_size];
-  char *contents = (char *)self->contents;
-  for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) {
-    size_t offset = i * element_size;
-    size_t reverse_offset = (self->size - 1 - i) * element_size;
-    memcpy(&swap, contents + offset, element_size);
-    memcpy(contents + offset, contents + reverse_offset, element_size);
-    memcpy(contents + reverse_offset, &swap, element_size);
-  }
-}
-
 #ifdef __cplusplus
 }
 #endif

data/ext/tree-sitter/tree-sitter/src/runtime/document.c

@@ -4,30 +4,31 @@
 #include "runtime/parser.h"
 #include "runtime/string_input.h"
 #include "runtime/document.h"
-#include "runtime/tree_path.h"
+#include "runtime/get_changed_ranges.h"
+
+#define LOG(...)                                                                                                   \
+  snprintf(self->parser.lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__);                   \
+  self->parser.lexer.logger.log(self->parser.lexer.logger.payload, TSLogTypeLex, self->parser.lexer.debug_buffer); \
 
 TSDocument *ts_document_new() {
   TSDocument *self = ts_calloc(1, sizeof(TSDocument));
-  if (!self)
-    goto error;
-
-  if (!parser_init(&self->parser))
-    goto error;
-
+  parser_init(&self->parser);
+  array_init(&self->tree_path1);
+  array_init(&self->tree_path2);
   return self;
-
-error:
-  if (self)
-    ts_free(self);
-  return NULL;
 }
 
 void ts_document_free(TSDocument *self) {
+  if (self->tree) ts_tree_release(&self->parser.tree_pool, self->tree);
+  if (self->tree_path1.contents) array_delete(&self->tree_path1);
+  if (self->tree_path2.contents) array_delete(&self->tree_path2);
   parser_destroy(&self->parser);
-  if (self->tree)
-    ts_tree_release(self->tree);
-  ts_document_set_input(self,
-                        (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 });
+  ts_document_set_input(self, (TSInput){
+    NULL,
+    NULL,
+    NULL,
+    TSInputEncodingUTF8,
+  });
   ts_free(self);
 }
 
@@ -40,7 +41,7 @@ void ts_document_set_language(TSDocument *self, const TSLanguage *language) {
   ts_document_invalidate(self);
   parser_set_language(&self->parser, language);
   if (self->tree) {
-    ts_tree_release(self->tree);
+    ts_tree_release(&self->parser.tree_pool, self->tree);
     self->tree = NULL;
   }
 }
@@ -97,12 +98,34 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) {
     edit.bytes_removed = max_bytes - edit.start_byte;
 
   ts_tree_edit(self->tree, &edit);
+
+  if (self->parser.print_debugging_graphs) {
+    ts_tree_print_dot_graph(self->tree, self->parser.language, stderr);
+  }
+}
+
+void ts_document_parse(TSDocument *self) {
+  ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = NULL,
+    .changed_range_count = NULL,
+  });
 }
 
 void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges,
                                               uint32_t *range_count) {
-  if (ranges) *ranges = NULL;
-  if (range_count) *range_count = 0;
+  ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = ranges,
+    .changed_range_count = range_count,
+  });
+}
+
+void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) {
+  if (options.changed_ranges && options.changed_range_count) {
+    *options.changed_ranges = NULL;
+    *options.changed_range_count = 0;
+  }
 
   if (!self->input.read || !self->parser.language)
     return;
@@ -111,20 +134,31 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   if (reusable_tree && !reusable_tree->has_changes)
     return;
 
-  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree);
+  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error);
 
   if (self->tree) {
     Tree *old_tree = self->tree;
     self->tree = tree;
 
-    if (ranges && range_count) {
-      tree_path_init(&self->parser.tree_path1, old_tree);
-      tree_path_init(&self->parser.tree_path2, tree);
-      tree_path_get_changes(&self->parser.tree_path1, &self->parser.tree_path2,
-                            ranges, range_count);
+    if (options.changed_ranges && options.changed_range_count) {
+      *options.changed_range_count = ts_tree_get_changed_ranges(
+        old_tree, tree, &self->tree_path1, &self->tree_path2,
+        self->parser.language, options.changed_ranges
+      );
+
+      if (self->parser.lexer.logger.log) {
+        for (unsigned i = 0; i < *options.changed_range_count; i++) {
+          TSRange range = (*options.changed_ranges)[i];
+          LOG(
+            "changed_range start:[%u %u], end:[%u %u]",
+            range.start.row, range.start.column,
+            range.end.row, range.end.column
+          );
+        }
+      }
     }
 
-    ts_tree_release(old_tree);
+    ts_tree_release(&self->parser.tree_pool, old_tree);
   }
 
   self->tree = tree;
@@ -132,19 +166,12 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   self->valid = true;
 }
 
-void ts_document_parse(TSDocument *self) {
-  ts_document_parse_and_get_changed_ranges(self, NULL, NULL);
-}
-
 void ts_document_invalidate(TSDocument *self) {
   self->valid = false;
 }
 
 TSNode ts_document_root_node(const TSDocument *self) {
-  TSNode result = ts_node_make(self->tree, 0, 0, 0);
-  while (result.data && !((Tree *)result.data)->visible)
-    result = ts_node_named_child(result, 0);
-  return result;
+  return ts_node_make(self->tree, 0, 0);
 }
 
 uint32_t ts_document_parse_count(const TSDocument *self) {