tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/test/helpers/load_language.h

@@ -0,0 +1,14 @@
+#ifndef HELPERS_LOAD_LANGUAGE_H_
+#define HELPERS_LOAD_LANGUAGE_H_
+
+#include "tree_sitter/compiler.h"
+#include "tree_sitter/runtime.h"
+#include <string>
+
+const TSLanguage *load_real_language(const std::string &name);
+
+const TSLanguage *load_test_language(const std::string &name,
+                                     const TSCompileResult &compile_result,
+                                     std::string external_scanner_path = "");
+
+#endif  // HELPERS_LOAD_LANGUAGE_H_

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/point_helpers.cc

@@ -15,9 +15,7 @@ bool operator==(const TSRange &left, const TSRange &right) {
 }
 
 bool operator==(const Length &left, const Length &right) {
-  return left.bytes == right.bytes &&
-    left.chars == right.chars &&
-    left.extent == right.extent;
+  return left.bytes == right.bytes && left.extent == right.extent;
 }
 
 bool operator<(const TSPoint &left, const TSPoint &right) {
@@ -40,6 +38,5 @@ std::ostream &operator<<(std::ostream &stream, const TSRange &range) {
 }
 
 ostream &operator<<(ostream &stream, const Length &length) {
-  return stream << "{chars:" << length.chars << ", bytes:" <<
-    length.bytes << ", extent:" << length.extent << "}";
+  return stream << "{bytes:" << length.bytes << ", extent:" << length.extent << "}";
 }

data/ext/tree-sitter/tree-sitter/test/helpers/random_helpers.cc

@@ -0,0 +1,59 @@
+#include <string>
+#include <vector>
+#include <random>
+#include <time.h>
+
+using std::string;
+using std::vector;
+
+static std::default_random_engine engine;
+
+unsigned get_time_as_seed() {
+  return time(nullptr);
+}
+
+void random_reseed(unsigned seed) {
+  engine.seed(seed);
+}
+
+unsigned random_unsigned() {
+  return std::uniform_int_distribution<unsigned>()(engine);
+}
+
+unsigned random_unsigned(unsigned max) {
+  return std::uniform_int_distribution<unsigned>(0, max - 1)(engine);
+}
+
+static string random_string(char min, char max) {
+  string result;
+  size_t length = random_unsigned(12);
+  for (size_t i = 0; i < length; i++) {
+    result += (min + random_unsigned(max - min));
+  }
+  return result;
+}
+
+static string random_char(string characters) {
+  size_t index = random_unsigned(characters.size());
+  return string() + characters[index];
+}
+
+string random_words(size_t count) {
+  string result;
+  bool just_inserted_word = false;
+  for (size_t i = 0; i < count; i++) {
+    if (random_unsigned(10) < 6) {
+      result += random_char("!(){}[]<>+-=");
+    } else {
+      if (just_inserted_word)
+        result += " ";
+      result += random_string('a', 'z');
+      just_inserted_word = true;
+    }
+  }
+  return result;
+}
+
+string select_random(const vector<string> &list) {
+  return list[random_unsigned(list.size())];
+}

data/ext/tree-sitter/tree-sitter/test/helpers/random_helpers.h

@@ -0,0 +1,14 @@
+#ifndef HELPERS_RANDOM_HELPERS_H_
+#define HELPERS_RANDOM_HELPERS_H_
+
+#include <string>
+#include <vector>
+
+unsigned get_time_as_seed();
+void random_reseed(unsigned);
+unsigned random_unsigned();
+unsigned random_unsigned(unsigned max);
+std::string random_words(size_t count);
+std::string select_random(const std::vector<std::string> &);
+
+#endif  // HELPERS_RANDOM_HELPERS_H_

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/read_test_entries.cc

@@ -1,20 +1,17 @@
 #include "helpers/read_test_entries.h"
+#include <assert.h>
 #include <string>
-#include <fstream>
-#include <streambuf>
-#include <dirent.h>
-
 #include <regex>
+#include "helpers/file_helpers.h"
+
+using std::move;
 using std::regex;
 using std::regex_search;
 using std::regex_replace;
-using std::smatch;
 using std::regex_constants::extended;
-
+using std::smatch;
 using std::string;
 using std::vector;
-using std::ifstream;
-using std::istreambuf_iterator;
 
 static string trim_output(const string &input) {
   string result(input);
@@ -26,8 +23,8 @@ static string trim_output(const string &input) {
 }
 
 static vector<TestEntry> parse_test_entries(string content) {
-  regex header_pattern("===+\n"  "([^=]+)\n"  "===+\n", extended);
-  regex separator_pattern("---+\n", extended);
+  regex header_pattern("(^|\n)===+\n"  "([^=]+)\n"  "===+\n", extended);
+  regex separator_pattern("---+\r?\n", extended);
   vector<string> descriptions;
   vector<string> bodies;
 
@@ -36,7 +33,7 @@ static vector<TestEntry> parse_test_entries(string content) {
     if (!regex_search(content, matches, header_pattern) || matches.empty())
       break;
 
-    string description = matches[1].str();
+    string description = matches[2].str();
     descriptions.push_back(description);
 
     if (!bodies.empty())
@@ -55,51 +52,54 @@ static vector<TestEntry> parse_test_entries(string content) {
         body.substr(0, matches.position() - 1),
         trim_output(body.substr(matches.position() + matches[0].length()))
       });
+    } else {
+      puts(("Invalid corpus entry with description: " + descriptions[i]).c_str());
+      abort();
     }
   }
 
   return result;
 }
 
-static vector<string> list_directory(string dir_name) {
-  vector<string> result;
+vector<TestEntry> read_real_language_corpus(string language_name) {
+  vector<TestEntry> result;
 
-  DIR *dir = opendir(dir_name.c_str());
-  if (!dir) {
-    printf("\nTest error - no such directory '%s'", dir_name.c_str());
-    return result;
+  string corpus_directory = join_path({"test", "fixtures", "grammars", language_name, "corpus"});
+  for (string &test_filename : list_directory(corpus_directory)) {
+    for (TestEntry &entry : parse_test_entries(read_file(join_path({corpus_directory, test_filename})))) {
+      result.push_back(entry);
+    }
   }
 
-  struct dirent *dir_entry;
-  while ((dir_entry = readdir(dir))) {
-    string name(dir_entry->d_name);
-    if (name != "." && name != "..")
-      result.push_back(dir_name + "/" + name);
+  string error_test_filename = join_path({"test", "fixtures", "error_corpus", language_name + "_errors.txt"});
+  for (TestEntry &entry : parse_test_entries(read_file(error_test_filename))) {
+    result.push_back(entry);
   }
 
-  closedir(dir);
-  return result;
-}
-
-static string read_file(string filename) {
-  ifstream file(filename);
-  string result((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
   return result;
 }
 
-vector<TestEntry> read_corpus_entries(string language_name) {
+vector<TestEntry> read_test_language_corpus(string language_name) {
   vector<TestEntry> result;
 
-  string fixtures_dir = "spec/fixtures/";
-
-  string test_directory = fixtures_dir + "grammars/" + language_name + "/grammar_test";
-  for (string &test_filename : list_directory(test_directory))
-    for (TestEntry &entry : parse_test_entries(read_file(test_filename)))
+  string test_directory = join_path({"test", "fixtures", "test_grammars", language_name});
+  for (string &test_filename : list_directory(test_directory)) {
+    for (TestEntry &entry : parse_test_entries(read_file(join_path({test_directory, test_filename})))) {
       result.push_back(entry);
+    }
+  }
 
-  string error_test_filename = fixtures_dir + "/error_corpus/" + language_name + "_errors.txt";
-  for (TestEntry &entry : parse_test_entries(read_file(error_test_filename)))
-    result.push_back(entry);
+  return result;
+}
 
+vector<ExampleEntry> examples_for_language(string language_name) {
+  vector<ExampleEntry> result;
+  string examples_directory = join_path({"test", "fixtures", "grammars", language_name, "examples"});
+  for (string &filename : list_directory(examples_directory)) {
+    auto content = read_file(join_path({examples_directory, filename}));
+    if (!content.empty()) {
+      result.push_back({filename, move(content)});
+    }
+  }
   return result;
 }

data/ext/tree-sitter/tree-sitter/test/helpers/read_test_entries.h

@@ -0,0 +1,22 @@
+#ifndef HELPERS_READ_TEST_ENTRIES_H_
+#define HELPERS_READ_TEST_ENTRIES_H_
+
+#include <string>
+#include <vector>
+
+struct TestEntry {
+  std::string description;
+	std::string input;
+	std::string tree_string;
+};
+
+struct ExampleEntry {
+  std::string file_name;
+	std::string input;
+};
+
+std::vector<TestEntry> read_real_language_corpus(std::string name);
+std::vector<TestEntry> read_test_language_corpus(std::string name);
+std::vector<ExampleEntry> examples_for_language(std::string name);
+
+#endif

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/record_alloc.cc

@@ -1,10 +1,9 @@
 #include <stdlib.h>
 #include <map>
-#include <set>
-#include "bandit/bandit.h"
+#include <vector>
 
 using std::map;
-using std::set;
+using std::vector;
 
 static bool _enabled = false;
 static size_t _allocation_count = 0;
@@ -22,10 +21,10 @@ void stop() {
   _enabled = false;
 }
 
-set<size_t> outstanding_allocation_indices() {
-  set<size_t> result;
+vector<size_t> outstanding_allocation_indices() {
+  vector<size_t> result;
   for (const auto &entry : _outstanding_allocations) {
-    result.insert(entry.second);
+    result.push_back(entry.second);
   }
   return result;
 }
@@ -39,18 +38,13 @@ size_t allocation_count() {
 extern "C" {
 
 static void *record_allocation(void *result) {
-  if (!_enabled)
-    return result;
-
+  if (!_enabled) return result;
   _outstanding_allocations[result] = _allocation_count;
   _allocation_count++;
   return result;
 }
 
 static void record_deallocation(void *pointer) {
-  if (!_enabled)
-    return;
-
   auto entry = _outstanding_allocations.find(pointer);
   if (entry != _outstanding_allocations.end()) {
     _outstanding_allocations.erase(entry);
@@ -71,8 +65,8 @@ void *ts_record_calloc(size_t count, size_t size) {
 }
 
 void ts_record_free(void *pointer) {
-  free(pointer);
   record_deallocation(pointer);
+  free(pointer);
 }
 
 bool ts_record_allocations_toggle(bool value) {

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/record_alloc.h

@@ -1,14 +1,14 @@
 #ifndef HELPERS_RECORD_ALLOC_H_
 #define HELPERS_RECORD_ALLOC_H_
 
-#include <set>
+#include <vector>
 
 namespace record_alloc {
 
 void start();
 void stop();
 void fail_at_allocation_index(size_t failure_index);
-std::set<size_t> outstanding_allocation_indices();
+std::vector<size_t> outstanding_allocation_indices();
 size_t allocation_count();
 
 }  // namespace record_alloc

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/spy_input.cc

@@ -6,22 +6,33 @@
 
 using std::pair;
 using std::string;
-
-static const size_t UTF8_MAX_CHAR_SIZE = 4;
+using std::vector;
 
 SpyInput::SpyInput(string content, size_t chars_per_chunk) :
-  chars_per_chunk(chars_per_chunk),
-  buffer_size(UTF8_MAX_CHAR_SIZE * chars_per_chunk),
-  buffer(new char[buffer_size]),
+  buffer(nullptr),
   byte_offset(0),
+  chars_per_chunk(chars_per_chunk),
   content(content),
   encoding(TSInputEncodingUTF8),
-  strings_read({""}) {}
+  ranges_read({}) {}
 
 SpyInput::~SpyInput() {
   delete[] buffer;
 }
 
+static void add_byte_range(vector<pair<uint32_t, uint32_t>> *ranges,
+                           uint32_t start, uint32_t count) {
+  uint32_t end = start + count;
+  for (auto &range : *ranges) {
+    if (range.first <= start && start <= range.second) {
+      if (start < range.first) range.first = start;
+      if (end > range.second) range.second = end;
+      return;
+    }
+  }
+  ranges->push_back({start, end});
+}
+
 const char * SpyInput::read(void *payload, uint32_t *bytes_read) {
   auto spy = static_cast<SpyInput *>(payload);
 
@@ -36,30 +47,43 @@ const char * SpyInput::read(void *payload, uint32_t *bytes_read) {
 
   string result = spy->content.substr(spy->byte_offset, byte_count);
   *bytes_read = byte_count;
-  spy->strings_read.back() += result;
+  add_byte_range(&spy->ranges_read, spy->byte_offset, byte_count);
   spy->byte_offset += byte_count;
 
   /*
    * This class stores its entire `content` in a contiguous buffer, but we want
    * to ensure that the code under test cannot accidentally read more than
    * `*bytes_read` bytes past the returned pointer. To make sure that this type
-   * of error does not fly, we copy the chunk into a zeroed-out buffer and
+   * of error does not fly, we allocate a separate buffer for each request and
    * return a reference to that buffer, rather than a pointer into the main
-   * content.
+   * content. The temporary buffer only fits `*bytes_read` bytes so valgrind
+   * can detect code reading too many bytes from the buffer.
    */
-  memset(spy->buffer, 0, spy->buffer_size);
-  memcpy(spy->buffer, result.data(), byte_count);
+  delete[] spy->buffer;
+  if (byte_count) {
+    spy->buffer = new char[byte_count];
+    memcpy(spy->buffer, result.data(), byte_count);
+  } else {
+    spy->buffer = nullptr;
+  }
+
   return spy->buffer;
 }
 
-int SpyInput::seek(void *payload, uint32_t character, uint32_t byte) {
+int SpyInput::seek(void *payload, uint32_t byte, TSPoint position) {
   auto spy = static_cast<SpyInput *>(payload);
-  if (spy->strings_read.size() == 0 || spy->strings_read.back().size() > 0)
-    spy->strings_read.push_back("");
   spy->byte_offset = byte;
   return 0;
 }
 
+vector<string> SpyInput::strings_read() const {
+  vector<string> result;
+  for (auto &range : ranges_read) {
+    result.push_back(content.substr(range.first, range.second - range.first));
+  }
+  return result;
+}
+
 TSInput SpyInput::input() {
   TSInput result;
   result.payload = this;
@@ -129,5 +153,5 @@ pair<string, TSPoint> SpyInput::swap_substr(size_t start_byte, size_t bytes_remo
 }
 
 void SpyInput::clear() {
-  strings_read.clear();
+  ranges_read.clear();
 }

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/spy_input.h

@@ -12,14 +12,12 @@ struct SpyInputEdit {
 };
 
 class SpyInput {
-  uint32_t chars_per_chunk;
-  uint32_t buffer_size;
   char *buffer;
   uint32_t byte_offset;
   std::vector<SpyInputEdit> undo_stack;
 
   static const char * read(void *, uint32_t *);
-  static int seek(void *, uint32_t, uint32_t);
+  static int seek(void *, uint32_t, TSPoint);
   std::pair<std::string, TSPoint> swap_substr(size_t, size_t, std::string);
 
  public:
@@ -30,10 +28,12 @@ class SpyInput {
   void clear();
   TSInputEdit replace(size_t start_char, size_t chars_removed, std::string text);
   TSInputEdit undo();
+  std::vector<std::string> strings_read() const;
 
+  uint32_t chars_per_chunk;
   std::string content;
   TSInputEncoding encoding;
-  std::vector<std::string> strings_read;
+  std::vector<std::pair<uint32_t, uint32_t>> ranges_read;
 };
 
 #endif  // HELPERS_SPY_INPUT_H_