tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/test/integration/real_grammars.cc

@@ -0,0 +1,142 @@
+#include "test_helper.h"
+#include "runtime/alloc.h"
+#include "helpers/load_language.h"
+#include "helpers/read_test_entries.h"
+#include "helpers/spy_input.h"
+#include "helpers/stderr_logger.h"
+#include "helpers/point_helpers.h"
+#include "helpers/encoding_helpers.h"
+#include "helpers/record_alloc.h"
+#include "helpers/random_helpers.h"
+#include "helpers/scope_sequence.h"
+#include "helpers/tree_helpers.h"
+#include <set>
+
+static void assert_correct_tree_size(TSDocument *document, string content) {
+  TSNode root_node = ts_document_root_node(document);
+  AssertThat(ts_node_end_byte(root_node), Equals(content.size()));
+  assert_consistent_tree_sizes(root_node);
+}
+
+START_TEST
+
+vector<string> test_languages({
+  "javascript",
+  "json",
+  "c",
+  "cpp",
+  "python",
+  "bash",
+});
+
+for (auto &language_name : test_languages) {
+  describe(("the " + language_name + " language").c_str(), [&]() {
+    TSDocument *document;
+    const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS");
+
+    before_each([&]() {
+      record_alloc::start();
+      document = ts_document_new();
+      ts_document_set_language(document, load_real_language(language_name));
+
+      // ts_document_set_logger(document, stderr_logger_new(true));
+      if (debug_graphs_enabled) {
+        ts_document_print_debugging_graphs(document, true);
+      }
+    });
+
+    after_each([&]() {
+      ts_document_free(document);
+      AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
+    });
+
+    for (auto &entry : read_real_language_corpus(language_name)) {
+      SpyInput *input;
+
+      auto it_handles_edit_sequence = [&](string name, std::function<void()> edit_sequence){
+        it(("parses " + entry.description + ": " + name).c_str(), [&]() {
+          input = new SpyInput(entry.input, 3);
+          if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
+          ts_document_set_input(document, input->input());
+          edit_sequence();
+
+          TSNode root_node = ts_document_root_node(document);
+          const char *node_string = ts_node_string(root_node, document);
+          string result(node_string);
+          ts_free((void *)node_string);
+          AssertThat(result, Equals(entry.tree_string));
+
+          assert_correct_tree_size(document, input->content);
+          delete input;
+        });
+      };
+
+      it_handles_edit_sequence("initial parse", [&]() {
+        ts_document_parse(document);
+      });
+
+      set<pair<size_t, size_t>> deletions;
+      set<pair<size_t, string>> insertions;
+
+      for (size_t i = 0; i < 60; i++) {
+        size_t edit_position = random_unsigned(utf8_char_count(entry.input));
+        size_t deletion_size = random_unsigned(utf8_char_count(entry.input) - edit_position);
+        string inserted_text = random_words(random_unsigned(4) + 1);
+
+        if (insertions.insert({edit_position, inserted_text}).second) {
+          string description = "\"" + inserted_text + "\" at " + to_string(edit_position);
+
+          it_handles_edit_sequence("repairing an insertion of " + description, [&]() {
+            ts_document_edit(document, input->replace(edit_position, 0, inserted_text));
+            ts_document_parse(document);
+            assert_correct_tree_size(document, input->content);
+            if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
+
+            ts_document_edit(document, input->undo());
+            assert_correct_tree_size(document, input->content);
+            if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
+
+            TSRange *ranges;
+            uint32_t range_count;
+            ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content);
+            ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count);
+            assert_correct_tree_size(document, input->content);
+
+            ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content);
+            verify_changed_ranges(old_scope_sequence, new_scope_sequence,
+                                  input->content, ranges, range_count);
+            ts_free(ranges);
+          });
+        }
+
+        if (deletions.insert({edit_position, deletion_size}).second) {
+          string desription = to_string(edit_position) + "-" + to_string(edit_position + deletion_size);
+
+          it_handles_edit_sequence("repairing a deletion of " + desription, [&]() {
+            ts_document_edit(document, input->replace(edit_position, deletion_size, ""));
+            ts_document_parse(document);
+            assert_correct_tree_size(document, input->content);
+            if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
+
+            ts_document_edit(document, input->undo());
+            assert_correct_tree_size(document, input->content);
+            if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
+
+            TSRange *ranges;
+            uint32_t range_count;
+            ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content);
+            ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count);
+            assert_correct_tree_size(document, input->content);
+
+            ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content);
+            verify_changed_ranges(old_scope_sequence, new_scope_sequence,
+                                  input->content, ranges, range_count);
+            ts_free(ranges);
+          });
+        }
+      }
+    }
+  });
+}
+
+END_TEST

data/ext/tree-sitter/tree-sitter/test/integration/test_grammars.cc

@@ -0,0 +1,79 @@
+#include "test_helper.h"
+#include "helpers/read_test_entries.h"
+#include "helpers/load_language.h"
+#include "helpers/stderr_logger.h"
+#include "helpers/file_helpers.h"
+#include "helpers/tree_helpers.h"
+#include "runtime/alloc.h"
+#include "helpers/record_alloc.h"
+
+START_TEST
+
+string grammars_dir_path = join_path({"test", "fixtures", "test_grammars"});
+vector<string> test_languages = list_directory(grammars_dir_path);
+
+for (auto &language_name : test_languages) {
+  if (language_name == "readme.md") continue;
+
+  describe(("test grammar: " + language_name).c_str(), [&]() {
+    string directory_path = join_path({grammars_dir_path, language_name});
+    string grammar_path = join_path({directory_path, "grammar.json"});
+    string expected_error_path = join_path({directory_path, "expected_error.txt"});
+
+    string grammar_json = read_file(grammar_path);
+    const TSLanguage *language = nullptr;
+
+    if (file_exists(expected_error_path)) {
+      it("fails with the correct error message", [&]() {
+        TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str());
+        string expected_error = read_file(expected_error_path);
+        AssertThat((void *)compile_result.error_message, !Equals<void *>(nullptr));
+        AssertThat(compile_result.error_message, Equals(expected_error));
+      });
+      return;
+    }
+
+    for (auto &entry : read_test_language_corpus(language_name)) {
+      it(("parses " + entry.description).c_str(), [&]() {
+        record_alloc::start();
+
+        if (!language) {
+          string external_scanner_path = join_path({directory_path, "scanner.c"});
+          if (!file_exists(external_scanner_path)) external_scanner_path = "";
+
+          TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str());
+
+          language = load_test_language(
+            language_name,
+            compile_result,
+            external_scanner_path
+          );
+        }
+
+        TSDocument *document = ts_document_new();
+        ts_document_set_language(document, language);
+        ts_document_set_input_string_with_length(document, entry.input.c_str(), entry.input.size());
+
+        // ts_document_print_debugging_graphs(document, true);
+        if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
+          ts_document_print_debugging_graphs(document, true);
+        }
+
+        ts_document_parse(document);
+
+        TSNode root_node = ts_document_root_node(document);
+        AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size()));
+        assert_consistent_tree_sizes(root_node);
+        const char *node_string = ts_node_string(root_node, document);
+        string result(node_string);
+        ts_free((void *)node_string);
+        ts_document_free(document);
+
+        AssertThat(result, Equals(entry.tree_string));
+        AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
+      });
+    }
+  });
+}
+
+END_TEST

data/ext/tree-sitter/tree-sitter/{spec/runtime/document_spec.cc → test/runtime/document_test.cc}

@@ -1,4 +1,4 @@
-#include "spec_helper.h"
+#include "test_helper.h"
 #include "runtime/alloc.h"
 #include "helpers/record_alloc.h"
 #include "helpers/stream_methods.h"
@@ -22,6 +22,10 @@ describe("Document", [&]() {
   before_each([&]() {
     record_alloc::start();
     document = ts_document_new();
+
+    if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
+      ts_document_print_debugging_graphs(document, true);
+    }
   });
 
   after_each([&]() {
@@ -43,14 +47,14 @@ describe("Document", [&]() {
     before_each([&]() {
       spy_input = new SpyInput("{\"key\": [null, 2]}", 3);
 
-      ts_document_set_language(document, get_test_language("json"));
+      ts_document_set_language(document, load_real_language("json"));
       ts_document_set_input_string(document, "{\"key\": [1, 2]}");
       ts_document_parse(document);
 
       root = ts_document_root_node(document);
       assert_node_string_equals(
         root,
-        "(object (pair (string) (array (number) (number))))");
+        "(value (object (pair (string) (array (number) (number)))))");
     });
 
     after_each([&]() {
@@ -69,18 +73,30 @@ describe("Document", [&]() {
       root = ts_document_root_node(document);
       assert_node_string_equals(
         root,
-        "(array (true) (false))");
+        "(value (array (true) (false)))");
+    });
+
+    it("handles truncated UTF16 data", [&]() {
+      const char content[1] = { '\0' };
+      spy_input->content = string(content, sizeof(content));
+      spy_input->encoding = TSInputEncodingUTF16;
+
+      ts_document_set_input(document, spy_input->input());
+      ts_document_invalidate(document);
+      ts_document_parse(document);
     });
 
-    it("allows columns to be measured in either bytes or characters", [&]() {
+    it("measures columns in bytes", [&]() {
       const char16_t content[] = u"[true, false]";
       spy_input->content = string((const char *)content, sizeof(content));
       spy_input->encoding = TSInputEncodingUTF16;
-      // spy_input->measure_columns_in_bytes
+      TSInput input = spy_input->input();
 
-      ts_document_set_input(document, spy_input->input());
+      ts_document_set_input(document, input);
       ts_document_invalidate(document);
       ts_document_parse(document);
+      root = ts_document_root_node(document);
+      AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 28}));
     });
 
     it("allows the input to be retrieved later", [&]() {
@@ -94,7 +110,7 @@ describe("Document", [&]() {
       ts_document_set_input(document, spy_input->input());
       AssertThat(ts_document_root_node(document), Equals<TSNode>(root));
       AssertThat(ts_node_has_changes(root), IsFalse());
-      AssertThat(spy_input->strings_read, Equals(vector<string>({ "" })));
+      AssertThat(spy_input->strings_read(), IsEmpty());
     });
 
     it("reads text from the new input for future parses", [&]() {
@@ -112,8 +128,8 @@ describe("Document", [&]() {
       TSNode new_root = ts_document_root_node(document);
       assert_node_string_equals(
         new_root,
-        "(object (pair (string) (array (null) (number))))");
-      AssertThat(spy_input->strings_read, Equals(vector<string>({" [null, 2" })));
+        "(value (object (pair (string) (array (null) (number)))))");
+      AssertThat(spy_input->strings_read(), Equals(vector<string>({" [null, 2" })));
     });
 
     it("allows setting input string with length", [&]() {
@@ -121,17 +137,17 @@ describe("Document", [&]() {
       ts_document_set_input_string_with_length(document, content, 1);
       ts_document_parse(document);
       TSNode new_root = ts_document_root_node(document);
-      AssertThat(ts_node_end_char(new_root), Equals<size_t>(1));
+      AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
       assert_node_string_equals(
         new_root,
-        "(number)");
+        "(value (number))");
     });
 
     it("reads from the new input correctly when the old input was blank", [&]() {
       ts_document_set_input_string(document, "");
       ts_document_parse(document);
       TSNode new_root = ts_document_root_node(document);
-      AssertThat(ts_node_end_char(new_root), Equals<size_t>(0));
+      AssertThat(ts_node_end_byte(new_root), Equals<size_t>(0));
       assert_node_string_equals(
         new_root,
         "(ERROR)");
@@ -139,10 +155,10 @@ describe("Document", [&]() {
       ts_document_set_input_string(document, "1");
       ts_document_parse(document);
       new_root = ts_document_root_node(document);
-      AssertThat(ts_node_end_char(new_root), Equals<size_t>(1));
+      AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
       assert_node_string_equals(
         new_root,
-        "(number)");
+        "(value (number))");
     });
   });
 
@@ -152,20 +168,20 @@ describe("Document", [&]() {
     });
 
     it("uses the given language for future parses", [&]() {
-      ts_document_set_language(document, get_test_language("json"));
+      ts_document_set_language(document, load_real_language("json"));
       ts_document_parse(document);
 
       root = ts_document_root_node(document);
       assert_node_string_equals(
         root,
-        "(object (pair (string) (array (number) (number))))");
+        "(value (object (pair (string) (array (number) (number)))))");
     });
 
     it("clears out any previous tree", [&]() {
-      ts_document_set_language(document, get_test_language("json"));
+      ts_document_set_language(document, load_real_language("json"));
       ts_document_parse(document);
 
-      ts_document_set_language(document, get_test_language("javascript"));
+      ts_document_set_language(document, load_real_language("javascript"));
       AssertThat(ts_document_root_node(document).data, Equals<void *>(nullptr));
 
       ts_document_parse(document);
@@ -177,14 +193,14 @@ describe("Document", [&]() {
     });
 
     it("does not allow setting a language with a different version number", [&]() {
-      TSLanguage language = *get_test_language("json");
+      TSLanguage language = *load_real_language("json");
       AssertThat(ts_language_version(&language), Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
 
       language.version++;
       AssertThat(ts_language_version(&language), !Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
 
       ts_document_set_language(document, &language);
-      AssertThat(ts_document_language(document), IsNull());
+      AssertThat(ts_document_language(document), Equals<const TSLanguage *>(nullptr));
     });
   });
 
@@ -193,7 +209,7 @@ describe("Document", [&]() {
 
     before_each([&]() {
       logger = new SpyLogger();
-      ts_document_set_language(document, get_test_language("json"));
+      ts_document_set_language(document, load_real_language("json"));
       ts_document_set_input_string(document, "[1, 2]");
     });
 
@@ -235,13 +251,13 @@ describe("Document", [&]() {
     SpyInput *input;
 
     before_each([&]() {
-      ts_document_set_language(document, get_test_language("javascript"));
-      input = new SpyInput("{a: null};", 3);
+      ts_document_set_language(document, load_real_language("javascript"));
+      input = new SpyInput("{a: null};\n", 3);
       ts_document_set_input(document, input->input());
       ts_document_parse(document);
       assert_node_string_equals(
         ts_document_root_node(document),
-        "(program (expression_statement (object (pair (identifier) (null)))))");
+        "(program (expression_statement (object (pair (property_identifier) (null)))))");
     });
 
     after_each([&]() {
@@ -290,6 +306,37 @@ describe("Document", [&]() {
       })));
     });
 
+    it("reports no changes when leading whitespace has changed (regression)", [&]() {
+      input->chars_per_chunk = 80;
+
+      // Insert leading whitespace
+      auto ranges = get_invalidated_ranges_for_edit([&]() {
+        return input->replace(0, 0, "\n");
+      });
+      assert_node_string_equals(
+        ts_document_root_node(document),
+        "(program (expression_statement (object (pair (property_identifier) (null)))))");
+      AssertThat(ranges, Equals(vector<TSRange>({})));
+
+      // Remove leading whitespace
+      ranges = get_invalidated_ranges_for_edit([&]() {
+        return input->undo();
+      });
+      assert_node_string_equals(
+        ts_document_root_node(document),
+        "(program (expression_statement (object (pair (property_identifier) (null)))))");
+      AssertThat(ranges, Equals(vector<TSRange>({})));
+
+      // Insert leading whitespace again
+      ranges = get_invalidated_ranges_for_edit([&]() {
+        return input->replace(0, 0, "\n");
+      });
+      assert_node_string_equals(
+        ts_document_root_node(document),
+        "(program (expression_statement (object (pair (property_identifier) (null)))))");
+      AssertThat(ranges, Equals(vector<TSRange>({})));
+    });
+
     it("reports changes when tokens have been appended", [&]() {
       // Add a second key-value pair
       auto ranges = get_invalidated_ranges_for_edit([&]() {
@@ -311,9 +358,9 @@ describe("Document", [&]() {
       assert_node_string_equals(
         ts_document_root_node(document),
         "(program (expression_statement (object "
-          "(pair (identifier) (null)) "
-          "(pair (identifier) (number)) "
-          "(pair (identifier) (false)))))");
+          "(pair (property_identifier) (null)) "
+          "(pair (property_identifier) (number)) "
+          "(pair (property_identifier) (false)))))");
 
       AssertThat(ranges, Equals(vector<TSRange>({
         TSRange{
@@ -330,8 +377,8 @@ describe("Document", [&]() {
       assert_node_string_equals(
         ts_document_root_node(document),
         "(program (expression_statement (object "
-          "(pair (identifier) (null)) "
-          "(pair (identifier) (false)))))");
+          "(pair (property_identifier) (null)) "
+          "(pair (property_identifier) (false)))))");
 
       AssertThat(ranges, IsEmpty());
 
@@ -343,7 +390,7 @@ describe("Document", [&]() {
       assert_node_string_equals(
         ts_document_root_node(document),
         "(program (expression_statement (object "
-          "(pair (identifier) (null)))))");
+          "(pair (property_identifier) (null)))))");
 
       AssertThat(ranges, IsEmpty());
     });
@@ -357,7 +404,7 @@ describe("Document", [&]() {
       assert_node_string_equals(
         ts_document_root_node(document),
         "(program (expression_statement (object "
-          "(pair (identifier) (rel_op (identifier) (null))))))");
+          "(pair (property_identifier) (binary_expression (identifier) (null))))))");
 
       AssertThat(ranges, Equals(vector<TSRange>({
         TSRange{
@@ -367,6 +414,77 @@ describe("Document", [&]() {
       })));
     });
   });
+
+  describe("parse_with_options(options)", [&]() {
+    it("halts as soon as an error is found if the halt_on_error flag is set", [&]() {
+      string input_string = "[1, null, error, 3]";
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string(document, input_string.c_str());
+
+      TSParseOptions options = {};
+      options.changed_ranges = nullptr;
+
+      options.halt_on_error = false;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))");
+
+      ts_document_invalidate(document);
+
+      options.halt_on_error = true;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(ERROR (number) (null))");
+
+      AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
+    });
+
+    it("does not insert missing tokens if the halt_on_error flag is set", [&]() {
+      string input_string = "[1, null, 3";
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string(document, input_string.c_str());
+
+      TSParseOptions options = {};
+      options.changed_ranges = nullptr;
+
+      options.halt_on_error = false;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(value (array (number) (null) (number) (MISSING)))");
+
+      ts_document_invalidate(document);
+
+      options.halt_on_error = true;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(ERROR (number) (null) (number))");
+
+      AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
+    });
+
+    it("can parse valid code with the halt_on_error flag set", [&]() {
+      string input_string = "[1, null, 3]";
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string(document, input_string.c_str());
+
+      TSParseOptions options = {};
+      options.changed_ranges = nullptr;
+      options.halt_on_error = true;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(value (array (number) (null) (number)))");
+    });
+  });
 });
 
 END_TEST