tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/{spec/runtime/parser_spec.cc → test/runtime/parser_test.cc}

@@ -1,4 +1,4 @@
-#include "spec_helper.h"
+#include "test_helper.h"
 #include "runtime/alloc.h"
 #include "helpers/record_alloc.h"
 #include "helpers/spy_input.h"
@@ -22,6 +22,9 @@ describe("Parser", [&]() {
     chunk_size = 3;
     input = nullptr;
     document = ts_document_new();
+    if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
+      ts_document_print_debugging_graphs(document, true);
+    }
   });
 
   after_each([&]() {
@@ -83,24 +86,24 @@ describe("Parser", [&]() {
   describe("handling errors", [&]() {
     describe("when there is an invalid substring right before a valid token", [&]() {
       it("computes the error node's size and position correctly", [&]() {
-        ts_document_set_language(document, get_test_language("json"));
+        ts_document_set_language(document, load_real_language("json"));
         set_text("  [123,  @@@@@,   true]");
 
         assert_root_node(
-          "(array (number) (ERROR (UNEXPECTED '@')) (true))");
+          "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))");
 
-        TSNode error = ts_node_named_child(root, 1);
+        TSNode error = ts_node_named_child(ts_node_child(root, 0), 1);
         AssertThat(ts_node_type(error, document), Equals("ERROR"));
-        AssertThat(get_node_text(error), Equals(",  @@@@@"));
+        AssertThat(get_node_text(error), Equals("@@@@@,"));
         AssertThat(ts_node_child_count(error), Equals<size_t>(2));
 
-        TSNode comma = ts_node_child(error, 0);
-        AssertThat(get_node_text(comma), Equals(","));
-
-        TSNode garbage = ts_node_child(error, 1);
+        TSNode garbage = ts_node_child(error, 0);
         AssertThat(get_node_text(garbage), Equals("@@@@@"));
 
-        TSNode node_after_error = ts_node_named_child(root, 2);
+        TSNode comma = ts_node_child(error, 1);
+        AssertThat(get_node_text(comma), Equals(","));
+
+        TSNode node_after_error = ts_node_next_named_sibling(error);
         AssertThat(ts_node_type(node_after_error, document), Equals("true"));
         AssertThat(get_node_text(node_after_error), Equals("true"));
       });
@@ -108,25 +111,26 @@ describe("Parser", [&]() {
 
     describe("when there is an unexpected string in the middle of a token", [&]() {
       it("computes the error node's size and position correctly", [&]() {
-        ts_document_set_language(document, get_test_language("json"));
+        ts_document_set_language(document, load_real_language("json"));
         set_text("  [123, faaaaalse, true]");
 
         assert_root_node(
-          "(array (number) (ERROR (UNEXPECTED 'a')) (true))");
+          "(value (array (number) (ERROR (UNEXPECTED 'a')) (true)))");
 
-        TSNode error = ts_node_named_child(root, 1);
+        TSNode error = ts_node_named_child(ts_node_child(root, 0), 1);
         AssertThat(ts_node_type(error, document), Equals("ERROR"));
+        AssertThat(get_node_text(error), Equals("faaaaalse,"));
         AssertThat(ts_node_child_count(error), Equals<size_t>(2));
 
-        TSNode comma = ts_node_child(error, 0);
-        AssertThat(ts_node_type(comma, document), Equals(","));
-        AssertThat(get_node_text(comma), Equals(","));
-
-        TSNode garbage = ts_node_child(error, 1);
+        TSNode garbage = ts_node_child(error, 0);
         AssertThat(ts_node_type(garbage, document), Equals("ERROR"));
         AssertThat(get_node_text(garbage), Equals("faaaaalse"));
 
-        TSNode last = ts_node_named_child(root, 2);
+        TSNode comma = ts_node_child(error, 1);
+        AssertThat(ts_node_type(comma, document), Equals(","));
+        AssertThat(get_node_text(comma), Equals(","));
+
+        TSNode last = ts_node_next_named_sibling(error);
         AssertThat(ts_node_type(last, document), Equals("true"));
         AssertThat(ts_node_start_byte(last), Equals(strlen("  [123, faaaaalse, ")));
       });
@@ -134,18 +138,18 @@ describe("Parser", [&]() {
 
     describe("when there is one unexpected token between two valid tokens", [&]() {
       it("computes the error node's size and position correctly", [&]() {
-        ts_document_set_language(document, get_test_language("json"));
+        ts_document_set_language(document, load_real_language("json"));
         set_text("  [123, true false, true]");
 
         assert_root_node(
-          "(array (number) (true) (ERROR (false)) (true))");
+          "(value (array (number) (true) (ERROR (false)) (true)))");
 
-        TSNode error = ts_node_named_child(root, 2);
+        TSNode error = ts_node_named_child(ts_node_child(root, 0), 2);
         AssertThat(ts_node_type(error, document), Equals("ERROR"));
         AssertThat(get_node_text(error), Equals("false"));
         AssertThat(ts_node_child_count(error), Equals<size_t>(1));
 
-        TSNode last = ts_node_named_child(root, 1);
+        TSNode last = ts_node_next_named_sibling(error);
         AssertThat(ts_node_type(last, document), Equals("true"));
         AssertThat(get_node_text(last), Equals("true"));
       });
@@ -153,26 +157,26 @@ describe("Parser", [&]() {
 
     describe("when there is an unexpected string at the end of a token", [&]() {
       it("computes the error's size and position correctly", [&]() {
-        ts_document_set_language(document, get_test_language("json"));
+        ts_document_set_language(document, load_real_language("json"));
         set_text("  [123, \"hi\n, true]");
 
         assert_root_node(
-          "(array (number) (ERROR (UNEXPECTED '\\n')) (true))");
+          "(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))");
       });
     });
 
     describe("when there is an unterminated error", [&]() {
       it("maintains a consistent tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
-        set_text("a; /* b");
+        ts_document_set_language(document, load_real_language("javascript"));
+        set_text("a; ' this string never ends");
         assert_root_node(
-          "(ERROR (program (expression_statement (identifier))) (UNEXPECTED EOF))");
+          "(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))");
       });
     });
 
     describe("when there are extra tokens at the end of the viable prefix", [&]() {
       it("does not include them in the error node", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text(
           "var x;\n"
           "\n"
@@ -187,38 +191,51 @@ describe("Parser", [&]() {
         AssertThat(ts_node_end_point(error), Equals<TSPoint>({2, 2}));
       });
     });
+
+    it("handles invalid UTF8 characters at EOF", [&]() {
+      char *string = (char *)malloc(1);
+      string[0] = '\xdf';
+
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string_with_length(document, string, 1);
+      ts_document_parse(document);
+
+      free(string);
+
+      assert_root_node("(ERROR (UNEXPECTED INVALID))");
+    });
   });
 
   describe("handling extra tokens", [&]() {
     describe("when the token appears as part of a grammar rule", [&]() {
       it("incorporates it into the tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("fn()\n");
 
         assert_root_node(
-          "(program (expression_statement (function_call (identifier) (arguments))))");
+          "(program (expression_statement (call_expression (identifier) (arguments))))");
       });
     });
 
     describe("when the token appears somewhere else", [&]() {
       it("incorporates it into the tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text(
           "fn()\n"
           "  .otherFn();");
 
         assert_root_node(
-          "(program (expression_statement (function_call "
-            "(member_access "
-              "(function_call (identifier) (arguments)) "
-              "(identifier)) "
+          "(program (expression_statement (call_expression "
+            "(member_expression "
+              "(call_expression (identifier) (arguments)) "
+              "(property_identifier)) "
             "(arguments))))");
       });
     });
 
     describe("when several extra tokens appear in a row", [&]() {
       it("incorporates them into the tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text(
           "fn()\n\n"
           "// This is a comment"
@@ -226,11 +243,11 @@ describe("Parser", [&]() {
           ".otherFn();");
 
         assert_root_node(
-          "(program (expression_statement (function_call "
-            "(member_access "
-              "(function_call (identifier) (arguments)) "
+          "(program (expression_statement (call_expression "
+            "(member_expression "
+              "(call_expression (identifier) (arguments)) "
               "(comment) "
-              "(identifier)) "
+              "(property_identifier)) "
             "(arguments))))");
       });
     });
@@ -239,22 +256,32 @@ describe("Parser", [&]() {
   describe("editing", [&]() {
     describe("creating new tokens near the end of the input", [&]() {
       it("updates the parse tree and re-reads only the changed portion of the text", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("x * (100 + abc);");
 
         assert_root_node(
-          "(program (expression_statement (math_op "
+          "(program (expression_statement (binary_expression "
             "(identifier) "
-            "(math_op (number) (identifier)))))");
+            "(parenthesized_expression "
+              "(binary_expression (number) (identifier))))))");
 
         insert_text(strlen("x * (100 + abc"), ".d");
 
         assert_root_node(
-          "(program (expression_statement (math_op "
+          "(program (expression_statement (binary_expression "
             "(identifier) "
-            "(math_op (number) (member_access (identifier) (identifier))))))");
-
-        AssertThat(input->strings_read, Equals(vector<string>({ " + abc.d)" })));
+            "(parenthesized_expression "
+              "(binary_expression (number) (member_expression (identifier) (property_identifier)))))))");
+
+        AssertThat(input->strings_read(), Equals(vector<string>({
+          // The '*' is not reused because the preceding `x` expression is reused, which
+          // puts the parser into a different state than when the `*` was initially tokenized.
+          // When the `*` was initially tokenized, `x` was just an identifier. In both of these
+          // states, external tokens are valid so we don't reuse tokens unless the lex states
+          // match. This could probably be improved somehow.
+          " * ",
+          " abc.d)"
+        })));
       });
     });
 
@@ -262,64 +289,67 @@ describe("Parser", [&]() {
       it("updates the parse tree and re-reads only the changed portion of the input", [&]() {
         chunk_size = 2;
 
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("123 + 456 * (10 + x);");
 
         assert_root_node(
-          "(program (expression_statement (math_op "
+          "(program (expression_statement (binary_expression "
             "(number) "
-            "(math_op (number) (math_op (number) (identifier))))))");
+            "(binary_expression (number) (parenthesized_expression (binary_expression (number) (identifier)))))))");
 
         insert_text(strlen("123"), " || 5");
 
         assert_root_node(
-          "(program (expression_statement (bool_op "
+          "(program (expression_statement (binary_expression "
             "(number) "
-            "(math_op "
+            "(binary_expression "
               "(number) "
-              "(math_op (number) (math_op (number) (identifier)))))))");
+              "(binary_expression (number) (parenthesized_expression (binary_expression (number) (identifier))))))))");
 
-        AssertThat(input->strings_read, Equals(vector<string>({ "123 || 5 +" })));
+        AssertThat(input->strings_read(), Equals(vector<string>({
+          "123 || 5 ",
+          ";"
+        })));
       });
     });
 
     describe("introducing an error", [&]() {
       it("gives the error the right size", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("var x = y;");
 
         assert_root_node(
-          "(program (var_declaration (var_assignment "
+          "(program (variable_declaration (variable_declarator "
             "(identifier) (identifier))))");
 
         insert_text(strlen("var x = y"), " *");
 
         assert_root_node(
-          "(program (var_declaration (var_assignment "
+          "(program (variable_declaration (variable_declarator "
             "(identifier) (identifier)) (ERROR)))");
 
         insert_text(strlen("var x = y *"), " z");
 
         assert_root_node(
-          "(program (var_declaration (var_assignment "
-            "(identifier) (math_op (identifier) (identifier)))))");
+          "(program (variable_declaration (variable_declarator "
+            "(identifier) (binary_expression (identifier) (identifier)))))");
       });
     });
 
     describe("into the middle of an existing token", [&]() {
       it("updates the parse tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("abc * 123;");
 
         assert_root_node(
-          "(program (expression_statement (math_op (identifier) (number))))");
+          "(program (expression_statement (binary_expression (identifier) (number))))");
 
         insert_text(strlen("ab"), "XYZ");
 
         assert_root_node(
-          "(program (expression_statement (math_op (identifier) (number))))");
+          "(program (expression_statement (binary_expression (identifier) (number))))");
 
-        TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1);
+        TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1);
         AssertThat(ts_node_type(node, document), Equals("identifier"));
         AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc")));
       });
@@ -327,18 +357,18 @@ describe("Parser", [&]() {
 
     describe("at the end of an existing token", [&]() {
       it("updates the parse tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("abc * 123;");
 
         assert_root_node(
-          "(program (expression_statement (math_op (identifier) (number))))");
+          "(program (expression_statement (binary_expression (identifier) (number))))");
 
         insert_text(strlen("abc"), "XYZ");
 
         assert_root_node(
-          "(program (expression_statement (math_op (identifier) (number))))");
+          "(program (expression_statement (binary_expression (identifier) (number))))");
 
-        TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1);
+        TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1);
         AssertThat(ts_node_type(node, document), Equals("identifier"));
         AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ")));
       });
@@ -346,13 +376,13 @@ describe("Parser", [&]() {
 
     describe("inserting text into a node containing a extra token", [&]() {
       it("updates the parse tree", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("123 *\n"
           "// a-comment\n"
           "abc;");
 
         assert_root_node(
-          "(program (expression_statement (math_op "
+          "(program (expression_statement (binary_expression "
             "(number) "
             "(comment) "
             "(identifier))))");
@@ -364,7 +394,7 @@ describe("Parser", [&]() {
           "XYZ");
 
         assert_root_node(
-          "(program (expression_statement (math_op "
+          "(program (expression_statement (binary_expression "
             "(number) "
             "(comment) "
             "(identifier))))");
@@ -373,26 +403,27 @@ describe("Parser", [&]() {
 
     describe("when a critical token is removed", [&]() {
       it("updates the parse tree, creating an error", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("123 * 456; 789 * 123;");
 
         assert_root_node(
           "(program "
-            "(expression_statement (math_op (number) (number))) "
-            "(expression_statement (math_op (number) (number))))");
+            "(expression_statement (binary_expression (number) (number))) "
+            "(expression_statement (binary_expression (number) (number))))");
 
         delete_text(strlen("123 "), 2);
 
         assert_root_node(
           "(program "
-            "(expression_statement (number) (ERROR (number))) "
-            "(expression_statement (math_op (number) (number))))");
+            "(ERROR (number)) "
+            "(expression_statement (number)) "
+            "(expression_statement (binary_expression (number) (number))))");
       });
     });
 
     describe("with external tokens", [&]() {
       it("maintains the external scanner's state during incremental parsing", [&]() {
-        ts_document_set_language(document, get_test_language("python"));
+        ts_document_set_language(document, load_real_language("python"));
         string text = dedent(R"PYTHON(
           if a:
               print b
@@ -419,23 +450,23 @@ describe("Parser", [&]() {
       });
     });
 
-    it("does not try to re-use nodes that are within the edited region", [&]() {
-      ts_document_set_language(document, get_test_language("javascript"));
+    it("does not try to reuse nodes that are within the edited region", [&]() {
+      ts_document_set_language(document, load_real_language("javascript"));
       set_text("{ x: (b.c) };");
 
       assert_root_node(
         "(program (expression_statement (object (pair "
-          "(identifier) (member_access (identifier) (identifier))))))");
+          "(property_identifier) (parenthesized_expression (member_expression (identifier) (property_identifier)))))))");
 
       replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c");
 
       assert_root_node(
         "(program (expression_statement (object (pair "
-          "(identifier) (member_access (identifier) (identifier))))))");
+          "(property_identifier) (member_expression (identifier) (property_identifier))))))");
     });
 
     it("updates the document's parse count", [&]() {
-      ts_document_set_language(document, get_test_language("javascript"));
+      ts_document_set_language(document, load_real_language("javascript"));
       AssertThat(ts_document_parse_count(document), Equals<size_t>(0));
 
       set_text("{ x: (b.c) };");
@@ -449,7 +480,7 @@ describe("Parser", [&]() {
   describe("lexing", [&]() {
     describe("handling tokens containing wildcard patterns (e.g. comments)", [&]() {
       it("terminates them at the end of the document", [&]() {
-        ts_document_set_language(document, get_test_language("javascript"));
+        ts_document_set_language(document, load_real_language("javascript"));
         set_text("x; // this is a comment");
 
         assert_root_node(
@@ -464,15 +495,25 @@ describe("Parser", [&]() {
 
     it("recognizes UTF8 characters as single characters", [&]() {
       // 'ΩΩΩ — ΔΔ';
-      ts_document_set_language(document, get_test_language("javascript"));
+      ts_document_set_language(document, load_real_language("javascript"));
       set_text("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';");
 
       assert_root_node(
         "(program (expression_statement (string)))");
 
-      AssertThat(ts_node_end_char(root), Equals(strlen("'OOO - DD';")));
       AssertThat(ts_node_end_byte(root), Equals(strlen("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';")));
     });
+
+    it("handles non-UTF8 characters", [&]() {
+      const char *string = "cons\xeb\x00e=ls\x83l6hi');\x0a";
+
+      ts_document_set_language(document, load_real_language("javascript"));
+      ts_document_set_input_string(document, string);
+      ts_document_parse(document);
+
+      TSNode root = ts_document_root_node(document);
+      AssertThat(ts_node_end_byte(root), Equals(strlen(string)));
+    });
   });
 });