tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/{spec/compiler/rules/character_set_spec.cc → test/compiler/rules/character_set_test.cc}

@@ -1,5 +1,5 @@
-#include "spec_helper.h"
-#include "compiler/rules/character_set.h"
+#include "test_helper.h"
+#include "compiler/rule.h"
 
 using namespace rules;
 
@@ -66,7 +66,7 @@ describe("CharacterSet", []() {
         .include('a', 'd')
         .include('f', 'm');
 
-      AssertThat(set1.hash_code(), Equals(set2.hash_code()));
+      AssertThat(hash<CharacterSet>()(set1), Equals(hash<CharacterSet>()(set2)));
     });
 
     it("returns different numbers for character sets that include different ranges", [&]() {
@@ -78,8 +78,8 @@ describe("CharacterSet", []() {
         .include('a', 'c')
         .include('f', 'm');
 
-      AssertThat(set1.hash_code(), !Equals(set2.hash_code()));
-      AssertThat(set2.hash_code(), !Equals(set1.hash_code()));
+      AssertThat(hash<CharacterSet>()(set1), !Equals(hash<CharacterSet>()(set2)));
+      AssertThat(hash<CharacterSet>()(set2), !Equals(hash<CharacterSet>()(set1)));
     });
 
     it("returns different numbers for character sets that exclude different ranges", [&]() {
@@ -93,16 +93,16 @@ describe("CharacterSet", []() {
         .exclude('a', 'c')
         .exclude('f', 'm');
 
-      AssertThat(set1.hash_code(), !Equals(set2.hash_code()));
-      AssertThat(set2.hash_code(), !Equals(set1.hash_code()));
+      AssertThat(hash<CharacterSet>()(set1), !Equals(hash<CharacterSet>()(set2)));
+      AssertThat(hash<CharacterSet>()(set2), !Equals(hash<CharacterSet>()(set1)));
     });
 
     it("returns different numbers for character sets with different sign", [&]() {
       CharacterSet set1 = CharacterSet().include_all();
       CharacterSet set2 = CharacterSet();
 
-      AssertThat(set1.hash_code(), !Equals(set2.hash_code()));
-      AssertThat(set2.hash_code(), !Equals(set1.hash_code()));
+      AssertThat(hash<CharacterSet>()(set1), !Equals(hash<CharacterSet>()(set2)));
+      AssertThat(hash<CharacterSet>()(set2), !Equals(hash<CharacterSet>()(set1)));
     });
   });
 
@@ -305,29 +305,17 @@ describe("CharacterSet", []() {
   });
 
   describe("::included_ranges", [&]() {
-    it("consolidates sequences of 3 or more consecutive characters into ranges", [&]() {
+    it("consolidates consecutive sequences of characters into ranges", [&]() {
       CharacterSet set1 = CharacterSet()
         .include('a', 'c')
-        .include('g')
+        .include('e', 'j')
+        .include('m')
         .include('z');
 
       AssertThat(set1.included_ranges(), Equals(vector<CharacterRange>({
-        CharacterRange('a', 'c'),
-        CharacterRange('g'),
-        CharacterRange('z'),
-      })));
-    });
-
-    it("doesn't consolidate sequences of 2 consecutive characters", [&]() {
-      CharacterSet set1 = CharacterSet()
-        .include('a', 'b')
-        .include('g')
-        .include('z');
-
-      AssertThat(set1.included_ranges(), Equals(vector<CharacterRange>({
-        CharacterRange('a'),
-        CharacterRange('b'),
-        CharacterRange('g'),
+        CharacterRange{'a', 'c'},
+        CharacterRange{'e', 'j'},
+        CharacterRange('m'),
         CharacterRange('z'),
       })));
     });

data/ext/tree-sitter/tree-sitter/test/compiler/rules/rule_test.cc

@@ -0,0 +1,90 @@
+#include "test_helper.h"
+#include "compiler/rule.h"
+
+using namespace rules;
+
+START_TEST
+
+describe("Repeat", []() {
+  describe("constructing repeats", [&]() {
+    it("doesn't create redundant repeats", [&]() {
+      Rule symbol = Symbol::non_terminal(1);
+      Rule repeat = Rule::repeat(Rule(symbol));
+      Rule outer_repeat = Rule::repeat(Rule(repeat));
+
+      AssertThat(repeat, !Equals(symbol));
+      AssertThat(outer_repeat, Equals(repeat));
+    });
+  });
+
+  describe("adding metadata to rules", [&]() {
+    it("doesn't create redundant metadata rules", [&]() {
+      Rule symbol = Symbol::non_terminal(1);
+      Rule outer_rule = Metadata::prec(2, Metadata::prec(1, Rule(symbol)));
+      AssertThat(outer_rule, Equals(Rule(Metadata::prec(1, Rule(symbol)))));
+    });
+  });
+
+  describe("constructing choices", [&]() {
+    it("eliminates duplicate members", [&]() {
+      Rule rule = Rule::choice({
+        Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
+        NamedSymbol{"three"},
+        Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} })
+      });
+
+      AssertThat(rule, Equals(Rule(Choice{{
+        Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
+        NamedSymbol{"three"},
+      }})));
+
+      rule = Rule::choice({
+        Blank{},
+        Blank{},
+        Rule::choice({
+          Blank{},
+          NamedSymbol{"four"}
+        })
+      });
+
+      AssertThat(rule, Equals(Rule::choice({Blank{}, NamedSymbol{"four"}})));
+    });
+
+    it("eliminates duplicates within nested choices", [&]() {
+      Rule rule = Rule::choice({
+        Rule::seq({
+          NamedSymbol{"one"},
+          NamedSymbol{"two"}
+        }),
+        Rule::choice({
+          NamedSymbol{"three"},
+          Rule::seq({
+            NamedSymbol{"one"},
+            NamedSymbol{"two"}
+          })
+        })
+      });
+
+      AssertThat(rule, Equals(Rule(Choice{{
+        Rule::seq({
+          NamedSymbol{"one"},
+          NamedSymbol{"two"},
+        }),
+        NamedSymbol{"three"},
+      }})));
+    });
+
+    it("doesn't construct a choice if there's only one unique member", [&]() {
+      Rule rule = Rule::choice({
+        NamedSymbol{"one"},
+        Rule::choice({
+          NamedSymbol{"one"},
+        })
+      });
+
+      AssertThat(rule, Equals(Rule(NamedSymbol{"one"})));
+    });
+  });
+});
+
+END_TEST

data/ext/tree-sitter/tree-sitter/{spec/compiler/util/string_helpers_spec.cc → test/compiler/util/string_helpers_test.cc}

@@ -1,4 +1,4 @@
-#include "spec_helper.h"
+#include "test_helper.h"
 #include "compiler/util/string_helpers.h"
 
 using util::escape_char;

data/ext/tree-sitter/tree-sitter/test/fixtures/error_corpus/c_errors.txt

@@ -0,0 +1,166 @@
+=======================================
+Statements with missing semicolons
+=======================================
+
+int main() {
+  puts("hello")
+  puts("world")
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator (identifier) (parameter_list))
+    (compound_statement
+      (expression_statement (call_expression (identifier) (argument_list (string_literal))) (MISSING))
+      (expression_statement (call_expression (identifier) (argument_list (string_literal))) (MISSING)))))
+
+==============================================
+Top-level declarations with missing semicolons
+==============================================
+
+int x
+static int b
+
+---
+
+(translation_unit
+  (declaration (primitive_type) (identifier) (MISSING))
+  (declaration (storage_class_specifier) (primitive_type) (identifier) (MISSING)))
+
+==========================================
+Partial declaration lists inside ifdefs
+==========================================
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ok
+int b;
+
+int c() {
+  return 5;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+---
+
+(translation_unit
+  (preproc_ifdef (identifier)
+    (linkage_specification (string_literal) (declaration_list
+      (preproc_call (preproc_directive))
+      (comment)
+      (declaration (primitive_type) (identifier))
+      (function_definition (primitive_type) (function_declarator (identifier) (parameter_list)) (compound_statement (return_statement (number_literal))))
+      (preproc_ifdef (identifier) (MISSING))))))
+
+==========================================
+If statements with incomplete expressions
+==========================================
+
+int main() {
+  if (a.) {
+    b();
+    c();
+
+    if () d();
+  }
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator (identifier) (parameter_list))
+    (compound_statement
+      (if_statement
+        (field_expression
+          (identifier)
+          (MISSING))
+        (compound_statement
+          (expression_statement (call_expression (identifier) (argument_list)))
+          (expression_statement (call_expression (identifier) (argument_list)))
+          (if_statement
+            (MISSING)
+            (expression_statement (call_expression (identifier) (argument_list)))))))))
+
+====================================
+Invalid characters in declarations
+====================================
+
+int main() {
+  int x;
+  int %$#@
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator (identifier) (parameter_list))
+    (compound_statement
+      (declaration (primitive_type) (identifier))
+      (ERROR (primitive_type) (UNEXPECTED '$')))))
+
+=========================================
+Extra values in parenthesized expressions
+=========================================
+
+int main() {
+  int x = (123 123);
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator (identifier) (parameter_list))
+    (compound_statement
+      (declaration (primitive_type) (init_declarator
+        (identifier)
+        (parenthesized_expression
+          (number_literal)
+          (ERROR (number_literal))))))))
+
+========================================
+Extra identifiers in declarations
+========================================
+
+float x WTF;
+int y = 5;
+
+---
+
+(translation_unit
+  (declaration (primitive_type) (ERROR (identifier)) (identifier))
+  (declaration (primitive_type) (init_declarator (identifier) (number_literal))))
+
+==========================================
+Declarations with missing variable names
+==========================================
+
+int a() {
+  struct x = 1;
+  int = 2;
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator (identifier) (parameter_list))
+    (compound_statement
+      (struct_specifier (type_identifier))
+      (ERROR (number_literal))
+      (primitive_type)
+      (ERROR (number_literal)))))

data/ext/tree-sitter/tree-sitter/test/fixtures/error_corpus/javascript_errors.txt

@@ -0,0 +1,179 @@
+===================================================
+Missing default values for function parameters
+===================================================
+
+class A {
+  constructor (a, b = ) {
+    this.a = a
+  }
+
+  foo() {}
+}
+
+---
+
+(program
+  (class (identifier) (class_body
+    (method_definition
+      (property_identifier)
+      (formal_parameters (identifier) (identifier) (ERROR))
+      (statement_block (expression_statement (assignment_expression (member_expression (this) (property_identifier)) (identifier)))))
+    (method_definition
+      (property_identifier)
+      (formal_parameters)
+      (statement_block)))))
+
+===================================================
+Missing object-literal values
+===================================================
+
+{
+  a: b,
+  c:
+}
+
+---
+
+(program (expression_statement (object
+  (pair (property_identifier) (identifier))
+  (pair (property_identifier) (yield_expression (MISSING))))))
+
+===================================================
+Extra identifiers in expressions
+===================================================
+
+if (a b) {
+  c d;
+}
+e f;
+
+---
+
+(program
+  (if_statement
+    (parenthesized_expression
+      (identifier)
+      (ERROR (identifier)))
+    (statement_block
+      (ERROR (identifier))
+      (expression_statement (identifier))))
+  (expression_statement
+    (identifier)
+    (ERROR (identifier))))
+
+===================================================
+Extra complex literals in expressions
+===================================================
+
+if ({a: 'b'} {c: 'd'}) {
+  x = function(a) { b; } function(c) { d; }
+}
+
+---
+
+(program
+  (if_statement
+    (parenthesized_expression
+      (ERROR (object (pair (property_identifier) (string))))
+      (object (pair (property_identifier) (string))))
+    (statement_block
+      (expression_statement
+        (assignment_expression
+          (identifier)
+          (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))
+        (MISSING))
+      (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))))
+
+===================================================
+Extra tokens at the end of the file
+===================================================
+
+// skip the equals sign
+a.b =
+---
+
+(program
+  (comment)
+  (ERROR (member_expression (identifier) (property_identifier))))
+
+===================================================
+Errors after a sequence of function declarations
+===================================================
+
+/*
+ * The JS grammar has an ambiguity such that these functions
+ * can be parsed either as function declarations or as
+ * function expressions. This ambiguity causes a lot of
+ * splitting and merging in the parse stack. When iterating
+ * the parse stack during an error repair, there would then
+ * be a very large number (> 2^16) of paths through the parse
+ * stack.
+ */
+function a() {}
+function b() {}
+function c() {}
+function e() {}
+function f() {}
+function g() {}
+function h() {}
+function i() {}
+
+var x = !!!
+
+---
+
+(program
+  (comment)
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (function (identifier) (formal_parameters) (statement_block))
+  (ERROR (identifier)))
+
+=========================================================
+Errors inside of a template string substitution
+=========================================================
+
+const a = `b c ${d +} f g`
+
+---
+
+(program
+  (lexical_declaration
+    (variable_declarator
+      (identifier)
+      (template_string (template_substitution (identifier) (ERROR))))))
+
+=========================================================
+Long sequences of invalid tokens
+=========================================================
+
+function main(x) {
+  console.log('a');
+  what??????????????????????????????????????????????????
+  console.log('b');
+  return {};
+}
+
+---
+
+(program
+  (function
+    (identifier)
+    (formal_parameters (identifier))
+    (statement_block
+      (expression_statement
+        (call_expression
+          (member_expression (identifier) (property_identifier))
+          (arguments (string))))
+      (expression_statement
+        (identifier)
+        (ERROR
+          (call_expression
+            (member_expression (identifier) (property_identifier))
+            (arguments (string)))))
+      (return_statement (object)))))