tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/runtime/tree.h

@@ -14,37 +14,37 @@ extern "C" {
 
 extern TSStateId TS_TREE_STATE_NONE;
 
-typedef struct Tree {
+typedef struct {
+  union {
+    char *long_data;
+    char short_data[sizeof(char *) + sizeof(uint32_t)];
+  };
+  uint32_t length;
+} TSExternalTokenState;
+
+typedef struct Tree Tree;
+
+typedef Array(Tree *) TreeArray;
+
+struct Tree {
   struct {
     struct Tree *parent;
     uint32_t index;
     Length offset;
+    TSSymbol alias_symbol : 15;
+    bool alias_is_named : 1;
   } context;
 
-  uint32_t child_count;
-  union {
-    struct {
-      uint32_t visible_child_count;
-      uint32_t named_child_count;
-      struct Tree **children;
-    };
-    TSExternalTokenState external_token_state;
-    int32_t lookahead_char;
-  };
-
   Length padding;
   Length size;
+  uint32_t ref_count;
+  uint32_t bytes_scanned;
+  uint32_t error_cost;
+  uint32_t node_count;
+  uint32_t repeat_depth;
+  uint32_t child_count;
+  int32_t dynamic_precedence;
 
-  TSSymbol symbol;
-  TSStateId parse_state;
-  unsigned error_cost;
-
-  struct {
-    TSSymbol symbol;
-    TSLexMode lex_mode;
-  } first_leaf;
-
-  unsigned short ref_count;
   bool visible : 1;
   bool named : 1;
   bool extra : 1;
@@ -52,44 +52,69 @@ typedef struct Tree {
   bool fragile_right : 1;
   bool has_changes : 1;
   bool has_external_tokens : 1;
-  bool has_external_token_state : 1;
-} Tree;
+  bool is_missing : 1;
+  TSSymbol symbol;
+  TSStateId parse_state;
+  struct {
+    TSSymbol symbol;
+    TSLexMode lex_mode;
+  } first_leaf;
 
-typedef struct {
-  Tree *tree;
-  Length position;
-  uint32_t child_index;
-} TreePathEntry;
+  union {
+    struct {
+      TreeArray children;
+      uint32_t visible_child_count;
+      uint32_t named_child_count;
+      uint16_t alias_sequence_id;
+    };
+    struct {
+      uint32_t _2;
+      TSExternalTokenState external_token_state;
+    };
+    struct {
+      uint32_t _1;
+      int32_t lookahead_char;
+    };
+  };
+};
 
-typedef Array(Tree *) TreeArray;
+typedef struct {
+  TreeArray free_trees;
+  TreeArray tree_stack;
+} TreePool;
 
-typedef Array(TreePathEntry) TreePath;
+void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned);
+const char *ts_external_token_state_data(const TSExternalTokenState *);
 
 bool ts_tree_array_copy(TreeArray, TreeArray *);
-void ts_tree_array_delete(TreeArray *);
-uint32_t ts_tree_array_essential_count(const TreeArray *);
-TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
+void ts_tree_array_delete(TreePool *, TreeArray *);
 TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
-
-Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
-Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata);
-Tree *ts_tree_make_copy(Tree *child);
-Tree *ts_tree_make_error_node(TreeArray *);
-Tree *ts_tree_make_error(Length, Length, char);
+void ts_tree_array_reverse(TreeArray *);
+
+void ts_tree_pool_init(TreePool *);
+void ts_tree_pool_delete(TreePool *);
+Tree *ts_tree_pool_allocate(TreePool *);
+void ts_tree_pool_free(TreePool *, Tree *);
+
+Tree *ts_tree_make_leaf(TreePool *, TSSymbol, Length, Length, const TSLanguage *);
+Tree *ts_tree_make_node(TreePool *, TSSymbol, TreeArray *, unsigned, const TSLanguage *);
+Tree *ts_tree_make_copy(TreePool *, Tree *child);
+Tree *ts_tree_make_error_node(TreePool *, TreeArray *, const TSLanguage *);
+Tree *ts_tree_make_error(TreePool *, Length, Length, int32_t, const TSLanguage *);
+Tree *ts_tree_make_missing_leaf(TreePool *, TSSymbol, const TSLanguage *);
 void ts_tree_retain(Tree *tree);
-void ts_tree_release(Tree *tree);
+void ts_tree_release(TreePool *, Tree *tree);
 bool ts_tree_eq(const Tree *tree1, const Tree *tree2);
-bool ts_tree_tokens_eq(const Tree *, const Tree *);
 int ts_tree_compare(const Tree *tree1, const Tree *tree2);
-
 uint32_t ts_tree_start_column(const Tree *self);
 uint32_t ts_tree_end_column(const Tree *self);
-void ts_tree_set_children(Tree *, uint32_t, Tree **);
-void ts_tree_assign_parents(Tree *, TreePath *);
+void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *);
+void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *);
 void ts_tree_edit(Tree *, const TSInputEdit *edit);
 char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);
 void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *);
-const TSExternalTokenState *ts_tree_last_external_token_state(const Tree *);
+Tree *ts_tree_last_external_token(Tree *);
+bool ts_tree_external_token_state_eq(const Tree *, const Tree *);
 
 static inline uint32_t ts_tree_total_bytes(const Tree *self) {
   return self->padding.bytes + self->size.bytes;
@@ -99,17 +124,6 @@ static inline Length ts_tree_total_size(const Tree *self) {
   return length_add(self->padding, self->size);
 }
 
-static inline TSPoint ts_tree_total_extent(const Tree *self) {
-  return point_add(self->padding.extent, self->size.extent);
-}
-
-static inline bool ts_tree_is_fragile(const Tree *tree) {
-  return tree->fragile_left || tree->fragile_right ||
-         ts_tree_total_bytes(tree) == 0;
-}
-
-bool ts_external_token_state_eq(const TSExternalTokenState *, const TSExternalTokenState *);
-
 #ifdef __cplusplus
 }
 #endif

data/ext/tree-sitter/tree-sitter/src/runtime/utf16.c

@@ -1,6 +1,11 @@
 #include "runtime/utf16.h"
 
 int utf16_iterate(const uint8_t *string, size_t length, int32_t *code_point) {
+  if (length < 2) {
+    *code_point = -1;
+    return 0;
+  }
+
   uint16_t *units = (uint16_t *)string;
   uint16_t unit = units[0];
 

data/ext/tree-sitter/tree-sitter/test/benchmarks.cc

@@ -0,0 +1,122 @@
+#include <cassert>
+#include <map>
+#include <vector>
+#include <ctime>
+#include <string>
+#include "tree_sitter/runtime.h"
+#include "helpers/load_language.h"
+#include "helpers/stderr_logger.h"
+#include "helpers/read_test_entries.h"
+
+using std::map;
+using std::vector;
+using std::string;
+
+vector<string> language_names({
+  "c",
+  "cpp",
+  "javascript",
+  "python",
+  "bash",
+});
+
+size_t mean(const vector<size_t> &values) {
+  if (values.empty()) return 0;
+  size_t result = 0;
+  for (size_t value : values) {
+    result += value;
+  }
+  return result / values.size();
+}
+
+size_t min(const vector<size_t> &values) {
+  size_t result = 0;
+  for (unsigned i = 0; i < values.size(); i++) {
+    size_t value = values[i];
+    if (i == 0 || value < result) result = value;
+  }
+  return result;
+}
+
+int main(int argc, char *arg[]) {
+  map<string, vector<ExampleEntry>> example_entries_by_language_name;
+  vector<size_t> error_speeds;
+  vector<size_t> non_error_speeds;
+
+  auto document = ts_document_new();
+
+  if (getenv("TREE_SITTER_BENCHMARK_SVG")) {
+    ts_document_print_debugging_graphs(document, true);
+  } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) {
+    ts_document_set_logger(document, stderr_logger_new(false));
+  }
+
+  auto language_filter = getenv("TREE_SITTER_BENCHMARK_LANGUAGE");
+  auto file_name_filter = getenv("TREE_SITTER_BENCHMARK_FILE_NAME");
+
+  for (auto &language_name : language_names) {
+    example_entries_by_language_name[language_name] = examples_for_language(language_name);
+  }
+
+  for (auto &language_name : language_names) {
+    if (language_filter && language_name != language_filter) continue;
+
+    ts_document_set_language(document, load_real_language(language_name));
+
+    printf("%s\n", language_name.c_str());
+
+    for (auto &example : example_entries_by_language_name[language_name]) {
+      if (file_name_filter && example.file_name != file_name_filter) continue;
+      if (example.input.size() < 256) continue;
+
+      ts_document_invalidate(document);
+      ts_document_set_input_string(document, "");
+      ts_document_parse(document);
+
+      ts_document_invalidate(document);
+      ts_document_set_input_string(document, example.input.c_str());
+
+      clock_t start_time = clock();
+      ts_document_parse(document);
+      clock_t end_time = clock();
+      unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC;
+      assert(!ts_node_has_error(ts_document_root_node(document)));
+      size_t speed = static_cast<double>(example.input.size()) / duration;
+      printf("  %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed);
+      if (speed != 0) non_error_speeds.push_back(speed);
+    }
+
+    for (auto &other_language_name : language_names) {
+      if (other_language_name == language_name) continue;
+
+      for (auto &example : example_entries_by_language_name[other_language_name]) {
+        if (file_name_filter && example.file_name != file_name_filter) continue;
+        if (example.input.size() < 256) continue;
+
+        ts_document_invalidate(document);
+        ts_document_set_input_string(document, example.input.c_str());
+
+        clock_t start_time = clock();
+        ts_document_parse(document);
+        clock_t end_time = clock();
+        unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC;
+        size_t speed = static_cast<double>(example.input.size()) / duration;
+        printf("  %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed);
+        if (speed != 0) error_speeds.push_back(speed);
+      }
+    }
+
+    puts("");
+  }
+
+  puts("without errors:");
+  printf("  %-30s\t%lu bytes/ms\n", "average speed", mean(non_error_speeds));
+  printf("  %-30s\t%lu bytes/ms\n", "worst speed", min(non_error_speeds));
+  puts("");
+
+  puts("with errors:");
+  printf("  %-30s\t%lu bytes/ms\n", "average speed", mean(error_speeds));
+  printf("  %-30s\t%lu bytes/ms\n", "worst speed", min(error_speeds));
+
+  return 0;
+}

data/ext/tree-sitter/tree-sitter/{spec/compiler/build_tables/lex_item_spec.cc → test/compiler/build_tables/lex_item_test.cc}

@@ -1,8 +1,6 @@
-#include "spec_helper.h"
+#include "test_helper.h"
 #include "compiler/build_tables/lex_item.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules.h"
-#include "helpers/rule_helpers.h"
+#include "compiler/rule.h"
 #include "helpers/stream_methods.h"
 
 using namespace rules;
@@ -13,29 +11,29 @@ START_TEST
 
 describe("LexItem", []() {
   describe("completion_status()", [&]() {
-    it("indicates whether the item is done, its precedence, and whether it is a string", [&]() {
-      LexItem item1(Symbol(0, Symbol::Terminal), character({ 'a', 'b', 'c' }));
+    it("indicates whether the item is done and its precedence", [&]() {
+      LexItem item1(Symbol::terminal(0), CharacterSet({'a', 'b', 'c'}));
       AssertThat(item1.completion_status().is_done, IsFalse());
       AssertThat(item1.completion_status().precedence, Equals(PrecedenceRange()));
-      AssertThat(item1.completion_status().is_string, IsFalse());
 
       MetadataParams params;
       params.precedence = 3;
       params.has_precedence = true;
       params.is_string = 1;
-      LexItem item2(Symbol(0, Symbol::Terminal), choice({
-        metadata(blank(), params),
-        character({ 'a', 'b', 'c' })
-      }));
+      LexItem item2(Symbol::terminal(0), Choice{{
+        Metadata{Blank{}, params},
+        CharacterSet({'a', 'b', 'c'}),
+      }});
 
       AssertThat(item2.completion_status().is_done, IsTrue());
       AssertThat(item2.completion_status().precedence, Equals(PrecedenceRange(3)));
-      AssertThat(item2.completion_status().is_string, IsTrue());
 
-      LexItem item3(Symbol(0, Symbol::Terminal), repeat(character({ ' ', '\t' })));
+      LexItem item3(Symbol::terminal(0), Choice{{
+        Blank{},
+        Repeat{CharacterSet({' ', '\t'})},
+      }});
       AssertThat(item3.completion_status().is_done, IsTrue());
       AssertThat(item3.completion_status().precedence, Equals(PrecedenceRange()));
-      AssertThat(item3.completion_status().is_string, IsFalse());
     });
   });
 });
@@ -43,17 +41,17 @@ describe("LexItem", []() {
 describe("LexItemSet::transitions()", [&]() {
   it("handles single characters", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), character({ 'x' })),
+      LexItem(Symbol::non_terminal(1), CharacterSet({'x'})),
     });
 
     AssertThat(
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('x'),
+          CharacterSet{{'x'}},
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), blank()),
+              LexItem(Symbol::non_terminal(1), Blank{}),
             }),
             PrecedenceRange(),
             false
@@ -63,21 +61,18 @@ describe("LexItemSet::transitions()", [&]() {
   });
 
   it("marks transitions that are within the main token (as opposed to separators)", [&]() {
-    MetadataParams params;
-    params.is_main_token = true;
-
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), metadata(character({ 'x' }), params)),
+      LexItem(Symbol::non_terminal(1), Metadata::main_token(CharacterSet{{'x'}})),
     });
 
     AssertThat(
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('x'),
+          CharacterSet({'x'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), metadata(blank(), params)),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(0, Metadata::main_token(Blank{}))),
             }),
             PrecedenceRange(),
             true
@@ -88,11 +83,11 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles sequences", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), seq({
-        character({ 'w' }),
-        character({ 'x' }),
-        character({ 'y' }),
-        character({ 'z' }),
+      LexItem(Symbol::non_terminal(1), Rule::seq({
+        CharacterSet({'w'}),
+        CharacterSet({'x'}),
+        CharacterSet({'y'}),
+        CharacterSet({'z'}),
       })),
     });
 
@@ -100,13 +95,13 @@ describe("LexItemSet::transitions()", [&]() {
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('w'),
+          CharacterSet({'w'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                character({ 'x' }),
-                character({ 'y' }),
-                character({ 'z' }),
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                CharacterSet({'x'}),
+                CharacterSet({'y'}),
+                CharacterSet({'z'}),
               })),
             }),
             PrecedenceRange(),
@@ -118,14 +113,14 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles sequences with nested precedence", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), seq({
-        prec(3, seq({
-          character({ 'v' }),
-          prec(4, seq({
-            character({ 'w' }),
-            character({ 'x' }) })),
-          character({ 'y' }) })),
-        character({ 'z' }),
+      LexItem(Symbol::non_terminal(1), Rule::seq({
+        Metadata::prec(3, Rule::seq({
+          CharacterSet({'v'}),
+          Metadata::prec(4, Rule::seq({
+            CharacterSet({'w'}),
+            CharacterSet({'x'}) })),
+          CharacterSet({'y'}) })),
+        CharacterSet({'z'}),
       })),
     });
 
@@ -135,18 +130,20 @@ describe("LexItemSet::transitions()", [&]() {
       transitions,
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('v'),
+          CharacterSet({'v'}),
           Transition{
             // The outer precedence is now 'active', because we are within its
             // contained rule.
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                active_prec(3, seq({
-                  prec(4, seq({
-                    character({ 'w' }),
-                    character({ 'x' }) })),
-                  character({ 'y' }) })),
-                character({ 'z' }),
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                Metadata::active_prec(3, Rule::seq({
+                  Metadata::prec(4, Rule::seq({
+                    CharacterSet({'w'}),
+                    CharacterSet({'x'})
+                  })),
+                  CharacterSet({'y'})
+                })),
+                CharacterSet({'z'}),
               })),
             }),
 
@@ -157,22 +154,23 @@ describe("LexItemSet::transitions()", [&]() {
         }
       })));
 
-    LexItemSet item_set2 = transitions[CharacterSet().include('v')].destination;
+    LexItemSet item_set2 = transitions[CharacterSet({'v'})].destination;
     transitions = item_set2.transitions();
 
     AssertThat(
       transitions,
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('w'),
+          CharacterSet({'w'}),
           Transition{
             // The inner precedence is now 'active'
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                active_prec(3, seq({
-                  active_prec(4, character({ 'x' })),
-                  character({ 'y' }) })),
-                character({ 'z' }),
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                Metadata::active_prec(3, Rule::seq({
+                  Metadata::active_prec(4, CharacterSet{{'x'}}),
+                  CharacterSet{{'y'}}
+                })),
+                CharacterSet{{'z'}},
               })),
             }),
 
@@ -183,19 +181,19 @@ describe("LexItemSet::transitions()", [&]() {
         }
       })));
 
-    LexItemSet item_set3 = transitions[CharacterSet().include('w')].destination;
+    LexItemSet item_set3 = transitions[CharacterSet({'w'})].destination;
     transitions = item_set3.transitions();
 
     AssertThat(
       transitions,
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('x'),
+          CharacterSet({'x'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                active_prec(3, character({ 'y' })),
-                character({ 'z' }),
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                Metadata::active_prec(3, CharacterSet{{'y'}}),
+                CharacterSet{{'z'}},
               })),
             }),
 
@@ -206,17 +204,17 @@ describe("LexItemSet::transitions()", [&]() {
         }
       })));
 
-    LexItemSet item_set4 = transitions[CharacterSet().include('x')].destination;
+    LexItemSet item_set4 = transitions[CharacterSet({'x'})].destination;
     transitions = item_set4.transitions();
 
     AssertThat(
       transitions,
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('y'),
+          CharacterSet({'y'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'z' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'z'})),
             }),
             PrecedenceRange(3),
             false
@@ -227,13 +225,13 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles sequences where the left hand side can be blank", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), seq({
-        choice({
-          character({ 'x' }),
-          blank(),
+      LexItem(Symbol::non_terminal(1), Rule::seq({
+        Rule::choice({
+          CharacterSet({'x'}),
+          Blank{},
         }),
-        character({ 'y' }),
-        character({ 'z' }),
+        CharacterSet({'y'}),
+        CharacterSet({'z'}),
       })),
     });
 
@@ -241,12 +239,12 @@ describe("LexItemSet::transitions()", [&]() {
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('x'),
+          CharacterSet({'x'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                character({ 'y' }),
-                character({ 'z' }),
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                CharacterSet({'y'}),
+                CharacterSet({'z'}),
               })),
             }),
             PrecedenceRange(),
@@ -254,10 +252,10 @@ describe("LexItemSet::transitions()", [&]() {
           }
         },
         {
-          CharacterSet().include('y'),
+          CharacterSet({'y'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'z' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'z'})),
             }),
             PrecedenceRange(),
             false
@@ -268,7 +266,7 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles blanks", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), blank()),
+      LexItem(Symbol::non_terminal(1), Blank{}),
     });
 
     AssertThat(item_set.transitions(), IsEmpty());
@@ -276,39 +274,39 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles repeats", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), repeat1(seq({
-        character({ 'a' }),
-        character({ 'b' }),
-      }))),
-      LexItem(Symbol(2, Symbol::NonTerminal), repeat1(character({ 'c' }))),
+      LexItem(Symbol::non_terminal(1), Repeat{Rule::seq({
+        CharacterSet({'a'}),
+        CharacterSet({'b'}),
+      })}),
+      LexItem(Symbol::non_terminal(2), Repeat{CharacterSet{{'c'}}}),
     });
 
     AssertThat(
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('a'),
+          CharacterSet({'a'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({
-                character({ 'b' }),
-                repeat1(seq({
-                  character({ 'a' }),
-                  character({ 'b' }),
-                }))
+              LexItem(Symbol::non_terminal(1), Rule::seq({
+                CharacterSet({'b'}),
+                Repeat{Rule::seq({
+                  CharacterSet({'a'}),
+                  CharacterSet({'b'}),
+                })}
               })),
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'b' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'b'})),
             }),
             PrecedenceRange(),
             false
           }
         },
         {
-          CharacterSet().include('c'),
+          CharacterSet({'c'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(2, Symbol::NonTerminal), repeat1(character({ 'c' }))),
-              LexItem(Symbol(2, Symbol::NonTerminal), blank()),
+              LexItem(Symbol::non_terminal(2), Repeat{CharacterSet({'c'})}),
+              LexItem(Symbol::non_terminal(2), Blank{}),
             }),
             PrecedenceRange(),
             false
@@ -319,18 +317,18 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles repeats with precedence", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), active_prec(-1, repeat1(character({ 'a' }))))
+      LexItem(Symbol::non_terminal(1), Metadata::active_prec(-1, Repeat{CharacterSet({'a'})}))
     });
 
     AssertThat(
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('a'),
+          CharacterSet({'a'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(-1, repeat1(character({ 'a' })))),
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(-1, blank())),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(-1, Repeat{CharacterSet({'a'})})),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(-1, Blank{})),
             }),
             PrecedenceRange(-1),
             false
@@ -341,14 +339,14 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles choices between overlapping character sets", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), choice({
-        active_prec(2, seq({
-          character({ 'a', 'b', 'c', 'd'  }),
-          character({ 'x' }),
+      LexItem(Symbol::non_terminal(1), Rule::choice({
+        Metadata::active_prec(2, Rule::seq({
+          CharacterSet({'a', 'b', 'c', 'd'}),
+          CharacterSet({'x'}),
         })),
-        active_prec(3, seq({
-          character({ 'c', 'd', 'e', 'f' }),
-          character({ 'y' }),
+        Metadata::active_prec(3, Rule::seq({
+          CharacterSet({'c', 'd', 'e', 'f'}),
+          CharacterSet({'y'}),
         })),
       }))
     });
@@ -357,31 +355,31 @@ describe("LexItemSet::transitions()", [&]() {
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('a', 'b'),
+          CharacterSet({'a', 'b'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(2, character({ 'x' }))),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(2, CharacterSet({'x'}))),
             }),
             PrecedenceRange(2),
             false
           }
         },
         {
-          CharacterSet().include('c', 'd'),
+          CharacterSet({'c', 'd'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(2, character({ 'x' }))),
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(3, character({ 'y' }))),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(2, CharacterSet({'x'}))),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(3, CharacterSet({'y'}))),
             }),
             PrecedenceRange(2, 3),
             false
           }
         },
         {
-          CharacterSet().include('e', 'f'),
+          CharacterSet({'e', 'f'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), active_prec(3, character({ 'y' }))),
+              LexItem(Symbol::non_terminal(1), Metadata::active_prec(3, CharacterSet({'y'}))),
             }),
             PrecedenceRange(3),
             false
@@ -392,14 +390,14 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles choices between a subset and a superset of characters", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), choice({
-        seq({
-          character({ 'b', 'c', 'd' }),
-          character({ 'x' }),
+      LexItem(Symbol::non_terminal(1), Rule::choice({
+        Rule::seq({
+          CharacterSet({'b', 'c', 'd'}),
+          CharacterSet({'x'}),
         }),
-        seq({
-          character({ 'a', 'b', 'c', 'd', 'e', 'f' }),
-          character({ 'y' }),
+        Rule::seq({
+          CharacterSet({'a', 'b', 'c', 'd', 'e', 'f'}),
+          CharacterSet({'y'}),
         }),
       })),
     });
@@ -408,21 +406,21 @@ describe("LexItemSet::transitions()", [&]() {
       item_set.transitions(),
       Equals(LexItemSet::TransitionMap({
         {
-          CharacterSet().include('a').include('e', 'f'),
+          CharacterSet({'a', 'e', 'f'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'y' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'y'})),
             }),
             PrecedenceRange(),
             false
           }
         },
         {
-          CharacterSet().include('b', 'd'),
+          CharacterSet({'b', 'c', 'd'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'x' })),
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ 'y' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'x'})),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'y'})),
             }),
             PrecedenceRange(),
             false
@@ -433,15 +431,15 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles choices between whitelisted and blacklisted character sets", [&]() {
     LexItemSet item_set({
-      LexItem(Symbol(1, Symbol::NonTerminal), seq({
-        choice({
-          character({ '/' }, false),
-          seq({
-            character({ '\\' }),
-            character({ '/' }),
+      LexItem(Symbol::non_terminal(1), Rule::seq({
+        Rule::choice({
+          CharacterSet().include_all().exclude('/'),
+          Rule::seq({
+            CharacterSet({'\\'}),
+            CharacterSet({'/'}),
           }),
         }),
-        character({ '/' }),
+        CharacterSet({'/'}),
       }))
     });
 
@@ -452,18 +450,18 @@ describe("LexItemSet::transitions()", [&]() {
           CharacterSet().include_all().exclude('/').exclude('\\'),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ '/' })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'/'})),
             }),
             PrecedenceRange(),
             false
           }
         },
         {
-          CharacterSet().include('\\'),
+          CharacterSet({'\\'}),
           Transition{
             LexItemSet({
-              LexItem(Symbol(1, Symbol::NonTerminal), character({ '/' })),
-              LexItem(Symbol(1, Symbol::NonTerminal), seq({ character({ '/' }), character({ '/' }) })),
+              LexItem(Symbol::non_terminal(1), CharacterSet({'/'})),
+              LexItem(Symbol::non_terminal(1), Rule::seq({CharacterSet({'/'}), CharacterSet({'/'})})),
             }),
             PrecedenceRange(),
             false
@@ -474,37 +472,37 @@ describe("LexItemSet::transitions()", [&]() {
 
   it("handles different items with overlapping character sets", [&]() {
     LexItemSet set1({
-      LexItem(Symbol(1, Symbol::NonTerminal), character({ 'a', 'b', 'c', 'd', 'e', 'f' })),
-      LexItem(Symbol(2, Symbol::NonTerminal), character({ 'e', 'f', 'g', 'h', 'i' }))
+      LexItem(Symbol::non_terminal(1), CharacterSet({'a', 'b', 'c', 'd', 'e', 'f'})),
+      LexItem(Symbol::non_terminal(2), CharacterSet({'e', 'f', 'g', 'h', 'i'}))
     });
 
     AssertThat(set1.transitions(), Equals(LexItemSet::TransitionMap({
       {
-        CharacterSet().include('a', 'd'),
+        CharacterSet({'a', 'b', 'c', 'd'}),
         Transition{
           LexItemSet({
-            LexItem(Symbol(1, Symbol::NonTerminal), blank()),
+            LexItem(Symbol::non_terminal(1), Blank{}),
           }),
           PrecedenceRange(),
           false
         }
       },
       {
-        CharacterSet().include('e', 'f'),
+        CharacterSet({'e', 'f'}),
         Transition{
           LexItemSet({
-            LexItem(Symbol(1, Symbol::NonTerminal), blank()),
-            LexItem(Symbol(2, Symbol::NonTerminal), blank()),
+            LexItem(Symbol::non_terminal(1), Blank{}),
+            LexItem(Symbol::non_terminal(2), Blank{}),
           }),
           PrecedenceRange(),
           false
         }
       },
       {
-        CharacterSet().include('g', 'i'),
+        CharacterSet({'g', 'h', 'i'}),
         Transition{
           LexItemSet({
-            LexItem(Symbol(2, Symbol::NonTerminal), blank()),
+            LexItem(Symbol::non_terminal(2), Blank{}),
           }),
           PrecedenceRange(),
           false