tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/include/tree_sitter/compiler.h

@@ -15,6 +15,8 @@ typedef enum {
   TSCompileErrorTypeLexConflict,
   TSCompileErrorTypeParseConflict,
   TSCompileErrorTypeEpsilonRule,
+  TSCompileErrorTypeInvalidTokenContents,
+  TSCompileErrorTypeInvalidRuleName,
 } TSCompileErrorType;
 
 typedef struct {

data/ext/tree-sitter/tree-sitter/include/tree_sitter/parser.h

@@ -9,23 +9,22 @@ extern "C" {
 #include <stdint.h>
 #include <stdlib.h>
 
-typedef unsigned short TSSymbol;
-typedef unsigned short TSStateId;
-
-typedef uint8_t TSExternalTokenState[16];
+typedef uint16_t TSSymbol;
+typedef uint16_t TSStateId;
 
 #define ts_builtin_sym_error ((TSSymbol)-1)
 #define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
 typedef struct {
   bool visible : 1;
   bool named : 1;
-  bool extra : 1;
-  bool structural : 1;
 } TSSymbolMetadata;
 
 typedef struct {
   void (*advance)(void *, bool);
+  void (*mark_end)(void *);
+  uint32_t (*get_column)(void *);
   int32_t lookahead;
   TSSymbol result_symbol;
 } TSLexer;
@@ -39,15 +38,19 @@ typedef enum {
 
 typedef struct {
   union {
-    TSStateId to_state;
+    struct {
+      TSStateId state;
+      bool extra : 1;
+      bool repetition : 1;
+    };
     struct {
       TSSymbol symbol;
-      unsigned short child_count;
+      int16_t dynamic_precedence;
+      uint8_t child_count;
+      uint8_t alias_sequence_id;
     };
   } params;
   TSParseActionType type : 4;
-  bool extra : 1;
-  bool fragile : 1;
 } TSParseAction;
 
 typedef struct {
@@ -58,32 +61,35 @@ typedef struct {
 typedef union {
   TSParseAction action;
   struct {
-    unsigned short count;
+    uint8_t count;
     bool reusable : 1;
-    bool depends_on_lookahead : 1;
   };
 } TSParseActionEntry;
 
 typedef struct TSLanguage {
   uint32_t version;
   uint32_t symbol_count;
+  uint32_t alias_count;
   uint32_t token_count;
   uint32_t external_token_count;
   const char **symbol_names;
   const TSSymbolMetadata *symbol_metadata;
-  const unsigned short *parse_table;
+  const uint16_t *parse_table;
   const TSParseActionEntry *parse_actions;
   const TSLexMode *lex_modes;
+  const TSSymbol *alias_sequences;
+  uint16_t max_alias_sequence_length;
   bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
   struct {
     const bool *states;
     const TSSymbol *symbol_map;
     void *(*create)();
     void (*destroy)(void *);
-    void (*reset)(void *);
     bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
-    bool (*serialize)(void *, TSExternalTokenState);
-    void (*deserialize)(void *, const TSExternalTokenState);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
 } TSLanguage;
 
@@ -91,32 +97,32 @@ typedef struct TSLanguage {
  *  Lexer Macros
  */
 
-#define START_LEXER() \
-  int32_t lookahead;  \
-  next_state:         \
+#define START_LEXER()           \
+  bool result = false;          \
+  int32_t lookahead;            \
+  next_state:                   \
   lookahead = lexer->lookahead;
 
-#define ADVANCE(state_value)                   \
-  {                                            \
+#define ADVANCE(state_value)      \
+  {                               \
     lexer->advance(lexer, false); \
-    state = state_value;                       \
-    goto next_state;                           \
+    state = state_value;          \
+    goto next_state;              \
   }
 
-#define SKIP(state_value)                     \
-  {                                           \
+#define SKIP(state_value)        \
+  {                              \
     lexer->advance(lexer, true); \
-    state = state_value;                      \
-    goto next_state;                          \
+    state = state_value;         \
+    goto next_state;             \
   }
 
-#define ACCEPT_TOKEN(symbol_value)       \
-  {                                      \
-    lexer->result_symbol = symbol_value; \
-    return true;                         \
-  }
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
 
-#define LEX_ERROR() return false
+#define END_STATE() return result;
 
 /*
  *  Parse Table Macros
@@ -125,39 +131,48 @@ typedef struct TSLanguage {
 #define STATE(id) id
 #define ACTIONS(id) id
 
-#define SHIFT(to_state_value)                                                 \
-  {                                                                           \
-    {                                                                         \
-      .type = TSParseActionTypeShift, .params = {.to_state = to_state_value } \
-    }                                                                         \
+#define SHIFT(state_value)              \
+  {                                     \
+    {                                   \
+      .type = TSParseActionTypeShift,   \
+      .params = {.state = state_value}, \
+    }                                   \
   }
 
-#define RECOVER(to_state_value)                                                 \
-  {                                                                             \
-    {                                                                           \
-      .type = TSParseActionTypeRecover, .params = {.to_state = to_state_value } \
-    }                                                                           \
+#define SHIFT_REPEAT(state_value)     \
+  {                                   \
+    {                                 \
+      .type = TSParseActionTypeShift, \
+      .params = {                     \
+        .state = state_value,         \
+        .repetition = true            \
+      },                              \
+    }                                 \
   }
 
-#define SHIFT_EXTRA()                                 \
-  {                                                   \
-    { .type = TSParseActionTypeShift, .extra = true } \
+#define RECOVER()                        \
+  {                                      \
+    { .type = TSParseActionTypeRecover } \
   }
 
-#define REDUCE(symbol_val, child_count_val)                             \
-  {                                                                     \
-    {                                                                   \
-      .type = TSParseActionTypeReduce,                                  \
-      .params = {.symbol = symbol_val, .child_count = child_count_val } \
-    }                                                                   \
+#define SHIFT_EXTRA()                 \
+  {                                   \
+    {                                 \
+      .type = TSParseActionTypeShift, \
+      .params = {.extra = true}       \
+    }                                 \
   }
 
-#define REDUCE_FRAGILE(symbol_val, child_count_val)                     \
-  {                                                                     \
-    {                                                                   \
-      .type = TSParseActionTypeReduce, .fragile = true,                 \
-      .params = {.symbol = symbol_val, .child_count = child_count_val } \
-    }                                                                   \
+#define REDUCE(symbol_val, child_count_val, ...) \
+  {                                              \
+    {                                            \
+      .type = TSParseActionTypeReduce,           \
+      .params = {                                \
+        .symbol = symbol_val,                    \
+        .child_count = child_count_val,          \
+        __VA_ARGS__                              \
+      }                                          \
+    }                                            \
   }
 
 #define ACCEPT_INPUT()                  \
@@ -165,22 +180,6 @@ typedef struct TSLanguage {
     { .type = TSParseActionTypeAccept } \
   }
 
-#define GET_LANGUAGE(...)                                          \
-  static TSLanguage language = {                                   \
-    .version = LANGUAGE_VERSION,                                   \
-    .symbol_count = SYMBOL_COUNT,                                  \
-    .token_count = TOKEN_COUNT,                                    \
-    .symbol_metadata = ts_symbol_metadata,                         \
-    .parse_table = (const unsigned short *)ts_parse_table,         \
-    .parse_actions = ts_parse_actions,                             \
-    .lex_modes = ts_lex_modes,                                     \
-    .symbol_names = ts_symbol_names,                               \
-    .lex_fn = ts_lex,                                              \
-    .external_token_count = EXTERNAL_TOKEN_COUNT,                  \
-    .external_scanner = {__VA_ARGS__}                              \
-  };                                                               \
-  return &language                                                 \
-
 #ifdef __cplusplus
 }
 #endif

data/ext/tree-sitter/tree-sitter/include/tree_sitter/runtime.h

@@ -9,7 +9,7 @@ extern "C" {
 #include <stdint.h>
 #include <stdbool.h>
 
-#define TREE_SITTER_LANGUAGE_VERSION 1
+#define TREE_SITTER_LANGUAGE_VERSION 8
 
 typedef unsigned short TSSymbol;
 typedef struct TSLanguage TSLanguage;
@@ -20,10 +20,21 @@ typedef enum {
   TSInputEncodingUTF16,
 } TSInputEncoding;
 
+typedef enum {
+  TSSymbolTypeRegular,
+  TSSymbolTypeAnonymous,
+  TSSymbolTypeAuxiliary,
+} TSSymbolType;
+
+typedef struct {
+  uint32_t row;
+  uint32_t column;
+} TSPoint;
+
 typedef struct {
   void *payload;
   const char *(*read)(void *payload, uint32_t *bytes_read);
-  int (*seek)(void *payload, uint32_t character_index, uint32_t byte_index);
+  int (*seek)(void *payload, uint32_t byte_index, TSPoint position);
   TSInputEncoding encoding;
 } TSInput;
 
@@ -37,11 +48,6 @@ typedef struct {
   void (*log)(void *payload, TSLogType, const char *);
 } TSLogger;
 
-typedef struct {
-  uint32_t row;
-  uint32_t column;
-} TSPoint;
-
 typedef struct {
   uint32_t start_byte;
   uint32_t bytes_removed;
@@ -58,40 +64,33 @@ typedef struct {
 
 typedef struct {
   const void *data;
-  uint32_t offset[3];
+  uint32_t offset[2];
 } TSNode;
 
-typedef struct {
-  TSSymbol value;
-  bool done;
-  void *data;
-} TSSymbolIterator;
-
-uint32_t ts_node_start_char(TSNode);
 uint32_t ts_node_start_byte(TSNode);
 TSPoint ts_node_start_point(TSNode);
-uint32_t ts_node_end_char(TSNode);
 uint32_t ts_node_end_byte(TSNode);
 TSPoint ts_node_end_point(TSNode);
 TSSymbol ts_node_symbol(TSNode);
-TSSymbolIterator ts_node_symbols(TSNode);
-void ts_symbol_iterator_next(TSSymbolIterator *);
 const char *ts_node_type(TSNode, const TSDocument *);
 char *ts_node_string(TSNode, const TSDocument *);
 bool ts_node_eq(TSNode, TSNode);
 bool ts_node_is_named(TSNode);
+bool ts_node_is_missing(TSNode);
 bool ts_node_has_changes(TSNode);
+bool ts_node_has_error(TSNode);
 TSNode ts_node_parent(TSNode);
 TSNode ts_node_child(TSNode, uint32_t);
 TSNode ts_node_named_child(TSNode, uint32_t);
 uint32_t ts_node_child_count(TSNode);
 uint32_t ts_node_named_child_count(TSNode);
+uint32_t ts_node_child_index(TSNode);
 TSNode ts_node_next_sibling(TSNode);
 TSNode ts_node_next_named_sibling(TSNode);
 TSNode ts_node_prev_sibling(TSNode);
 TSNode ts_node_prev_named_sibling(TSNode);
-TSNode ts_node_descendant_for_char_range(TSNode, uint32_t, uint32_t);
-TSNode ts_node_named_descendant_for_char_range(TSNode, uint32_t, uint32_t);
+TSNode ts_node_first_child_for_byte(TSNode, uint32_t);
+TSNode ts_node_first_named_child_for_byte(TSNode, uint32_t);
 TSNode ts_node_descendant_for_byte_range(TSNode, uint32_t, uint32_t);
 TSNode ts_node_named_descendant_for_byte_range(TSNode, uint32_t, uint32_t);
 TSNode ts_node_descendant_for_point_range(TSNode, TSPoint, TSPoint);
@@ -111,12 +110,22 @@ void ts_document_print_debugging_graphs(TSDocument *, bool);
 void ts_document_edit(TSDocument *, TSInputEdit);
 void ts_document_parse(TSDocument *);
 void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *);
+
+typedef struct {
+  TSRange **changed_ranges;
+  uint32_t *changed_range_count;
+  bool halt_on_error;
+} TSParseOptions;
+
+void ts_document_parse_with_options(TSDocument *, TSParseOptions);
+
 void ts_document_invalidate(TSDocument *);
 TSNode ts_document_root_node(const TSDocument *);
 uint32_t ts_document_parse_count(const TSDocument *);
 
 uint32_t ts_language_symbol_count(const TSLanguage *);
 const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);
+TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol);
 uint32_t ts_language_version(const TSLanguage *);
 
 #ifdef __cplusplus

data/ext/tree-sitter/tree-sitter/project.gyp

@@ -11,16 +11,13 @@
         'externals/json-parser',
       ],
       'sources': [
-        'src/compiler/build_tables/build_lex_table.cc',
-        'src/compiler/build_tables/build_parse_table.cc',
-        'src/compiler/build_tables/build_tables.cc',
-        'src/compiler/build_tables/recovery_tokens.cc',
         'src/compiler/build_tables/lex_item.cc',
         'src/compiler/build_tables/lex_item_transitions.cc',
-        'src/compiler/build_tables/lex_conflict_manager.cc',
+        'src/compiler/build_tables/lex_table_builder.cc',
         'src/compiler/build_tables/lookahead_set.cc',
         'src/compiler/build_tables/parse_item.cc',
         'src/compiler/build_tables/parse_item_set_builder.cc',
+        'src/compiler/build_tables/parse_table_builder.cc',
         'src/compiler/build_tables/rule_can_be_blank.cc',
         'src/compiler/compile.cc',
         'src/compiler/generate_code/c_code.cc',
@@ -34,41 +31,26 @@
         'src/compiler/prepare_grammar/extract_tokens.cc',
         'src/compiler/prepare_grammar/flatten_grammar.cc',
         'src/compiler/prepare_grammar/intern_symbols.cc',
-        'src/compiler/prepare_grammar/is_token.cc',
         'src/compiler/prepare_grammar/normalize_rules.cc',
         'src/compiler/prepare_grammar/parse_regex.cc',
         'src/compiler/prepare_grammar/prepare_grammar.cc',
         'src/compiler/prepare_grammar/token_description.cc',
         'src/compiler/rule.cc',
         'src/compiler/syntax_grammar.cc',
-        'src/compiler/variable.cc',
-        'src/compiler/rules/blank.cc',
-        'src/compiler/rules/built_in_symbols.cc',
-        'src/compiler/rules/character_range.cc',
         'src/compiler/rules/character_set.cc',
         'src/compiler/rules/choice.cc',
         'src/compiler/rules/metadata.cc',
-        'src/compiler/rules/named_symbol.cc',
-        'src/compiler/rules/pattern.cc',
         'src/compiler/rules/repeat.cc',
-        'src/compiler/rules/rules.cc',
         'src/compiler/rules/seq.cc',
-        'src/compiler/rules/string.cc',
-        'src/compiler/rules/symbol.cc',
-        'src/compiler/rules/visitor.cc',
         'src/compiler/util/string_helpers.cc',
         'externals/utf8proc/utf8proc.c',
         'externals/json-parser/json.c',
       ],
       'cflags_cc': [
-        '-std=c++0x',
-      ],
-      'cflags_cc!': [
-        '-fno-rtti'
+        '-std=c++14',
       ],
       'xcode_settings': {
-        'CLANG_CXX_LANGUAGE_STANDARD': 'c++11',
-        'GCC_ENABLE_CPP_RTTI': 'YES',
+        'CLANG_CXX_LANGUAGE_STANDARD': 'c++14',
         'GCC_ENABLE_CPP_EXCEPTIONS': 'NO',
       },
       'direct_dependent_settings': {
@@ -106,7 +88,7 @@
       ],
       'sources': [
         'src/runtime/document.c',
-        'src/runtime/error_costs.c',
+        'src/runtime/get_changed_ranges.c',
         'src/runtime/language.c',
         'src/runtime/lexer.c',
         'src/runtime/node.c',
@@ -118,7 +100,7 @@
         'externals/utf8proc/utf8proc.c',
       ],
       'cflags_c': [
-        '-std=c99'
+        '-std=c99', '-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=1'
       ],
       'ldflags': [
         '-g',
@@ -149,8 +131,13 @@
         'xcode_settings': {
           'OTHER_LDFLAGS': ['-g'],
           'GCC_OPTIMIZATION_LEVEL': '0',
+          'OTHER_CPLUSPLUSFLAGS': ['-fsanitize=address'],
         },
       },
+      'Fuzz': {
+        'cflags': ['<!@(echo $CFLAGS)'],
+        'ldflags': ['<!@(echo $CFLAGS)'],
+      },
       'Release': {
         'cflags': [ '-O2', '-fno-strict-aliasing' ],
         'cflags!': [ '-O3', '-fstrict-aliasing' ],
@@ -163,6 +150,8 @@
       '-Wno-unused-parameter'
     ],
 
+    'defines': ['UTF8PROC_EXPORTS'],
+
     'xcode_settings': {
       'ALWAYS_SEARCH_USER_PATHS': 'NO',
       'WARNING_CFLAGS': [