@ast-grep/lang-python 0.0.4 → 0.0.5

package/src/scanner.c

@@ -85,7 +85,7 @@ static inline void set_end_character(Delimiter *delimiter, int32_t character) {
 typedef struct {
     Array(uint16_t) indents;
     Array(Delimiter) delimiters;
-    bool inside_f_string;
+    bool inside_interpolated_string;
 } Scanner;
 
 static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
@@ -177,7 +177,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
                                 lexer->mark_end(lexer);
                                 array_pop(&scanner->delimiters);
                                 lexer->result_symbol = STRING_END;
-                                scanner->inside_f_string = false;
+                                scanner->inside_interpolated_string = false;
                             }
                             return true;
                         }
@@ -195,7 +195,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
                     advance(lexer);
                     array_pop(&scanner->delimiters);
                     lexer->result_symbol = STRING_END;
-                    scanner->inside_f_string = false;
+                    scanner->inside_interpolated_string = false;
                 }
                 lexer->mark_end(lexer);
                 return true;
@@ -211,7 +211,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
     lexer->mark_end(lexer);
 
     bool found_end_of_line = false;
-    uint32_t indent_length = 0;
+    uint16_t indent_length = 0;
     int32_t first_comment_indent_length = -1;
     for (;;) {
         if (lexer->lookahead == '\n') {
@@ -280,7 +280,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
             if ((valid_symbols[DEDENT] ||
                  (!valid_symbols[NEWLINE] && !(valid_symbols[STRING_START] && next_tok_is_string_start) &&
                   !within_brackets)) &&
-                indent_length < current_indent_length && !scanner->inside_f_string &&
+                indent_length < current_indent_length && !scanner->inside_interpolated_string &&
 
                 // Wait to create a dedent token until we've consumed any
                 // comments
@@ -303,7 +303,8 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
 
         bool has_flags = false;
         while (lexer->lookahead) {
-            if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
+            if (lexer->lookahead == 'f' || lexer->lookahead == 'F' || lexer->lookahead == 't' ||
+                lexer->lookahead == 'T') {
                 set_format(&delimiter);
             } else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
                 set_raw(&delimiter);
@@ -349,7 +350,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, con
         if (end_character(&delimiter)) {
             array_push(&scanner->delimiters, delimiter);
             lexer->result_symbol = STRING_START;
-            scanner->inside_f_string = is_format(&delimiter);
+            scanner->inside_interpolated_string = is_format(&delimiter);
             return true;
         }
         if (has_flags) {
@@ -365,7 +366,7 @@ unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buff
 
     size_t size = 0;
 
-    buffer[size++] = (char)scanner->inside_f_string;
+    buffer[size++] = (char)scanner->inside_interpolated_string;
 
     size_t delimiter_count = scanner->delimiters.size;
     if (delimiter_count > UINT8_MAX) {
@@ -380,7 +381,9 @@ unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buff
 
     uint32_t iter = 1;
     for (; iter < scanner->indents.size && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) {
-        buffer[size++] = (char)*array_get(&scanner->indents, iter);
+        uint16_t indent_value = *array_get(&scanner->indents, iter);
+        buffer[size++] = (char)(indent_value & 0xFF);
+        buffer[size++] = (char)((indent_value >> 8) & 0xFF);
     }
 
     return size;
@@ -396,7 +399,7 @@ void tree_sitter_python_external_scanner_deserialize(void *payload, const char *
     if (length > 0) {
         size_t size = 0;
 
-        scanner->inside_f_string = (bool)buffer[size++];
+        scanner->inside_interpolated_string = (bool)buffer[size++];
 
         size_t delimiter_count = (uint8_t)buffer[size++];
         if (delimiter_count > 0) {
@@ -406,8 +409,9 @@ void tree_sitter_python_external_scanner_deserialize(void *payload, const char *
             size += delimiter_count;
         }
 
-        for (; size < length; size++) {
-            array_push(&scanner->indents, (unsigned char)buffer[size]);
+        for (; size + 1 < length; size += 2) {
+            uint16_t indent_value = (unsigned char)buffer[size] | ((unsigned char)buffer[size + 1] << 8);
+            array_push(&scanner->indents, indent_value);
         }
     }
 }

package/src/tree_sitter/parser.h

@@ -18,6 +18,11 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
 #endif
 
 typedef struct {
@@ -26,10 +31,11 @@ typedef struct {
   bool inherited;
 } TSFieldMapEntry;
 
+// Used to index the field and supertype maps.
 typedef struct {
   uint16_t index;
   uint16_t length;
-} TSFieldMapSlice;
+} TSMapSlice;
 
 typedef struct {
   bool visible;
@@ -79,6 +85,12 @@ typedef struct {
   uint16_t external_lex_state;
 } TSLexMode;
 
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
 typedef union {
   TSParseAction action;
   struct {
@@ -93,7 +105,7 @@ typedef struct {
 } TSCharacterRange;
 
 struct TSLanguage {
-  uint32_t version;
+  uint32_t abi_version;
   uint32_t symbol_count;
   uint32_t alias_count;
   uint32_t token_count;
@@ -109,13 +121,13 @@ struct TSLanguage {
   const TSParseActionEntry *parse_actions;
   const char * const *symbol_names;
   const char * const *field_names;
-  const TSFieldMapSlice *field_map_slices;
+  const TSMapSlice *field_map_slices;
   const TSFieldMapEntry *field_map_entries;
   const TSSymbolMetadata *symbol_metadata;
   const TSSymbol *public_symbol_map;
   const uint16_t *alias_map;
   const TSSymbol *alias_sequences;
-  const TSLexMode *lex_modes;
+  const TSLexerMode *lex_modes;
   bool (*lex_fn)(TSLexer *, TSStateId);
   bool (*keyword_lex_fn)(TSLexer *, TSStateId);
   TSSymbol keyword_capture_token;
@@ -129,15 +141,23 @@ struct TSLanguage {
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
   const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
 };
 
-static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
   uint32_t index = 0;
   uint32_t size = len - index;
   while (size > 1) {
     uint32_t half_size = size / 2;
     uint32_t mid_index = index + half_size;
-    TSCharacterRange *range = &ranges[mid_index];
+    const TSCharacterRange *range = &ranges[mid_index];
     if (lookahead >= range->start && lookahead <= range->end) {
       return true;
     } else if (lookahead > range->end) {
@@ -145,7 +165,7 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
     }
     size -= half_size;
   }
-  TSCharacterRange *range = &ranges[index];
+  const TSCharacterRange *range = &ranges[index];
   return (lookahead >= range->start && lookahead <= range->end);
 }
 

package/type.d.ts

@@ -1347,7 +1347,7 @@ type pythonTypes = {
         ]
       },
       "value": {
-        "multiple": false,
+        "multiple": true,
         "required": false,
         "types": [
           {
@@ -1368,25 +1368,6 @@ type pythonTypes = {
       ]
     }
   },
-  "except_group_clause": {
-    "type": "except_group_clause",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": true,
-      "required": true,
-      "types": [
-        {
-          "type": "block",
-          "named": true
-        },
-        {
-          "type": "expression",
-          "named": true
-        }
-      ]
-    }
-  },
   "exec_statement": {
     "type": "exec_statement",
     "named": true,
@@ -2729,7 +2710,7 @@ type pythonTypes = {
     },
     "children": {
       "multiple": true,
-      "required": true,
+      "required": false,
       "types": [
         {
           "type": "else_clause",
@@ -2739,10 +2720,6 @@ type pythonTypes = {
           "type": "except_clause",
           "named": true
         },
-        {
-          "type": "except_group_clause",
-          "named": true
-        },
         {
           "type": "finally_clause",
           "named": true
@@ -3168,7 +3145,8 @@ type pythonTypes = {
   },
   "comment": {
     "type": "comment",
-    "named": true
+    "named": true,
+    "extra": true
   },
   "ellipsis": {
     "type": "ellipsis",
@@ -3200,7 +3178,8 @@ type pythonTypes = {
   },
   "line_continuation": {
     "type": "line_continuation",
-    "named": true
+    "named": true,
+    "extra": true
   },
   "none": {
     "type": "none",