@apple/tree-sitter-pkl 0.17.0 → 0.18.0

package/src/scanner.c

@@ -1,3 +1,19 @@
+/*
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #include <tree_sitter/parser.h>
 #include <stdio.h>
 
@@ -34,6 +50,8 @@ enum TokenType {
   OPEN_SUBSCRIPT_BRACKET,
   // '(' at the start of a method call, or a type constraint
   OPEN_ARGUMENT_PAREN,
+  // binary minus ('-') operator
+  BINARY_MINUS
 };
 
 void *tree_sitter_pkl_external_scanner_create() { return NULL; }
@@ -214,7 +232,7 @@ static bool parse_mlx_string_chars(TSLexer *lexer, int num_pounds) {
   }
 }
 
-static bool parse_open_subscript_or_argument(TSLexer *lexer, bool open_subscript_bracket, bool open_argument_paren) {
+static bool parse_symbol_no_preceding_newline_or_semicolon(TSLexer *lexer, bool open_subscript_bracket, bool open_argument_paren, bool binary_minus) {
   if (lexer->eof(lexer)) {
     return false;
   }
@@ -231,12 +249,29 @@ static bool parse_open_subscript_or_argument(TSLexer *lexer, bool open_subscript
           advance(lexer);
           lexer->result_symbol = OPEN_SUBSCRIPT_BRACKET;
           return true;
+        } else {
+          return false;
         }
       case '(':
         if (open_argument_paren) {
           advance(lexer);
           lexer->result_symbol = OPEN_ARGUMENT_PAREN;
           return true;
+        } else {
+          return false;
+        }
+      case '-':
+        if (binary_minus) {
+          advance(lexer);
+          lexer->mark_end(lexer);
+          // avoid parsing `->` as binary minus
+          if (lexer->lookahead == '>') {
+            return false;
+          }
+          lexer->result_symbol = BINARY_MINUS;
+          return true;
+        } else {
+          return false;
         }
       default:
         return false;
@@ -261,8 +296,9 @@ bool tree_sitter_pkl_external_scanner_scan(void *payload, TSLexer *lexer, const
   bool ml6 = valid_symbols[ML6_STRING_CHARS];
   bool osb = valid_symbols[OPEN_SUBSCRIPT_BRACKET];
   bool oap = valid_symbols[OPEN_ARGUMENT_PAREN];
-  
-  if (sl && sl1 && sl2 && sl3 && sl4 && sl5 && sl6 && ml && ml1 && ml2 && ml3 && ml4 && ml5 && ml6 && osb && oap) {
+  bool bminus = valid_symbols[BINARY_MINUS];
+
+  if (sl && sl1 && sl2 && sl3 && sl4 && sl5 && sl6 && ml && ml1 && ml2 && ml3 && ml4 && ml5 && ml6 && osb && oap && bminus) {
     // error recovery mode -> don't match any string chars
     return false;
   }
@@ -309,9 +345,9 @@ bool tree_sitter_pkl_external_scanner_scan(void *payload, TSLexer *lexer, const
   if (ml6) {
     return parse_mlx_string_chars(lexer, 6);
   }
-  // both can possibly be true
-  if (osb || oap) {
-    return parse_open_subscript_or_argument(lexer, osb, oap);
+  // either possibly be true
+  if (osb || oap || bminus) {
+    return parse_symbol_no_preceding_newline_or_semicolon(lexer, osb, oap, bminus);
   }
   return false;
 }

package/src/tree_sitter/alloc.h

@@ -12,10 +12,10 @@ extern "C" {
 // Allow clients to override allocation functions
 #ifdef TREE_SITTER_REUSE_ALLOCATOR
 
-extern void *(*ts_current_malloc)(size_t);
-extern void *(*ts_current_calloc)(size_t, size_t);
-extern void *(*ts_current_realloc)(void *, size_t);
-extern void (*ts_current_free)(void *);
+extern void *(*ts_current_malloc)(size_t size);
+extern void *(*ts_current_calloc)(size_t count, size_t size);
+extern void *(*ts_current_realloc)(void *ptr, size_t size);
+extern void (*ts_current_free)(void *ptr);
 
 #ifndef ts_malloc
 #define ts_malloc  ts_current_malloc

package/src/tree_sitter/array.h

@@ -14,6 +14,7 @@ extern "C" {
 #include <string.h>
 
 #ifdef _MSC_VER
+#pragma warning(push)
 #pragma warning(disable : 4101)
 #elif defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic push
@@ -278,7 +279,7 @@ static inline void _array__splice(Array *self, size_t element_size,
 #define _compare_int(a, b) ((int)*(a) - (int)(b))
 
 #ifdef _MSC_VER
-#pragma warning(default : 4101)
+#pragma warning(pop)
 #elif defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic pop
 #endif

package/src/tree_sitter/parser.h

@@ -18,6 +18,12 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata TSLanguageMetadata;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
 #endif
 
 typedef struct {
@@ -26,10 +32,11 @@ typedef struct {
   bool inherited;
 } TSFieldMapEntry;
 
+// Used to index the field and supertype maps.
 typedef struct {
   uint16_t index;
   uint16_t length;
-} TSFieldMapSlice;
+} TSMapSlice;
 
 typedef struct {
   bool visible;
@@ -47,6 +54,7 @@ struct TSLexer {
   uint32_t (*get_column)(TSLexer *);
   bool (*is_at_included_range_start)(const TSLexer *);
   bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
 };
 
 typedef enum {
@@ -78,6 +86,12 @@ typedef struct {
   uint16_t external_lex_state;
 } TSLexMode;
 
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
 typedef union {
   TSParseAction action;
   struct {
@@ -92,7 +106,7 @@ typedef struct {
 } TSCharacterRange;
 
 struct TSLanguage {
-  uint32_t version;
+  uint32_t abi_version;
   uint32_t symbol_count;
   uint32_t alias_count;
   uint32_t token_count;
@@ -108,13 +122,13 @@ struct TSLanguage {
   const TSParseActionEntry *parse_actions;
   const char * const *symbol_names;
   const char * const *field_names;
-  const TSFieldMapSlice *field_map_slices;
+  const TSMapSlice *field_map_slices;
   const TSFieldMapEntry *field_map_entries;
   const TSSymbolMetadata *symbol_metadata;
   const TSSymbol *public_symbol_map;
   const uint16_t *alias_map;
   const TSSymbol *alias_sequences;
-  const TSLexMode *lex_modes;
+  const TSLexerMode *lex_modes;
   bool (*lex_fn)(TSLexer *, TSStateId);
   bool (*keyword_lex_fn)(TSLexer *, TSStateId);
   TSSymbol keyword_capture_token;
@@ -128,15 +142,23 @@ struct TSLanguage {
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
   const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
 };
 
-static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
   uint32_t index = 0;
   uint32_t size = len - index;
   while (size > 1) {
     uint32_t half_size = size / 2;
     uint32_t mid_index = index + half_size;
-    TSCharacterRange *range = &ranges[mid_index];
+    const TSCharacterRange *range = &ranges[mid_index];
     if (lookahead >= range->start && lookahead <= range->end) {
       return true;
     } else if (lookahead > range->end) {
@@ -144,7 +166,7 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
     }
     size -= half_size;
   }
-  TSCharacterRange *range = &ranges[index];
+  const TSCharacterRange *range = &ranges[index];
   return (lookahead >= range->start && lookahead <= range->end);
 }
 

package/test/corpus/basic/annotation.txt

@@ -76,7 +76,7 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))))
     (annotation
       (qualifiedIdentifier
@@ -84,11 +84,11 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))
         (objectProperty
           (identifier)
-          (intLiteral))))
+          (intLiteralExpr))))
     (moduleClause
       (qualifiedIdentifier
         (identifier)
@@ -104,7 +104,7 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))))
     (annotation
       (qualifiedIdentifier
@@ -112,11 +112,11 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))
         (objectProperty
           (identifier)
-          (intLiteral))))
+          (intLiteralExpr))))
     (identifier)
     (classBody
       (classMethod
@@ -130,7 +130,7 @@ someProperty: String = "hi"
           (objectBody
             (objectProperty
               (identifier)
-              (slStringLiteral
+              (slStringLiteralExpr
                 (slStringLiteralPart)))))
         (annotation
           (qualifiedIdentifier
@@ -138,15 +138,15 @@ someProperty: String = "hi"
           (objectBody
             (objectProperty
               (identifier)
-              (slStringLiteral
+              (slStringLiteralExpr
                 (slStringLiteralPart)))
             (objectProperty
               (identifier)
-              (intLiteral))))
+              (intLiteralExpr))))
         (methodHeader
           (identifier)
           (parameterList))
-        (slStringLiteral
+        (slStringLiteralExpr
           (slStringLiteralPart)))
       (classProperty
         (docComment)
@@ -159,7 +159,7 @@ someProperty: String = "hi"
           (objectBody
             (objectProperty
               (identifier)
-              (slStringLiteral
+              (slStringLiteralExpr
                 (slStringLiteralPart)))))
         (annotation
           (qualifiedIdentifier
@@ -167,18 +167,17 @@ someProperty: String = "hi"
           (objectBody
             (objectProperty
               (identifier)
-              (slStringLiteral
+              (slStringLiteralExpr
                 (slStringLiteralPart)))
             (objectProperty
               (identifier)
-              (intLiteral))))
+              (intLiteralExpr))))
         (identifier)
         (typeAnnotation
-          (type
-            (declaredType
-              (qualifiedIdentifier
-                (identifier)))))
-        (slStringLiteral
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))
+        (slStringLiteralExpr
           (slStringLiteralPart)))))
   (classMethod
     (docComment)
@@ -191,7 +190,7 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))))
     (annotation
       (qualifiedIdentifier
@@ -199,15 +198,15 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))
         (objectProperty
           (identifier)
-          (intLiteral))))
+          (intLiteralExpr))))
     (methodHeader
       (identifier)
       (parameterList))
-    (slStringLiteral
+    (slStringLiteralExpr
       (slStringLiteralPart)))
   (classProperty
     (docComment)
@@ -220,7 +219,7 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))))
     (annotation
       (qualifiedIdentifier
@@ -228,16 +227,15 @@ someProperty: String = "hi"
       (objectBody
         (objectProperty
           (identifier)
-          (slStringLiteral
+          (slStringLiteralExpr
             (slStringLiteralPart)))
         (objectProperty
           (identifier)
-          (intLiteral))))
+          (intLiteralExpr))))
     (identifier)
     (typeAnnotation
-      (type
-        (declaredType
-          (qualifiedIdentifier
-            (identifier)))))
-    (slStringLiteral
+      (declaredType
+        (qualifiedIdentifier
+          (identifier))))
+    (slStringLiteralExpr
       (slStringLiteralPart))))

package/test/corpus/basic/comments.txt

@@ -20,22 +20,22 @@ a = 20 // single-line comment running until EOF rather than newline
 (module
   (classProperty
     (identifier)
-    (intLiteral))
+    (intLiteralExpr))
   (lineComment)
   (classProperty
     (identifier)
-    (intLiteral))
+    (intLiteralExpr))
   (blockComment)
   (classProperty
     (identifier)
-    (intLiteral))
+    (intLiteralExpr))
   (classMethod
     (docComment)
     (methodHeader
       (identifier)
       (parameterList))
-    (intLiteral))
+    (intLiteralExpr))
   (classProperty
     (identifier)
-    (intLiteral))
+    (intLiteralExpr))
   (lineComment))

package/test/corpus/basic/shebangComment.txt

@@ -12,4 +12,4 @@ foo = 1
   (shebangComment)
   (classProperty
     (identifier)
-    (intLiteral)))
+    (intLiteralExpr)))