@apple/tree-sitter-pkl 0.16.0 → 0.18.0

package/src/tree_sitter/parser.h

@@ -13,12 +13,17 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-typedef uint16_t TSStateId;
-
 #ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata TSLanguageMetadata;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
 #endif
 
 typedef struct {
@@ -27,10 +32,11 @@ typedef struct {
   bool inherited;
 } TSFieldMapEntry;
 
+// Used to index the field and supertype maps.
 typedef struct {
   uint16_t index;
   uint16_t length;
-} TSFieldMapSlice;
+} TSMapSlice;
 
 typedef struct {
   bool visible;
@@ -48,6 +54,7 @@ struct TSLexer {
   uint32_t (*get_column)(TSLexer *);
   bool (*is_at_included_range_start)(const TSLexer *);
   bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
 };
 
 typedef enum {
@@ -79,6 +86,12 @@ typedef struct {
   uint16_t external_lex_state;
 } TSLexMode;
 
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
 typedef union {
   TSParseAction action;
   struct {
@@ -87,8 +100,13 @@ typedef union {
   } entry;
 } TSParseActionEntry;
 
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
 struct TSLanguage {
-  uint32_t version;
+  uint32_t abi_version;
   uint32_t symbol_count;
   uint32_t alias_count;
   uint32_t token_count;
@@ -104,13 +122,13 @@ struct TSLanguage {
   const TSParseActionEntry *parse_actions;
   const char * const *symbol_names;
   const char * const *field_names;
-  const TSFieldMapSlice *field_map_slices;
+  const TSMapSlice *field_map_slices;
   const TSFieldMapEntry *field_map_entries;
   const TSSymbolMetadata *symbol_metadata;
   const TSSymbol *public_symbol_map;
   const uint16_t *alias_map;
   const TSSymbol *alias_sequences;
-  const TSLexMode *lex_modes;
+  const TSLexerMode *lex_modes;
   bool (*lex_fn)(TSLexer *, TSStateId);
   bool (*keyword_lex_fn)(TSLexer *, TSStateId);
   TSSymbol keyword_capture_token;
@@ -124,15 +142,48 @@ struct TSLanguage {
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
   const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
 };
 
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    const TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  const TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
 /*
  *  Lexer Macros
  */
 
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
+  UNUSED                        \
   bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
@@ -148,6 +199,17 @@ struct TSLanguage {
     goto next_state;         \
   }
 
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
 #define SKIP(state_value) \
   {                       \
     skip = true;          \
@@ -166,7 +228,7 @@ struct TSLanguage {
  *  Parse Table Macros
  */
 
-#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
 
 #define STATE(id) id
 
@@ -176,7 +238,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value            \
+      .state = (state_value)          \
     }                                 \
   }}
 
@@ -184,7 +246,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value,           \
+      .state = (state_value),         \
       .repetition = true              \
     }                                 \
   }}
@@ -197,14 +259,15 @@ struct TSLanguage {
     }                                 \
   }}
 
-#define REDUCE(symbol_val, child_count_val, ...) \
-  {{                                             \
-    .reduce = {                                  \
-      .type = TSParseActionTypeReduce,           \
-      .symbol = symbol_val,                      \
-      .child_count = child_count_val,            \
-      __VA_ARGS__                                \
-    },                                           \
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
   }}
 
 #define RECOVER()                    \

package/test/corpus/basic/annotation.txt

@@ -0,0 +1,241 @@
+================
+annotation
+================
+
+/// doc comment
+/// doc comment
+@Ann
+@Ann2 { x = "foo" }
+@Ann3 {
+  x = "foo"
+  y = 1
+}
+module some.mod
+
+/// doc comment
+/// doc comment
+@Ann
+@Ann2 { x = "foo" }
+@Ann3 {
+  x = "foo"
+  y = 1
+}
+class SomeClass {
+  /// doc comment
+  /// doc comment
+  @Ann
+  @Ann2 { x = "foo" }
+  @Ann3 {
+    x = "foo"
+    y = 1
+  }
+  function someMethod() = "hi"
+
+  /// doc comment
+  /// doc comment
+  @Ann
+  @Ann2 { x = "foo" }
+  @Ann3 {
+    x = "foo"
+    y = 1
+  }
+  someProperty: String = "hi"
+}
+
+/// doc comment
+/// doc comment
+@Ann
+@Ann2 { x = "foo" }
+@Ann3 {
+  x = "foo"
+  y = 1
+}
+function someMethod() = "hi"
+
+/// doc comment
+/// doc comment
+@Ann
+@Ann2 { x = "foo" }
+@Ann3 {
+  x = "foo"
+  y = 1
+}
+someProperty: String = "hi"
+
+----------------
+
+(module
+  (moduleHeader
+    (docComment)
+    (annotation
+      (qualifiedIdentifier
+        (identifier)))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))
+        (objectProperty
+          (identifier)
+          (intLiteralExpr))))
+    (moduleClause
+      (qualifiedIdentifier
+        (identifier)
+        (identifier))))
+  (clazz
+    (docComment)
+    (annotation
+      (qualifiedIdentifier
+        (identifier)))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))
+        (objectProperty
+          (identifier)
+          (intLiteralExpr))))
+    (identifier)
+    (classBody
+      (classMethod
+        (docComment)
+        (annotation
+          (qualifiedIdentifier
+            (identifier)))
+        (annotation
+          (qualifiedIdentifier
+            (identifier))
+          (objectBody
+            (objectProperty
+              (identifier)
+              (slStringLiteralExpr
+                (slStringLiteralPart)))))
+        (annotation
+          (qualifiedIdentifier
+            (identifier))
+          (objectBody
+            (objectProperty
+              (identifier)
+              (slStringLiteralExpr
+                (slStringLiteralPart)))
+            (objectProperty
+              (identifier)
+              (intLiteralExpr))))
+        (methodHeader
+          (identifier)
+          (parameterList))
+        (slStringLiteralExpr
+          (slStringLiteralPart)))
+      (classProperty
+        (docComment)
+        (annotation
+          (qualifiedIdentifier
+            (identifier)))
+        (annotation
+          (qualifiedIdentifier
+            (identifier))
+          (objectBody
+            (objectProperty
+              (identifier)
+              (slStringLiteralExpr
+                (slStringLiteralPart)))))
+        (annotation
+          (qualifiedIdentifier
+            (identifier))
+          (objectBody
+            (objectProperty
+              (identifier)
+              (slStringLiteralExpr
+                (slStringLiteralPart)))
+            (objectProperty
+              (identifier)
+              (intLiteralExpr))))
+        (identifier)
+        (typeAnnotation
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))
+        (slStringLiteralExpr
+          (slStringLiteralPart)))))
+  (classMethod
+    (docComment)
+    (annotation
+      (qualifiedIdentifier
+        (identifier)))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))
+        (objectProperty
+          (identifier)
+          (intLiteralExpr))))
+    (methodHeader
+      (identifier)
+      (parameterList))
+    (slStringLiteralExpr
+      (slStringLiteralPart)))
+  (classProperty
+    (docComment)
+    (annotation
+      (qualifiedIdentifier
+        (identifier)))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))))
+    (annotation
+      (qualifiedIdentifier
+        (identifier))
+      (objectBody
+        (objectProperty
+          (identifier)
+          (slStringLiteralExpr
+            (slStringLiteralPart)))
+        (objectProperty
+          (identifier)
+          (intLiteralExpr))))
+    (identifier)
+    (typeAnnotation
+      (declaredType
+        (qualifiedIdentifier
+          (identifier))))
+    (slStringLiteralExpr
+      (slStringLiteralPart))))

package/test/corpus/basic/comments.txt

@@ -0,0 +1,41 @@
+========
+comments
+========
+
+x = 10 // end-of-line comment
+
+y = 20 /*
+multi
+line
+comment
+*/ z = 30
+
+/// documentation comment
+function myFun() = 0
+
+a = 20 // single-line comment running until EOF rather than newline
+
+--------
+
+(module
+  (classProperty
+    (identifier)
+    (intLiteralExpr))
+  (lineComment)
+  (classProperty
+    (identifier)
+    (intLiteralExpr))
+  (blockComment)
+  (classProperty
+    (identifier)
+    (intLiteralExpr))
+  (classMethod
+    (docComment)
+    (methodHeader
+      (identifier)
+      (parameterList))
+    (intLiteralExpr))
+  (classProperty
+    (identifier)
+    (intLiteralExpr))
+  (lineComment))

package/test/corpus/basic/shebangComment.txt

@@ -0,0 +1,15 @@
+===
+shebangComment
+===
+
+#!/usr/bin/env pkl
+
+foo = 1
+
+---
+
+(module
+  (shebangComment)
+  (classProperty
+    (identifier)
+    (intLiteralExpr)))

package/test/corpus/basic/types.txt

@@ -0,0 +1,187 @@
+===
+types
+===
+
+res1: String
+
+res2: String?
+
+res3: String|Int
+
+res4: *String|Int
+
+res5: String?
+
+res6: String?(this == null)
+
+res7: String(this == null)?
+
+res8: String(this == null)?(!isEmpty)
+
+res9: module
+
+res10: module?
+
+res11: nothing
+
+res12: nothing?
+
+res13: unknown
+
+res14: unknown?
+
+res15: "string literal type"
+
+res16: (Int) -> Boolean
+
+res17: (module)
+
+res18: (String|Boolean)|(Int) -> Boolean
+---
+
+(module
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (declaredType
+        (qualifiedIdentifier
+          (identifier)))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (declaredType
+          (qualifiedIdentifier
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (unionType
+        (declaredType
+          (qualifiedIdentifier
+            (identifier)))
+        (declaredType
+          (qualifiedIdentifier
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (unionType
+        (defaultUnionType
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))
+        (declaredType
+          (qualifiedIdentifier
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (declaredType
+          (qualifiedIdentifier
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (constrainedType
+        (nullableType
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))
+        (equalityExpr
+          (thisExpr)
+          (nullLiteralExpr)))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (constrainedType
+          (declaredType
+            (qualifiedIdentifier
+              (identifier)))
+          (equalityExpr
+            (thisExpr)
+            (nullLiteralExpr))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (constrainedType
+        (nullableType
+          (constrainedType
+            (declaredType
+              (qualifiedIdentifier
+                (identifier)))
+            (equalityExpr
+              (thisExpr)
+              (nullLiteralExpr))))
+        (logicalNotExpr
+          (unqualifiedAccessExpr
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (moduleType)))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (moduleType))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nothingType)))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (nothingType))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (unknownType)))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (nullableType
+        (unknownType))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (stringLiteralType
+        (stringConstant
+          (slStringLiteralPart)))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (functionLiteralType
+        (declaredType
+          (qualifiedIdentifier
+            (identifier)))
+        (declaredType
+          (qualifiedIdentifier
+            (identifier))))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (parenthesizedType
+        (moduleType))))
+  (classProperty
+    (identifier)
+    (typeAnnotation
+      (unionType
+        (parenthesizedType
+          (unionType
+            (declaredType
+              (qualifiedIdentifier
+                (identifier)))
+            (declaredType
+              (qualifiedIdentifier
+                (identifier)))))
+        (functionLiteralType
+          (declaredType
+            (qualifiedIdentifier
+              (identifier)))
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))))))

package/test/corpus/class/constModifier.txt

@@ -0,0 +1,23 @@
+================================================================================
+constModifier
+================================================================================
+
+class MyClass {
+    const myProp: String = "hello"
+}
+
+--------------------------------------------------------------------------------
+
+(module
+  (clazz
+    (identifier)
+    (classBody
+      (classProperty
+        (modifier)
+        (identifier)
+        (typeAnnotation
+          (declaredType
+            (qualifiedIdentifier
+              (identifier))))
+        (slStringLiteralExpr
+          (slStringLiteralPart))))))