@apple/tree-sitter-pkl 0.16.0 → 0.17.0

package/CHANGELOG.adoc

@@ -0,0 +1,40 @@
+= Changelog
+
+[[release-0.17.0]]
+== 0.17.0 (2024-10-10)
+
+=== Breaking changes
+
+* Give type nodes explicit names (e.g. `"unknown"` -> `unknownTypeNode`)
+* Remove support for legacy syntax in for and when generators
+
+=== Additions
+
+* Add support for Swift Package Manager
+* Add support for C bindings
+* Add doc comments to the parse tree
+* Add `slStringLiteralPart` and `mlStringLiteralPart` to the parse tree
+* Add field names to various nodes
+* Add support for shebang comments
+
+=== Fixes
+
+* Fix parsing of parenthesized expressions in object 
+elements
+* Fix parsing of union types with default markers
+* Add missing `else` in when generators
+* Add missing module type
+* Fix parsing of strings that start with two slashes
+
+=== Contributors ❤️
+
+Thanks to all the contributors for this release!
+
+* https://github.com/ObserverOfTime[@ObserverOfTime]
+* https://github.com/jayadamsmorgan[jayadamsmorgan]
+* https://github.com/thomaspurchas[@thomaspurchas]
+
+[[release-0.16.0]]
+== 0.16.0 (2024-02-02)
+
+Initial release

package/Cargo.lock

@@ -73,7 +73,7 @@ dependencies = [
 
 [[package]]
 name = "tree-sitter-pkl"
-version = "0.16.0"
+version = "0.17.0"
 dependencies = [
  "cc",
  "tree-sitter",

package/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-pkl"
 description = "Pkl grammar for the tree-sitter parsing library"
-version = "0.16.0"
+version = "0.17.0"
 keywords = ["incremental", "parsing", "pkl"]
 categories = ["parsing", "text-editors"]
 repository = "https://github.com/apple/tree-sitter-pkl"

package/Package.swift

@@ -0,0 +1,24 @@
+// swift-tools-version:5.9
+import PackageDescription
+
+let package = Package(
+    name: "TreeSitterPkl",
+    products: [
+        .library(name: "TreeSitterPkl", targets: ["TreeSitterPkl"]),
+    ],
+    dependencies: [],
+    targets: [
+        .target(name: "TreeSitterPkl",
+                path: ".",
+                sources: [
+                    "src/parser.c",
+                    "src/scanner.c",
+                ],
+                resources: [
+                    .copy("queries"),
+                ],
+                publicHeadersPath: "bindings/swift",
+                cSettings: [.headerSearchPath("src")]
+        )
+    ]
+)

package/README.md

@@ -0,0 +1,42 @@
+A tree-sitter grammar for Pkl.
+
+    # install dependencies
+    $ npm install
+
+    # build parser
+    $ npm run build
+
+    # parse some code
+    $ ./node_modules/.bin/tree-sitter parse test.pkl
+
+# Tests
+
+Tree sitter comes with its own test framework. Files in `test/corpus/`
+describe one test each. All tests in `test/corpus/` are performed by the
+command
+
+    $ tree-sitter test
+
+# Upgrading tree-sitter
+
+Upgrading tree-sitter involves upgrading the NPM package.
+
+1.  Run `npm update tree-sitter` to install the newer version of tree-sitter.
+2.  Commit to main, and push.
+
+# Releasing
+
+1.  Run the build & test to make sure everything is up-to-date and passes (check 0 diff).
+2.  Create a `Prepare 1.2.3 release` (with appropriate version number) commit where
+  - Versions are bumped in `package.json`, `Cargo.toml`
+  - Lockfiles are updated (`npm install`, `cargo check`)
+  - You have checked the previous release PR for any changes in process not described in this `README.adoc`; if any
+    - Adopt the changes accordingly
+    - Update this description to capture the changed process
+3. Merge into `main` & push
+4. Check that CI release succeeded ([release pipeline](https://app.circleci.com/pipelines/github/apple/tree-sitter-pkl))
+5. Check the publication is reachable, [on NPM](https://www.npmjs.com/package/@apple/tree-sitter-pkl)
+
+# Resources
+-   [Tree-sitter docs](https://tree-sitter.github.io/tree-sitter/)
+-   [Guide to your first Tree-sitter grammar](https://gist.github.com/Aerijo/df27228d70c633e088b0591b8857eeef)

package/bindings/c/tree-sitter-pkl.h

@@ -0,0 +1,16 @@
+#ifndef TREE_SITTER_PKL_H
+#define TREE_SITTER_PKL_H
+
+typedef struct TSLanguage TSLanguage;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const TSLanguage *tree_sitter_pkl();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PKL_H

package/bindings/swift/TreeSitterPkl/pkl.h

@@ -0,0 +1,16 @@
+#ifndef TREE_SITTER_PKL_H
+#define TREE_SITTER_PKL_H
+
+typedef struct TSLanguage TSLanguage;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern TSLanguage *tree_sitter_pkl();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PKL_H

package/grammar.js

@@ -45,8 +45,8 @@ const PREC = {
 
   NULLABLE_TYPE: 5,
   FUN_TYPE: -5,
-  UNION_TYPE: -6,
-  DEFAULT_TYPE: -7,
+  UNION_DEFAULT_TYPE: -6,
+  UNION_TYPE: -7,
 
   VAR_OBJ_LITERAL: 2,
   OBJ_LITERAL: 1,
@@ -59,6 +59,7 @@ module.exports = grammar({
   name: 'pkl',
 
   externals: $ => [
+    $._sl_string_chars,
     $._sl1_string_chars,
     $._sl2_string_chars,
     $._sl3_string_chars,
@@ -72,46 +73,47 @@ module.exports = grammar({
     $._ml4_string_chars,
     $._ml5_string_chars,
     $._ml6_string_chars,
-    $._open_square_bracket,
-    $._open_entry_bracket,
+    // '[' without preceding newline or semicolon
+    $._open_subscript_bracket,
+    // '(' without preceding newline or semicolon
+    $._open_argument_paren,
   ],
 
   extras: $ => [
     $.lineComment,
-    $.docComment,
     $.blockComment,
+    $.shebangComment,
     /[ \t\f\r\n;]/
   ],
 
   word: $ => $.identifier,
 
   conflicts: $ => [
-    // in ANTLR we deal with these by not allowing a newline or semicolon before subscript
-    [$.objectProperty, $.subscriptExpr],
-    [$.objectMethod, $.subscriptExpr],
-    [$.objectEntry, $.subscriptExpr],
-    [$.objectPredicate, $.subscriptExpr],
-
     // these should be fixable in some other way (perhaps with prec)
-    [$.propertyCallExpr, $.methodCallExpr],
-    [$.variableExpr, $.methodCallExpr],
     [$.variableExpr, $.typedIdentifier],
 
     // not sure what this one is about
     [$.objectElement, $._expr],
 
     [$.qualifiedIdentifier],
-    [$.type]
+    [$.declaredType]
   ],
 
   rules: {
     module: $ => seq(
+      optional($.shebangComment),
       optional($.moduleHeader),
       repeat(choice($.importClause, $.importGlobClause)),
       repeat($._moduleMember)
     ),
 
+    shebangComment: $ => seq(
+      "#!",
+      /.*/
+    ),
+
     moduleHeader: $ => seq(
+      optional($.docComment),
       repeat($.annotation),
       choice(
         seq($.moduleClause, optional($.extendsOrAmendsClause)),
@@ -150,6 +152,7 @@ module.exports = grammar({
     ),
 
     clazz: $ => seq(
+      optional($.docComment),
       repeat($.annotation),
       repeat($.modifier),
       "class",
@@ -172,6 +175,7 @@ module.exports = grammar({
     ),
 
     typeAlias: $ => seq(
+      optional($.docComment),
       repeat($.annotation),
       repeat($.modifier),
       "typealias",
@@ -182,6 +186,7 @@ module.exports = grammar({
     ),
 
     classProperty: $ => seq(
+      optional($.docComment),
       repeat($.annotation),
       repeat($.modifier),
       $.identifier,
@@ -198,6 +203,7 @@ module.exports = grammar({
     ),
 
     classMethod: $ => seq(
+      optional($.docComment),
       repeat($.annotation),
       $.methodHeader,
       optional(seq("=", $._expr))
@@ -252,11 +258,11 @@ module.exports = grammar({
     objectMethod: $ => seq($.methodHeader, "=", $._expr),
 
     objectEntry: $ => seq(
-      $._open_entry_bracket,
-      $._expr,
+      "[",
+      field("key", $._expr),
       "]",
       choice(
-        seq("=", $._expr),
+        seq("=", field("valueExpr", $._expr)),
         repeat1($.objectBody)
       )
     ),
@@ -265,10 +271,10 @@ module.exports = grammar({
 
     objectPredicate: $ => seq(
       "[[",
-      $._expr,
+      field("conditionExpr", $._expr),
       "]]",
       choice(
-        seq("=", $._expr),
+        seq("=", field("valueExpr", $._expr)),
         repeat1($.objectBody)
       )
     ),
@@ -284,20 +290,20 @@ module.exports = grammar({
       "in",
       $._expr,
       ")",
-      choice(
-        $.objectBody,
-        $._objectMember // deprecated in 0.15
-      )
+      $.objectBody
     ),
 
     whenGenerator: $ => seq(
       "when",
       "(",
-      $._expr,
+      field("conditionExpr", $._expr),
       ")",
-      choice(
-        $.objectBody,
-        $._objectMember // deprecated in 0.15
+      field("thenBody", $.objectBody),
+      optional(
+        seq(
+          "else",
+          field("elseBody", $.objectBody)
+        )
       )
     ),
 
@@ -317,18 +323,35 @@ module.exports = grammar({
     typeAnnotation: $ => seq(":", $.type),
 
     type: $ => choice(
-      "unknown",
-      "nothing",
-      $.stringConstant,
-      seq($.qualifiedIdentifier, optional($.typeArgumentList)),
-      seq("(", $.type, ")"),
-      prec(PREC.NULLABLE_TYPE, seq($.type, "?")),
-      seq($.type, "(", commaSep1($._expr), ")"),
-      prec.left(PREC.UNION_TYPE, seq($.type, "|", $.type)),
-      prec(PREC.DEFAULT_TYPE, seq("*", $.type)),
-      prec(PREC.FUN_TYPE, seq("(", commaSep($.type), ")", "->", $.type))
+      alias("unknown", $.unknownType),
+      alias("nothing", $.nothingType),
+      alias("module", $.moduleType),
+      $.stringLiteralType,
+      $.declaredType,
+      $.parenthesizedType,
+      $.nullableType,
+      $.constrainedType,
+      $.unionType,
+      $.unionDefaultType,
+      $.functionLiteralType
     ),
 
+    stringLiteralType: $ => $.stringConstant,
+
+    declaredType: $ => seq($.qualifiedIdentifier, optional($.typeArgumentList)),
+
+    parenthesizedType: $ => seq("(", $.type, ")"),
+
+    nullableType: $ => prec(PREC.NULLABLE_TYPE, seq($.type, "?")),
+
+    constrainedType: $ => seq($.type, "(", commaSep1($._expr), ")"),
+
+    unionType: $ => prec.left(PREC.UNION_TYPE, seq($.type, "|", $.type)),
+
+    unionDefaultType: $ => prec(PREC.UNION_DEFAULT_TYPE, seq("*", $.type)),
+
+    functionLiteralType: $ => prec(PREC.FUN_TYPE, seq("(", commaSep($.type), ")", "->", $.type)),
+
     typeArgumentList: $ => seq(
       "<",
       commaSep1($.type),
@@ -353,7 +376,7 @@ module.exports = grammar({
     ),
 
     argumentList: $ => seq(
-      '(',
+      alias($._open_argument_paren, '('),
       commaSep($._expr),
       ')'
     ),
@@ -450,7 +473,7 @@ module.exports = grammar({
       seq(
         '"',
         repeat(choice(
-          token.immediate(/[^"\\\n\r]+/),
+          $.slStringLiteralPart,
           $.escapeSequence
         )),
         '"'
@@ -458,7 +481,7 @@ module.exports = grammar({
       seq(
         '#"',
         repeat(choice(
-          $._sl1_string_chars,
+          alias($.slStringLiteralPart1, $.slStringLiteralPart),
           alias($.escapeSequence1, $.escapeSequence),
         )),
         '"#'
@@ -469,7 +492,7 @@ module.exports = grammar({
       seq(
         '"',
         repeat(choice(
-          token.immediate(/[^"\\\n\r]+/),
+          $.slStringLiteralPart,
           $.escapeSequence,
           $.interpolationExpr,
         )),
@@ -478,7 +501,7 @@ module.exports = grammar({
       seq(
         '#"',
         repeat(choice(
-          $._sl1_string_chars,
+          alias($.slStringLiteralPart1, $.slStringLiteralPart),
           alias($.escapeSequence1, $.escapeSequence),
           alias($.interpolationExpr1, $.interpolationExpr)
         )),
@@ -487,7 +510,7 @@ module.exports = grammar({
       seq(
         '##"',
         repeat(choice(
-          $._sl2_string_chars,
+          alias($.slStringLiteralPart2, $.slStringLiteralPart),
           alias($.escapeSequence2, $.escapeSequence),
           alias($.interpolationExpr2, $.interpolationExpr)
         )),
@@ -496,7 +519,7 @@ module.exports = grammar({
       seq(
         '###"',
         repeat(choice(
-          $._sl3_string_chars,
+          alias($.slStringLiteralPart3, $.slStringLiteralPart),
           alias($.escapeSequence3, $.escapeSequence),
           alias($.interpolationExpr3, $.interpolationExpr)
         )),
@@ -505,7 +528,7 @@ module.exports = grammar({
       seq(
         '####"',
         repeat(choice(
-          $._sl4_string_chars,
+          alias($.slStringLiteralPart4, $.slStringLiteralPart),
           alias($.escapeSequence4, $.escapeSequence),
           alias($.interpolationExpr4, $.interpolationExpr)
         )),
@@ -514,7 +537,7 @@ module.exports = grammar({
       seq(
         '#####"',
         repeat(choice(
-          $._sl5_string_chars,
+          alias($.slStringLiteralPart5, $.slStringLiteralPart),
           alias($.escapeSequence5, $.escapeSequence),
           alias($.interpolationExpr5, $.interpolationExpr)
         )),
@@ -523,7 +546,7 @@ module.exports = grammar({
       seq(
         '######"',
         repeat(choice(
-          $._sl6_string_chars,
+          alias($.slStringLiteralPart6, $.slStringLiteralPart),
           alias($.escapeSequence6, $.escapeSequence),
           alias($.interpolationExpr6, $.interpolationExpr)
         )),
@@ -531,11 +554,25 @@ module.exports = grammar({
       ),
     ),
 
+    slStringLiteralPart: $ => $._sl_string_chars,
+
+    slStringLiteralPart1: $ => $._sl1_string_chars,
+
+    slStringLiteralPart2: $ => $._sl2_string_chars,
+
+    slStringLiteralPart3: $ => $._sl3_string_chars,
+
+    slStringLiteralPart4: $ => $._sl4_string_chars,
+
+    slStringLiteralPart5: $ => $._sl5_string_chars,
+
+    slStringLiteralPart6: $ => $._sl6_string_chars,
+
     mlStringLiteral: $ => choice(
       seq(
         '"""',
         repeat(choice(
-          $._ml_string_chars,
+          $.mlStringLiteralPart,
           $.escapeSequence,
           $.interpolationExpr
         )),
@@ -544,7 +581,7 @@ module.exports = grammar({
       seq(
         '#"""',
         repeat(choice(
-          $._ml1_string_chars,
+          alias($.mlStringLiteralPart1, $.mlStringLiteralPart),
           alias($.escapeSequence1, $.escapeSequence),
           alias($.interpolationExpr1, $.interpolationExpr)
         )),
@@ -553,7 +590,7 @@ module.exports = grammar({
       seq(
         '##"""',
         repeat(choice(
-          $._ml2_string_chars,
+          alias($.mlStringLiteralPart2, $.mlStringLiteralPart),
           alias($.escapeSequence2, $.escapeSequence),
           alias($.interpolationExpr2, $.interpolationExpr)
         )),
@@ -562,7 +599,7 @@ module.exports = grammar({
       seq(
         '###"""',
         repeat(choice(
-          $._ml3_string_chars,
+          alias($.mlStringLiteralPart3, $.mlStringLiteralPart),
           alias($.escapeSequence3, $.escapeSequence),
           alias($.interpolationExpr3, $.interpolationExpr)
         )),
@@ -571,7 +608,7 @@ module.exports = grammar({
       seq(
         '####"""',
         repeat(choice(
-          $._ml4_string_chars,
+          alias($.mlStringLiteralPart4, $.mlStringLiteralPart),
           alias($.escapeSequence4, $.escapeSequence),
           alias($.interpolationExpr4, $.interpolationExpr)
         )),
@@ -580,7 +617,7 @@ module.exports = grammar({
       seq(
         '#####"""',
         repeat(choice(
-          $._ml5_string_chars,
+          alias($.mlStringLiteralPart5, $.mlStringLiteralPart),
           alias($.escapeSequence5, $.escapeSequence),
           alias($.interpolationExpr5, $.interpolationExpr)
         )),
@@ -589,7 +626,7 @@ module.exports = grammar({
       seq(
         '######"""',
         repeat(choice(
-          $._ml6_string_chars,
+          alias($.mlStringLiteralPart6, $.mlStringLiteralPart),
           alias($.escapeSequence6, $.escapeSequence),
           alias($.interpolationExpr6, $.interpolationExpr)
         )),
@@ -597,6 +634,20 @@ module.exports = grammar({
       ),
     ),
 
+    mlStringLiteralPart: $ => $._ml_string_chars,
+
+    mlStringLiteralPart1: $ => $._ml1_string_chars,
+
+    mlStringLiteralPart2: $ => $._ml2_string_chars,
+
+    mlStringLiteralPart3: $ => $._ml3_string_chars,
+
+    mlStringLiteralPart4: $ => $._ml4_string_chars,
+
+    mlStringLiteralPart5: $ => $._ml5_string_chars,
+
+    mlStringLiteralPart6: $ => $._ml6_string_chars,
+
     escapeSequence: $ => token.immediate(seq(
       '\\',
       choice(
@@ -671,11 +722,38 @@ module.exports = grammar({
 
     objectLiteral: $ => prec(PREC.OBJ_LITERAL, seq($._expr2, $.objectBody)),
 
-    methodCallExpr: $ => seq(optional(seq(choice("super", $._expr), choice(".", "?."))), $.identifier, $.argumentList),
-
-    propertyCallExpr: $ => seq(choice("super", $._expr), choice(".", "?."), $.identifier),
-
-    subscriptExpr: $ => seq(choice("super", $._expr), $._open_square_bracket, $._expr, "]"),
+    methodCallExpr: $ => seq(
+        optional(
+            seq(
+                field("receiver", choice("super", $._expr)),
+                choice(".", "?.")
+            )
+        ),
+        $.identifier,
+        $.argumentList
+    ),
+
+    propertyCallExpr: $ => seq(
+        field("receiver", choice("super", $._expr)),
+        choice(".", "?."),
+        // allow newline as a possible token because tree-sitter otherwise doesn't handle error recovery correctly.
+        // E.g.
+        // This gets parsed as a single class property, where `bar.baz { foo = bar }` is the property's value
+        // ```
+        // foo = bar.
+        // baz { foo = bar }
+        // ```
+        //
+        // This gets parsed as two class properties; `foo = bar.typealias` and `Baz = String`
+        // ```
+        // foo = bar.
+        //
+        // typealias Baz = String
+        // ```
+        choice($.identifier, field("err", "\n"))
+    ),
+
+    subscriptExpr: $ => seq(field("receiver", choice("super", $._expr)), alias($._open_subscript_bracket, "["), $._expr, "]"),
 
     unaryExpr: $ => choice(
       prec.left(PREC.NEG, seq($._expr, '!!')),
@@ -687,7 +765,7 @@ module.exports = grammar({
       ['**', PREC.EXP],
       ['??', PREC.COALESCE]
     ].map(([operator, precedence]) =>
-      prec.right(precedence, seq($._expr, operator, $._expr))
+      prec.right(precedence, seq($._expr, field('operator', operator), $._expr))
     )),
 
     binaryExpr: $ => choice(...[
@@ -707,7 +785,7 @@ module.exports = grammar({
       ['||', PREC.OR],
       ['|>', PREC.PIPE]
     ].map(([operator, precedence]) =>
-      prec.left(precedence, seq($._expr, operator, $._expr))
+      prec.left(precedence, seq($._expr, field('operator', operator), $._expr))
     )),
 
     isExpr: $ => prec(PREC.IS, seq($._expr, "is", $.type)),
@@ -749,10 +827,16 @@ module.exports = grammar({
       return token(choice(seq(alpha, repeat(alpha_numeric)), /`[^`]*`/))
     },
 
-    lineComment: $ => token(seq('//', /.*/)),
+    lineComment: $ => choice(
+      token(seq(/\/\/[^\/]/, /.*/)),
+      '//$',
+    ),
 
     // TODO
-    docComment: $ => token(seq('///', /.*/)),
+    docComment: $ => seq(
+      token(seq('///', /.*/)),
+      repeat(token(seq('///', /.*/)))
+    ),
 
     blockComment: $ => token(seq(
       '/*',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apple/tree-sitter-pkl",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Pkl grammar for node-tree-sitter",
   "repository": {
     "type": "git",
@@ -27,14 +27,13 @@
     "@types/glob": "^8.1.0",
     "@types/node": "^20.4.5",
     "node-gyp": "^9.1.0",
-    "tree-sitter-cli": "^0.20.0",
-    "ts-node": "^10.0.0",
+    "tree-sitter-cli": "^0.22.6",
     "typescript": "^4.9.0"
   },
   "scripts": {
     "build": "tree-sitter generate && node-gyp configure && node-gyp build",
+    "generate": "tree-sitter generate --no-bindings",
     "test": "tree-sitter test",
-    "synctests": "ts-node src/synctests.ts",
     "getVersion": "node -p \"require('./package.json').version\""
   },
   "tree-sitter": [