@eagleoutice/flowr 2.8.3 → 2.8.4

package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.js

@@ -6,6 +6,7 @@ exports.makeTreeSitterStrict = makeTreeSitterStrict;
 const normalizer_data_1 = require("../ast/parser/main/normalizer-data");
 const tree_sitter_types_1 = require("./tree-sitter-types");
 const type_1 = require("../ast/model/type");
+const range_1 = require("../../../util/range");
 const retriever_1 = require("../../retriever");
 const convert_values_1 = require("../convert-values");
 const normalize_meta_1 = require("../ast/parser/main/normalize-meta");
@@ -13,6 +14,7 @@ const arrays_1 = require("../../../util/collections/arrays");
 const r_function_call_1 = require("../ast/model/nodes/r-function-call");
 const strings_1 = require("../../../util/text/strings");
 const log_1 = require("../../../util/log");
+const parser_1 = require("../ast/parser/json/parser");
 /**
  * @param tree - The tree to normalize
  * @param lax - Whether to use lax parsing (i.e., ignore errors) or strict parsing (i.e., fail on errors)
@@ -63,427 +65,453 @@ function makeTreeSitterStrict() {
     nonErrorChildren = nonErrorChildrenStrict;
 }
 function convertTreeNode(node) {
-    // generally, the grammar source file dictates what children a node has in what order:
-    // https://github.com/r-lib/tree-sitter-r/blob/main/grammar.js
-    const range = makeSourceRange(node);
-    const defaultInfo = {
-        info: {
-            fullRange: range,
-            additionalTokens: [],
-            fullLexeme: node.text,
-            treeSitterId: node.id
-        }
-    };
-    switch (node.type) {
-        case tree_sitter_types_1.TreeSitterType.Program: {
-            const [comments, children] = splitComments(nonErrorChildren(node));
-            const body = children.map(n => [n, convertTreeNode(n)]);
-            const remainingComments = linkCommentsToNextNodes(body, comments);
-            return {
-                type: type_1.RType.ExpressionList,
-                children: body.map(n => n[1]),
-                grouping: undefined,
-                lexeme: undefined,
-                info: {
-                    additionalTokens: remainingComments.map(c => c[1]),
-                    treeSitterId: node.id
-                }
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.BracedExpression:
-        case tree_sitter_types_1.TreeSitterType.ParenthesizedExpression: {
-            const [comments, children] = splitComments(nonErrorChildren(node));
-            const opening = children[0];
-            const body = children.slice(1, -1).map(n => [n, convertTreeNode(n)]);
-            const remainingComments = linkCommentsToNextNodes(body, comments);
-            const closing = children[children.length - 1];
-            return {
-                type: type_1.RType.ExpressionList,
-                location: undefined,
-                lexeme: undefined,
-                children: body.map(n => n[1]),
-                grouping: [
-                    {
-                        type: type_1.RType.Symbol,
-                        location: makeSourceRange(opening),
-                        content: (0, retriever_1.removeRQuotes)(opening.text),
-                        lexeme: opening.text,
-                        namespace: undefined,
-                        ...defaultInfo
-                    }, {
-                        type: type_1.RType.Symbol,
-                        location: makeSourceRange(closing),
-                        content: (0, retriever_1.removeRQuotes)(closing.text),
-                        lexeme: closing.text,
-                        namespace: undefined,
-                        ...defaultInfo
-                    }
-                ],
-                info: {
-                    additionalTokens: remainingComments.map(c => c[1]),
-                    treeSitterId: node.id
-                }
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.BinaryOperator: {
-            const children = nonErrorChildren(node);
-            const lhs = convertTreeNode(children[0]);
-            const rhs = convertTreeNode(children[children.length - 1]);
-            const [commentsBoth, [op]] = splitComments(children.slice(1, -1));
-            const comments = commentsBoth.map(c => c[1]);
-            const opSource = makeSourceRange(op);
-            const lhsAsArg = {
-                type: type_1.RType.Argument,
-                location: lhs.location,
-                value: lhs,
-                name: undefined,
-                lexeme: lhs.lexeme,
-                info: {
-                    treeSitterId: lhs.info.treeSitterId
-                }
-            };
-            if (op.type == 'special') {
+    if (!node) {
+        return {
+            type: type_1.RType.ExpressionList,
+            location: undefined,
+            lexeme: undefined,
+            children: [],
+            grouping: undefined,
+            info: {
+                fullRange: (0, range_1.invalidRange)(),
+                additionalTokens: [],
+                treeSitterId: -1,
+            }
+        };
+    }
+    try {
+        // generally, the grammar source file dictates what children a node has in what order:
+        // https://github.com/r-lib/tree-sitter-r/blob/main/grammar.js
+        const range = makeSourceRange(node);
+        const defaultInfo = {
+            info: {
+                fullRange: range,
+                additionalTokens: [],
+                fullLexeme: node.text,
+                treeSitterId: node.id
+            }
+        };
+        switch (node.type) {
+            case tree_sitter_types_1.TreeSitterType.Program: {
+                const [comments, children] = splitComments(nonErrorChildren(node));
+                const body = children.map(n => [n, convertTreeNode(n)]);
+                const remainingComments = linkCommentsToNextNodes(body, comments);
                 return {
-                    type: type_1.RType.FunctionCall,
-                    location: opSource,
-                    lexeme: node.text,
-                    functionName: {
-                        type: type_1.RType.Symbol,
-                        location: opSource,
-                        lexeme: op.text,
-                        content: op.text,
-                        namespace: undefined,
-                        info: {
-                            treeSitterId: op.id
-                        }
-                    },
-                    arguments: [lhsAsArg, {
-                            type: type_1.RType.Argument,
-                            location: rhs.location,
-                            value: rhs,
-                            name: undefined,
-                            lexeme: rhs.lexeme,
-                            info: {
-                                treeSitterId: rhs.info.treeSitterId
-                            }
-                        }],
-                    named: true,
-                    infixSpecial: true,
+                    type: type_1.RType.ExpressionList,
+                    children: body.map(n => n[1]),
+                    grouping: undefined,
+                    lexeme: undefined,
                     info: {
-                        additionalTokens: comments,
+                        additionalTokens: remainingComments.map(c => c[1]),
                         treeSitterId: node.id
                     }
                 };
             }
-            else if (op.text === '|>') {
+            case tree_sitter_types_1.TreeSitterType.BracedExpression:
+            case tree_sitter_types_1.TreeSitterType.ParenthesizedExpression: {
+                const [comments, children] = splitComments(nonErrorChildren(node));
+                const opening = children[0];
+                const body = children.slice(1, -1).map(n => [n, convertTreeNode(n)]);
+                const remainingComments = linkCommentsToNextNodes(body, comments);
+                const closing = children[children.length - 1];
                 return {
-                    type: type_1.RType.Pipe,
-                    location: opSource,
-                    lhs: lhsAsArg,
-                    rhs,
-                    lexeme: op.text,
-                    ...defaultInfo,
+                    type: type_1.RType.ExpressionList,
+                    location: undefined,
+                    lexeme: undefined,
+                    children: body.map(n => n[1]),
+                    grouping: [
+                        {
+                            type: type_1.RType.Symbol,
+                            location: makeSourceRange(opening),
+                            content: (0, retriever_1.removeRQuotes)(opening.text),
+                            lexeme: opening.text,
+                            namespace: undefined,
+                            ...defaultInfo
+                        }, {
+                            type: type_1.RType.Symbol,
+                            location: makeSourceRange(closing),
+                            content: (0, retriever_1.removeRQuotes)(closing.text),
+                            lexeme: closing.text,
+                            namespace: undefined,
+                            ...defaultInfo
+                        }
+                    ],
                     info: {
-                        fullRange: range,
-                        additionalTokens: comments,
-                        fullLexeme: node.text,
+                        additionalTokens: remainingComments.map(c => c[1]),
                         treeSitterId: node.id
                     }
                 };
             }
-            else {
+            case tree_sitter_types_1.TreeSitterType.BinaryOperator: {
+                const children = nonErrorChildren(node);
+                const lhs = convertTreeNode(children[0]);
+                const rhs = convertTreeNode(children[children.length - 1]);
+                const [commentsBoth, [op]] = splitComments(children.slice(1, -1));
+                const comments = commentsBoth.map(c => c[1]);
+                const opSource = makeSourceRange(op);
+                const lhsAsArg = {
+                    type: type_1.RType.Argument,
+                    location: lhs.location,
+                    value: lhs,
+                    name: undefined,
+                    lexeme: lhs.lexeme,
+                    info: {
+                        treeSitterId: lhs.info.treeSitterId
+                    }
+                };
+                if (op.type == 'special') {
+                    return {
+                        type: type_1.RType.FunctionCall,
+                        location: opSource,
+                        lexeme: node.text,
+                        functionName: {
+                            type: type_1.RType.Symbol,
+                            location: opSource,
+                            lexeme: op.text,
+                            content: op.text,
+                            namespace: undefined,
+                            info: {
+                                treeSitterId: op.id
+                            }
+                        },
+                        arguments: [lhsAsArg, {
+                                type: type_1.RType.Argument,
+                                location: rhs.location,
+                                value: rhs,
+                                name: undefined,
+                                lexeme: rhs.lexeme,
+                                info: {
+                                    treeSitterId: rhs.info.treeSitterId
+                                }
+                            }],
+                        named: true,
+                        infixSpecial: true,
+                        info: {
+                            additionalTokens: comments,
+                            treeSitterId: node.id
+                        }
+                    };
+                }
+                else if (op.text === '|>') {
+                    return {
+                        type: type_1.RType.Pipe,
+                        location: opSource,
+                        lhs: lhsAsArg,
+                        rhs,
+                        lexeme: op.text,
+                        ...defaultInfo,
+                        info: {
+                            fullRange: range,
+                            additionalTokens: comments,
+                            fullLexeme: node.text,
+                            treeSitterId: node.id
+                        }
+                    };
+                }
+                else {
+                    return {
+                        type: type_1.RType.BinaryOp,
+                        location: opSource,
+                        lhs, rhs,
+                        operator: op.text,
+                        lexeme: op.text,
+                        info: {
+                            fullRange: range,
+                            additionalTokens: comments,
+                            fullLexeme: node.text,
+                            treeSitterId: node.id
+                        }
+                    };
+                }
+            }
+            case tree_sitter_types_1.TreeSitterType.UnaryOperator: {
+                const [op, operand] = nonErrorChildren(node);
                 return {
-                    type: type_1.RType.BinaryOp,
-                    location: opSource,
-                    lhs, rhs,
+                    type: type_1.RType.UnaryOp,
+                    operand: convertTreeNode(operand),
+                    location: makeSourceRange(op),
                     operator: op.text,
                     lexeme: op.text,
-                    info: {
-                        fullRange: range,
-                        additionalTokens: comments,
-                        fullLexeme: node.text,
-                        treeSitterId: node.id
-                    }
+                    ...defaultInfo
                 };
             }
-        }
-        case tree_sitter_types_1.TreeSitterType.UnaryOperator: {
-            const [op, operand] = nonErrorChildren(node);
-            return {
-                type: type_1.RType.UnaryOp,
-                operand: convertTreeNode(operand),
-                location: makeSourceRange(op),
-                operator: op.text,
-                lexeme: op.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.NamespaceOperator: {
-            const [lhs, /* :: or ::: */ , rhs] = nonErrorChildren(node);
-            return {
-                type: type_1.RType.Symbol,
-                location: makeSourceRange(rhs),
-                content: rhs.text,
-                lexeme: rhs.text,
-                namespace: lhs.text,
-                ...defaultInfo
-            };
-        }
-        case '(':
-        case ')':
-        case tree_sitter_types_1.TreeSitterType.Na:
-        case tree_sitter_types_1.TreeSitterType.Null:
-        case tree_sitter_types_1.TreeSitterType.Dots:
-        case tree_sitter_types_1.TreeSitterType.DotDotI:
-        case tree_sitter_types_1.TreeSitterType.Identifier:
-        case tree_sitter_types_1.TreeSitterType.Return:
-            return {
-                type: type_1.RType.Symbol,
-                location: range,
-                content: (0, strings_1.startAndEndsWith)(node.text, '`') ? node.text.slice(1, -1) : (0, retriever_1.removeRQuotes)(node.text),
-                lexeme: node.text,
-                namespace: undefined,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.IfStatement: {
-            const [ifNode, /* ( */ , condition, /* ) */ , then, /* else */ , ...otherwise] = nonErrorChildren(node);
-            const filteredOtherwise = otherwise.filter(n => n.type !== tree_sitter_types_1.TreeSitterType.ElseStatement);
-            return {
-                type: type_1.RType.IfThenElse,
-                condition: convertTreeNode(condition),
-                then: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(then)),
-                otherwise: filteredOtherwise.length > 0 ? (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(filteredOtherwise[0])) : undefined,
-                location: makeSourceRange(ifNode),
-                lexeme: ifNode.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.ForStatement: {
-            const children = nonErrorChildren(node);
-            const forNode = children[0]; // we follow with a (
-            const variable = getNodesUntil(children, 'in', 2); // we follow with the "in"
-            const sequence = getNodesUntil(children, ')', 2 + variable.length + 1); // we follow with a (
-            const body = children[2 + variable.length + 1 + sequence.length + 1];
-            const [variableComments, [variableNode]] = splitComments(variable);
-            const [sequenceComments, [sequenceNode]] = splitComments(sequence);
-            return {
-                type: type_1.RType.ForLoop,
-                variable: {
+            case tree_sitter_types_1.TreeSitterType.NamespaceOperator: {
+                const [lhs, /* :: or ::: */ , rhs] = nonErrorChildren(node);
+                return {
                     type: type_1.RType.Symbol,
-                    location: makeSourceRange(variableNode),
-                    content: (0, retriever_1.removeRQuotes)(variableNode.text),
-                    lexeme: variableNode.text,
+                    location: makeSourceRange(rhs),
+                    content: rhs.text,
+                    lexeme: rhs.text,
+                    namespace: lhs.text,
+                    ...defaultInfo
+                };
+            }
+            case '(':
+            case ')':
+            case tree_sitter_types_1.TreeSitterType.Na:
+            case tree_sitter_types_1.TreeSitterType.Null:
+            case tree_sitter_types_1.TreeSitterType.Dots:
+            case tree_sitter_types_1.TreeSitterType.DotDotI:
+            case tree_sitter_types_1.TreeSitterType.Identifier:
+            case tree_sitter_types_1.TreeSitterType.Return:
+                return {
+                    type: type_1.RType.Symbol,
+                    location: range,
+                    content: (0, strings_1.startAndEndsWith)(node.text, '`') ? node.text.slice(1, -1) : (0, retriever_1.removeRQuotes)(node.text),
+                    lexeme: node.text,
                     namespace: undefined,
-                    info: {
-                        fullRange: undefined,
-                        additionalTokens: [],
-                        fullLexeme: undefined,
-                        treeSitterId: variableNode.id
-                    }
-                },
-                vector: convertTreeNode(sequenceNode),
-                body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
-                location: makeSourceRange(forNode),
-                lexeme: forNode.text,
-                info: {
-                    fullRange: range,
-                    additionalTokens: variableComments.concat(sequenceComments).map(c => c[1]),
-                    fullLexeme: node.text,
-                    treeSitterId: node.id
-                }
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.WhileStatement: {
-            const [whileNode, /* ( */ , condition, /* ) */ , body] = nonErrorChildren(node);
-            return {
-                type: type_1.RType.WhileLoop,
-                condition: convertTreeNode(condition),
-                body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
-                location: makeSourceRange(whileNode),
-                lexeme: whileNode.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.RepeatStatement: {
-            const [repeatNode, body] = nonErrorChildren(node);
-            return {
-                type: type_1.RType.RepeatLoop,
-                body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
-                location: makeSourceRange(repeatNode),
-                lexeme: repeatNode.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.Call: {
-            const [func, argsParentheses] = nonErrorChildren(node);
-            // tree-sitter wraps next and break in a function call, but we don't, so unwrap
-            if (func.type === tree_sitter_types_1.TreeSitterType.Next || func.type == tree_sitter_types_1.TreeSitterType.Break) {
+                    ...defaultInfo
+                };
+            case tree_sitter_types_1.TreeSitterType.IfStatement: {
+                const [ifNode, /* ( */ , condition, /* ) */ , then, /* else */ , ...otherwise] = nonErrorChildren(node);
+                const filteredOtherwise = otherwise.filter(n => n.type !== tree_sitter_types_1.TreeSitterType.ElseStatement);
                 return {
-                    ...convertTreeNode(func),
+                    type: type_1.RType.IfThenElse,
+                    condition: convertTreeNode(condition),
+                    then: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(then)),
+                    otherwise: filteredOtherwise.length > 0 ? (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(filteredOtherwise[0])) : undefined,
+                    location: makeSourceRange(ifNode),
+                    lexeme: ifNode.text,
                     ...defaultInfo
                 };
             }
-            const rawArgs = nonErrorChildren(argsParentheses);
-            const [comments, noCommentrawArgs] = splitComments(rawArgs);
-            const args = (0, arrays_1.splitArrayOn)(noCommentrawArgs.slice(1, -1), x => x.type === 'comma');
-            const funcRange = makeSourceRange(func);
-            const mappedArgs = args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0]));
-            const call = {
-                arguments: mappedArgs,
-                location: funcRange,
-                lexeme: func.text,
-                ...defaultInfo,
-                info: {
-                    ...defaultInfo.info,
-                    additionalTokens: comments.map(c => c[1]),
-                }
-            };
-            if (func.type === tree_sitter_types_1.TreeSitterType.Identifier || func.type === tree_sitter_types_1.TreeSitterType.String || func.type === tree_sitter_types_1.TreeSitterType.NamespaceOperator || func.type === tree_sitter_types_1.TreeSitterType.Return) {
-                let funcNode = convertTreeNode(func);
-                if (funcNode.type === type_1.RType.String) {
-                    funcNode = {
-                        ...funcNode,
+            case tree_sitter_types_1.TreeSitterType.ForStatement: {
+                const children = nonErrorChildren(node);
+                const forNode = children[0]; // we follow with a (
+                const variable = getNodesUntil(children, 'in', 2); // we follow with the "in"
+                const sequence = getNodesUntil(children, ')', 2 + variable.length + 1); // we follow with a (
+                const body = children[2 + variable.length + 1 + sequence.length + 1];
+                const [variableComments, [variableNode]] = splitComments(variable);
+                const [sequenceComments, [sequenceNode]] = splitComments(sequence);
+                return {
+                    type: type_1.RType.ForLoop,
+                    variable: {
                         type: type_1.RType.Symbol,
+                        location: makeSourceRange(variableNode),
+                        content: (0, retriever_1.removeRQuotes)(variableNode.text),
+                        lexeme: variableNode.text,
                         namespace: undefined,
-                        content: (0, retriever_1.removeRQuotes)(func.text)
-                    };
-                }
-                return {
-                    ...call,
-                    type: type_1.RType.FunctionCall,
-                    functionName: {
-                        ...funcNode,
                         info: {
-                            fullRange: range,
+                            fullRange: undefined,
                             additionalTokens: [],
-                            fullLexeme: node.text,
-                            treeSitterId: node.id
+                            fullLexeme: undefined,
+                            treeSitterId: variableNode.id
                         }
                     },
-                    named: true
+                    vector: convertTreeNode(sequenceNode),
+                    body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
+                    location: makeSourceRange(forNode),
+                    lexeme: forNode.text,
+                    info: {
+                        fullRange: range,
+                        additionalTokens: variableComments.concat(sequenceComments).map(c => c[1]),
+                        fullLexeme: node.text,
+                        treeSitterId: node.id
+                    }
                 };
             }
-            else {
+            case tree_sitter_types_1.TreeSitterType.WhileStatement: {
+                const [whileNode, /* ( */ , condition, /* ) */ , body] = nonErrorChildren(node);
                 return {
-                    ...call,
-                    type: type_1.RType.FunctionCall,
-                    calledFunction: convertTreeNode(func),
-                    named: undefined
+                    type: type_1.RType.WhileLoop,
+                    condition: convertTreeNode(condition),
+                    body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
+                    location: makeSourceRange(whileNode),
+                    lexeme: whileNode.text,
+                    ...defaultInfo
                 };
             }
-        }
-        case tree_sitter_types_1.TreeSitterType.FunctionDefinition: {
-            const [name, paramsParens, body] = nonErrorChildren(node);
-            const [comments, noCommentRawParams] = splitComments(paramsParens.children.slice(1, -1));
-            const params = (0, arrays_1.splitArrayOn)(noCommentRawParams, x => x.type === 'comma');
-            return {
-                type: type_1.RType.FunctionDefinition,
-                parameters: params.map(n => convertTreeNode(n[0])),
-                body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
-                location: makeSourceRange(name),
-                lexeme: name.text,
-                info: {
-                    ...defaultInfo.info,
-                    additionalTokens: comments.map(c => c[1]),
+            case tree_sitter_types_1.TreeSitterType.RepeatStatement: {
+                const [repeatNode, body] = nonErrorChildren(node);
+                return {
+                    type: type_1.RType.RepeatLoop,
+                    body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
+                    location: makeSourceRange(repeatNode),
+                    lexeme: repeatNode.text,
+                    ...defaultInfo
+                };
+            }
+            case tree_sitter_types_1.TreeSitterType.Call: {
+                const [func, argsParentheses] = nonErrorChildren(node);
+                // tree-sitter wraps next and break in a function call, but we don't, so unwrap
+                if (func.type === tree_sitter_types_1.TreeSitterType.Next || func.type == tree_sitter_types_1.TreeSitterType.Break) {
+                    return {
+                        ...convertTreeNode(func),
+                        ...defaultInfo
+                    };
                 }
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.String:
-            return {
-                type: type_1.RType.String,
-                location: range,
-                content: (0, convert_values_1.string2ts)(node.text),
-                lexeme: node.text,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.Float:
-        case tree_sitter_types_1.TreeSitterType.Integer:
-        case tree_sitter_types_1.TreeSitterType.Complex:
-        case tree_sitter_types_1.TreeSitterType.Inf:
-        case tree_sitter_types_1.TreeSitterType.Nan:
-            return {
-                type: type_1.RType.Number,
-                location: range,
-                content: (0, convert_values_1.number2ts)(node.text),
-                lexeme: node.text,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.True:
-        case tree_sitter_types_1.TreeSitterType.False:
-            return {
-                type: type_1.RType.Logical,
-                location: range,
-                content: (0, convert_values_1.boolean2ts)(node.text),
-                lexeme: node.text,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.Break:
-        case tree_sitter_types_1.TreeSitterType.Next:
-            return {
-                type: node.type == tree_sitter_types_1.TreeSitterType.Break ? type_1.RType.Break : type_1.RType.Next,
-                location: range,
-                lexeme: node.text,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.Subset:
-        case tree_sitter_types_1.TreeSitterType.Subset2: {
-            // subset has children like a and [x]
-            const [func, content] = nonErrorChildren(node);
-            // bracket is now [ or [[ and argsClosing is x] or x]]
-            const [bracket, ...argsClosing] = nonErrorChildren(content);
-            const args = (0, arrays_1.splitArrayOn)(argsClosing.slice(0, -1), x => x.type === 'comma');
-            return {
-                type: type_1.RType.Access,
-                operator: bracket.text,
-                accessed: convertTreeNode(func),
-                access: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
-                location: makeSourceRange(bracket),
-                lexeme: bracket.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.ExtractOperator: {
-            const [lhs, operator, rhs] = nonErrorChildren(node);
-            const rhsRange = makeSourceRange(rhs);
-            return {
-                type: type_1.RType.Access,
-                operator: operator.text,
-                accessed: convertTreeNode(lhs),
-                access: [{
-                        type: type_1.RType.Argument,
-                        name: undefined,
-                        value: {
-                            ...convertTreeNode(rhs),
-                            ...defaultInfo
+                const rawArgs = nonErrorChildren(argsParentheses);
+                const [comments, noCommentrawArgs] = splitComments(rawArgs);
+                const args = (0, arrays_1.splitArrayOn)(noCommentrawArgs.slice(1, -1), x => x.type === 'comma');
+                const funcRange = makeSourceRange(func);
+                const mappedArgs = args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0]));
+                const call = {
+                    arguments: mappedArgs,
+                    location: funcRange,
+                    lexeme: func.text,
+                    ...defaultInfo,
+                    info: {
+                        ...defaultInfo.info,
+                        additionalTokens: comments.map(c => c[1]),
+                    }
+                };
+                if (func.type === tree_sitter_types_1.TreeSitterType.Identifier || func.type === tree_sitter_types_1.TreeSitterType.String || func.type === tree_sitter_types_1.TreeSitterType.NamespaceOperator || func.type === tree_sitter_types_1.TreeSitterType.Return) {
+                    let funcNode = convertTreeNode(func);
+                    if (funcNode.type === type_1.RType.String) {
+                        funcNode = {
+                            ...funcNode,
+                            type: type_1.RType.Symbol,
+                            namespace: undefined,
+                            content: (0, retriever_1.removeRQuotes)(func.text)
+                        };
+                    }
+                    return {
+                        ...call,
+                        type: type_1.RType.FunctionCall,
+                        functionName: {
+                            ...funcNode,
+                            info: {
+                                fullRange: range,
+                                additionalTokens: [],
+                                fullLexeme: node.text,
+                                treeSitterId: node.id
+                            }
                         },
-                        location: rhsRange,
-                        lexeme: rhs.text,
+                        named: true
+                    };
+                }
+                else {
+                    return {
+                        ...call,
+                        type: type_1.RType.FunctionCall,
+                        calledFunction: convertTreeNode(func),
+                        named: undefined
+                    };
+                }
+            }
+            case tree_sitter_types_1.TreeSitterType.FunctionDefinition: {
+                const [name, paramsParens, body] = nonErrorChildren(node);
+                const [comments, noCommentRawParams] = splitComments(paramsParens.children.slice(1, -1));
+                const params = (0, arrays_1.splitArrayOn)(noCommentRawParams, x => x.type === 'comma');
+                return {
+                    type: type_1.RType.FunctionDefinition,
+                    parameters: params.map(n => convertTreeNode(n[0])),
+                    body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
+                    location: makeSourceRange(name),
+                    lexeme: name.text,
+                    info: {
+                        ...defaultInfo.info,
+                        additionalTokens: comments.map(c => c[1]),
+                    }
+                };
+            }
+            case tree_sitter_types_1.TreeSitterType.String:
+                return {
+                    type: type_1.RType.String,
+                    location: range,
+                    content: (0, convert_values_1.string2ts)(node.text),
+                    lexeme: node.text,
+                    ...defaultInfo
+                };
+            case tree_sitter_types_1.TreeSitterType.Float:
+            case tree_sitter_types_1.TreeSitterType.Integer:
+            case tree_sitter_types_1.TreeSitterType.Complex:
+            case tree_sitter_types_1.TreeSitterType.Inf:
+            case tree_sitter_types_1.TreeSitterType.Nan:
+                return {
+                    type: type_1.RType.Number,
+                    location: range,
+                    content: (0, convert_values_1.number2ts)(node.text),
+                    lexeme: node.text,
+                    ...defaultInfo
+                };
+            case tree_sitter_types_1.TreeSitterType.True:
+            case tree_sitter_types_1.TreeSitterType.False:
+                return {
+                    type: type_1.RType.Logical,
+                    location: range,
+                    content: (0, convert_values_1.boolean2ts)(node.text),
+                    lexeme: node.text,
+                    ...defaultInfo
+                };
+            case tree_sitter_types_1.TreeSitterType.Break:
+            case tree_sitter_types_1.TreeSitterType.Next:
+                return {
+                    type: node.type == tree_sitter_types_1.TreeSitterType.Break ? type_1.RType.Break : type_1.RType.Next,
+                    location: range,
+                    lexeme: node.text,
+                    ...defaultInfo
+                };
+            case tree_sitter_types_1.TreeSitterType.Subset:
+            case tree_sitter_types_1.TreeSitterType.Subset2: {
+                // subset has children like a and [x]
+                const [func, content] = nonErrorChildren(node);
+                // bracket is now [ or [[ and argsClosing is x] or x]]
+                const [bracket, ...argsClosing] = nonErrorChildren(content);
+                const args = (0, arrays_1.splitArrayOn)(argsClosing.slice(0, -1), x => x.type === 'comma');
+                return {
+                    type: type_1.RType.Access,
+                    operator: bracket.text,
+                    accessed: convertTreeNode(func),
+                    access: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
+                    location: makeSourceRange(bracket),
+                    lexeme: bracket.text,
+                    ...defaultInfo
+                };
+            }
+            case tree_sitter_types_1.TreeSitterType.ExtractOperator: {
+                const [lhs, operator, rhs] = nonErrorChildren(node);
+                const rhsRange = makeSourceRange(rhs);
+                return {
+                    type: type_1.RType.Access,
+                    operator: operator.text,
+                    accessed: convertTreeNode(lhs),
+                    access: [{
+                            type: type_1.RType.Argument,
+                            name: undefined,
+                            value: {
+                                ...convertTreeNode(rhs),
+                                ...defaultInfo
+                            },
+                            location: rhsRange,
+                            lexeme: rhs?.text,
+                            info: {
+                                fullRange: rhsRange,
+                                additionalTokens: [],
+                                fullLexeme: rhs?.text,
+                                treeSitterId: rhs?.id
+                            }
+                        }],
+                    location: makeSourceRange(operator),
+                    lexeme: operator.text,
+                    ...defaultInfo
+                };
+            }
+            case tree_sitter_types_1.TreeSitterType.Parameter: {
+                const children = nonErrorChildren(node);
+                const name = children[0];
+                const nameRange = makeSourceRange(name);
+                let defaultValue = undefined;
+                if (children.length == 3) {
+                    defaultValue = convertTreeNode(children[2]);
+                }
+                return {
+                    type: type_1.RType.Parameter,
+                    name: {
+                        type: type_1.RType.Symbol,
+                        location: nameRange,
+                        content: name.text,
+                        lexeme: name.text,
                         info: {
-                            fullRange: rhsRange,
+                            fullRange: range,
                             additionalTokens: [],
-                            fullLexeme: rhs.text,
-                            treeSitterId: rhs.id
+                            fullLexeme: name.text,
+                            treeSitterId: name.id
                         }
-                    }],
-                location: makeSourceRange(operator),
-                lexeme: operator.text,
-                ...defaultInfo
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.Parameter: {
-            const children = nonErrorChildren(node);
-            const name = children[0];
-            const nameRange = makeSourceRange(name);
-            let defaultValue = undefined;
-            if (children.length == 3) {
-                defaultValue = convertTreeNode(children[2]);
-            }
-            return {
-                type: type_1.RType.Parameter,
-                name: {
-                    type: type_1.RType.Symbol,
+                    },
+                    special: name.text === '...',
+                    defaultValue,
                     location: nameRange,
-                    content: name.text,
                     lexeme: name.text,
                     info: {
                         fullRange: range,
@@ -491,84 +519,90 @@ function convertTreeNode(node) {
                         fullLexeme: name.text,
                         treeSitterId: name.id
                     }
-                },
-                special: name.text === '...',
-                defaultValue,
-                location: nameRange,
-                lexeme: name.text,
-                info: {
-                    fullRange: range,
-                    additionalTokens: [],
-                    fullLexeme: name.text,
-                    treeSitterId: name.id
+                };
+            }
+            case tree_sitter_types_1.TreeSitterType.Argument: {
+                const children = nonErrorChildren(node);
+                if (children.length == 1) {
+                    const [arg] = children;
+                    return {
+                        type: type_1.RType.Argument,
+                        name: undefined,
+                        value: convertTreeNode(arg),
+                        location: range,
+                        lexeme: node.text,
+                        ...defaultInfo
+                    };
                 }
-            };
-        }
-        case tree_sitter_types_1.TreeSitterType.Argument: {
-            const children = nonErrorChildren(node);
-            if (children.length == 1) {
-                const [arg] = children;
+                else {
+                    const [nameNode, /* = */ , valueNode] = children;
+                    let name = convertTreeNode(nameNode);
+                    // unescape argument names
+                    if (name.type === type_1.RType.String) {
+                        name = {
+                            ...name,
+                            type: type_1.RType.Symbol,
+                            content: name.content.str,
+                            namespace: undefined
+                        };
+                    }
+                    else if ((0, strings_1.startAndEndsWith)(name.content, '`')) {
+                        name.content = name.content.slice(1, -1);
+                    }
+                    const nameRange = makeSourceRange(nameNode);
+                    return {
+                        type: type_1.RType.Argument,
+                        name,
+                        value: valueNode ? convertTreeNode(valueNode) : undefined,
+                        location: nameRange,
+                        lexeme: nameNode.text,
+                        info: {
+                            fullRange: nameRange,
+                            additionalTokens: [],
+                            fullLexeme: nameNode.text,
+                            treeSitterId: nameNode.id
+                        }
+                    };
+                }
+            }
+            case tree_sitter_types_1.TreeSitterType.Comment:
                 return {
-                    type: type_1.RType.Argument,
-                    name: undefined,
-                    value: convertTreeNode(arg),
+                    type: type_1.RType.Comment,
                     location: range,
                     lexeme: node.text,
                     ...defaultInfo
                 };
-            }
-            else {
-                const [nameNode, /* = */ , valueNode] = children;
-                let name = convertTreeNode(nameNode);
-                // unescape argument names
-                if (name.type === type_1.RType.String) {
-                    name = {
-                        ...name,
-                        type: type_1.RType.Symbol,
-                        content: name.content.str,
-                        namespace: undefined
-                    };
-                }
-                else if ((0, strings_1.startAndEndsWith)(name.content, '`')) {
-                    name.content = name.content.slice(1, -1);
-                }
-                const nameRange = makeSourceRange(nameNode);
+            case tree_sitter_types_1.TreeSitterType.Error:
                 return {
-                    type: type_1.RType.Argument,
-                    name,
-                    value: valueNode ? convertTreeNode(valueNode) : undefined,
-                    location: nameRange,
-                    lexeme: nameNode.text,
-                    info: {
-                        fullRange: nameRange,
-                        additionalTokens: [],
-                        fullLexeme: nameNode.text,
-                        treeSitterId: nameNode.id
-                    }
+                    type: type_1.RType.ExpressionList,
+                    location: undefined,
+                    lexeme: undefined,
+                    children: [],
+                    grouping: undefined,
+                    ...defaultInfo
                 };
-            }
         }
-        case tree_sitter_types_1.TreeSitterType.Comment:
-            return {
-                type: type_1.RType.Comment,
-                location: range,
-                lexeme: node.text,
-                ...defaultInfo
-            };
-        case tree_sitter_types_1.TreeSitterType.Error:
-            return {
-                type: type_1.RType.ExpressionList,
-                location: undefined,
-                lexeme: undefined,
-                children: [],
-                grouping: undefined,
-                ...defaultInfo
-            };
-        default:
-            throw new normalizer_data_1.ParseError(`unexpected node type ${node.type} at ${JSON.stringify(range)}`);
     }
+    catch {
+        parser_1.parseLog.error(`[Tree-Sitter] Failed to convert node of type ${node.type} at ${JSON.stringify(makeSourceRange(node))}`);
+    }
+    return {
+        type: type_1.RType.ExpressionList,
+        location: undefined,
+        lexeme: undefined,
+        children: [],
+        grouping: undefined,
+        info: {
+            fullRange: (0, range_1.invalidRange)(),
+            additionalTokens: [],
+            treeSitterId: -1,
+        }
+    };
 }
 function makeSourceRange(node) {
+    if (!node) {
+        return (0, range_1.invalidRange)();
+    }
     if (node.startPosition && node.endPosition) {
         return [
             // tree-sitter is 0-based but we want 1-based