@eagleoutice/flowr 2.6.0 → 2.6.2

package/r-bridge/lang-4.x/ast/parser/main/internal/other/normalize-line-directive.d.ts

@@ -6,7 +6,6 @@ import type { JsonEntry } from '../../../json/format';
  * Normalize the given object as an R line directive (`#line <number> "<file>"`).
  * This requires you to check the corresponding name beforehand.
  * If the given object turns out to be no line directive, this returns a normal comment instead.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/other/normalize-line-directive.js

@@ -10,7 +10,6 @@ const LineDirectiveRegex = /^#line\s+(\d+)\s+"([^"]+)"\s*$/;
  * Normalize the given object as an R line directive (`#line <number> "<file>"`).
  * This requires you to check the corresponding name beforehand.
  * If the given object turns out to be no line directive, this returns a normal comment instead.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-delimiter.d.ts

@@ -1,3 +1,6 @@
 import type { RDelimiter } from '../../../../model/nodes/info/r-delimiter';
 import type { NamedJsonEntry } from '../../../json/format';
+/**
+ * Normalizes a delimiter node like `,`.
+ */
 export declare function normalizeDelimiter(elem: NamedJsonEntry): RDelimiter;

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-delimiter.js

@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.normalizeDelimiter = normalizeDelimiter;
 const normalize_meta_1 = require("../../normalize-meta");
 const type_1 = require("../../../../model/type");
+/**
+ * Normalizes a delimiter node like `,`.
+ */
 function normalizeDelimiter(elem) {
     const { location, content } = (0, normalize_meta_1.retrieveMetaStructure)(elem.content);
     return {

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.d.ts

@@ -2,9 +2,18 @@ import type { NormalizerData } from '../../normalizer-data';
 import type { RDelimiter } from '../../../../model/nodes/info/r-delimiter';
 import type { RNode } from '../../../../model/model';
 import type { JsonEntry, NamedJsonEntry } from '../../../json/format';
+/**
+ * Split a token collection into comments tokens and "other" tokens
+ */
 export declare function splitComments(tokens: readonly NamedJsonEntry[]): {
     comments: NamedJsonEntry[];
     others: NamedJsonEntry[];
 };
+/**
+ * Normalizes a list of expressions, handling semicolons and braces
+ */
 export declare function normalizeExpressions(data: NormalizerData, tokens: readonly JsonEntry[] | readonly NamedJsonEntry[]): (RNode | RDelimiter)[];
+/**
+ * Parses nodes when their type is unknown
+ */
 export declare function parseNodesWithUnknownType(data: NormalizerData, mappedWithName: readonly NamedJsonEntry[] | undefined): (RNode | RDelimiter)[];

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.js

@@ -45,6 +45,9 @@ function normalizeMappedWithoutSemicolonBasedOnType(mappedWithName, data) {
     // otherwise perform default parsing
     return result !== undefined ? [result] : parseNodesWithUnknownType(data, mappedWithName);
 }
+/**
+ * Split a token collection into comments tokens and "other" tokens
+ */
 function splitComments(tokens) {
     const comments = [];
     const others = [];
@@ -132,6 +135,9 @@ function processBraces([start, end], processed, comments, data) {
         }
     };
 }
+/**
+ * Normalizes a list of expressions, handling semicolons and braces
+ */
 function normalizeExpressions(data, tokens) {
     if (tokens.length === 0) {
         parser_1.parseLog.warn('no children received, skipping');
@@ -170,6 +176,9 @@ function normalizeExpressions(data, tokens) {
     }
     return [...parsedComments, ...normalizeMappedWithoutSemicolonBasedOnType(mappedWithName, data)];
 }
+/**
+ * Parses nodes when their type is unknown
+ */
 function parseNodesWithUnknownType(data, mappedWithName) {
     const parsedNodes = [];
     // used to indicate the new root node of this set of nodes

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-single-node.d.ts

@@ -1,13 +1,11 @@
-import type { NormalizerData } from '../../normalizer-data';
+import { type NormalizerData } from '../../normalizer-data';
 import type { RNode } from '../../../../model/model';
 import type { RDelimiter } from '../../../../model/nodes/info/r-delimiter';
 import type { NamedJsonEntry } from '../../../json/format';
 /**
  * Parses a single structure in the ast based on its type (e.g., a string, a number, a symbol, ...)
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param elem - The element to parse
- *
  * @returns The parsed element as an `RNode` or an `RDelimiter` if it is such.
  */
 export declare function normalizeSingleNode(data: NormalizerData, elem: NamedJsonEntry): RNode | RDelimiter;

package/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-single-node.js

@@ -16,10 +16,8 @@ const normalize_next_1 = require("../loops/normalize-next");
 const normalize_symbol_1 = require("../values/normalize-symbol");
 /**
  * Parses a single structure in the ast based on its type (e.g., a string, a number, a symbol, ...)
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param elem - The element to parse
- *
  * @returns The parsed element as an `RNode` or an `RDelimiter` if it is such.
  */
 function normalizeSingleNode(data, elem) {

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-number.d.ts

@@ -1,5 +1,5 @@
 import type { NormalizerData } from '../../normalizer-data';
-import type { RNa } from '../../../../../convert-values';
+import { type RNa } from '../../../../../convert-values';
 import type { RNumber } from '../../../../model/nodes/r-number';
 import type { RLogical } from '../../../../model/nodes/r-logical';
 import type { RSymbol } from '../../../../model/nodes/r-symbol';
@@ -9,7 +9,6 @@ import type { JsonEntry } from '../../../json/format';
  * Normalize the given object as a R number (see {@link number2ts}), supporting booleans (see {@link boolean2ts}),
  * and special values.
  * This requires you to check the corresponding name beforehand.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-number.js

@@ -8,7 +8,6 @@ const type_1 = require("../../../../model/type");
  * Normalize the given object as a R number (see {@link number2ts}), supporting booleans (see {@link boolean2ts}),
  * and special values.
  * This requires you to check the corresponding name beforehand.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-string.d.ts

@@ -4,7 +4,6 @@ import type { JsonEntry } from '../../../json/format';
 /**
  * Normalize the given object as a R string (see {@link string2ts}).
  * This requires you to check the corresponding name beforehand.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-string.js

@@ -8,7 +8,6 @@ const type_1 = require("../../../../model/type");
 /**
  * Normalize the given object as a R string (see {@link string2ts}).
  * This requires you to check the corresponding name beforehand.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param obj  - The JSON object to extract the meta-information from
  */

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-symbol.d.ts

@@ -5,10 +5,8 @@ import type { NamedJsonEntry } from '../../../json/format';
  * Normalize the given object as an R symbol (incorporating namespace information).
  * <p>
  * The special symbols `T` and `F` are parsed as logic values.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param objs - The JSON object to extract the meta-information from
- *
  * @returns The parsed symbol (with populated namespace information) or `undefined` if the given object is not a symbol.
  */
 export declare function tryNormalizeSymbol(data: NormalizerData, objs: readonly NamedJsonEntry[]): RSymbol | undefined;

package/r-bridge/lang-4.x/ast/parser/main/internal/values/normalize-symbol.js

@@ -9,10 +9,8 @@ const type_1 = require("../../../../model/type");
  * Normalize the given object as an R symbol (incorporating namespace information).
  * <p>
  * The special symbols `T` and `F` are parsed as logic values.
- *
  * @param data - The data used by the parser (see {@link NormalizerData})
  * @param objs - The JSON object to extract the meta-information from
- *
  * @returns The parsed symbol (with populated namespace information) or `undefined` if the given object is not a symbol.
  */
 function tryNormalizeSymbol(data, objs) {

package/r-bridge/lang-4.x/ast/parser/main/normalize-meta.d.ts

@@ -1,6 +1,6 @@
 import type { JsonEntry, NamedJsonEntry } from '../json/format';
-import type { SourceRange } from '../../../../../util/range';
-import type { RawRType } from '../../model/type';
+import { type SourceRange } from '../../../../../util/range';
+import { type RawRType } from '../../model/type';
 import type { RNode } from '../../model/model';
 import type { RExpressionList } from '../../model/nodes/r-expression-list';
 /**
@@ -10,7 +10,6 @@ export declare function extractLocation(ast: JsonEntry): SourceRange;
 /**
  * The JSON object that represents the input contains various meta-information.
  * This function extracts the meta-information and returns it.
- *
  * @param entry - The JSON object to extract the meta-information from
  * @returns An object containing the passed entry, the location of the corresponding R-ast element, and the content of the passed entry
  */
@@ -18,24 +17,34 @@ export declare function retrieveMetaStructure(entry: JsonEntry): {
     location: SourceRange;
     content: string;
 };
+/**
+ * Ensure that the given token is of the expected type.
+ */
 export declare function assureTokenType(token: string, expectedName: RawRType): void;
 /**
  * Extract the token-type of the given object. This is based on the knowledge, that all JSON objects created
  * from the R XML have a name attached.
- *
  * @param content  - the JSON object to extract the token-type from
  */
 export declare function getTokenType(content: JsonEntry): RawRType;
+/**
+ * Given a list of JSON objects, extract their token-types and return them alongside the original objects.
+ */
 export declare function getWithTokenType(obj: JsonEntry[]): {
     name: RawRType;
     content: JsonEntry;
 }[];
+/**
+ * Extract the operation name from the given operator node.
+ */
 export declare function retrieveOpName(operator: NamedJsonEntry): string;
 /**
  * Ensure that the first child is completely before the second child.
- *
  * @param first  - the first child which should be the lhs
  * @param second - the second child which should be the rhs
  */
 export declare function ensureChildrenAreLhsAndRhsOrdered(first: JsonEntry, second: JsonEntry): void;
+/**
+ * Ensure that the given node is an expression list. If it is not, wrap it in an expression list.
+ */
 export declare function ensureExpressionList<Info>(node: RNode<Info>): RExpressionList<Info>;

package/r-bridge/lang-4.x/ast/parser/main/normalize-meta.js

@@ -20,7 +20,6 @@ function extractLocation(ast) {
 /**
  * The JSON object that represents the input contains various meta-information.
  * This function extracts the meta-information and returns it.
- *
  * @param entry - The JSON object to extract the meta-information from
  * @returns An object containing the passed entry, the location of the corresponding R-ast element, and the content of the passed entry
  */
@@ -30,6 +29,9 @@ function retrieveMetaStructure(entry) {
         content: entry.text
     };
 }
+/**
+ * Ensure that the given token is of the expected type.
+ */
 function assureTokenType(token, expectedName) {
     if (token !== expectedName) {
         throw new normalizer_data_1.ParseError(`expected name to be ${expectedName}, yet received ${token}`);
@@ -38,27 +40,29 @@ function assureTokenType(token, expectedName) {
 /**
  * Extract the token-type of the given object. This is based on the knowledge, that all JSON objects created
  * from the R XML have a name attached.
- *
  * @param content  - the JSON object to extract the token-type from
  */
 function getTokenType(content) {
     return content.token;
 }
+/**
+ * Given a list of JSON objects, extract their token-types and return them alongside the original objects.
+ */
 function getWithTokenType(obj) {
     return obj.map((content) => ({
         name: getTokenType(content),
         content
     }));
 }
+/**
+ * Extract the operation name from the given operator node.
+ */
 function retrieveOpName(operator) {
-    /*
-   * only real arithmetic ops have their operation as their own name, the others identify via content/text
-   */
+    /* only real arithmetic ops have their operation as their own name, the others identify via content/text */
     return operator.content.text;
 }
 /**
  * Ensure that the first child is completely before the second child.
- *
  * @param first  - the first child which should be the lhs
  * @param second - the second child which should be the rhs
  */
@@ -69,6 +73,9 @@ function ensureChildrenAreLhsAndRhsOrdered(first, second) {
         throw new normalizer_data_1.ParseError(`expected the first child to be the lhs, yet received ${JSON.stringify(first)} & ${JSON.stringify(second)}`);
     }
 }
+/**
+ * Ensure that the given node is an expression list. If it is not, wrap it in an expression list.
+ */
 function ensureExpressionList(node) {
     if (node.type !== type_1.RType.ExpressionList) {
         return {

package/r-bridge/lang-4.x/convert-values.d.ts

@@ -6,7 +6,13 @@ export declare function ts2r<T>(value: T): string;
 export declare const RTrue = "TRUE";
 /** The R literal for the logical false */
 export declare const RFalse = "FALSE";
+/**
+ * Checks whether the given string is an R boolean (logical) literal.
+ */
 export declare function isBoolean(value: string): boolean;
+/**
+ * Converts an R boolean (logical) literal to a TypeScript boolean.
+ */
 export declare function boolean2ts(value: string): boolean;
 export declare const RNumHexFloatRegex: RegExp;
 export declare const RImaginaryMarker = "i";
@@ -19,6 +25,9 @@ export interface RNumberValue {
     /** see {@link RImaginaryMarker}, compound imaginary numbers are expressions in R */
     complexNumber: boolean;
 }
+/**
+ * Convert a valid R number into a {@link RNumberValue}.
+ */
 export declare function number2ts(value: string): RNumberValue;
 export interface RStringValue {
     str: string;
@@ -29,10 +38,12 @@ export interface RStringValue {
 }
 /**
  * Convert a valid R string into a {@link RStringValue}.
- *
  * @throws {@link ValueConversionError} if the string has an unknown starting quote
  */
 export declare function string2ts(value: string): RStringValue;
 export declare const RNa = "NA";
 export declare const RNull = "NULL";
+/**
+ * Checks whether the given string is the R NA literal (untyped).
+ */
 export declare function isNA(value: string): value is (typeof RNa);

package/r-bridge/lang-4.x/convert-values.js

@@ -54,9 +54,15 @@ function ts2r(value) {
 exports.RTrue = 'TRUE';
 /** The R literal for the logical false */
 exports.RFalse = 'FALSE';
+/**
+ * Checks whether the given string is an R boolean (logical) literal.
+ */
 function isBoolean(value) {
     return value === exports.RTrue || value === exports.RFalse;
 }
+/**
+ * Converts an R boolean (logical) literal to a TypeScript boolean.
+ */
 function boolean2ts(value) {
     if (value === exports.RTrue) {
         return true;
@@ -73,6 +79,9 @@ function getDecimalPlacesWithRadix(floatPart, radix) {
 exports.RImaginaryMarker = 'i';
 exports.RIntegerMarker = 'L';
 exports.RInf = 'Inf';
+/**
+ * Convert a valid R number into a {@link RNumberValue}.
+ */
 function number2ts(value) {
     // check for hexadecimal number with floating point addon which is supported by R but not by JS :/
     let lcValue = value.toLowerCase();
@@ -119,7 +128,6 @@ function number2ts(value) {
 }
 /**
  * Convert a valid R string into a {@link RStringValue}.
- *
  * @throws {@link ValueConversionError} if the string has an unknown starting quote
  */
 function string2ts(value) {
@@ -152,6 +160,9 @@ function string2ts(value) {
 }
 exports.RNa = 'NA';
 exports.RNull = 'NULL';
+/**
+ * Checks whether the given string is the R NA literal (untyped).
+ */
 function isNA(value) {
     return value === exports.RNa;
 }

package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.d.ts

@@ -1,7 +1,9 @@
 import Parser from 'web-tree-sitter';
+import type { Query, QueryCapture } from 'web-tree-sitter';
 import type { RParseRequest } from '../../retriever';
-import type { SyncParser } from '../../parser';
+import type { SyncParser, TreeSitterInformation } from '../../parser';
 import type { TreeSitterEngineConfig } from '../../../config';
+import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer';
 export declare const DEFAULT_TREE_SITTER_R_WASM_PATH = "./node_modules/@eagleoutice/tree-sitter-r/tree-sitter-r.wasm";
 export declare const DEFAULT_TREE_SITTER_WASM_PATH = "./node_modules/web-tree-sitter/tree-sitter.wasm";
 /**
@@ -9,7 +11,7 @@ export declare const DEFAULT_TREE_SITTER_WASM_PATH = "./node_modules/web-tree-si
  */
 export declare class TreeSitterExecutor implements SyncParser<Parser.Tree> {
     readonly name = "tree-sitter";
-    readonly parser: Parser;
+    private readonly parser;
     private static language;
     /**
      * Initializes the underlying tree-sitter parser. This only needs to be called once globally.
@@ -20,7 +22,10 @@ export declare class TreeSitterExecutor implements SyncParser<Parser.Tree> {
     static initTreeSitter(config?: TreeSitterEngineConfig, overrideWasmPath?: string, overrideTreeSitterWasmPath?: string): Promise<void>;
     constructor();
     rVersion(): Promise<string | 'unknown' | 'none'>;
+    information(analyzer: FlowrAnalysisProvider): TreeSitterInformation;
     treeSitterVersion(): number;
     parse(request: RParseRequest): Parser.Tree;
+    createQuery(source: string): Query;
+    query(source: Query | string, tree: Parser.Tree): QueryCapture[];
     close(): void;
 }

package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.js

@@ -48,6 +48,15 @@ class TreeSitterExecutor {
     rVersion() {
         return Promise.resolve('none');
     }
+    information(analyzer) {
+        return {
+            name: 'tree-sitter',
+            grammarVersion: this.treeSitterVersion(),
+            treeSitterQuery: async (source, force) => {
+                return this.query(source, (await analyzer.parse(force)).parsed);
+            }
+        };
+    }
     treeSitterVersion() {
         return this.parser.getLanguage().version;
     }
@@ -61,6 +70,14 @@ class TreeSitterExecutor {
         }
         return this.parser.parse(sourceCode);
     }
+    createQuery(source) {
+        return this.parser.getLanguage().query(source);
+    }
+    query(source, tree) {
+        const query = typeof source === 'string' ? this.createQuery(source) : source;
+        const matches = query.matches(tree.rootNode);
+        return matches.flatMap(m => m.captures);
+    }
     close() {
         this.parser.delete();
     }

package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.d.ts

@@ -1,8 +1,20 @@
 import type { RExpressionList } from '../ast/model/nodes/r-expression-list';
 import type { Tree } from 'web-tree-sitter';
+export interface TreeSitterInfo {
+    treeSitterId: number;
+}
 /**
  * @param tree - The tree to normalize
+ * @param lax - Whether to use lax parsing (i.e., ignore errors) or strict parsing (i.e., fail on errors)
+ */
+export declare function normalizeTreeSitterTreeToAst(tree: Tree, lax?: boolean): RExpressionList<TreeSitterInfo>;
+/**
+ * Globally switch to lax parsing mode for tree-sitter normalization
+ * @see {@link makeTreeSitterStrict}
  */
-export declare function normalizeTreeSitterTreeToAst(tree: Tree, lax?: boolean): RExpressionList;
 export declare function makeTreeSitterLax(): void;
+/**
+ * Globally switch to strict parsing mode for tree-sitter normalization
+ * @see {@link makeTreeSitterLax}
+ */
 export declare function makeTreeSitterStrict(): void;

package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.js

@@ -15,6 +15,7 @@ const strings_1 = require("../../../util/text/strings");
 const log_1 = require("../../../util/log");
 /**
  * @param tree - The tree to normalize
+ * @param lax - Whether to use lax parsing (i.e., ignore errors) or strict parsing (i.e., fail on errors)
  */
 function normalizeTreeSitterTreeToAst(tree, lax) {
     if (lax) {
@@ -36,10 +37,18 @@ function nonErrorChildrenLax(node) {
     return node.hasError ? node.children.filter(n => n.type !== tree_sitter_types_1.TreeSitterType.Error) : node.children;
 }
 let nonErrorChildren = nonErrorChildrenStrict;
+/**
+ * Globally switch to lax parsing mode for tree-sitter normalization
+ * @see {@link makeTreeSitterStrict}
+ */
 function makeTreeSitterLax() {
     log_1.log.info('[Tree-Sitter] Lax parsing active');
     nonErrorChildren = nonErrorChildrenLax;
 }
+/**
+ * Globally switch to strict parsing mode for tree-sitter normalization
+ * @see {@link makeTreeSitterLax}
+ */
 function makeTreeSitterStrict() {
     log_1.log.info('[Tree-Sitter] Strict parsing active');
     nonErrorChildren = nonErrorChildrenStrict;
@@ -52,7 +61,8 @@ function convertTreeNode(node) {
         info: {
             fullRange: range,
             additionalTokens: [],
-            fullLexeme: node.text
+            fullLexeme: node.text,
+            treeSitterId: node.id
         }
     };
     switch (node.type) {
@@ -66,7 +76,8 @@ function convertTreeNode(node) {
                 grouping: undefined,
                 lexeme: undefined,
                 info: {
-                    additionalTokens: remainingComments.map(c => c[1])
+                    additionalTokens: remainingComments.map(c => c[1]),
+                    treeSitterId: node.id
                 }
             };
         }
@@ -100,7 +111,8 @@ function convertTreeNode(node) {
                     }
                 ],
                 info: {
-                    additionalTokens: remainingComments.map(c => c[1])
+                    additionalTokens: remainingComments.map(c => c[1]),
+                    treeSitterId: node.id
                 }
             };
         }
@@ -117,7 +129,9 @@ function convertTreeNode(node) {
                 value: lhs,
                 name: undefined,
                 lexeme: lhs.lexeme,
-                info: {}
+                info: {
+                    treeSitterId: lhs.info.treeSitterId
+                }
             };
             if (op.type == 'special') {
                 return {
@@ -130,7 +144,9 @@ function convertTreeNode(node) {
                         lexeme: op.text,
                         content: op.text,
                         namespace: undefined,
-                        info: {}
+                        info: {
+                            treeSitterId: op.id
+                        }
                     },
                     arguments: [lhsAsArg, {
                             type: type_1.RType.Argument,
@@ -138,12 +154,15 @@ function convertTreeNode(node) {
                             value: rhs,
                             name: undefined,
                             lexeme: rhs.lexeme,
-                            info: {}
+                            info: {
+                                treeSitterId: rhs.info.treeSitterId
+                            }
                         }],
                     named: true,
                     infixSpecial: true,
                     info: {
-                        additionalTokens: comments
+                        additionalTokens: comments,
+                        treeSitterId: node.id
                     }
                 };
             }
@@ -158,7 +177,8 @@ function convertTreeNode(node) {
                     info: {
                         fullRange: range,
                         additionalTokens: comments,
-                        fullLexeme: node.text
+                        fullLexeme: node.text,
+                        treeSitterId: node.id
                     }
                 };
             }
@@ -172,7 +192,8 @@ function convertTreeNode(node) {
                     info: {
                         fullRange: range,
                         additionalTokens: comments,
-                        fullLexeme: node.text
+                        fullLexeme: node.text,
+                        treeSitterId: node.id
                     }
                 };
             }
@@ -247,7 +268,8 @@ function convertTreeNode(node) {
                     info: {
                         fullRange: undefined,
                         additionalTokens: [],
-                        fullLexeme: undefined
+                        fullLexeme: undefined,
+                        treeSitterId: variableNode.id
                     }
                 },
                 vector: convertTreeNode(sequenceNode),
@@ -256,8 +278,9 @@ function convertTreeNode(node) {
                 lexeme: forNode.text,
                 info: {
                     fullRange: range,
-                    additionalTokens: [...variableComments, ...sequenceComments].map(c => c[1]),
-                    fullLexeme: node.text
+                    additionalTokens: variableComments.concat(sequenceComments).map(c => c[1]),
+                    fullLexeme: node.text,
+                    treeSitterId: node.id
                 }
             };
         }
@@ -324,7 +347,8 @@ function convertTreeNode(node) {
                         info: {
                             fullRange: range,
                             additionalTokens: [],
-                            fullLexeme: node.text
+                            fullLexeme: node.text,
+                            treeSitterId: node.id
                         }
                     },
                     named: true
@@ -428,7 +452,8 @@ function convertTreeNode(node) {
                         info: {
                             fullRange: rhsRange,
                             additionalTokens: [],
-                            fullLexeme: rhs.text
+                            fullLexeme: rhs.text,
+                            treeSitterId: rhs.id
                         }
                     }],
                 location: makeSourceRange(operator),
@@ -455,7 +480,8 @@ function convertTreeNode(node) {
                     info: {
                         fullRange: range,
                         additionalTokens: [],
-                        fullLexeme: name.text
+                        fullLexeme: name.text,
+                        treeSitterId: name.id
                     }
                 },
                 special: name.text === '...',
@@ -465,7 +491,8 @@ function convertTreeNode(node) {
                 info: {
                     fullRange: range,
                     additionalTokens: [],
-                    fullLexeme: name.text
+                    fullLexeme: name.text,
+                    treeSitterId: name.id
                 }
             };
         }
@@ -507,7 +534,8 @@ function convertTreeNode(node) {
                     info: {
                         fullRange: nameRange,
                         additionalTokens: [],
-                        fullLexeme: nameNode.text
+                        fullLexeme: nameNode.text,
+                        treeSitterId: nameNode.id
                     }
                 };
             }
@@ -527,7 +555,7 @@ function convertTreeNode(node) {
                 lexeme: undefined,
                 children: [],
                 grouping: undefined,
-                info: defaultInfo
+                ...defaultInfo
             };
         default:
             throw new normalizer_data_1.ParseError(`unexpected node type ${node.type} at ${JSON.stringify(range)}`);
@@ -562,7 +590,8 @@ function splitComments(nodes) {
                     lexeme: node.text,
                     info: {
                         additionalTokens: [],
-                        fullLexeme: node.text
+                        fullLexeme: node.text,
+                        treeSitterId: node.id
                     }
                 }]);
         }