@eagleoutice/flowr 2.1.12 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +0 -1
  2. package/benchmark/slicer.d.ts +5 -12
  3. package/benchmark/slicer.js +46 -28
  4. package/cli/benchmark-app.d.ts +2 -0
  5. package/cli/benchmark-app.js +2 -1
  6. package/cli/benchmark-helper-app.d.ts +2 -0
  7. package/cli/benchmark-helper-app.js +2 -2
  8. package/cli/common/options.js +3 -1
  9. package/cli/flowr-main-options.js +36 -2
  10. package/cli/flowr.d.ts +6 -0
  11. package/cli/flowr.js +51 -24
  12. package/cli/repl/commands/repl-cfg.js +2 -4
  13. package/cli/repl/commands/repl-dataflow.js +2 -4
  14. package/cli/repl/commands/repl-execute.d.ts +2 -2
  15. package/cli/repl/commands/repl-execute.js +15 -5
  16. package/cli/repl/commands/repl-lineage.js +2 -4
  17. package/cli/repl/commands/repl-main.d.ts +2 -2
  18. package/cli/repl/commands/repl-normalize.js +2 -4
  19. package/cli/repl/commands/repl-parse.js +2 -4
  20. package/cli/repl/commands/repl-query.js +6 -8
  21. package/cli/repl/commands/repl-version.d.ts +5 -4
  22. package/cli/repl/commands/repl-version.js +10 -9
  23. package/cli/repl/core.d.ts +5 -5
  24. package/cli/repl/core.js +8 -12
  25. package/cli/repl/print-version.d.ts +2 -2
  26. package/cli/repl/print-version.js +3 -3
  27. package/cli/repl/server/connection.d.ts +3 -3
  28. package/cli/repl/server/connection.js +5 -7
  29. package/cli/repl/server/messages/message-hello.js +2 -1
  30. package/cli/repl/server/server.d.ts +4 -3
  31. package/cli/repl/server/server.js +7 -5
  32. package/cli/slicer-app.js +1 -1
  33. package/config.d.ts +36 -4
  34. package/config.js +30 -1
  35. package/core/pipeline-executor.d.ts +1 -1
  36. package/core/pipeline-executor.js +1 -1
  37. package/core/steps/all/core/00-parse.d.ts +4 -18
  38. package/core/steps/all/core/00-parse.js +2 -11
  39. package/core/steps/all/core/01-parse-tree-sitter.d.ts +23 -0
  40. package/core/steps/all/core/01-parse-tree-sitter.js +19 -0
  41. package/core/steps/all/core/10-normalize.d.ts +3 -2
  42. package/core/steps/all/core/10-normalize.js +1 -0
  43. package/core/steps/all/core/11-normalize-tree-sitter.d.ts +25 -0
  44. package/core/steps/all/core/11-normalize-tree-sitter.js +27 -0
  45. package/core/steps/all/core/20-dataflow.d.ts +2 -0
  46. package/core/steps/all/core/20-dataflow.js +1 -1
  47. package/core/steps/pipeline/default-pipelines.d.ts +368 -23
  48. package/core/steps/pipeline/default-pipelines.js +42 -4
  49. package/dataflow/extractor.d.ts +2 -1
  50. package/dataflow/extractor.js +2 -1
  51. package/dataflow/internal/process/functions/call/built-in/built-in-source.js +6 -5
  52. package/dataflow/processor.d.ts +2 -0
  53. package/documentation/doc-util/doc-auto-gen.js +2 -1
  54. package/documentation/doc-util/doc-cfg.js +1 -1
  55. package/documentation/doc-util/doc-dfg.js +2 -2
  56. package/documentation/doc-util/doc-files.d.ts +1 -0
  57. package/documentation/doc-util/doc-files.js +4 -0
  58. package/documentation/doc-util/doc-normalized-ast.js +2 -3
  59. package/documentation/doc-util/doc-query.js +1 -1
  60. package/documentation/doc-util/doc-search.js +1 -1
  61. package/documentation/doc-util/doc-types.js +2 -2
  62. package/documentation/print-dataflow-graph-wiki.js +15 -15
  63. package/documentation/print-engines-wiki.d.ts +1 -0
  64. package/documentation/print-engines-wiki.js +82 -0
  65. package/documentation/print-interface-wiki.js +6 -7
  66. package/documentation/print-normalized-ast-wiki.js +1 -1
  67. package/package.json +9 -5
  68. package/queries/catalog/cluster-query/cluster-query-format.d.ts +5 -4
  69. package/queries/catalog/dataflow-query/dataflow-query-format.d.ts +5 -4
  70. package/queries/catalog/dependencies-query/dependencies-query-format.d.ts +5 -4
  71. package/queries/catalog/id-map-query/id-map-query-format.d.ts +5 -4
  72. package/queries/catalog/lineage-query/lineage-query-format.d.ts +5 -4
  73. package/queries/catalog/normalized-ast-query/normalized-ast-query-format.d.ts +5 -4
  74. package/queries/catalog/search-query/search-query-format.d.ts +5 -4
  75. package/queries/catalog/static-slice-query/static-slice-query-format.d.ts +5 -4
  76. package/queries/query.d.ts +40 -32
  77. package/r-bridge/lang-4.x/ast/parser/json/parser.d.ts +4 -2
  78. package/r-bridge/lang-4.x/ast/parser/json/parser.js +5 -0
  79. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.d.ts +18 -0
  80. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.js +57 -0
  81. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.d.ts +3 -0
  82. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.js +541 -0
  83. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-types.d.ts +35 -0
  84. package/r-bridge/lang-4.x/tree-sitter/tree-sitter-types.js +40 -0
  85. package/r-bridge/parser.d.ts +32 -0
  86. package/r-bridge/parser.js +14 -0
  87. package/r-bridge/shell-executor.d.ts +37 -1
  88. package/r-bridge/shell-executor.js +39 -0
  89. package/r-bridge/shell.d.ts +12 -6
  90. package/r-bridge/shell.js +15 -6
  91. package/search/search-optimizer/search-optimizer.js +1 -1
  92. package/statistics/statistics.js +1 -1
  93. package/util/version.js +1 -1
@@ -0,0 +1,541 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.normalizeTreeSitterTreeToAst = normalizeTreeSitterTreeToAst;
4
+ const normalizer_data_1 = require("../ast/parser/main/normalizer-data");
5
+ const tree_sitter_types_1 = require("./tree-sitter-types");
6
+ const type_1 = require("../ast/model/type");
7
+ const retriever_1 = require("../../retriever");
8
+ const convert_values_1 = require("../convert-values");
9
+ const normalize_meta_1 = require("../ast/parser/main/normalize-meta");
10
+ const arrays_1 = require("../../../util/arrays");
11
+ const r_function_call_1 = require("../ast/model/nodes/r-function-call");
12
+ const strings_1 = require("../../../util/strings");
13
+ function normalizeTreeSitterTreeToAst(tree) {
14
+ const root = convertTreeNode(tree.rootNode);
15
+ if (root.type !== type_1.RType.ExpressionList) {
16
+ throw new normalizer_data_1.ParseError(`expected root to resolve to an expression list, got a ${root.type}`);
17
+ }
18
+ return root;
19
+ }
20
+ function convertTreeNode(node) {
21
+ // generally, the grammar source file dictates what children a node has in what order:
22
+ // https://github.com/r-lib/tree-sitter-r/blob/main/grammar.js
23
+ const range = makeSourceRange(node);
24
+ const defaultInfo = {
25
+ info: {
26
+ fullRange: range,
27
+ additionalTokens: [],
28
+ fullLexeme: node.text
29
+ }
30
+ };
31
+ switch (node.type) {
32
+ case tree_sitter_types_1.TreeSitterType.Program: {
33
+ const [comments, children] = splitComments(node.children);
34
+ const body = children.map(n => [n, convertTreeNode(n)]);
35
+ const remainingComments = linkCommentsToNextNodes(body, comments);
36
+ return {
37
+ type: type_1.RType.ExpressionList,
38
+ children: body.map(n => n[1]),
39
+ grouping: undefined,
40
+ lexeme: undefined,
41
+ info: {
42
+ additionalTokens: remainingComments.map(c => c[1])
43
+ }
44
+ };
45
+ }
46
+ case tree_sitter_types_1.TreeSitterType.BracedExpression:
47
+ case tree_sitter_types_1.TreeSitterType.ParenthesizedExpression: {
48
+ const [comments, children] = splitComments(node.children);
49
+ const opening = children[0];
50
+ const body = children.slice(1, -1).map(n => [n, convertTreeNode(n)]);
51
+ const remainingComments = linkCommentsToNextNodes(body, comments);
52
+ const closing = children[children.length - 1];
53
+ return {
54
+ type: type_1.RType.ExpressionList,
55
+ location: undefined,
56
+ lexeme: undefined,
57
+ children: body.map(n => n[1]),
58
+ grouping: [
59
+ {
60
+ type: type_1.RType.Symbol,
61
+ location: makeSourceRange(opening),
62
+ content: (0, retriever_1.removeRQuotes)(opening.text),
63
+ lexeme: opening.text,
64
+ namespace: undefined,
65
+ ...defaultInfo
66
+ }, {
67
+ type: type_1.RType.Symbol,
68
+ location: makeSourceRange(closing),
69
+ content: (0, retriever_1.removeRQuotes)(closing.text),
70
+ lexeme: closing.text,
71
+ namespace: undefined,
72
+ ...defaultInfo
73
+ }
74
+ ],
75
+ info: {
76
+ additionalTokens: remainingComments.map(c => c[1])
77
+ }
78
+ };
79
+ }
80
+ case tree_sitter_types_1.TreeSitterType.BinaryOperator: {
81
+ const lhs = convertTreeNode(node.children[0]);
82
+ const rhs = convertTreeNode(node.children[node.children.length - 1]);
83
+ const [commentsBoth, [op]] = splitComments(node.children.slice(1, -1));
84
+ const comments = commentsBoth.map(c => c[1]);
85
+ const opSource = makeSourceRange(op);
86
+ const lhsAsArg = {
87
+ type: type_1.RType.Argument,
88
+ location: lhs.location,
89
+ value: lhs,
90
+ name: undefined,
91
+ lexeme: lhs.lexeme,
92
+ info: {}
93
+ };
94
+ if (op.type == 'special') {
95
+ return {
96
+ type: type_1.RType.FunctionCall,
97
+ location: opSource,
98
+ lexeme: node.text,
99
+ functionName: {
100
+ type: type_1.RType.Symbol,
101
+ location: opSource,
102
+ lexeme: op.text,
103
+ content: op.text,
104
+ namespace: undefined,
105
+ info: {}
106
+ },
107
+ arguments: [lhsAsArg, {
108
+ type: type_1.RType.Argument,
109
+ location: rhs.location,
110
+ value: rhs,
111
+ name: undefined,
112
+ lexeme: rhs.lexeme,
113
+ info: {}
114
+ }],
115
+ named: true,
116
+ infixSpecial: true,
117
+ info: {
118
+ additionalTokens: comments
119
+ }
120
+ };
121
+ }
122
+ else if (op.text === '|>') {
123
+ return {
124
+ type: type_1.RType.Pipe,
125
+ location: opSource,
126
+ lhs: lhsAsArg,
127
+ rhs,
128
+ lexeme: op.text,
129
+ ...defaultInfo,
130
+ info: {
131
+ fullRange: range,
132
+ additionalTokens: comments,
133
+ fullLexeme: node.text
134
+ }
135
+ };
136
+ }
137
+ else {
138
+ return {
139
+ type: type_1.RType.BinaryOp,
140
+ location: opSource,
141
+ lhs, rhs,
142
+ operator: op.text,
143
+ lexeme: op.text,
144
+ info: {
145
+ fullRange: range,
146
+ additionalTokens: comments,
147
+ fullLexeme: node.text
148
+ }
149
+ };
150
+ }
151
+ }
152
+ case tree_sitter_types_1.TreeSitterType.UnaryOperator: {
153
+ const [op, operand] = node.children;
154
+ return {
155
+ type: type_1.RType.UnaryOp,
156
+ operand: convertTreeNode(operand),
157
+ location: makeSourceRange(op),
158
+ operator: op.text,
159
+ lexeme: op.text,
160
+ ...defaultInfo
161
+ };
162
+ }
163
+ case tree_sitter_types_1.TreeSitterType.NamespaceOperator: {
164
+ const [lhs, /* :: or ::: */ , rhs] = node.children;
165
+ return {
166
+ type: type_1.RType.Symbol,
167
+ location: makeSourceRange(rhs),
168
+ content: rhs.text,
169
+ lexeme: rhs.text,
170
+ namespace: lhs.text,
171
+ ...defaultInfo
172
+ };
173
+ }
174
+ case '(':
175
+ case ')':
176
+ case tree_sitter_types_1.TreeSitterType.Na:
177
+ case tree_sitter_types_1.TreeSitterType.Null:
178
+ case tree_sitter_types_1.TreeSitterType.Dots:
179
+ case tree_sitter_types_1.TreeSitterType.Identifier:
180
+ case tree_sitter_types_1.TreeSitterType.Return:
181
+ return {
182
+ type: type_1.RType.Symbol,
183
+ location: range,
184
+ content: (0, strings_1.startAndEndsWith)(node.text, '`') ? node.text.slice(1, -1) : (0, retriever_1.removeRQuotes)(node.text),
185
+ lexeme: node.text,
186
+ namespace: undefined,
187
+ ...defaultInfo
188
+ };
189
+ case tree_sitter_types_1.TreeSitterType.IfStatement: {
190
+ const [ifNode, /* ( */ , condition, /* ) */ , then, /* else */ , ...otherwise] = node.children;
191
+ return {
192
+ type: type_1.RType.IfThenElse,
193
+ condition: convertTreeNode(condition),
194
+ then: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(then)),
195
+ otherwise: otherwise.length > 0 ? (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(otherwise[0])) : undefined,
196
+ location: makeSourceRange(ifNode),
197
+ lexeme: ifNode.text,
198
+ ...defaultInfo
199
+ };
200
+ }
201
+ case tree_sitter_types_1.TreeSitterType.ForStatement: {
202
+ const forNode = node.children[0]; // we follow with a (
203
+ const variable = getNodesUntil(node.children, 'in', 2); // we follow with the "in"
204
+ const sequence = getNodesUntil(node.children, ')', 2 + variable.length + 1); // we follow with a (
205
+ const body = node.children[2 + variable.length + 1 + sequence.length + 1];
206
+ const [variableComments, [variableNode]] = splitComments(variable);
207
+ const [sequenceComments, [sequenceNode]] = splitComments(sequence);
208
+ return {
209
+ type: type_1.RType.ForLoop,
210
+ variable: {
211
+ type: type_1.RType.Symbol,
212
+ location: makeSourceRange(variableNode),
213
+ content: (0, retriever_1.removeRQuotes)(variableNode.text),
214
+ lexeme: variableNode.text,
215
+ namespace: undefined,
216
+ info: {
217
+ fullRange: undefined,
218
+ additionalTokens: [],
219
+ fullLexeme: undefined
220
+ }
221
+ },
222
+ vector: convertTreeNode(sequenceNode),
223
+ body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
224
+ location: makeSourceRange(forNode),
225
+ lexeme: forNode.text,
226
+ info: {
227
+ fullRange: range,
228
+ additionalTokens: [...variableComments, ...sequenceComments].map(c => c[1]),
229
+ fullLexeme: node.text
230
+ }
231
+ };
232
+ }
233
+ case tree_sitter_types_1.TreeSitterType.WhileStatement: {
234
+ const [whileNode, /* ( */ , condition, /* ) */ , body] = node.children;
235
+ return {
236
+ type: type_1.RType.WhileLoop,
237
+ condition: convertTreeNode(condition),
238
+ body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
239
+ location: makeSourceRange(whileNode),
240
+ lexeme: whileNode.text,
241
+ ...defaultInfo
242
+ };
243
+ }
244
+ case tree_sitter_types_1.TreeSitterType.RepeatStatement: {
245
+ const [repeatNode, body] = node.children;
246
+ return {
247
+ type: type_1.RType.RepeatLoop,
248
+ body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
249
+ location: makeSourceRange(repeatNode),
250
+ lexeme: repeatNode.text,
251
+ ...defaultInfo
252
+ };
253
+ }
254
+ case tree_sitter_types_1.TreeSitterType.Call: {
255
+ const [func, argsParentheses] = node.children;
256
+ // tree-sitter wraps next and break in a function call, but we don't, so unwrap
257
+ if (func.type === tree_sitter_types_1.TreeSitterType.Next || func.type == tree_sitter_types_1.TreeSitterType.Break) {
258
+ return {
259
+ ...convertTreeNode(func),
260
+ ...defaultInfo
261
+ };
262
+ }
263
+ const args = (0, arrays_1.splitArrayOn)(argsParentheses.children.slice(1, -1), x => x.type === 'comma');
264
+ const funcRange = makeSourceRange(func);
265
+ const call = {
266
+ arguments: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
267
+ location: funcRange,
268
+ lexeme: func.text,
269
+ ...defaultInfo
270
+ };
271
+ if (func.type === tree_sitter_types_1.TreeSitterType.Identifier || func.type === tree_sitter_types_1.TreeSitterType.String || func.type === tree_sitter_types_1.TreeSitterType.NamespaceOperator || func.type === tree_sitter_types_1.TreeSitterType.Return) {
272
+ let funcNode = convertTreeNode(func);
273
+ if (funcNode.type === type_1.RType.String) {
274
+ funcNode = {
275
+ ...funcNode,
276
+ type: type_1.RType.Symbol,
277
+ namespace: undefined,
278
+ content: (0, retriever_1.removeRQuotes)(func.text)
279
+ };
280
+ }
281
+ return {
282
+ ...call,
283
+ type: type_1.RType.FunctionCall,
284
+ functionName: {
285
+ ...funcNode,
286
+ info: {
287
+ fullRange: range,
288
+ additionalTokens: [],
289
+ fullLexeme: node.text
290
+ }
291
+ },
292
+ named: true
293
+ };
294
+ }
295
+ else {
296
+ return {
297
+ ...call,
298
+ type: type_1.RType.FunctionCall,
299
+ calledFunction: convertTreeNode(func),
300
+ named: undefined
301
+ };
302
+ }
303
+ }
304
+ case tree_sitter_types_1.TreeSitterType.FunctionDefinition: {
305
+ const [name, paramsParens, body] = node.children;
306
+ const params = (0, arrays_1.splitArrayOn)(paramsParens.children.slice(1, -1), x => x.type === 'comma');
307
+ return {
308
+ type: type_1.RType.FunctionDefinition,
309
+ parameters: params.map(n => convertTreeNode(n[0])),
310
+ body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
311
+ location: makeSourceRange(name),
312
+ lexeme: name.text,
313
+ ...defaultInfo
314
+ };
315
+ }
316
+ case tree_sitter_types_1.TreeSitterType.String:
317
+ return {
318
+ type: type_1.RType.String,
319
+ location: range,
320
+ content: (0, convert_values_1.string2ts)(node.text),
321
+ lexeme: node.text,
322
+ ...defaultInfo
323
+ };
324
+ case tree_sitter_types_1.TreeSitterType.Float:
325
+ case tree_sitter_types_1.TreeSitterType.Integer:
326
+ case tree_sitter_types_1.TreeSitterType.Complex:
327
+ case tree_sitter_types_1.TreeSitterType.Inf:
328
+ case tree_sitter_types_1.TreeSitterType.Nan:
329
+ return {
330
+ type: type_1.RType.Number,
331
+ location: range,
332
+ content: (0, convert_values_1.number2ts)(node.text),
333
+ lexeme: node.text,
334
+ ...defaultInfo
335
+ };
336
+ case tree_sitter_types_1.TreeSitterType.True:
337
+ case tree_sitter_types_1.TreeSitterType.False:
338
+ return {
339
+ type: type_1.RType.Logical,
340
+ location: range,
341
+ content: (0, convert_values_1.boolean2ts)(node.text),
342
+ lexeme: node.text,
343
+ ...defaultInfo
344
+ };
345
+ case tree_sitter_types_1.TreeSitterType.Break:
346
+ case tree_sitter_types_1.TreeSitterType.Next:
347
+ return {
348
+ type: node.type == tree_sitter_types_1.TreeSitterType.Break ? type_1.RType.Break : type_1.RType.Next,
349
+ location: range,
350
+ lexeme: node.text,
351
+ ...defaultInfo
352
+ };
353
+ case tree_sitter_types_1.TreeSitterType.Subset:
354
+ case tree_sitter_types_1.TreeSitterType.Subset2: {
355
+ // subset has children like a and [x]
356
+ const [func, content] = node.children;
357
+ // bracket is now [ or [[ and argsClosing is x] or x]]
358
+ const [bracket, ...argsClosing] = content.children;
359
+ const args = (0, arrays_1.splitArrayOn)(argsClosing.slice(0, -1), x => x.type === 'comma');
360
+ return {
361
+ type: type_1.RType.Access,
362
+ operator: bracket.text,
363
+ accessed: convertTreeNode(func),
364
+ access: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
365
+ location: makeSourceRange(bracket),
366
+ lexeme: bracket.text,
367
+ ...defaultInfo
368
+ };
369
+ }
370
+ case tree_sitter_types_1.TreeSitterType.ExtractOperator: {
371
+ const [lhs, operator, rhs] = node.children;
372
+ const rhsRange = makeSourceRange(rhs);
373
+ return {
374
+ type: type_1.RType.Access,
375
+ operator: operator.text,
376
+ accessed: convertTreeNode(lhs),
377
+ access: [{
378
+ type: type_1.RType.Argument,
379
+ name: undefined,
380
+ value: {
381
+ ...convertTreeNode(rhs),
382
+ ...defaultInfo
383
+ },
384
+ location: rhsRange,
385
+ lexeme: rhs.text,
386
+ info: {
387
+ fullRange: rhsRange,
388
+ additionalTokens: [],
389
+ fullLexeme: rhs.text
390
+ }
391
+ }],
392
+ location: makeSourceRange(operator),
393
+ lexeme: operator.text,
394
+ ...defaultInfo
395
+ };
396
+ }
397
+ case tree_sitter_types_1.TreeSitterType.Parameter: {
398
+ const name = node.children[0];
399
+ const nameRange = makeSourceRange(name);
400
+ let defaultValue = undefined;
401
+ if (node.children.length == 3) {
402
+ defaultValue = convertTreeNode(node.children[2]);
403
+ }
404
+ return {
405
+ type: type_1.RType.Parameter,
406
+ name: {
407
+ type: type_1.RType.Symbol,
408
+ location: nameRange,
409
+ namespace: undefined,
410
+ content: name.text,
411
+ lexeme: name.text,
412
+ info: {
413
+ fullRange: range,
414
+ additionalTokens: [],
415
+ fullLexeme: name.text
416
+ }
417
+ },
418
+ special: name.text === '...',
419
+ defaultValue,
420
+ location: nameRange,
421
+ lexeme: name.text,
422
+ info: {
423
+ fullRange: range,
424
+ additionalTokens: [],
425
+ fullLexeme: name.text
426
+ }
427
+ };
428
+ }
429
+ case tree_sitter_types_1.TreeSitterType.Argument: {
430
+ if (node.children.length == 1) {
431
+ const [arg] = node.children;
432
+ return {
433
+ type: type_1.RType.Argument,
434
+ name: undefined,
435
+ value: convertTreeNode(arg),
436
+ location: range,
437
+ lexeme: node.text,
438
+ ...defaultInfo
439
+ };
440
+ }
441
+ else {
442
+ const [nameNode, /* = */ , valueNode] = node.children;
443
+ let name = convertTreeNode(nameNode);
444
+ // unescape argument names
445
+ if (name.type === type_1.RType.String) {
446
+ name = {
447
+ ...name,
448
+ type: type_1.RType.Symbol,
449
+ content: name.content.str,
450
+ namespace: undefined
451
+ };
452
+ }
453
+ else if ((0, strings_1.startAndEndsWith)(name.content, '`')) {
454
+ name.content = name.content.slice(1, -1);
455
+ }
456
+ const nameRange = makeSourceRange(nameNode);
457
+ return {
458
+ type: type_1.RType.Argument,
459
+ name: name,
460
+ value: convertTreeNode(valueNode),
461
+ location: nameRange,
462
+ lexeme: nameNode.text,
463
+ info: {
464
+ fullRange: nameRange,
465
+ additionalTokens: [],
466
+ fullLexeme: nameNode.text
467
+ }
468
+ };
469
+ }
470
+ }
471
+ default:
472
+ throw new normalizer_data_1.ParseError(`unexpected node type ${node.type}`);
473
+ }
474
+ }
475
+ function makeSourceRange(node) {
476
+ return [
477
+ // tree-sitter is 0-based but we want 1-based
478
+ node.startPosition.row + 1, node.startPosition.column + 1,
479
+ // tree-sitter's end position is one off from ours, so we don't add 1 here
480
+ node.endPosition.row + 1, node.endPosition.column
481
+ ];
482
+ }
483
+ function splitComments(nodes) {
484
+ const comments = [];
485
+ const others = [];
486
+ for (const node of nodes) {
487
+ if (node.type === tree_sitter_types_1.TreeSitterType.Comment) {
488
+ comments.push([node, {
489
+ type: type_1.RType.Comment,
490
+ location: makeSourceRange(node),
491
+ content: node.text.slice(1),
492
+ lexeme: node.text,
493
+ info: {
494
+ additionalTokens: [],
495
+ fullLexeme: node.text
496
+ }
497
+ }]);
498
+ }
499
+ else {
500
+ others.push(node);
501
+ }
502
+ }
503
+ return [comments, others];
504
+ }
505
+ function linkCommentsToNextNodes(nodes, comments) {
506
+ const remain = [];
507
+ for (const [commentSyntaxNode, commentNode] of comments) {
508
+ let sibling;
509
+ if (commentSyntaxNode.previousSibling?.endIndex === commentSyntaxNode.startIndex) {
510
+ // if there is a sibling on the same line, we link the comment to that node
511
+ sibling = commentSyntaxNode.previousSibling;
512
+ }
513
+ else {
514
+ sibling = commentSyntaxNode.nextSibling;
515
+ while (sibling && sibling.type === tree_sitter_types_1.TreeSitterType.Comment) {
516
+ sibling = sibling.nextSibling;
517
+ }
518
+ }
519
+ // if there is no valid sibling, we just link the comment to the first node (see normalize-expressions.ts)
520
+ const [, node] = (sibling ? nodes.find(([s]) => s.equals(sibling)) : undefined) ?? nodes[0] ?? [];
521
+ if (node) {
522
+ node.info.additionalTokens ??= [];
523
+ node.info.additionalTokens.push(commentNode);
524
+ }
525
+ else {
526
+ remain.push([commentSyntaxNode, commentNode]);
527
+ }
528
+ }
529
+ return remain;
530
+ }
531
+ function getNodesUntil(nodes, type, startIndex = 0) {
532
+ const ret = [];
533
+ for (let i = startIndex; i < nodes.length; i++) {
534
+ if (nodes[i].type === type) {
535
+ break;
536
+ }
537
+ ret.push(nodes[i]);
538
+ }
539
+ return ret;
540
+ }
541
+ //# sourceMappingURL=tree-sitter-normalize.js.map
@@ -0,0 +1,35 @@
1
+ export declare enum TreeSitterType {
2
+ Program = "program",
3
+ BracedExpression = "braced_expression",
4
+ ParenthesizedExpression = "parenthesized_expression",
5
+ BinaryOperator = "binary_operator",
6
+ UnaryOperator = "unary_operator",
7
+ NamespaceOperator = "namespace_operator",
8
+ Identifier = "identifier",
9
+ IfStatement = "if_statement",
10
+ ForStatement = "for_statement",
11
+ WhileStatement = "while_statement",
12
+ RepeatStatement = "repeat_statement",
13
+ Call = "call",
14
+ Return = "return",
15
+ FunctionDefinition = "function_definition",
16
+ String = "string",
17
+ Float = "float",
18
+ Integer = "integer",
19
+ Complex = "complex",
20
+ Inf = "inf",
21
+ Nan = "nan",
22
+ Na = "na",
23
+ Null = "null",
24
+ True = "true",
25
+ False = "false",
26
+ Break = "break",
27
+ Next = "next",
28
+ Subset = "subset",
29
+ Subset2 = "subset2",
30
+ ExtractOperator = "extract_operator",
31
+ Parameter = "parameter",
32
+ Argument = "argument",
33
+ Dots = "dots",
34
+ Comment = "comment"
35
+ }
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.TreeSitterType = void 0;
4
+ var TreeSitterType;
5
+ (function (TreeSitterType) {
6
+ TreeSitterType["Program"] = "program";
7
+ TreeSitterType["BracedExpression"] = "braced_expression";
8
+ TreeSitterType["ParenthesizedExpression"] = "parenthesized_expression";
9
+ TreeSitterType["BinaryOperator"] = "binary_operator";
10
+ TreeSitterType["UnaryOperator"] = "unary_operator";
11
+ TreeSitterType["NamespaceOperator"] = "namespace_operator";
12
+ TreeSitterType["Identifier"] = "identifier";
13
+ TreeSitterType["IfStatement"] = "if_statement";
14
+ TreeSitterType["ForStatement"] = "for_statement";
15
+ TreeSitterType["WhileStatement"] = "while_statement";
16
+ TreeSitterType["RepeatStatement"] = "repeat_statement";
17
+ TreeSitterType["Call"] = "call";
18
+ TreeSitterType["Return"] = "return";
19
+ TreeSitterType["FunctionDefinition"] = "function_definition";
20
+ TreeSitterType["String"] = "string";
21
+ TreeSitterType["Float"] = "float";
22
+ TreeSitterType["Integer"] = "integer";
23
+ TreeSitterType["Complex"] = "complex";
24
+ TreeSitterType["Inf"] = "inf";
25
+ TreeSitterType["Nan"] = "nan";
26
+ TreeSitterType["Na"] = "na";
27
+ TreeSitterType["Null"] = "null";
28
+ TreeSitterType["True"] = "true";
29
+ TreeSitterType["False"] = "false";
30
+ TreeSitterType["Break"] = "break";
31
+ TreeSitterType["Next"] = "next";
32
+ TreeSitterType["Subset"] = "subset";
33
+ TreeSitterType["Subset2"] = "subset2";
34
+ TreeSitterType["ExtractOperator"] = "extract_operator";
35
+ TreeSitterType["Parameter"] = "parameter";
36
+ TreeSitterType["Argument"] = "argument";
37
+ TreeSitterType["Dots"] = "dots";
38
+ TreeSitterType["Comment"] = "comment";
39
+ })(TreeSitterType || (exports.TreeSitterType = TreeSitterType = {}));
40
+ //# sourceMappingURL=tree-sitter-types.js.map
@@ -0,0 +1,32 @@
1
+ import type { RParseRequest, RParseRequests } from './retriever';
2
+ import type { RShell } from './shell';
3
+ import type { RShellExecutor } from './shell-executor';
4
+ import type { TreeSitterExecutor } from './lang-4.x/tree-sitter/tree-sitter-executor';
5
+ interface ParserContent<T> {
6
+ readonly name: string;
7
+ rVersion(): Promise<string | 'unknown' | 'none'>;
8
+ parse(request: RParseRequest): T;
9
+ close(): void;
10
+ }
11
+ export type SyncParser<T> = ParserContent<Awaited<T>> & {
12
+ readonly async?: false;
13
+ };
14
+ export type AsyncParser<T> = ParserContent<Promise<T>> & {
15
+ readonly async: true;
16
+ };
17
+ export type Parser<T> = SyncParser<T> | AsyncParser<T>;
18
+ export type KnownParser = RShell | RShellExecutor | TreeSitterExecutor;
19
+ export type KnownParserType = Awaited<ReturnType<KnownParser['parse']>>;
20
+ export type KnownParserName = KnownParser['name'];
21
+ export interface ParseRequiredInput<T> {
22
+ /** This is the {@link RShell}, {@link RShellExecutor} or {@link TreeSitterExecutor} connection to be used to obtain the original parses AST of the R code */
23
+ readonly parser: Parser<T>;
24
+ /** The request which essentially indicates the input to extract the AST from */
25
+ readonly request: RParseRequests;
26
+ }
27
+ export interface ParseStepOutput<T> {
28
+ /** The parsed AST of the R code as given by the R parse side */
29
+ readonly parsed: T;
30
+ }
31
+ export declare function parseRequests<T extends KnownParserType>(_results: unknown, input: Partial<ParseRequiredInput<T>>): Promise<ParseStepOutput<T>>;
32
+ export {};
@@ -0,0 +1,14 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.parseRequests = parseRequests;
4
+ async function parseRequests(_results, input) {
5
+ /* in the future, we want to expose all cases */
6
+ const request = (Array.isArray(input.request) ? input.request[0] : input.request);
7
+ if (input.parser?.async) {
8
+ return { parsed: await input.parser.parse(request) };
9
+ }
10
+ else {
11
+ return { parsed: input.parser.parse(request) };
12
+ }
13
+ }
14
+ //# sourceMappingURL=parser.js.map