@eagleoutice/flowr 2.1.12 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/benchmark/slicer.d.ts +5 -12
- package/benchmark/slicer.js +46 -28
- package/cli/benchmark-app.d.ts +2 -0
- package/cli/benchmark-app.js +2 -1
- package/cli/benchmark-helper-app.d.ts +2 -0
- package/cli/benchmark-helper-app.js +2 -2
- package/cli/common/options.js +3 -1
- package/cli/flowr-main-options.js +36 -2
- package/cli/flowr.d.ts +6 -0
- package/cli/flowr.js +51 -24
- package/cli/repl/commands/repl-cfg.js +2 -4
- package/cli/repl/commands/repl-dataflow.js +2 -4
- package/cli/repl/commands/repl-execute.d.ts +2 -2
- package/cli/repl/commands/repl-execute.js +15 -5
- package/cli/repl/commands/repl-lineage.js +2 -4
- package/cli/repl/commands/repl-main.d.ts +2 -2
- package/cli/repl/commands/repl-normalize.js +2 -4
- package/cli/repl/commands/repl-parse.js +2 -4
- package/cli/repl/commands/repl-query.js +6 -8
- package/cli/repl/commands/repl-version.d.ts +5 -4
- package/cli/repl/commands/repl-version.js +10 -9
- package/cli/repl/core.d.ts +5 -5
- package/cli/repl/core.js +8 -12
- package/cli/repl/print-version.d.ts +2 -2
- package/cli/repl/print-version.js +3 -3
- package/cli/repl/server/connection.d.ts +3 -3
- package/cli/repl/server/connection.js +5 -7
- package/cli/repl/server/messages/message-hello.js +2 -1
- package/cli/repl/server/server.d.ts +4 -3
- package/cli/repl/server/server.js +7 -5
- package/cli/slicer-app.js +1 -1
- package/config.d.ts +36 -4
- package/config.js +30 -1
- package/core/pipeline-executor.d.ts +1 -1
- package/core/pipeline-executor.js +1 -1
- package/core/steps/all/core/00-parse.d.ts +4 -18
- package/core/steps/all/core/00-parse.js +2 -11
- package/core/steps/all/core/01-parse-tree-sitter.d.ts +23 -0
- package/core/steps/all/core/01-parse-tree-sitter.js +19 -0
- package/core/steps/all/core/10-normalize.d.ts +3 -2
- package/core/steps/all/core/10-normalize.js +1 -0
- package/core/steps/all/core/11-normalize-tree-sitter.d.ts +25 -0
- package/core/steps/all/core/11-normalize-tree-sitter.js +27 -0
- package/core/steps/all/core/20-dataflow.d.ts +2 -0
- package/core/steps/all/core/20-dataflow.js +1 -1
- package/core/steps/pipeline/default-pipelines.d.ts +368 -23
- package/core/steps/pipeline/default-pipelines.js +42 -4
- package/dataflow/extractor.d.ts +2 -1
- package/dataflow/extractor.js +2 -1
- package/dataflow/internal/process/functions/call/built-in/built-in-source.js +6 -5
- package/dataflow/processor.d.ts +2 -0
- package/documentation/doc-util/doc-auto-gen.js +2 -1
- package/documentation/doc-util/doc-cfg.js +1 -1
- package/documentation/doc-util/doc-dfg.js +2 -2
- package/documentation/doc-util/doc-files.d.ts +1 -0
- package/documentation/doc-util/doc-files.js +4 -0
- package/documentation/doc-util/doc-normalized-ast.js +2 -3
- package/documentation/doc-util/doc-query.js +1 -1
- package/documentation/doc-util/doc-search.js +1 -1
- package/documentation/doc-util/doc-types.js +2 -2
- package/documentation/print-dataflow-graph-wiki.js +15 -15
- package/documentation/print-engines-wiki.d.ts +1 -0
- package/documentation/print-engines-wiki.js +82 -0
- package/documentation/print-interface-wiki.js +6 -7
- package/documentation/print-normalized-ast-wiki.js +1 -1
- package/package.json +9 -5
- package/queries/catalog/cluster-query/cluster-query-format.d.ts +5 -4
- package/queries/catalog/dataflow-query/dataflow-query-format.d.ts +5 -4
- package/queries/catalog/dependencies-query/dependencies-query-format.d.ts +5 -4
- package/queries/catalog/id-map-query/id-map-query-format.d.ts +5 -4
- package/queries/catalog/lineage-query/lineage-query-format.d.ts +5 -4
- package/queries/catalog/normalized-ast-query/normalized-ast-query-format.d.ts +5 -4
- package/queries/catalog/search-query/search-query-format.d.ts +5 -4
- package/queries/catalog/static-slice-query/static-slice-query-format.d.ts +5 -4
- package/queries/query.d.ts +40 -32
- package/r-bridge/lang-4.x/ast/parser/json/parser.d.ts +4 -2
- package/r-bridge/lang-4.x/ast/parser/json/parser.js +5 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.d.ts +18 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.js +57 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.d.ts +3 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-normalize.js +541 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-types.d.ts +35 -0
- package/r-bridge/lang-4.x/tree-sitter/tree-sitter-types.js +40 -0
- package/r-bridge/parser.d.ts +32 -0
- package/r-bridge/parser.js +14 -0
- package/r-bridge/shell-executor.d.ts +37 -1
- package/r-bridge/shell-executor.js +39 -0
- package/r-bridge/shell.d.ts +12 -6
- package/r-bridge/shell.js +15 -6
- package/search/search-optimizer/search-optimizer.js +1 -1
- package/statistics/statistics.js +1 -1
- package/util/version.js +1 -1
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.normalizeTreeSitterTreeToAst = normalizeTreeSitterTreeToAst;
|
|
4
|
+
const normalizer_data_1 = require("../ast/parser/main/normalizer-data");
|
|
5
|
+
const tree_sitter_types_1 = require("./tree-sitter-types");
|
|
6
|
+
const type_1 = require("../ast/model/type");
|
|
7
|
+
const retriever_1 = require("../../retriever");
|
|
8
|
+
const convert_values_1 = require("../convert-values");
|
|
9
|
+
const normalize_meta_1 = require("../ast/parser/main/normalize-meta");
|
|
10
|
+
const arrays_1 = require("../../../util/arrays");
|
|
11
|
+
const r_function_call_1 = require("../ast/model/nodes/r-function-call");
|
|
12
|
+
const strings_1 = require("../../../util/strings");
|
|
13
|
+
function normalizeTreeSitterTreeToAst(tree) {
|
|
14
|
+
const root = convertTreeNode(tree.rootNode);
|
|
15
|
+
if (root.type !== type_1.RType.ExpressionList) {
|
|
16
|
+
throw new normalizer_data_1.ParseError(`expected root to resolve to an expression list, got a ${root.type}`);
|
|
17
|
+
}
|
|
18
|
+
return root;
|
|
19
|
+
}
|
|
20
|
+
function convertTreeNode(node) {
|
|
21
|
+
// generally, the grammar source file dictates what children a node has in what order:
|
|
22
|
+
// https://github.com/r-lib/tree-sitter-r/blob/main/grammar.js
|
|
23
|
+
const range = makeSourceRange(node);
|
|
24
|
+
const defaultInfo = {
|
|
25
|
+
info: {
|
|
26
|
+
fullRange: range,
|
|
27
|
+
additionalTokens: [],
|
|
28
|
+
fullLexeme: node.text
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
switch (node.type) {
|
|
32
|
+
case tree_sitter_types_1.TreeSitterType.Program: {
|
|
33
|
+
const [comments, children] = splitComments(node.children);
|
|
34
|
+
const body = children.map(n => [n, convertTreeNode(n)]);
|
|
35
|
+
const remainingComments = linkCommentsToNextNodes(body, comments);
|
|
36
|
+
return {
|
|
37
|
+
type: type_1.RType.ExpressionList,
|
|
38
|
+
children: body.map(n => n[1]),
|
|
39
|
+
grouping: undefined,
|
|
40
|
+
lexeme: undefined,
|
|
41
|
+
info: {
|
|
42
|
+
additionalTokens: remainingComments.map(c => c[1])
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
case tree_sitter_types_1.TreeSitterType.BracedExpression:
|
|
47
|
+
case tree_sitter_types_1.TreeSitterType.ParenthesizedExpression: {
|
|
48
|
+
const [comments, children] = splitComments(node.children);
|
|
49
|
+
const opening = children[0];
|
|
50
|
+
const body = children.slice(1, -1).map(n => [n, convertTreeNode(n)]);
|
|
51
|
+
const remainingComments = linkCommentsToNextNodes(body, comments);
|
|
52
|
+
const closing = children[children.length - 1];
|
|
53
|
+
return {
|
|
54
|
+
type: type_1.RType.ExpressionList,
|
|
55
|
+
location: undefined,
|
|
56
|
+
lexeme: undefined,
|
|
57
|
+
children: body.map(n => n[1]),
|
|
58
|
+
grouping: [
|
|
59
|
+
{
|
|
60
|
+
type: type_1.RType.Symbol,
|
|
61
|
+
location: makeSourceRange(opening),
|
|
62
|
+
content: (0, retriever_1.removeRQuotes)(opening.text),
|
|
63
|
+
lexeme: opening.text,
|
|
64
|
+
namespace: undefined,
|
|
65
|
+
...defaultInfo
|
|
66
|
+
}, {
|
|
67
|
+
type: type_1.RType.Symbol,
|
|
68
|
+
location: makeSourceRange(closing),
|
|
69
|
+
content: (0, retriever_1.removeRQuotes)(closing.text),
|
|
70
|
+
lexeme: closing.text,
|
|
71
|
+
namespace: undefined,
|
|
72
|
+
...defaultInfo
|
|
73
|
+
}
|
|
74
|
+
],
|
|
75
|
+
info: {
|
|
76
|
+
additionalTokens: remainingComments.map(c => c[1])
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
case tree_sitter_types_1.TreeSitterType.BinaryOperator: {
|
|
81
|
+
const lhs = convertTreeNode(node.children[0]);
|
|
82
|
+
const rhs = convertTreeNode(node.children[node.children.length - 1]);
|
|
83
|
+
const [commentsBoth, [op]] = splitComments(node.children.slice(1, -1));
|
|
84
|
+
const comments = commentsBoth.map(c => c[1]);
|
|
85
|
+
const opSource = makeSourceRange(op);
|
|
86
|
+
const lhsAsArg = {
|
|
87
|
+
type: type_1.RType.Argument,
|
|
88
|
+
location: lhs.location,
|
|
89
|
+
value: lhs,
|
|
90
|
+
name: undefined,
|
|
91
|
+
lexeme: lhs.lexeme,
|
|
92
|
+
info: {}
|
|
93
|
+
};
|
|
94
|
+
if (op.type == 'special') {
|
|
95
|
+
return {
|
|
96
|
+
type: type_1.RType.FunctionCall,
|
|
97
|
+
location: opSource,
|
|
98
|
+
lexeme: node.text,
|
|
99
|
+
functionName: {
|
|
100
|
+
type: type_1.RType.Symbol,
|
|
101
|
+
location: opSource,
|
|
102
|
+
lexeme: op.text,
|
|
103
|
+
content: op.text,
|
|
104
|
+
namespace: undefined,
|
|
105
|
+
info: {}
|
|
106
|
+
},
|
|
107
|
+
arguments: [lhsAsArg, {
|
|
108
|
+
type: type_1.RType.Argument,
|
|
109
|
+
location: rhs.location,
|
|
110
|
+
value: rhs,
|
|
111
|
+
name: undefined,
|
|
112
|
+
lexeme: rhs.lexeme,
|
|
113
|
+
info: {}
|
|
114
|
+
}],
|
|
115
|
+
named: true,
|
|
116
|
+
infixSpecial: true,
|
|
117
|
+
info: {
|
|
118
|
+
additionalTokens: comments
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
else if (op.text === '|>') {
|
|
123
|
+
return {
|
|
124
|
+
type: type_1.RType.Pipe,
|
|
125
|
+
location: opSource,
|
|
126
|
+
lhs: lhsAsArg,
|
|
127
|
+
rhs,
|
|
128
|
+
lexeme: op.text,
|
|
129
|
+
...defaultInfo,
|
|
130
|
+
info: {
|
|
131
|
+
fullRange: range,
|
|
132
|
+
additionalTokens: comments,
|
|
133
|
+
fullLexeme: node.text
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
return {
|
|
139
|
+
type: type_1.RType.BinaryOp,
|
|
140
|
+
location: opSource,
|
|
141
|
+
lhs, rhs,
|
|
142
|
+
operator: op.text,
|
|
143
|
+
lexeme: op.text,
|
|
144
|
+
info: {
|
|
145
|
+
fullRange: range,
|
|
146
|
+
additionalTokens: comments,
|
|
147
|
+
fullLexeme: node.text
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
case tree_sitter_types_1.TreeSitterType.UnaryOperator: {
|
|
153
|
+
const [op, operand] = node.children;
|
|
154
|
+
return {
|
|
155
|
+
type: type_1.RType.UnaryOp,
|
|
156
|
+
operand: convertTreeNode(operand),
|
|
157
|
+
location: makeSourceRange(op),
|
|
158
|
+
operator: op.text,
|
|
159
|
+
lexeme: op.text,
|
|
160
|
+
...defaultInfo
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
case tree_sitter_types_1.TreeSitterType.NamespaceOperator: {
|
|
164
|
+
const [lhs, /* :: or ::: */ , rhs] = node.children;
|
|
165
|
+
return {
|
|
166
|
+
type: type_1.RType.Symbol,
|
|
167
|
+
location: makeSourceRange(rhs),
|
|
168
|
+
content: rhs.text,
|
|
169
|
+
lexeme: rhs.text,
|
|
170
|
+
namespace: lhs.text,
|
|
171
|
+
...defaultInfo
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
case '(':
|
|
175
|
+
case ')':
|
|
176
|
+
case tree_sitter_types_1.TreeSitterType.Na:
|
|
177
|
+
case tree_sitter_types_1.TreeSitterType.Null:
|
|
178
|
+
case tree_sitter_types_1.TreeSitterType.Dots:
|
|
179
|
+
case tree_sitter_types_1.TreeSitterType.Identifier:
|
|
180
|
+
case tree_sitter_types_1.TreeSitterType.Return:
|
|
181
|
+
return {
|
|
182
|
+
type: type_1.RType.Symbol,
|
|
183
|
+
location: range,
|
|
184
|
+
content: (0, strings_1.startAndEndsWith)(node.text, '`') ? node.text.slice(1, -1) : (0, retriever_1.removeRQuotes)(node.text),
|
|
185
|
+
lexeme: node.text,
|
|
186
|
+
namespace: undefined,
|
|
187
|
+
...defaultInfo
|
|
188
|
+
};
|
|
189
|
+
case tree_sitter_types_1.TreeSitterType.IfStatement: {
|
|
190
|
+
const [ifNode, /* ( */ , condition, /* ) */ , then, /* else */ , ...otherwise] = node.children;
|
|
191
|
+
return {
|
|
192
|
+
type: type_1.RType.IfThenElse,
|
|
193
|
+
condition: convertTreeNode(condition),
|
|
194
|
+
then: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(then)),
|
|
195
|
+
otherwise: otherwise.length > 0 ? (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(otherwise[0])) : undefined,
|
|
196
|
+
location: makeSourceRange(ifNode),
|
|
197
|
+
lexeme: ifNode.text,
|
|
198
|
+
...defaultInfo
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
case tree_sitter_types_1.TreeSitterType.ForStatement: {
|
|
202
|
+
const forNode = node.children[0]; // we follow with a (
|
|
203
|
+
const variable = getNodesUntil(node.children, 'in', 2); // we follow with the "in"
|
|
204
|
+
const sequence = getNodesUntil(node.children, ')', 2 + variable.length + 1); // we follow with a (
|
|
205
|
+
const body = node.children[2 + variable.length + 1 + sequence.length + 1];
|
|
206
|
+
const [variableComments, [variableNode]] = splitComments(variable);
|
|
207
|
+
const [sequenceComments, [sequenceNode]] = splitComments(sequence);
|
|
208
|
+
return {
|
|
209
|
+
type: type_1.RType.ForLoop,
|
|
210
|
+
variable: {
|
|
211
|
+
type: type_1.RType.Symbol,
|
|
212
|
+
location: makeSourceRange(variableNode),
|
|
213
|
+
content: (0, retriever_1.removeRQuotes)(variableNode.text),
|
|
214
|
+
lexeme: variableNode.text,
|
|
215
|
+
namespace: undefined,
|
|
216
|
+
info: {
|
|
217
|
+
fullRange: undefined,
|
|
218
|
+
additionalTokens: [],
|
|
219
|
+
fullLexeme: undefined
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
vector: convertTreeNode(sequenceNode),
|
|
223
|
+
body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
|
|
224
|
+
location: makeSourceRange(forNode),
|
|
225
|
+
lexeme: forNode.text,
|
|
226
|
+
info: {
|
|
227
|
+
fullRange: range,
|
|
228
|
+
additionalTokens: [...variableComments, ...sequenceComments].map(c => c[1]),
|
|
229
|
+
fullLexeme: node.text
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
case tree_sitter_types_1.TreeSitterType.WhileStatement: {
|
|
234
|
+
const [whileNode, /* ( */ , condition, /* ) */ , body] = node.children;
|
|
235
|
+
return {
|
|
236
|
+
type: type_1.RType.WhileLoop,
|
|
237
|
+
condition: convertTreeNode(condition),
|
|
238
|
+
body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
|
|
239
|
+
location: makeSourceRange(whileNode),
|
|
240
|
+
lexeme: whileNode.text,
|
|
241
|
+
...defaultInfo
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
case tree_sitter_types_1.TreeSitterType.RepeatStatement: {
|
|
245
|
+
const [repeatNode, body] = node.children;
|
|
246
|
+
return {
|
|
247
|
+
type: type_1.RType.RepeatLoop,
|
|
248
|
+
body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
|
|
249
|
+
location: makeSourceRange(repeatNode),
|
|
250
|
+
lexeme: repeatNode.text,
|
|
251
|
+
...defaultInfo
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
case tree_sitter_types_1.TreeSitterType.Call: {
|
|
255
|
+
const [func, argsParentheses] = node.children;
|
|
256
|
+
// tree-sitter wraps next and break in a function call, but we don't, so unwrap
|
|
257
|
+
if (func.type === tree_sitter_types_1.TreeSitterType.Next || func.type == tree_sitter_types_1.TreeSitterType.Break) {
|
|
258
|
+
return {
|
|
259
|
+
...convertTreeNode(func),
|
|
260
|
+
...defaultInfo
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
const args = (0, arrays_1.splitArrayOn)(argsParentheses.children.slice(1, -1), x => x.type === 'comma');
|
|
264
|
+
const funcRange = makeSourceRange(func);
|
|
265
|
+
const call = {
|
|
266
|
+
arguments: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
|
|
267
|
+
location: funcRange,
|
|
268
|
+
lexeme: func.text,
|
|
269
|
+
...defaultInfo
|
|
270
|
+
};
|
|
271
|
+
if (func.type === tree_sitter_types_1.TreeSitterType.Identifier || func.type === tree_sitter_types_1.TreeSitterType.String || func.type === tree_sitter_types_1.TreeSitterType.NamespaceOperator || func.type === tree_sitter_types_1.TreeSitterType.Return) {
|
|
272
|
+
let funcNode = convertTreeNode(func);
|
|
273
|
+
if (funcNode.type === type_1.RType.String) {
|
|
274
|
+
funcNode = {
|
|
275
|
+
...funcNode,
|
|
276
|
+
type: type_1.RType.Symbol,
|
|
277
|
+
namespace: undefined,
|
|
278
|
+
content: (0, retriever_1.removeRQuotes)(func.text)
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
...call,
|
|
283
|
+
type: type_1.RType.FunctionCall,
|
|
284
|
+
functionName: {
|
|
285
|
+
...funcNode,
|
|
286
|
+
info: {
|
|
287
|
+
fullRange: range,
|
|
288
|
+
additionalTokens: [],
|
|
289
|
+
fullLexeme: node.text
|
|
290
|
+
}
|
|
291
|
+
},
|
|
292
|
+
named: true
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
return {
|
|
297
|
+
...call,
|
|
298
|
+
type: type_1.RType.FunctionCall,
|
|
299
|
+
calledFunction: convertTreeNode(func),
|
|
300
|
+
named: undefined
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
case tree_sitter_types_1.TreeSitterType.FunctionDefinition: {
|
|
305
|
+
const [name, paramsParens, body] = node.children;
|
|
306
|
+
const params = (0, arrays_1.splitArrayOn)(paramsParens.children.slice(1, -1), x => x.type === 'comma');
|
|
307
|
+
return {
|
|
308
|
+
type: type_1.RType.FunctionDefinition,
|
|
309
|
+
parameters: params.map(n => convertTreeNode(n[0])),
|
|
310
|
+
body: (0, normalize_meta_1.ensureExpressionList)(convertTreeNode(body)),
|
|
311
|
+
location: makeSourceRange(name),
|
|
312
|
+
lexeme: name.text,
|
|
313
|
+
...defaultInfo
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
case tree_sitter_types_1.TreeSitterType.String:
|
|
317
|
+
return {
|
|
318
|
+
type: type_1.RType.String,
|
|
319
|
+
location: range,
|
|
320
|
+
content: (0, convert_values_1.string2ts)(node.text),
|
|
321
|
+
lexeme: node.text,
|
|
322
|
+
...defaultInfo
|
|
323
|
+
};
|
|
324
|
+
case tree_sitter_types_1.TreeSitterType.Float:
|
|
325
|
+
case tree_sitter_types_1.TreeSitterType.Integer:
|
|
326
|
+
case tree_sitter_types_1.TreeSitterType.Complex:
|
|
327
|
+
case tree_sitter_types_1.TreeSitterType.Inf:
|
|
328
|
+
case tree_sitter_types_1.TreeSitterType.Nan:
|
|
329
|
+
return {
|
|
330
|
+
type: type_1.RType.Number,
|
|
331
|
+
location: range,
|
|
332
|
+
content: (0, convert_values_1.number2ts)(node.text),
|
|
333
|
+
lexeme: node.text,
|
|
334
|
+
...defaultInfo
|
|
335
|
+
};
|
|
336
|
+
case tree_sitter_types_1.TreeSitterType.True:
|
|
337
|
+
case tree_sitter_types_1.TreeSitterType.False:
|
|
338
|
+
return {
|
|
339
|
+
type: type_1.RType.Logical,
|
|
340
|
+
location: range,
|
|
341
|
+
content: (0, convert_values_1.boolean2ts)(node.text),
|
|
342
|
+
lexeme: node.text,
|
|
343
|
+
...defaultInfo
|
|
344
|
+
};
|
|
345
|
+
case tree_sitter_types_1.TreeSitterType.Break:
|
|
346
|
+
case tree_sitter_types_1.TreeSitterType.Next:
|
|
347
|
+
return {
|
|
348
|
+
type: node.type == tree_sitter_types_1.TreeSitterType.Break ? type_1.RType.Break : type_1.RType.Next,
|
|
349
|
+
location: range,
|
|
350
|
+
lexeme: node.text,
|
|
351
|
+
...defaultInfo
|
|
352
|
+
};
|
|
353
|
+
case tree_sitter_types_1.TreeSitterType.Subset:
|
|
354
|
+
case tree_sitter_types_1.TreeSitterType.Subset2: {
|
|
355
|
+
// subset has children like a and [x]
|
|
356
|
+
const [func, content] = node.children;
|
|
357
|
+
// bracket is now [ or [[ and argsClosing is x] or x]]
|
|
358
|
+
const [bracket, ...argsClosing] = content.children;
|
|
359
|
+
const args = (0, arrays_1.splitArrayOn)(argsClosing.slice(0, -1), x => x.type === 'comma');
|
|
360
|
+
return {
|
|
361
|
+
type: type_1.RType.Access,
|
|
362
|
+
operator: bracket.text,
|
|
363
|
+
accessed: convertTreeNode(func),
|
|
364
|
+
access: args.map(n => n.length == 0 ? r_function_call_1.EmptyArgument : convertTreeNode(n[0])),
|
|
365
|
+
location: makeSourceRange(bracket),
|
|
366
|
+
lexeme: bracket.text,
|
|
367
|
+
...defaultInfo
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
case tree_sitter_types_1.TreeSitterType.ExtractOperator: {
|
|
371
|
+
const [lhs, operator, rhs] = node.children;
|
|
372
|
+
const rhsRange = makeSourceRange(rhs);
|
|
373
|
+
return {
|
|
374
|
+
type: type_1.RType.Access,
|
|
375
|
+
operator: operator.text,
|
|
376
|
+
accessed: convertTreeNode(lhs),
|
|
377
|
+
access: [{
|
|
378
|
+
type: type_1.RType.Argument,
|
|
379
|
+
name: undefined,
|
|
380
|
+
value: {
|
|
381
|
+
...convertTreeNode(rhs),
|
|
382
|
+
...defaultInfo
|
|
383
|
+
},
|
|
384
|
+
location: rhsRange,
|
|
385
|
+
lexeme: rhs.text,
|
|
386
|
+
info: {
|
|
387
|
+
fullRange: rhsRange,
|
|
388
|
+
additionalTokens: [],
|
|
389
|
+
fullLexeme: rhs.text
|
|
390
|
+
}
|
|
391
|
+
}],
|
|
392
|
+
location: makeSourceRange(operator),
|
|
393
|
+
lexeme: operator.text,
|
|
394
|
+
...defaultInfo
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
case tree_sitter_types_1.TreeSitterType.Parameter: {
|
|
398
|
+
const name = node.children[0];
|
|
399
|
+
const nameRange = makeSourceRange(name);
|
|
400
|
+
let defaultValue = undefined;
|
|
401
|
+
if (node.children.length == 3) {
|
|
402
|
+
defaultValue = convertTreeNode(node.children[2]);
|
|
403
|
+
}
|
|
404
|
+
return {
|
|
405
|
+
type: type_1.RType.Parameter,
|
|
406
|
+
name: {
|
|
407
|
+
type: type_1.RType.Symbol,
|
|
408
|
+
location: nameRange,
|
|
409
|
+
namespace: undefined,
|
|
410
|
+
content: name.text,
|
|
411
|
+
lexeme: name.text,
|
|
412
|
+
info: {
|
|
413
|
+
fullRange: range,
|
|
414
|
+
additionalTokens: [],
|
|
415
|
+
fullLexeme: name.text
|
|
416
|
+
}
|
|
417
|
+
},
|
|
418
|
+
special: name.text === '...',
|
|
419
|
+
defaultValue,
|
|
420
|
+
location: nameRange,
|
|
421
|
+
lexeme: name.text,
|
|
422
|
+
info: {
|
|
423
|
+
fullRange: range,
|
|
424
|
+
additionalTokens: [],
|
|
425
|
+
fullLexeme: name.text
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
case tree_sitter_types_1.TreeSitterType.Argument: {
|
|
430
|
+
if (node.children.length == 1) {
|
|
431
|
+
const [arg] = node.children;
|
|
432
|
+
return {
|
|
433
|
+
type: type_1.RType.Argument,
|
|
434
|
+
name: undefined,
|
|
435
|
+
value: convertTreeNode(arg),
|
|
436
|
+
location: range,
|
|
437
|
+
lexeme: node.text,
|
|
438
|
+
...defaultInfo
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
const [nameNode, /* = */ , valueNode] = node.children;
|
|
443
|
+
let name = convertTreeNode(nameNode);
|
|
444
|
+
// unescape argument names
|
|
445
|
+
if (name.type === type_1.RType.String) {
|
|
446
|
+
name = {
|
|
447
|
+
...name,
|
|
448
|
+
type: type_1.RType.Symbol,
|
|
449
|
+
content: name.content.str,
|
|
450
|
+
namespace: undefined
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
else if ((0, strings_1.startAndEndsWith)(name.content, '`')) {
|
|
454
|
+
name.content = name.content.slice(1, -1);
|
|
455
|
+
}
|
|
456
|
+
const nameRange = makeSourceRange(nameNode);
|
|
457
|
+
return {
|
|
458
|
+
type: type_1.RType.Argument,
|
|
459
|
+
name: name,
|
|
460
|
+
value: convertTreeNode(valueNode),
|
|
461
|
+
location: nameRange,
|
|
462
|
+
lexeme: nameNode.text,
|
|
463
|
+
info: {
|
|
464
|
+
fullRange: nameRange,
|
|
465
|
+
additionalTokens: [],
|
|
466
|
+
fullLexeme: nameNode.text
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
default:
|
|
472
|
+
throw new normalizer_data_1.ParseError(`unexpected node type ${node.type}`);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
function makeSourceRange(node) {
|
|
476
|
+
return [
|
|
477
|
+
// tree-sitter is 0-based but we want 1-based
|
|
478
|
+
node.startPosition.row + 1, node.startPosition.column + 1,
|
|
479
|
+
// tree-sitter's end position is one off from ours, so we don't add 1 here
|
|
480
|
+
node.endPosition.row + 1, node.endPosition.column
|
|
481
|
+
];
|
|
482
|
+
}
|
|
483
|
+
function splitComments(nodes) {
|
|
484
|
+
const comments = [];
|
|
485
|
+
const others = [];
|
|
486
|
+
for (const node of nodes) {
|
|
487
|
+
if (node.type === tree_sitter_types_1.TreeSitterType.Comment) {
|
|
488
|
+
comments.push([node, {
|
|
489
|
+
type: type_1.RType.Comment,
|
|
490
|
+
location: makeSourceRange(node),
|
|
491
|
+
content: node.text.slice(1),
|
|
492
|
+
lexeme: node.text,
|
|
493
|
+
info: {
|
|
494
|
+
additionalTokens: [],
|
|
495
|
+
fullLexeme: node.text
|
|
496
|
+
}
|
|
497
|
+
}]);
|
|
498
|
+
}
|
|
499
|
+
else {
|
|
500
|
+
others.push(node);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
return [comments, others];
|
|
504
|
+
}
|
|
505
|
+
function linkCommentsToNextNodes(nodes, comments) {
|
|
506
|
+
const remain = [];
|
|
507
|
+
for (const [commentSyntaxNode, commentNode] of comments) {
|
|
508
|
+
let sibling;
|
|
509
|
+
if (commentSyntaxNode.previousSibling?.endIndex === commentSyntaxNode.startIndex) {
|
|
510
|
+
// if there is a sibling on the same line, we link the comment to that node
|
|
511
|
+
sibling = commentSyntaxNode.previousSibling;
|
|
512
|
+
}
|
|
513
|
+
else {
|
|
514
|
+
sibling = commentSyntaxNode.nextSibling;
|
|
515
|
+
while (sibling && sibling.type === tree_sitter_types_1.TreeSitterType.Comment) {
|
|
516
|
+
sibling = sibling.nextSibling;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
// if there is no valid sibling, we just link the comment to the first node (see normalize-expressions.ts)
|
|
520
|
+
const [, node] = (sibling ? nodes.find(([s]) => s.equals(sibling)) : undefined) ?? nodes[0] ?? [];
|
|
521
|
+
if (node) {
|
|
522
|
+
node.info.additionalTokens ??= [];
|
|
523
|
+
node.info.additionalTokens.push(commentNode);
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
remain.push([commentSyntaxNode, commentNode]);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
return remain;
|
|
530
|
+
}
|
|
531
|
+
function getNodesUntil(nodes, type, startIndex = 0) {
|
|
532
|
+
const ret = [];
|
|
533
|
+
for (let i = startIndex; i < nodes.length; i++) {
|
|
534
|
+
if (nodes[i].type === type) {
|
|
535
|
+
break;
|
|
536
|
+
}
|
|
537
|
+
ret.push(nodes[i]);
|
|
538
|
+
}
|
|
539
|
+
return ret;
|
|
540
|
+
}
|
|
541
|
+
//# sourceMappingURL=tree-sitter-normalize.js.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export declare enum TreeSitterType {
|
|
2
|
+
Program = "program",
|
|
3
|
+
BracedExpression = "braced_expression",
|
|
4
|
+
ParenthesizedExpression = "parenthesized_expression",
|
|
5
|
+
BinaryOperator = "binary_operator",
|
|
6
|
+
UnaryOperator = "unary_operator",
|
|
7
|
+
NamespaceOperator = "namespace_operator",
|
|
8
|
+
Identifier = "identifier",
|
|
9
|
+
IfStatement = "if_statement",
|
|
10
|
+
ForStatement = "for_statement",
|
|
11
|
+
WhileStatement = "while_statement",
|
|
12
|
+
RepeatStatement = "repeat_statement",
|
|
13
|
+
Call = "call",
|
|
14
|
+
Return = "return",
|
|
15
|
+
FunctionDefinition = "function_definition",
|
|
16
|
+
String = "string",
|
|
17
|
+
Float = "float",
|
|
18
|
+
Integer = "integer",
|
|
19
|
+
Complex = "complex",
|
|
20
|
+
Inf = "inf",
|
|
21
|
+
Nan = "nan",
|
|
22
|
+
Na = "na",
|
|
23
|
+
Null = "null",
|
|
24
|
+
True = "true",
|
|
25
|
+
False = "false",
|
|
26
|
+
Break = "break",
|
|
27
|
+
Next = "next",
|
|
28
|
+
Subset = "subset",
|
|
29
|
+
Subset2 = "subset2",
|
|
30
|
+
ExtractOperator = "extract_operator",
|
|
31
|
+
Parameter = "parameter",
|
|
32
|
+
Argument = "argument",
|
|
33
|
+
Dots = "dots",
|
|
34
|
+
Comment = "comment"
|
|
35
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TreeSitterType = void 0;
|
|
4
|
+
var TreeSitterType;
|
|
5
|
+
(function (TreeSitterType) {
|
|
6
|
+
TreeSitterType["Program"] = "program";
|
|
7
|
+
TreeSitterType["BracedExpression"] = "braced_expression";
|
|
8
|
+
TreeSitterType["ParenthesizedExpression"] = "parenthesized_expression";
|
|
9
|
+
TreeSitterType["BinaryOperator"] = "binary_operator";
|
|
10
|
+
TreeSitterType["UnaryOperator"] = "unary_operator";
|
|
11
|
+
TreeSitterType["NamespaceOperator"] = "namespace_operator";
|
|
12
|
+
TreeSitterType["Identifier"] = "identifier";
|
|
13
|
+
TreeSitterType["IfStatement"] = "if_statement";
|
|
14
|
+
TreeSitterType["ForStatement"] = "for_statement";
|
|
15
|
+
TreeSitterType["WhileStatement"] = "while_statement";
|
|
16
|
+
TreeSitterType["RepeatStatement"] = "repeat_statement";
|
|
17
|
+
TreeSitterType["Call"] = "call";
|
|
18
|
+
TreeSitterType["Return"] = "return";
|
|
19
|
+
TreeSitterType["FunctionDefinition"] = "function_definition";
|
|
20
|
+
TreeSitterType["String"] = "string";
|
|
21
|
+
TreeSitterType["Float"] = "float";
|
|
22
|
+
TreeSitterType["Integer"] = "integer";
|
|
23
|
+
TreeSitterType["Complex"] = "complex";
|
|
24
|
+
TreeSitterType["Inf"] = "inf";
|
|
25
|
+
TreeSitterType["Nan"] = "nan";
|
|
26
|
+
TreeSitterType["Na"] = "na";
|
|
27
|
+
TreeSitterType["Null"] = "null";
|
|
28
|
+
TreeSitterType["True"] = "true";
|
|
29
|
+
TreeSitterType["False"] = "false";
|
|
30
|
+
TreeSitterType["Break"] = "break";
|
|
31
|
+
TreeSitterType["Next"] = "next";
|
|
32
|
+
TreeSitterType["Subset"] = "subset";
|
|
33
|
+
TreeSitterType["Subset2"] = "subset2";
|
|
34
|
+
TreeSitterType["ExtractOperator"] = "extract_operator";
|
|
35
|
+
TreeSitterType["Parameter"] = "parameter";
|
|
36
|
+
TreeSitterType["Argument"] = "argument";
|
|
37
|
+
TreeSitterType["Dots"] = "dots";
|
|
38
|
+
TreeSitterType["Comment"] = "comment";
|
|
39
|
+
})(TreeSitterType || (exports.TreeSitterType = TreeSitterType = {}));
|
|
40
|
+
//# sourceMappingURL=tree-sitter-types.js.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { RParseRequest, RParseRequests } from './retriever';
|
|
2
|
+
import type { RShell } from './shell';
|
|
3
|
+
import type { RShellExecutor } from './shell-executor';
|
|
4
|
+
import type { TreeSitterExecutor } from './lang-4.x/tree-sitter/tree-sitter-executor';
|
|
5
|
+
interface ParserContent<T> {
|
|
6
|
+
readonly name: string;
|
|
7
|
+
rVersion(): Promise<string | 'unknown' | 'none'>;
|
|
8
|
+
parse(request: RParseRequest): T;
|
|
9
|
+
close(): void;
|
|
10
|
+
}
|
|
11
|
+
export type SyncParser<T> = ParserContent<Awaited<T>> & {
|
|
12
|
+
readonly async?: false;
|
|
13
|
+
};
|
|
14
|
+
export type AsyncParser<T> = ParserContent<Promise<T>> & {
|
|
15
|
+
readonly async: true;
|
|
16
|
+
};
|
|
17
|
+
export type Parser<T> = SyncParser<T> | AsyncParser<T>;
|
|
18
|
+
export type KnownParser = RShell | RShellExecutor | TreeSitterExecutor;
|
|
19
|
+
export type KnownParserType = Awaited<ReturnType<KnownParser['parse']>>;
|
|
20
|
+
export type KnownParserName = KnownParser['name'];
|
|
21
|
+
export interface ParseRequiredInput<T> {
|
|
22
|
+
/** This is the {@link RShell}, {@link RShellExecutor} or {@link TreeSitterExecutor} connection to be used to obtain the original parses AST of the R code */
|
|
23
|
+
readonly parser: Parser<T>;
|
|
24
|
+
/** The request which essentially indicates the input to extract the AST from */
|
|
25
|
+
readonly request: RParseRequests;
|
|
26
|
+
}
|
|
27
|
+
export interface ParseStepOutput<T> {
|
|
28
|
+
/** The parsed AST of the R code as given by the R parse side */
|
|
29
|
+
readonly parsed: T;
|
|
30
|
+
}
|
|
31
|
+
export declare function parseRequests<T extends KnownParserType>(_results: unknown, input: Partial<ParseRequiredInput<T>>): Promise<ParseStepOutput<T>>;
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseRequests = parseRequests;
|
|
4
|
+
async function parseRequests(_results, input) {
|
|
5
|
+
/* in the future, we want to expose all cases */
|
|
6
|
+
const request = (Array.isArray(input.request) ? input.request[0] : input.request);
|
|
7
|
+
if (input.parser?.async) {
|
|
8
|
+
return { parsed: await input.parser.parse(request) };
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
return { parsed: input.parser.parse(request) };
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=parser.js.map
|