@narrative.io/data-collaboration-sdk-ts 2.95.0-beta.0 → 2.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/apps/index.d.ts +0 -7
- package/build/apps/index.js +0 -9
- package/build/collaboration-policy/core/filter-builder.js +1 -7
- package/build/collaboration-policy/core/sql-builder.js +9 -11
- package/build/collaboration-policy/core/types.d.ts +10 -81
- package/build/collaboration-policy/index.d.ts +4 -5
- package/build/collaboration-policy/index.js +3 -4
- package/build/collaboration-policy/types/collaboration-policy.d.ts +1370 -0
- package/build/collaboration-policy/types/collaboration-policy.js +274 -0
- package/build/collaboration-policy/types/index.d.ts +2 -0
- package/build/collaboration-policy/types/index.js +1 -0
- package/build/collaboration-policy/useCollaborationPolicy.d.ts +4 -13
- package/build/collaboration-policy/useCollaborationPolicy.js +1 -23
- package/build/collaboration-policy/utils/path-helpers.d.ts +3 -7
- package/build/collaboration-policy/utils/path-helpers.js +22 -6
- package/build/data-planes/types.d.ts +1 -1
- package/build/datasets/index.d.ts +12 -12
- package/build/datasets/index.js +18 -13
- package/build/datasets/statistics-types.d.ts +81 -0
- package/build/datasets/types.d.ts +60 -13
- package/build/datasets/types.js +1 -0
- package/build/index.d.ts +1 -0
- package/build/index.js +1 -0
- package/build/jobs/types.d.ts +1 -0
- package/package.json +7 -8
- package/build/collaboration-policy/core/jsonschema/json-schema-types.d.ts +0 -45
- package/build/collaboration-policy/core/jsonschema/policy-branches.d.ts +0 -49
- package/build/collaboration-policy/core/jsonschema/policy-branches.js +0 -1
- package/build/collaboration-policy/core/jsonschema/policy-evaluator.d.ts +0 -22
- package/build/collaboration-policy/core/jsonschema/policy-evaluator.js +0 -260
- package/build/collaboration-policy/core/jsonschema/sql-builder.d.ts +0 -79
- package/build/collaboration-policy/core/jsonschema/sql-builder.js +0 -305
- package/build/collaboration-policy/core/jsonschema/sql-poc.d.ts +0 -79
- package/build/collaboration-policy/core/jsonschema/sql-poc.js +0 -305
- package/build/nql/SubstraitParser.d.ts +0 -771
- package/build/nql/SubstraitParser.js +0 -797
- /package/build/{collaboration-policy/core/jsonschema/json-schema-types.js → datasets/statistics-types.js} +0 -0
|
@@ -1,797 +0,0 @@
|
|
|
1
|
-
/*!
|
|
2
|
-
Copyright 2023 Narrative I/O, Inc.
|
|
3
|
-
|
|
4
|
-
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
you may not use this file except in compliance with the License.
|
|
6
|
-
You may obtain a copy of the License at
|
|
7
|
-
|
|
8
|
-
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
|
|
10
|
-
NOTICE: THIS FILE HAS BEEN MODIFIED BY NARRATIVE I/O UNDER COMPLIANCE WITH THE APACHE 2.0 LICENCE FROM THE ORIGINAL WORK
|
|
11
|
-
OF THE COMPANY Substrait (https://github.com/substrait-io/substrait-js).
|
|
12
|
-
*/
|
|
13
|
-
var AggregationInvocation;
|
|
14
|
-
(function (AggregationInvocation) {
|
|
15
|
-
AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_UNSPECIFIED"] = 0] = "AGGREGATION_INVOCATION_UNSPECIFIED";
|
|
16
|
-
AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_ALL"] = 1] = "AGGREGATION_INVOCATION_ALL";
|
|
17
|
-
AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_DISTINCT"] = 2] = "AGGREGATION_INVOCATION_DISTINCT";
|
|
18
|
-
})(AggregationInvocation || (AggregationInvocation = {}));
|
|
19
|
-
var AggregationPhase;
|
|
20
|
-
(function (AggregationPhase) {
|
|
21
|
-
AggregationPhase[AggregationPhase["AGGREGATION_PHASE_UNSPECIFIED"] = 0] = "AGGREGATION_PHASE_UNSPECIFIED";
|
|
22
|
-
AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE"] = 1] = "AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE";
|
|
23
|
-
AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE"] = 2] = "AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE";
|
|
24
|
-
AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INITIAL_TO_RESULT"] = 3] = "AGGREGATION_PHASE_INITIAL_TO_RESULT";
|
|
25
|
-
AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT"] = 4] = "AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT";
|
|
26
|
-
})(AggregationPhase || (AggregationPhase = {}));
|
|
27
|
-
var JoinType;
|
|
28
|
-
(function (JoinType) {
|
|
29
|
-
JoinType[JoinType["JOIN_TYPE_UNSPECIFIED"] = 0] = "JOIN_TYPE_UNSPECIFIED";
|
|
30
|
-
JoinType[JoinType["JOIN_TYPE_INNER"] = 1] = "JOIN_TYPE_INNER";
|
|
31
|
-
JoinType[JoinType["JOIN_TYPE_OUTER"] = 2] = "JOIN_TYPE_OUTER";
|
|
32
|
-
JoinType[JoinType["JOIN_TYPE_LEFT"] = 3] = "JOIN_TYPE_LEFT";
|
|
33
|
-
JoinType[JoinType["JOIN_TYPE_RIGHT"] = 4] = "JOIN_TYPE_RIGHT";
|
|
34
|
-
JoinType[JoinType["JOIN_TYPE_SEMI"] = 5] = "JOIN_TYPE_SEMI";
|
|
35
|
-
JoinType[JoinType["JOIN_TYPE_ANTI"] = 6] = "JOIN_TYPE_ANTI";
|
|
36
|
-
JoinType[JoinType["JOIN_TYPE_SINGLE"] = 7] = "JOIN_TYPE_SINGLE";
|
|
37
|
-
})(JoinType || (JoinType = {}));
|
|
38
|
-
var SetOp;
|
|
39
|
-
(function (SetOp) {
|
|
40
|
-
SetOp[SetOp["SET_OP_UNSPECIFIED"] = 0] = "SET_OP_UNSPECIFIED";
|
|
41
|
-
SetOp[SetOp["SET_OP_MINUS_PRIMARY"] = 1] = "SET_OP_MINUS_PRIMARY";
|
|
42
|
-
SetOp[SetOp["SET_OP_MINUS_MULTISET"] = 2] = "SET_OP_MINUS_MULTISET";
|
|
43
|
-
SetOp[SetOp["SET_OP_INTERSECTION_PRIMARY"] = 3] = "SET_OP_INTERSECTION_PRIMARY";
|
|
44
|
-
SetOp[SetOp["SET_OP_INTERSECTION_MULTISET"] = 4] = "SET_OP_INTERSECTION_MULTISET";
|
|
45
|
-
SetOp[SetOp["SET_OP_UNION_DISTINCT"] = 5] = "SET_OP_UNION_DISTINCT";
|
|
46
|
-
SetOp[SetOp["SET_OP_UNION_ALL"] = 6] = "SET_OP_UNION_ALL";
|
|
47
|
-
})(SetOp || (SetOp = {}));
|
|
48
|
-
var SortDirection;
|
|
49
|
-
(function (SortDirection) {
|
|
50
|
-
SortDirection[SortDirection["UNSPECIFIED"] = 0] = "UNSPECIFIED";
|
|
51
|
-
SortDirection[SortDirection["ASCENDING_NULLS_FIRST"] = 1] = "ASCENDING_NULLS_FIRST";
|
|
52
|
-
SortDirection[SortDirection["ASCENDING_NULLS_LAST"] = 2] = "ASCENDING_NULLS_LAST";
|
|
53
|
-
SortDirection[SortDirection["DESCENDING_NULLS_FIRST"] = 3] = "DESCENDING_NULLS_FIRST";
|
|
54
|
-
SortDirection[SortDirection["DESCENDING_NULLS_LAST"] = 4] = "DESCENDING_NULLS_LAST";
|
|
55
|
-
SortDirection[SortDirection["CLUSTERED"] = 5] = "CLUSTERED";
|
|
56
|
-
})(SortDirection || (SortDirection = {}));
|
|
57
|
-
class SubstraitParser {
|
|
58
|
-
idCounters;
|
|
59
|
-
extSet;
|
|
60
|
-
fieldNames = [];
|
|
61
|
-
selectedFields = [];
|
|
62
|
-
filters = [];
|
|
63
|
-
logicalOperators = [];
|
|
64
|
-
/**
|
|
65
|
-
* Constructs a SubstraitParser
|
|
66
|
-
* @param {SubstraitPlan} plan the Substrait plan
|
|
67
|
-
*/
|
|
68
|
-
constructor(plan) {
|
|
69
|
-
this.idCounters = new Map();
|
|
70
|
-
this.extSet = this.buildExtensionSet(plan);
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* Creates a map from anchor to function name
|
|
74
|
-
* @param {SubstraitPlan} plan the Substrait plan
|
|
75
|
-
* @return {Map<number, string>} the created mapping
|
|
76
|
-
*/
|
|
77
|
-
buildExtensionSet(plan) {
|
|
78
|
-
const extSet = new Map();
|
|
79
|
-
for (const extensionDecl of plan.extensions) {
|
|
80
|
-
const func = extensionDecl.extensionFunction;
|
|
81
|
-
extSet.set(func.functionAnchor, func.name.split(":")[0].toLowerCase());
|
|
82
|
-
}
|
|
83
|
-
return extSet;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Geneate a unique id for the node
|
|
87
|
-
* @param {string} type The type of the node
|
|
88
|
-
* @return {string} A unique id
|
|
89
|
-
*/
|
|
90
|
-
nextId(type) {
|
|
91
|
-
const value = this.idCounters.get(type) ?? 0;
|
|
92
|
-
this.idCounters.set(type, value + 1);
|
|
93
|
-
return `${type}_${value}`;
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Create a schema field
|
|
97
|
-
*
|
|
98
|
-
* This is a special field with an empty name and an empty type string
|
|
99
|
-
* and uknown nullability
|
|
100
|
-
*
|
|
101
|
-
* @param {Field[]} fields the fields of the schema
|
|
102
|
-
* @return {Field} a schema field
|
|
103
|
-
*/
|
|
104
|
-
makeSchemaField(fields) {
|
|
105
|
-
return {
|
|
106
|
-
name: "",
|
|
107
|
-
children: fields,
|
|
108
|
-
type: "",
|
|
109
|
-
nullability: "NULLABILITY_UNSPECIFIED",
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* Applies a Substrait emit to an output schema
|
|
114
|
-
* @param {Field} schema the original output schema
|
|
115
|
-
* @param {Object} emit a potential remapping of fields
|
|
116
|
-
* @return {Field} the remapped output schema
|
|
117
|
-
*/
|
|
118
|
-
applyEmit(schema, emit) {
|
|
119
|
-
let outputFields = schema.children;
|
|
120
|
-
if (emit?.outputMapping != null) {
|
|
121
|
-
outputFields = emit.outputMapping.map((idx) => schema.children[idx]);
|
|
122
|
-
}
|
|
123
|
-
return this.makeSchemaField(outputFields);
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* Create a print node, generating a new id
|
|
127
|
-
*
|
|
128
|
-
* @param {string} type the node type
|
|
129
|
-
* @param {PrintNode[]} inputs the inputs to the node
|
|
130
|
-
* @param {SimpleProperty[]} props properties describing the node
|
|
131
|
-
* @param {Field} schema the output schema of the node
|
|
132
|
-
* @param {Object} emit the Substrait emit for the node
|
|
133
|
-
* @return {PrintNode} a print node
|
|
134
|
-
*/
|
|
135
|
-
makePrintNode(type, inputs, props, schema, emit) {
|
|
136
|
-
const emitSchema = this.applyEmit(schema, emit);
|
|
137
|
-
return {
|
|
138
|
-
id: this.nextId(type),
|
|
139
|
-
type,
|
|
140
|
-
inputs,
|
|
141
|
-
props,
|
|
142
|
-
schema: emitSchema,
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* Recursive helper for structFieldToStr
|
|
147
|
-
* @param {*} field the Substrait field to convert
|
|
148
|
-
* @param {*} currentName the current name
|
|
149
|
-
* @param {Field} currentField the current input field
|
|
150
|
-
* @return {Field} a stringified represenstation
|
|
151
|
-
*/
|
|
152
|
-
structFieldHelper(field, currentName, currentField) {
|
|
153
|
-
const fieldNum = field.field;
|
|
154
|
-
const fieldName = currentField.name !== "" ? currentField.name : `$${fieldNum}`;
|
|
155
|
-
const nextName = `${currentName}${fieldName}`;
|
|
156
|
-
if (field.child != null) {
|
|
157
|
-
return this.structFieldHelper(field.child, `${nextName}.`, currentField.children[fieldNum]);
|
|
158
|
-
}
|
|
159
|
-
return currentField.children[fieldNum];
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* Convert a struct field to a stringified representation
|
|
163
|
-
*
|
|
164
|
-
* Struct fields are usually intuitively written with names.
|
|
165
|
-
* For example, a.b.c. However, fields in Substrait plans
|
|
166
|
-
* are not typically named. So instead we represent them as
|
|
167
|
-
* $0, $1, $2, ... where $N means the field at index N.
|
|
168
|
-
*
|
|
169
|
-
* If the field does happen to be named we will use that.
|
|
170
|
-
*
|
|
171
|
-
* @param {StructItem} field a Substrait struct field
|
|
172
|
-
* @param {Field} exprIn the input to the expression
|
|
173
|
-
* @return {Field} a stringified representation
|
|
174
|
-
*/
|
|
175
|
-
structFieldToField(field, exprIn) {
|
|
176
|
-
return this.structFieldHelper(field, "", exprIn);
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Convert a direct reference to a stringified representation
|
|
180
|
-
* @param {Object} ref A Substrait direct reference
|
|
181
|
-
* @param {Object} exprIn the input to the expression
|
|
182
|
-
* @return {Field} a string representation
|
|
183
|
-
*/
|
|
184
|
-
directReferenceToField(ref, exprIn) {
|
|
185
|
-
if (ref.structField != null) {
|
|
186
|
-
return this.structFieldToField(ref.structField, exprIn);
|
|
187
|
-
}
|
|
188
|
-
else {
|
|
189
|
-
throw new Error(`Unrecognized direct reference: ${JSON.stringify(ref)}`);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Convert a reference to a stringified representation
|
|
194
|
-
* @param {Object} ref A Substrait reference expression
|
|
195
|
-
* @param {Object} exprIn the input to the expression
|
|
196
|
-
* @return {Field} a string representation
|
|
197
|
-
*/
|
|
198
|
-
referenceToField(ref, exprIn) {
|
|
199
|
-
if (ref.directReference != null) {
|
|
200
|
-
return this.directReferenceToField(ref.directReference, exprIn);
|
|
201
|
-
}
|
|
202
|
-
else {
|
|
203
|
-
throw new Error(`Unrecognized reference: ${JSON.stringify(ref)}`);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
/**
|
|
207
|
-
* Convert a Substrait function argument to a string
|
|
208
|
-
* @param {Object} arg the argument to convert
|
|
209
|
-
* @param {Field} inp the input field
|
|
210
|
-
* @return {string} a string representation
|
|
211
|
-
*/
|
|
212
|
-
functionArgumentToStr(arg, inp) {
|
|
213
|
-
if (arg.enum != null) {
|
|
214
|
-
if (arg.enum.specified == null) {
|
|
215
|
-
return "unspecified";
|
|
216
|
-
}
|
|
217
|
-
return arg.enum.specified;
|
|
218
|
-
}
|
|
219
|
-
else if (arg.type != null) {
|
|
220
|
-
return this.typeToField(arg.type).type;
|
|
221
|
-
}
|
|
222
|
-
else if (arg.value != null) {
|
|
223
|
-
return this.expressionToStr(arg.value, inp).name;
|
|
224
|
-
}
|
|
225
|
-
else {
|
|
226
|
-
throw new Error("A FunctionArgument did not have an enum/type/value");
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Converts a Substrait function reference to a string
|
|
231
|
-
* @param {number} ref a Substrait function reference
|
|
232
|
-
* @return {string} a string version of the reference
|
|
233
|
-
*/
|
|
234
|
-
functionRefToName(ref) {
|
|
235
|
-
return this.extSet.get(ref) ?? "unknown";
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* Convert arguments to a joined string
|
|
239
|
-
* @param {FunctionTyp} func A Substrait function call
|
|
240
|
-
* @param {Field} schema The input type
|
|
241
|
-
* @return {string} A string representation
|
|
242
|
-
*/
|
|
243
|
-
argsToString(func, schema) {
|
|
244
|
-
const args = func.arguments.length > 0
|
|
245
|
-
? func.arguments.map((arg) => this.functionArgumentToStr(arg, schema))
|
|
246
|
-
: [];
|
|
247
|
-
return args.join(",");
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* Convert a scalar function to a stringified representation
|
|
251
|
-
* @param {Object} func the function to convert
|
|
252
|
-
* @param {Field} schema the input type info
|
|
253
|
-
* @return {Field} a string version of the function
|
|
254
|
-
*/
|
|
255
|
-
scalarFunctionToField(func, schema) {
|
|
256
|
-
const argsString = this.argsToString(func, schema);
|
|
257
|
-
const outputType = this.typeToField(func.outputType);
|
|
258
|
-
const functionName = this.functionRefToName(func.functionReference);
|
|
259
|
-
if (!["and", "or", "not", "cast"].includes(functionName)) {
|
|
260
|
-
const args = argsString.split(",");
|
|
261
|
-
if (args.length === 1) {
|
|
262
|
-
this.filters.push(`${args[0]} ${functionName} `);
|
|
263
|
-
}
|
|
264
|
-
else if (args.length === 2) {
|
|
265
|
-
this.filters.push(`${args[0]} ${functionName} ${args[1]}`);
|
|
266
|
-
}
|
|
267
|
-
else {
|
|
268
|
-
this.filters.push(`${functionName}(${args.join(", ")})`);
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
else {
|
|
272
|
-
this.logicalOperators.push(functionName);
|
|
273
|
-
}
|
|
274
|
-
const fieldName = `${functionName}(${argsString})`;
|
|
275
|
-
outputType.name = fieldName;
|
|
276
|
-
return outputType;
|
|
277
|
-
}
|
|
278
|
-
/**
|
|
279
|
-
* Converts a Substrait literal to a field
|
|
280
|
-
* @param {Object} lit the Substrait literal
|
|
281
|
-
* @return {Field} a field representation
|
|
282
|
-
*/
|
|
283
|
-
literalToField(lit) {
|
|
284
|
-
const nullability = lit.nullable
|
|
285
|
-
? "NULLABILITY_NULLABLE"
|
|
286
|
-
: "NULLABILITY_REQUIRED";
|
|
287
|
-
if (lit.bool != null) {
|
|
288
|
-
return {
|
|
289
|
-
name: String(lit.bool.value),
|
|
290
|
-
type: "boolean",
|
|
291
|
-
nullability,
|
|
292
|
-
children: [],
|
|
293
|
-
};
|
|
294
|
-
}
|
|
295
|
-
else if (lit.date != null) {
|
|
296
|
-
return {
|
|
297
|
-
name: new Date((lit.date?.value ?? 0) * 8.64e7).toISOString(),
|
|
298
|
-
type: "date",
|
|
299
|
-
nullability,
|
|
300
|
-
children: [],
|
|
301
|
-
};
|
|
302
|
-
}
|
|
303
|
-
else if (lit.decimal != null) {
|
|
304
|
-
return {
|
|
305
|
-
name: String(this.base64ToDecimal(lit.decimal.value ?? "", lit.decimal.precision, lit.decimal.scale)),
|
|
306
|
-
type: "decimal",
|
|
307
|
-
nullability,
|
|
308
|
-
children: [],
|
|
309
|
-
};
|
|
310
|
-
}
|
|
311
|
-
else if (lit.i32 != null) {
|
|
312
|
-
return {
|
|
313
|
-
name: lit.i32?.value?.toString() ?? "unspecified",
|
|
314
|
-
type: "i32",
|
|
315
|
-
nullability,
|
|
316
|
-
children: [],
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
else if (lit.intervalDayToSecond != null) {
|
|
320
|
-
const val = lit.intervalDayToSecond;
|
|
321
|
-
return {
|
|
322
|
-
name: `${val.days}d${val.seconds}s${val.microseconds}u`,
|
|
323
|
-
type: "interval_day",
|
|
324
|
-
nullability,
|
|
325
|
-
children: [],
|
|
326
|
-
};
|
|
327
|
-
}
|
|
328
|
-
else if (lit.fixedChar != null) {
|
|
329
|
-
const val = typeof lit.fixedChar === "string"
|
|
330
|
-
? lit.fixedChar
|
|
331
|
-
: lit.fixedChar.value ?? "unspecified";
|
|
332
|
-
return {
|
|
333
|
-
name: `${val}`,
|
|
334
|
-
type: "interval_day",
|
|
335
|
-
nullability,
|
|
336
|
-
children: [],
|
|
337
|
-
};
|
|
338
|
-
}
|
|
339
|
-
else {
|
|
340
|
-
throw new Error(`Unrecognized literal: ${JSON.stringify(lit)}`);
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
/**
|
|
344
|
-
* Convert a cast expression to a stringified representation
|
|
345
|
-
* @param {Object} cast A Substrait cast expression
|
|
346
|
-
* @param {Field} schema The input schema
|
|
347
|
-
* @return {Field} A field representation
|
|
348
|
-
*/
|
|
349
|
-
castToField(cast, schema) {
|
|
350
|
-
const input = this.expressionToStr(cast.input, schema);
|
|
351
|
-
const outputType = this.typeToField(cast.type);
|
|
352
|
-
let fb = "unspecified";
|
|
353
|
-
if (cast.failureBehavior === "FAILURE_BEHAVIOR_RETURN_NULL") {
|
|
354
|
-
fb = "or null";
|
|
355
|
-
}
|
|
356
|
-
else if (cast.failureBehavior === "THROW_EXCEPTION") {
|
|
357
|
-
fb = "or die";
|
|
358
|
-
}
|
|
359
|
-
const name = `cast(${input.name}, ${outputType.type}, ${fb})`;
|
|
360
|
-
outputType.name = name;
|
|
361
|
-
return outputType;
|
|
362
|
-
}
|
|
363
|
-
/**
|
|
364
|
-
* Convert an expression to a stringified representation
|
|
365
|
-
*
|
|
366
|
-
* The expression is returned as a field where the type
|
|
367
|
-
* of the field is the output type of the expression and the
|
|
368
|
-
* name of the field is a string representation of the expression
|
|
369
|
-
*
|
|
370
|
-
* @param {Object} expr A Substrait expression
|
|
371
|
-
* @param {Object} exprIn the input to the expression
|
|
372
|
-
* @return {Field} a string representation
|
|
373
|
-
*/
|
|
374
|
-
expressionToStr(expr, exprIn) {
|
|
375
|
-
if (expr.selection != null) {
|
|
376
|
-
return this.referenceToField(expr.selection, exprIn);
|
|
377
|
-
}
|
|
378
|
-
else if (expr.scalarFunction != null) {
|
|
379
|
-
return this.scalarFunctionToField(expr.scalarFunction, exprIn);
|
|
380
|
-
}
|
|
381
|
-
else if (expr.literal != null) {
|
|
382
|
-
return this.literalToField(expr.literal);
|
|
383
|
-
}
|
|
384
|
-
else if (expr.cast != null) {
|
|
385
|
-
return this.castToField(expr.cast, exprIn);
|
|
386
|
-
}
|
|
387
|
-
else {
|
|
388
|
-
throw new Error(`Unrecognized expression: ${JSON.stringify(expr)}`);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
/**
|
|
392
|
-
* Convert a project relation to a print node
|
|
393
|
-
* @param {Object} project A Substrait ProjectRel
|
|
394
|
-
* @return {PrintNode} a print node
|
|
395
|
-
*/
|
|
396
|
-
projectToNode(project) {
|
|
397
|
-
const input = this.relToNode(project.input);
|
|
398
|
-
const expressions = project.expressions.map((val, idx) => {
|
|
399
|
-
return {
|
|
400
|
-
name: `expressions[${idx}]`,
|
|
401
|
-
field: this.expressionToStr(val, input.schema),
|
|
402
|
-
};
|
|
403
|
-
});
|
|
404
|
-
const outFields = expressions.map((expr) => expr.field);
|
|
405
|
-
const fields = input.schema.children.concat(outFields);
|
|
406
|
-
const schema = this.makeSchemaField(fields);
|
|
407
|
-
const props = expressions.map((expr) => ({
|
|
408
|
-
name: expr.name,
|
|
409
|
-
value: expr.field.name,
|
|
410
|
-
}));
|
|
411
|
-
const inputs = [this.relToNode(project.input)];
|
|
412
|
-
return this.makePrintNode("project", inputs, props, schema, project.common?.emit);
|
|
413
|
-
}
|
|
414
|
-
/**
|
|
415
|
-
* Convert a Substrait data type to an unnamed Field
|
|
416
|
-
*
|
|
417
|
-
* The field's name will be an empty string
|
|
418
|
-
*
|
|
419
|
-
* @param {Object} type a Substrait data type
|
|
420
|
-
* @return {Field} an unnamed field
|
|
421
|
-
*/
|
|
422
|
-
typeToField(type) {
|
|
423
|
-
let typeStr = null;
|
|
424
|
-
let nullability = "NULLABILITY_UNSPECIFIED";
|
|
425
|
-
if (type.varchar != null) {
|
|
426
|
-
typeStr = `VARCHAR<${type.varchar.length}>`;
|
|
427
|
-
nullability = type.varchar.nullability;
|
|
428
|
-
}
|
|
429
|
-
else if (type.fixedChar != null) {
|
|
430
|
-
typeStr = `FIXEDCHAR<${type.fixedChar.length}>`;
|
|
431
|
-
nullability =
|
|
432
|
-
typeof type.fixedChar !== "string"
|
|
433
|
-
? type.fixedChar?.nullability
|
|
434
|
-
: "NULLABILITY_REQUIRED";
|
|
435
|
-
}
|
|
436
|
-
else if (type.date != null) {
|
|
437
|
-
typeStr = "date";
|
|
438
|
-
nullability = type.date.nullability;
|
|
439
|
-
}
|
|
440
|
-
else if (type.decimal != null) {
|
|
441
|
-
typeStr = `DECIMAL<${type.decimal.precision},${type.decimal.scale}>`;
|
|
442
|
-
nullability = type.decimal.nullability ?? "NULLABILITY_UNSPECIFIED";
|
|
443
|
-
}
|
|
444
|
-
else if (type.i32 != null) {
|
|
445
|
-
typeStr = "i32";
|
|
446
|
-
nullability = type.i32.nullability;
|
|
447
|
-
}
|
|
448
|
-
else if (type.i64 != null) {
|
|
449
|
-
typeStr = "i64";
|
|
450
|
-
nullability = type.i64.nullability;
|
|
451
|
-
}
|
|
452
|
-
else if (type.bool != null) {
|
|
453
|
-
typeStr = "boolean";
|
|
454
|
-
nullability = type.bool.nullability;
|
|
455
|
-
}
|
|
456
|
-
else if (type.string != null) {
|
|
457
|
-
typeStr = "string";
|
|
458
|
-
nullability = type.string.nullability;
|
|
459
|
-
}
|
|
460
|
-
else if (type.fp64 != null) {
|
|
461
|
-
typeStr = "fp64";
|
|
462
|
-
nullability = type.fp64.nullability;
|
|
463
|
-
}
|
|
464
|
-
else if (type.timestampTz != null) {
|
|
465
|
-
typeStr = "timestampTz";
|
|
466
|
-
nullability = type.timestampTz.nullability;
|
|
467
|
-
}
|
|
468
|
-
else if (type.struct != null) {
|
|
469
|
-
typeStr = "struct";
|
|
470
|
-
nullability = type.struct.nullability;
|
|
471
|
-
}
|
|
472
|
-
else if (type.fixedBinary != null) {
|
|
473
|
-
typeStr = "fixedBinary";
|
|
474
|
-
nullability = type.fixedBinary.nullability;
|
|
475
|
-
}
|
|
476
|
-
else {
|
|
477
|
-
throw new Error(`Unrecognized type: ${JSON.stringify(type)}`);
|
|
478
|
-
}
|
|
479
|
-
return {
|
|
480
|
-
name: "",
|
|
481
|
-
type: typeStr,
|
|
482
|
-
nullability,
|
|
483
|
-
children: [],
|
|
484
|
-
};
|
|
485
|
-
}
|
|
486
|
-
/**
|
|
487
|
-
* Convert a named struct to an array of Field
|
|
488
|
-
* @param {NamedStruct} struct a Substrait named struct
|
|
489
|
-
* @return {Field[]} named fields
|
|
490
|
-
*/
|
|
491
|
-
namedStructToSchema(struct) {
|
|
492
|
-
// const names = struct.names
|
|
493
|
-
// const types = this.flattenTypes(struct.struct)
|
|
494
|
-
// console.log('names', names.length)
|
|
495
|
-
// console.log('types', types.length)
|
|
496
|
-
// if (names.length !== types.length) {
|
|
497
|
-
// throw new Error('Mismatched names and types')
|
|
498
|
-
// }
|
|
499
|
-
return {
|
|
500
|
-
name: "test",
|
|
501
|
-
type: "struct",
|
|
502
|
-
nullability: "NULLABILITY_UNSPECIFIED",
|
|
503
|
-
children: [],
|
|
504
|
-
};
|
|
505
|
-
// return this.makeSchemaField(outTypes)
|
|
506
|
-
}
|
|
507
|
-
// mapNames (input: NamedStruct, prefix: string = ''): string[] {
|
|
508
|
-
// const output: string[] = []
|
|
509
|
-
// for (let i = 0; i < input.names.length; i++) {
|
|
510
|
-
// const name = input.names[i]
|
|
511
|
-
// const type = input.struct.types[i]
|
|
512
|
-
// if (type.struct != null) {
|
|
513
|
-
// output.push(...this.mapNames({ name, types: type.struct.types }, `${prefix}${name}.`))
|
|
514
|
-
// } else {
|
|
515
|
-
// output.push(`${prefix}${name}`)
|
|
516
|
-
// }
|
|
517
|
-
// }
|
|
518
|
-
// return output
|
|
519
|
-
// }
|
|
520
|
-
/**
|
|
521
|
-
* Extract the properties from a read relation
|
|
522
|
-
* @param {Object} read a Substrait read relation
|
|
523
|
-
* @return {SimpleProperty[]} properties for the relation
|
|
524
|
-
*/
|
|
525
|
-
readTypeToProps(read) {
|
|
526
|
-
return [
|
|
527
|
-
{ name: "namedTable.names", value: read.namedTable.names.join(".") },
|
|
528
|
-
];
|
|
529
|
-
}
|
|
530
|
-
/**
|
|
531
|
-
* Convert a Substrait read relation to a print node
|
|
532
|
-
* @param {Object} read a Substrait read relation
|
|
533
|
-
* @return {PrintNode} a print node
|
|
534
|
-
*/
|
|
535
|
-
readToNode(read) {
|
|
536
|
-
const schema = this.namedStructToSchema(read.baseSchema);
|
|
537
|
-
const props = this.readTypeToProps(read);
|
|
538
|
-
return this.makePrintNode("read", [], props, schema, read.common?.emit);
|
|
539
|
-
}
|
|
540
|
-
/**
|
|
541
|
-
* True if the object has a field with the given name
|
|
542
|
-
* @param {Object} obj the object to test
|
|
543
|
-
* @param {string} fieldName the field name to look for
|
|
544
|
-
* @return {boolean} the result of the test
|
|
545
|
-
*/
|
|
546
|
-
hasValue(obj, fieldName) {
|
|
547
|
-
return fieldName in obj;
|
|
548
|
-
}
|
|
549
|
-
/**
|
|
550
|
-
* Convert a Substrait fetch relation to a print node
|
|
551
|
-
* @param {Object} fetch a Substrait fetch relation
|
|
552
|
-
* @return {PrintNode} a print node
|
|
553
|
-
*/
|
|
554
|
-
fetchToNode(fetch) {
|
|
555
|
-
const input = this.relToNode(fetch.input);
|
|
556
|
-
const schema = input.schema;
|
|
557
|
-
const props = [];
|
|
558
|
-
if (this.hasValue(fetch, "offset")) {
|
|
559
|
-
props.push({ name: "offset", value: String(fetch.offset) });
|
|
560
|
-
}
|
|
561
|
-
if (this.hasValue(fetch, "count")) {
|
|
562
|
-
props.push({ name: "count", value: String(fetch.count) });
|
|
563
|
-
}
|
|
564
|
-
return this.makePrintNode("fetch", [input], props, schema, fetch.common.emit);
|
|
565
|
-
}
|
|
566
|
-
/**
|
|
567
|
-
* Convert a Substrait sort field to a property
|
|
568
|
-
* @param {Object} sortField a Substrait sort field
|
|
569
|
-
* @param {Field} schema the schema of the sort input
|
|
570
|
-
* @return {SimpleProperty} a property describing the sort
|
|
571
|
-
*/
|
|
572
|
-
sortFieldToStr(sortField, schema) {
|
|
573
|
-
const exprStr = this.expressionToStr(sortField.expr, schema).name;
|
|
574
|
-
if (sortField.direction !== null) {
|
|
575
|
-
const dir = sortField.direction;
|
|
576
|
-
let sortDesc = "";
|
|
577
|
-
if (dir === SortDirection.ASCENDING_NULLS_FIRST) {
|
|
578
|
-
sortDesc = "ascending (nulls first)";
|
|
579
|
-
}
|
|
580
|
-
else if (dir === SortDirection.ASCENDING_NULLS_LAST) {
|
|
581
|
-
sortDesc = "ascending (nulls last)";
|
|
582
|
-
}
|
|
583
|
-
else if (dir === SortDirection.DESCENDING_NULLS_FIRST) {
|
|
584
|
-
sortDesc = "descending (nulls first)";
|
|
585
|
-
}
|
|
586
|
-
else if (dir === SortDirection.DESCENDING_NULLS_LAST) {
|
|
587
|
-
sortDesc = "descending (nulls last)";
|
|
588
|
-
}
|
|
589
|
-
else if (dir === SortDirection.CLUSTERED) {
|
|
590
|
-
sortDesc = "clustered";
|
|
591
|
-
}
|
|
592
|
-
else {
|
|
593
|
-
sortDesc = "unspecified";
|
|
594
|
-
}
|
|
595
|
-
return `${exprStr} | ${sortDesc}`;
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
/**
|
|
599
|
-
* Convert a Substrait sort relation to a print node
|
|
600
|
-
* @param {Object} sort a Substrait sort relation
|
|
601
|
-
* @return {PrintNode} a print node
|
|
602
|
-
*/
|
|
603
|
-
sortToNode(sort) {
|
|
604
|
-
const input = this.relToNode(sort.input);
|
|
605
|
-
const schema = input.schema;
|
|
606
|
-
const props = sort.sorts.map((sort, idx) => {
|
|
607
|
-
return {
|
|
608
|
-
name: `sorts[${idx}]`,
|
|
609
|
-
value: this.sortFieldToStr(sort, schema) ?? "undefined",
|
|
610
|
-
};
|
|
611
|
-
});
|
|
612
|
-
return this.makePrintNode("sort", [input], props, schema, sort.common.emit);
|
|
613
|
-
}
|
|
614
|
-
/**
|
|
615
|
-
* Convert a Substrait aggregate function to a field
|
|
616
|
-
* @param {Object} func the Substrait aggregate function
|
|
617
|
-
* @param {Field} inp the input field
|
|
618
|
-
* @return {Field} a field representation of the function
|
|
619
|
-
*/
|
|
620
|
-
aggregateFunctionToField(func, inp) {
|
|
621
|
-
// TODO (weston) sorts, phases, invocation
|
|
622
|
-
return this.scalarFunctionToField(func, inp);
|
|
623
|
-
}
|
|
624
|
-
/**
|
|
625
|
-
* Convert a Substrait aggregate relation to a print node
|
|
626
|
-
* @param {Object} agg a Substrait aggregate relation
|
|
627
|
-
* @return {PrintNode} a print node
|
|
628
|
-
*/
|
|
629
|
-
aggToNode(agg) {
|
|
630
|
-
const input = this.relToNode(agg.input);
|
|
631
|
-
const props = [];
|
|
632
|
-
const fields = [];
|
|
633
|
-
agg.groupings.forEach((grouping, idx) => {
|
|
634
|
-
for (const groupingExpr of grouping.groupingExpressions) {
|
|
635
|
-
const groupingField = this.expressionToStr(groupingExpr, input.schema);
|
|
636
|
-
props.push({ name: `grouping[${idx}]`, value: groupingField.name });
|
|
637
|
-
fields.push(groupingField);
|
|
638
|
-
}
|
|
639
|
-
});
|
|
640
|
-
agg.measures.forEach((measure, idx) => {
|
|
641
|
-
const aggregate = this.aggregateFunctionToField(measure.measure, input.schema);
|
|
642
|
-
fields.push(aggregate);
|
|
643
|
-
const innerProps = [
|
|
644
|
-
{
|
|
645
|
-
name: "measure",
|
|
646
|
-
value: aggregate.name,
|
|
647
|
-
},
|
|
648
|
-
];
|
|
649
|
-
innerProps.push({
|
|
650
|
-
name: "filter",
|
|
651
|
-
value: this.expressionToStr(measure.filter, input.schema).name,
|
|
652
|
-
});
|
|
653
|
-
props.push({
|
|
654
|
-
name: `measures[${idx}]`,
|
|
655
|
-
value: JSON.stringify(innerProps), // TODO Is this right?
|
|
656
|
-
});
|
|
657
|
-
});
|
|
658
|
-
const schema = this.makeSchemaField(fields);
|
|
659
|
-
return this.makePrintNode("aggregate", [input], props, schema, agg.common.emit);
|
|
660
|
-
}
|
|
661
|
-
/**
|
|
662
|
-
* Convert a Substrait filter relation to a print node
|
|
663
|
-
* @param {Object} filt a Substrait filter relation
|
|
664
|
-
* @return {PrintNode} a print node
|
|
665
|
-
*/
|
|
666
|
-
filterToNode(filt) {
|
|
667
|
-
const input = this.relToNode(filt.input);
|
|
668
|
-
const props = [];
|
|
669
|
-
const schema = input.schema;
|
|
670
|
-
props.push({
|
|
671
|
-
name: "condition",
|
|
672
|
-
value: this.expressionToStr(filt.condition, schema).name,
|
|
673
|
-
});
|
|
674
|
-
const printNode = this.makePrintNode("filter", [input], props, schema, filt.common.emit);
|
|
675
|
-
return printNode;
|
|
676
|
-
}
|
|
677
|
-
/**
|
|
678
|
-
* Converts a Substrait join relation to a print node
|
|
679
|
-
* @param {Object} join a Substrait join relation
|
|
680
|
-
* @return {PrintNode} a print node
|
|
681
|
-
*/
|
|
682
|
-
joinToNode(join) {
|
|
683
|
-
const left = this.relToNode(join.left);
|
|
684
|
-
const right = this.relToNode(join.right);
|
|
685
|
-
const props = [];
|
|
686
|
-
const schema = this.makeSchemaField(left.schema.children.concat(right.schema.children));
|
|
687
|
-
let joinType = "unspecified";
|
|
688
|
-
if (join.type !== null) {
|
|
689
|
-
joinType = JoinType[join.type];
|
|
690
|
-
}
|
|
691
|
-
props.push({ name: "type", value: joinType });
|
|
692
|
-
if (join.expression !== null) {
|
|
693
|
-
const exprStr = this.expressionToStr(join.expression, schema).name;
|
|
694
|
-
props.push({ name: "expression", value: exprStr });
|
|
695
|
-
}
|
|
696
|
-
return this.makePrintNode("join", [left, right], props, schema, join.common.emit);
|
|
697
|
-
}
|
|
698
|
-
/**
|
|
699
|
-
* Converts a Substrait relation to a print node
|
|
700
|
-
* @param {Object} rel the Substrait relation to convert
|
|
701
|
-
* @return {PrintNode} a print node
|
|
702
|
-
*/
|
|
703
|
-
relToNode(rel) {
|
|
704
|
-
if (rel.project != null) {
|
|
705
|
-
return this.projectToNode(rel.project);
|
|
706
|
-
}
|
|
707
|
-
else if (rel.read != null) {
|
|
708
|
-
return this.readToNode(rel.read);
|
|
709
|
-
}
|
|
710
|
-
else if (rel.fetch != null) {
|
|
711
|
-
return this.fetchToNode(rel.fetch);
|
|
712
|
-
}
|
|
713
|
-
else if (rel.sort != null) {
|
|
714
|
-
return this.sortToNode(rel.sort);
|
|
715
|
-
}
|
|
716
|
-
else if (rel.aggregate != null) {
|
|
717
|
-
return this.aggToNode(rel.aggregate);
|
|
718
|
-
}
|
|
719
|
-
else if (rel.filter != null) {
|
|
720
|
-
return this.filterToNode(rel.filter);
|
|
721
|
-
}
|
|
722
|
-
else if (rel.join != null) {
|
|
723
|
-
return this.joinToNode(rel.join);
|
|
724
|
-
}
|
|
725
|
-
else {
|
|
726
|
-
throw new Error(`Unrecognized relation: ${JSON.stringify(rel)}`);
|
|
727
|
-
}
|
|
728
|
-
}
|
|
729
|
-
/**
|
|
730
|
-
* Converts a Substrait root relation to a print node
|
|
731
|
-
* @param {Object} rel the Substrait relation to convert
|
|
732
|
-
* @return {PrintNode} a print node
|
|
733
|
-
*/
|
|
734
|
-
rootRelToNode(rel) {
|
|
735
|
-
const props = rel.names.map((val, idx) => ({
|
|
736
|
-
name: `name[${idx}]`,
|
|
737
|
-
value: val,
|
|
738
|
-
}));
|
|
739
|
-
this.selectedFields = this.selectedFields.concat(rel.names);
|
|
740
|
-
const inputs = [this.relToNode(rel.input)];
|
|
741
|
-
const schema = this.makeSchemaField([]);
|
|
742
|
-
return this.makePrintNode("sink", inputs, props, schema);
|
|
743
|
-
}
|
|
744
|
-
/**
|
|
745
|
-
* Converts a Substrait plan to a print node
|
|
746
|
-
* @param {Object} plan the Substrait plan to convert
|
|
747
|
-
* @return {PrintNode} a print node
|
|
748
|
-
*/
|
|
749
|
-
planToNode(plan) {
|
|
750
|
-
const nodes = [];
|
|
751
|
-
for (const relation of plan.relations) {
|
|
752
|
-
if (relation.root != null) {
|
|
753
|
-
nodes.push(this.rootRelToNode(relation.root));
|
|
754
|
-
}
|
|
755
|
-
else {
|
|
756
|
-
throw new Error(`Unrecognized plan relation ${JSON.stringify(relation)}`);
|
|
757
|
-
}
|
|
758
|
-
}
|
|
759
|
-
const schema = this.makeSchemaField([]);
|
|
760
|
-
return this.makePrintNode("plan", nodes, [], schema);
|
|
761
|
-
}
|
|
762
|
-
parse(plan) {
|
|
763
|
-
this.selectedFields = [];
|
|
764
|
-
this.planToNode(plan);
|
|
765
|
-
return {
|
|
766
|
-
selection: this.selectedFields,
|
|
767
|
-
filters: this.deduplicate(this.filters),
|
|
768
|
-
logicalOperators: this.deduplicate(this.logicalOperators),
|
|
769
|
-
};
|
|
770
|
-
}
|
|
771
|
-
/**
|
|
772
|
-
* Converts a base64 string into a decimal number.
|
|
773
|
-
*
|
|
774
|
-
* @public
|
|
775
|
-
* @param {string} base64 - The base64 string to convert.
|
|
776
|
-
* @param {number} precision - The total number of digits in the output number.
|
|
777
|
-
* @param {number} scale - The number of digits after the decimal point in the output number.
|
|
778
|
-
* @throws {Error} Will throw an error if the conversion fails.
|
|
779
|
-
* @returns {number} The converted decimal number.
|
|
780
|
-
* @example
|
|
781
|
-
* // returns 123.45
|
|
782
|
-
* this.base64ToDecimal('MTIzNDU=', 5, 2);
|
|
783
|
-
*/
|
|
784
|
-
base64ToDecimal(base64, precision, scale) {
|
|
785
|
-
// Decode base64 string to a Buffer
|
|
786
|
-
const buffer = Buffer.from(base64, "base64");
|
|
787
|
-
let value = BigInt(0);
|
|
788
|
-
for (let i = buffer.length - 1; i >= 0; i--) {
|
|
789
|
-
value = (value << BigInt(8)) + BigInt(buffer[i]);
|
|
790
|
-
}
|
|
791
|
-
return Number(value) / Math.pow(10, scale);
|
|
792
|
-
}
|
|
793
|
-
deduplicate(array) {
|
|
794
|
-
return Array.from(new Set(array));
|
|
795
|
-
}
|
|
796
|
-
}
|
|
797
|
-
export { SubstraitParser };
|