@narrative.io/data-collaboration-sdk-ts 0.0.107 → 0.0.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/access-rules/index.d.ts +44 -0
- package/build/access-rules/index.js +51 -0
- package/build/access-rules/types.d.ts +29 -0
- package/build/access-rules/types.js +1 -0
- package/build/apps/index.d.ts +3 -3
- package/build/apps/index.js +4 -4
- package/build/apps/types.d.ts +1 -1
- package/build/apps/types.js +0 -1
- package/build/attributes/index.d.ts +9 -9
- package/build/attributes/index.js +8 -8
- package/build/attributes/types.d.ts +5 -5
- package/build/authentication/index.d.ts +3 -3
- package/build/authentication/index.js +3 -3
- package/build/authentication/types.d.ts +3 -3
- package/build/base-api.d.ts +42 -42
- package/build/base-api.js +51 -51
- package/build/company-info/index.d.ts +3 -3
- package/build/company-info/index.js +2 -2
- package/build/contracts/index.d.ts +18 -18
- package/build/contracts/index.js +18 -18
- package/build/contracts/types.d.ts +3 -3
- package/build/data-streams/index.d.ts +10 -10
- package/build/data-streams/index.js +9 -9
- package/build/data-streams/types.d.ts +7 -7
- package/build/datasets/index.d.ts +46 -30
- package/build/datasets/index.js +52 -32
- package/build/datasets/types.d.ts +15 -9
- package/build/forecast/index.d.ts +3 -3
- package/build/forecast/index.js +3 -3
- package/build/forecast/types.d.ts +7 -7
- package/build/health/index.d.ts +2 -2
- package/build/health/index.js +2 -2
- package/build/health/types.d.ts +1 -1
- package/build/index.d.ts +32 -31
- package/build/index.js +44 -24
- package/build/installations/index.d.ts +3 -3
- package/build/installations/index.js +3 -3
- package/build/installations/types.d.ts +1 -1
- package/build/mappings/index.d.ts +16 -16
- package/build/mappings/index.js +18 -18
- package/build/mappings/types.d.ts +6 -6
- package/build/nql/DataRulesConverter.d.ts +3 -3
- package/build/nql/DataRulesConverter.js +55 -31
- package/build/nql/NQLParser.d.ts +4 -4
- package/build/nql/NQLParser.js +74 -57
- package/build/nql/SubstraitParser.d.ts +173 -173
- package/build/nql/SubstraitParser.js +270 -261
- package/build/nql/index.d.ts +4 -4
- package/build/nql/index.js +3 -3
- package/build/nql/types.d.ts +6 -6
- package/build/nql/types.js +59 -29
- package/build/ping/index.d.ts +2 -2
- package/build/ping/index.js +2 -2
- package/build/ping/types.d.ts +1 -1
- package/build/products/index.d.ts +6 -6
- package/build/products/index.js +6 -6
- package/build/products/types.d.ts +4 -4
- package/build/resources/index.d.ts +4 -4
- package/build/resources/index.js +2 -2
- package/build/resources/types.d.ts +3 -3
- package/build/rosetta/types.d.ts +12 -12
- package/build/rosetta/types.js +69 -42
- package/build/rosetta-stone/index.d.ts +3 -3
- package/build/rosetta-stone/index.js +3 -3
- package/build/types.d.ts +2 -2
- package/build/uploads/index.d.ts +8 -8
- package/build/uploads/index.js +8 -8
- package/build/utils.js +1 -1
- package/package.json +3 -1
|
@@ -62,76 +62,76 @@ class SubstraitParser {
|
|
|
62
62
|
filters = [];
|
|
63
63
|
logicalOperators = [];
|
|
64
64
|
/**
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
65
|
+
* Constructs a SubstraitParser
|
|
66
|
+
* @param {SubstraitPlan} plan the Substrait plan
|
|
67
|
+
*/
|
|
68
68
|
constructor(plan) {
|
|
69
69
|
this.idCounters = new Map();
|
|
70
70
|
this.extSet = this.buildExtensionSet(plan);
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
73
|
+
* Creates a map from anchor to function name
|
|
74
|
+
* @param {SubstraitPlan} plan the Substrait plan
|
|
75
|
+
* @return {Map<number, string>} the created mapping
|
|
76
|
+
*/
|
|
77
77
|
buildExtensionSet(plan) {
|
|
78
78
|
const extSet = new Map();
|
|
79
79
|
for (const extensionDecl of plan.extensions) {
|
|
80
80
|
const func = extensionDecl.extensionFunction;
|
|
81
|
-
extSet.set(func.functionAnchor, func.name.split(
|
|
81
|
+
extSet.set(func.functionAnchor, func.name.split(":")[0].toLowerCase());
|
|
82
82
|
}
|
|
83
83
|
return extSet;
|
|
84
84
|
}
|
|
85
85
|
/**
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
86
|
+
* Geneate a unique id for the node
|
|
87
|
+
* @param {string} type The type of the node
|
|
88
|
+
* @return {string} A unique id
|
|
89
|
+
*/
|
|
90
90
|
nextId(type) {
|
|
91
91
|
const value = this.idCounters.get(type) ?? 0;
|
|
92
92
|
this.idCounters.set(type, value + 1);
|
|
93
93
|
return `${type}_${value}`;
|
|
94
94
|
}
|
|
95
95
|
/**
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
96
|
+
* Create a schema field
|
|
97
|
+
*
|
|
98
|
+
* This is a special field with an empty name and an empty type string
|
|
99
|
+
* and uknown nullability
|
|
100
|
+
*
|
|
101
|
+
* @param {Field[]} fields the fields of the schema
|
|
102
|
+
* @return {Field} a schema field
|
|
103
|
+
*/
|
|
104
104
|
makeSchemaField(fields) {
|
|
105
105
|
return {
|
|
106
|
-
name:
|
|
106
|
+
name: "",
|
|
107
107
|
children: fields,
|
|
108
|
-
type:
|
|
109
|
-
nullability:
|
|
108
|
+
type: "",
|
|
109
|
+
nullability: "NULLABILITY_UNSPECIFIED",
|
|
110
110
|
};
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
113
|
+
* Applies a Substrait emit to an output schema
|
|
114
|
+
* @param {Field} schema the original output schema
|
|
115
|
+
* @param {Object} emit a potential remapping of fields
|
|
116
|
+
* @return {Field} the remapped output schema
|
|
117
|
+
*/
|
|
118
118
|
applyEmit(schema, emit) {
|
|
119
119
|
let outputFields = schema.children;
|
|
120
|
-
if (
|
|
120
|
+
if (emit?.outputMapping != null) {
|
|
121
121
|
outputFields = emit.outputMapping.map((idx) => schema.children[idx]);
|
|
122
122
|
}
|
|
123
123
|
return this.makeSchemaField(outputFields);
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
126
|
+
* Create a print node, generating a new id
|
|
127
|
+
*
|
|
128
|
+
* @param {string} type the node type
|
|
129
|
+
* @param {PrintNode[]} inputs the inputs to the node
|
|
130
|
+
* @param {SimpleProperty[]} props properties describing the node
|
|
131
|
+
* @param {Field} schema the output schema of the node
|
|
132
|
+
* @param {Object} emit the Substrait emit for the node
|
|
133
|
+
* @return {PrintNode} a print node
|
|
134
|
+
*/
|
|
135
135
|
makePrintNode(type, inputs, props, schema, emit) {
|
|
136
136
|
const emitSchema = this.applyEmit(schema, emit);
|
|
137
137
|
return {
|
|
@@ -139,19 +139,19 @@ class SubstraitParser {
|
|
|
139
139
|
type,
|
|
140
140
|
inputs,
|
|
141
141
|
props,
|
|
142
|
-
schema: emitSchema
|
|
142
|
+
schema: emitSchema,
|
|
143
143
|
};
|
|
144
144
|
}
|
|
145
145
|
/**
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
146
|
+
* Recursive helper for structFieldToStr
|
|
147
|
+
* @param {*} field the Substrait field to convert
|
|
148
|
+
* @param {*} currentName the current name
|
|
149
|
+
* @param {Field} currentField the current input field
|
|
150
|
+
* @return {Field} a stringified represenstation
|
|
151
|
+
*/
|
|
152
152
|
structFieldHelper(field, currentName, currentField) {
|
|
153
153
|
const fieldNum = field.field;
|
|
154
|
-
const fieldName =
|
|
154
|
+
const fieldName = currentField.name !== "" ? currentField.name : `$${fieldNum}`;
|
|
155
155
|
const nextName = `${currentName}${fieldName}`;
|
|
156
156
|
if (field.child != null) {
|
|
157
157
|
return this.structFieldHelper(field.child, `${nextName}.`, currentField.children[fieldNum]);
|
|
@@ -159,28 +159,28 @@ class SubstraitParser {
|
|
|
159
159
|
return currentField.children[fieldNum];
|
|
160
160
|
}
|
|
161
161
|
/**
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
162
|
+
* Convert a struct field to a stringified representation
|
|
163
|
+
*
|
|
164
|
+
* Struct fields are usually intuitively written with names.
|
|
165
|
+
* For example, a.b.c. However, fields in Substrait plans
|
|
166
|
+
* are not typically named. So instead we represent them as
|
|
167
|
+
* $0, $1, $2, ... where $N means the field at index N.
|
|
168
|
+
*
|
|
169
|
+
* If the field does happen to be named we will use that.
|
|
170
|
+
*
|
|
171
|
+
* @param {StructItem} field a Substrait struct field
|
|
172
|
+
* @param {Field} exprIn the input to the expression
|
|
173
|
+
* @return {Field} a stringified representation
|
|
174
|
+
*/
|
|
175
175
|
structFieldToField(field, exprIn) {
|
|
176
|
-
return this.structFieldHelper(field,
|
|
176
|
+
return this.structFieldHelper(field, "", exprIn);
|
|
177
177
|
}
|
|
178
178
|
/**
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
179
|
+
* Convert a direct reference to a stringified representation
|
|
180
|
+
* @param {Object} ref A Substrait direct reference
|
|
181
|
+
* @param {Object} exprIn the input to the expression
|
|
182
|
+
* @return {Field} a string representation
|
|
183
|
+
*/
|
|
184
184
|
directReferenceToField(ref, exprIn) {
|
|
185
185
|
if (ref.structField != null) {
|
|
186
186
|
return this.structFieldToField(ref.structField, exprIn);
|
|
@@ -190,11 +190,11 @@ class SubstraitParser {
|
|
|
190
190
|
}
|
|
191
191
|
}
|
|
192
192
|
/**
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
193
|
+
* Convert a reference to a stringified representation
|
|
194
|
+
* @param {Object} ref A Substrait reference expression
|
|
195
|
+
* @param {Object} exprIn the input to the expression
|
|
196
|
+
* @return {Field} a string representation
|
|
197
|
+
*/
|
|
198
198
|
referenceToField(ref, exprIn) {
|
|
199
199
|
if (ref.directReference != null) {
|
|
200
200
|
return this.directReferenceToField(ref.directReference, exprIn);
|
|
@@ -204,15 +204,15 @@ class SubstraitParser {
|
|
|
204
204
|
}
|
|
205
205
|
}
|
|
206
206
|
/**
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
207
|
+
* Convert a Substrait function argument to a string
|
|
208
|
+
* @param {Object} arg the argument to convert
|
|
209
|
+
* @param {Field} inp the input field
|
|
210
|
+
* @return {string} a string representation
|
|
211
|
+
*/
|
|
212
212
|
functionArgumentToStr(arg, inp) {
|
|
213
213
|
if (arg.enum != null) {
|
|
214
214
|
if (arg.enum.specified == null) {
|
|
215
|
-
return
|
|
215
|
+
return "unspecified";
|
|
216
216
|
}
|
|
217
217
|
return arg.enum.specified;
|
|
218
218
|
}
|
|
@@ -223,41 +223,41 @@ class SubstraitParser {
|
|
|
223
223
|
return this.expressionToStr(arg.value, inp).name;
|
|
224
224
|
}
|
|
225
225
|
else {
|
|
226
|
-
throw new Error(
|
|
226
|
+
throw new Error("A FunctionArgument did not have an enum/type/value");
|
|
227
227
|
}
|
|
228
228
|
}
|
|
229
229
|
/**
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
* Converts a Substrait function reference to a string
|
|
231
|
+
* @param {number} ref a Substrait function reference
|
|
232
|
+
* @return {string} a string version of the reference
|
|
233
|
+
*/
|
|
234
234
|
functionRefToName(ref) {
|
|
235
|
-
return this.extSet.get(ref) ??
|
|
235
|
+
return this.extSet.get(ref) ?? "unknown";
|
|
236
236
|
}
|
|
237
237
|
/**
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
238
|
+
* Convert arguments to a joined string
|
|
239
|
+
* @param {FunctionTyp} func A Substrait function call
|
|
240
|
+
* @param {Field} schema The input type
|
|
241
|
+
* @return {string} A string representation
|
|
242
|
+
*/
|
|
243
243
|
argsToString(func, schema) {
|
|
244
244
|
const args = func.arguments.length > 0
|
|
245
245
|
? func.arguments.map((arg) => this.functionArgumentToStr(arg, schema))
|
|
246
246
|
: [];
|
|
247
|
-
return args.join(
|
|
247
|
+
return args.join(",");
|
|
248
248
|
}
|
|
249
249
|
/**
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
250
|
+
* Convert a scalar function to a stringified representation
|
|
251
|
+
* @param {Object} func the function to convert
|
|
252
|
+
* @param {Field} schema the input type info
|
|
253
|
+
* @return {Field} a string version of the function
|
|
254
|
+
*/
|
|
255
255
|
scalarFunctionToField(func, schema) {
|
|
256
256
|
const argsString = this.argsToString(func, schema);
|
|
257
257
|
const outputType = this.typeToField(func.outputType);
|
|
258
258
|
const functionName = this.functionRefToName(func.functionReference);
|
|
259
|
-
if (![
|
|
260
|
-
const args = argsString.split(
|
|
259
|
+
if (!["and", "or", "not", "cast"].includes(functionName)) {
|
|
260
|
+
const args = argsString.split(",");
|
|
261
261
|
if (args.length === 1) {
|
|
262
262
|
this.filters.push(`${args[0]} ${functionName} `);
|
|
263
263
|
}
|
|
@@ -265,7 +265,7 @@ class SubstraitParser {
|
|
|
265
265
|
this.filters.push(`${args[0]} ${functionName} ${args[1]}`);
|
|
266
266
|
}
|
|
267
267
|
else {
|
|
268
|
-
this.filters.push(`${functionName}(${args.join(
|
|
268
|
+
this.filters.push(`${functionName}(${args.join(", ")})`);
|
|
269
269
|
}
|
|
270
270
|
}
|
|
271
271
|
else {
|
|
@@ -276,60 +276,64 @@ class SubstraitParser {
|
|
|
276
276
|
return outputType;
|
|
277
277
|
}
|
|
278
278
|
/**
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
279
|
+
* Converts a Substrait literal to a field
|
|
280
|
+
* @param {Object} lit the Substrait literal
|
|
281
|
+
* @return {Field} a field representation
|
|
282
|
+
*/
|
|
283
283
|
literalToField(lit) {
|
|
284
|
-
const nullability = lit.nullable
|
|
284
|
+
const nullability = lit.nullable
|
|
285
|
+
? "NULLABILITY_NULLABLE"
|
|
286
|
+
: "NULLABILITY_REQUIRED";
|
|
285
287
|
if (lit.bool != null) {
|
|
286
288
|
return {
|
|
287
289
|
name: String(lit.bool.value),
|
|
288
|
-
type:
|
|
290
|
+
type: "boolean",
|
|
289
291
|
nullability,
|
|
290
|
-
children: []
|
|
292
|
+
children: [],
|
|
291
293
|
};
|
|
292
294
|
}
|
|
293
295
|
else if (lit.date != null) {
|
|
294
296
|
return {
|
|
295
297
|
name: new Date((lit.date?.value ?? 0) * 8.64e7).toISOString(),
|
|
296
|
-
type:
|
|
298
|
+
type: "date",
|
|
297
299
|
nullability,
|
|
298
|
-
children: []
|
|
300
|
+
children: [],
|
|
299
301
|
};
|
|
300
302
|
}
|
|
301
303
|
else if (lit.decimal != null) {
|
|
302
304
|
return {
|
|
303
305
|
name: String(this.base64ToDecimal(lit.decimal.value, lit.decimal.precision, lit.decimal.scale)),
|
|
304
|
-
type:
|
|
306
|
+
type: "decimal",
|
|
305
307
|
nullability,
|
|
306
|
-
children: []
|
|
308
|
+
children: [],
|
|
307
309
|
};
|
|
308
310
|
}
|
|
309
311
|
else if (lit.i32 != null) {
|
|
310
312
|
return {
|
|
311
|
-
name: lit.i32?.value?.toString() ??
|
|
312
|
-
type:
|
|
313
|
+
name: lit.i32?.value?.toString() ?? "unspecified",
|
|
314
|
+
type: "i32",
|
|
313
315
|
nullability,
|
|
314
|
-
children: []
|
|
316
|
+
children: [],
|
|
315
317
|
};
|
|
316
318
|
}
|
|
317
319
|
else if (lit.intervalDayToSecond != null) {
|
|
318
320
|
const val = lit.intervalDayToSecond;
|
|
319
321
|
return {
|
|
320
322
|
name: `${val.days}d${val.seconds}s${val.microseconds}u`,
|
|
321
|
-
type:
|
|
323
|
+
type: "interval_day",
|
|
322
324
|
nullability,
|
|
323
|
-
children: []
|
|
325
|
+
children: [],
|
|
324
326
|
};
|
|
325
327
|
}
|
|
326
328
|
else if (lit.fixedChar != null) {
|
|
327
|
-
const val = typeof lit.fixedChar ===
|
|
329
|
+
const val = typeof lit.fixedChar === "string"
|
|
330
|
+
? lit.fixedChar
|
|
331
|
+
: lit.fixedChar.value ?? "unspecified";
|
|
328
332
|
return {
|
|
329
333
|
name: `${val}`,
|
|
330
|
-
type:
|
|
334
|
+
type: "interval_day",
|
|
331
335
|
nullability,
|
|
332
|
-
children: []
|
|
336
|
+
children: [],
|
|
333
337
|
};
|
|
334
338
|
}
|
|
335
339
|
else {
|
|
@@ -337,36 +341,36 @@ class SubstraitParser {
|
|
|
337
341
|
}
|
|
338
342
|
}
|
|
339
343
|
/**
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
344
|
+
* Convert a cast expression to a stringified representation
|
|
345
|
+
* @param {Object} cast A Substrait cast expression
|
|
346
|
+
* @param {Field} schema The input schema
|
|
347
|
+
* @return {Field} A field representation
|
|
348
|
+
*/
|
|
345
349
|
castToField(cast, schema) {
|
|
346
350
|
const input = this.expressionToStr(cast.input, schema);
|
|
347
351
|
const outputType = this.typeToField(cast.type);
|
|
348
|
-
let fb =
|
|
349
|
-
if (cast.failureBehavior ===
|
|
350
|
-
fb =
|
|
352
|
+
let fb = "unspecified";
|
|
353
|
+
if (cast.failureBehavior === "FAILURE_BEHAVIOR_RETURN_NULL") {
|
|
354
|
+
fb = "or null";
|
|
351
355
|
}
|
|
352
|
-
else if (cast.failureBehavior ===
|
|
353
|
-
fb =
|
|
356
|
+
else if (cast.failureBehavior === "THROW_EXCEPTION") {
|
|
357
|
+
fb = "or die";
|
|
354
358
|
}
|
|
355
359
|
const name = `cast(${input.name}, ${outputType.type}, ${fb})`;
|
|
356
360
|
outputType.name = name;
|
|
357
361
|
return outputType;
|
|
358
362
|
}
|
|
359
363
|
/**
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
364
|
+
* Convert an expression to a stringified representation
|
|
365
|
+
*
|
|
366
|
+
* The expression is returned as a field where the type
|
|
367
|
+
* of the field is the output type of the expression and the
|
|
368
|
+
* name of the field is a string representation of the expression
|
|
369
|
+
*
|
|
370
|
+
* @param {Object} expr A Substrait expression
|
|
371
|
+
* @param {Object} exprIn the input to the expression
|
|
372
|
+
* @return {Field} a string representation
|
|
373
|
+
*/
|
|
370
374
|
expressionToStr(expr, exprIn) {
|
|
371
375
|
if (expr.selection != null) {
|
|
372
376
|
return this.referenceToField(expr.selection, exprIn);
|
|
@@ -385,16 +389,16 @@ class SubstraitParser {
|
|
|
385
389
|
}
|
|
386
390
|
}
|
|
387
391
|
/**
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
+
* Convert a project relation to a print node
|
|
393
|
+
* @param {Object} project A Substrait ProjectRel
|
|
394
|
+
* @return {PrintNode} a print node
|
|
395
|
+
*/
|
|
392
396
|
projectToNode(project) {
|
|
393
397
|
const input = this.relToNode(project.input);
|
|
394
398
|
const expressions = project.expressions.map((val, idx) => {
|
|
395
399
|
return {
|
|
396
400
|
name: `expressions[${idx}]`,
|
|
397
|
-
field: this.expressionToStr(val, input.schema)
|
|
401
|
+
field: this.expressionToStr(val, input.schema),
|
|
398
402
|
};
|
|
399
403
|
});
|
|
400
404
|
const outFields = expressions.map((expr) => expr.field);
|
|
@@ -402,85 +406,88 @@ class SubstraitParser {
|
|
|
402
406
|
const schema = this.makeSchemaField(fields);
|
|
403
407
|
const props = expressions.map((expr) => ({
|
|
404
408
|
name: expr.name,
|
|
405
|
-
value: expr.field.name
|
|
409
|
+
value: expr.field.name,
|
|
406
410
|
}));
|
|
407
411
|
const inputs = [this.relToNode(project.input)];
|
|
408
|
-
return this.makePrintNode(
|
|
412
|
+
return this.makePrintNode("project", inputs, props, schema, project.common?.emit);
|
|
409
413
|
}
|
|
410
414
|
/**
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
415
|
+
* Convert a Substrait data type to an unnamed Field
|
|
416
|
+
*
|
|
417
|
+
* The field's name will be an empty string
|
|
418
|
+
*
|
|
419
|
+
* @param {Object} type a Substrait data type
|
|
420
|
+
* @return {Field} an unnamed field
|
|
421
|
+
*/
|
|
418
422
|
typeToField(type) {
|
|
419
423
|
let typeStr = null;
|
|
420
|
-
let nullability =
|
|
424
|
+
let nullability = "NULLABILITY_UNSPECIFIED";
|
|
421
425
|
if (type.varchar != null) {
|
|
422
426
|
typeStr = `VARCHAR<${type.varchar.length}>`;
|
|
423
427
|
nullability = type.varchar.nullability;
|
|
424
428
|
}
|
|
425
429
|
else if (type.fixedChar != null) {
|
|
426
430
|
typeStr = `FIXEDCHAR<${type.fixedChar.length}>`;
|
|
427
|
-
nullability =
|
|
431
|
+
nullability =
|
|
432
|
+
typeof type.fixedChar !== "string"
|
|
433
|
+
? type.fixedChar?.nullability
|
|
434
|
+
: "NULLABILITY_REQUIRED";
|
|
428
435
|
}
|
|
429
436
|
else if (type.date != null) {
|
|
430
|
-
typeStr =
|
|
437
|
+
typeStr = "date";
|
|
431
438
|
nullability = type.date.nullability;
|
|
432
439
|
}
|
|
433
440
|
else if (type.decimal != null) {
|
|
434
441
|
typeStr = `DECIMAL<${type.decimal.precision},${type.decimal.scale}>`;
|
|
435
|
-
nullability = type.decimal.nullability ??
|
|
442
|
+
nullability = type.decimal.nullability ?? "NULLABILITY_UNSPECIFIED";
|
|
436
443
|
}
|
|
437
444
|
else if (type.i32 != null) {
|
|
438
|
-
typeStr =
|
|
445
|
+
typeStr = "i32";
|
|
439
446
|
nullability = type.i32.nullability;
|
|
440
447
|
}
|
|
441
448
|
else if (type.i64 != null) {
|
|
442
|
-
typeStr =
|
|
449
|
+
typeStr = "i64";
|
|
443
450
|
nullability = type.i64.nullability;
|
|
444
451
|
}
|
|
445
452
|
else if (type.bool != null) {
|
|
446
|
-
typeStr =
|
|
453
|
+
typeStr = "boolean";
|
|
447
454
|
nullability = type.bool.nullability;
|
|
448
455
|
}
|
|
449
456
|
else if (type.string != null) {
|
|
450
|
-
typeStr =
|
|
457
|
+
typeStr = "string";
|
|
451
458
|
nullability = type.string.nullability;
|
|
452
459
|
}
|
|
453
460
|
else if (type.fp64 != null) {
|
|
454
|
-
typeStr =
|
|
461
|
+
typeStr = "fp64";
|
|
455
462
|
nullability = type.fp64.nullability;
|
|
456
463
|
}
|
|
457
464
|
else if (type.timestampTz != null) {
|
|
458
|
-
typeStr =
|
|
465
|
+
typeStr = "timestampTz";
|
|
459
466
|
nullability = type.timestampTz.nullability;
|
|
460
467
|
}
|
|
461
468
|
else if (type.struct != null) {
|
|
462
|
-
typeStr =
|
|
469
|
+
typeStr = "struct";
|
|
463
470
|
nullability = type.struct.nullability;
|
|
464
471
|
}
|
|
465
472
|
else if (type.fixedBinary != null) {
|
|
466
|
-
typeStr =
|
|
473
|
+
typeStr = "fixedBinary";
|
|
467
474
|
nullability = type.fixedBinary.nullability;
|
|
468
475
|
}
|
|
469
476
|
else {
|
|
470
477
|
throw new Error(`Unrecognized type: ${JSON.stringify(type)}`);
|
|
471
478
|
}
|
|
472
479
|
return {
|
|
473
|
-
name:
|
|
480
|
+
name: "",
|
|
474
481
|
type: typeStr,
|
|
475
482
|
nullability,
|
|
476
|
-
children: []
|
|
483
|
+
children: [],
|
|
477
484
|
};
|
|
478
485
|
}
|
|
479
486
|
/**
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
487
|
+
* Convert a named struct to an array of Field
|
|
488
|
+
* @param {NamedStruct} struct a Substrait named struct
|
|
489
|
+
* @return {Field[]} named fields
|
|
490
|
+
*/
|
|
484
491
|
namedStructToSchema(struct) {
|
|
485
492
|
// const names = struct.names
|
|
486
493
|
// const types = this.flattenTypes(struct.struct)
|
|
@@ -490,10 +497,10 @@ class SubstraitParser {
|
|
|
490
497
|
// throw new Error('Mismatched names and types')
|
|
491
498
|
// }
|
|
492
499
|
return {
|
|
493
|
-
name:
|
|
494
|
-
type:
|
|
495
|
-
nullability:
|
|
496
|
-
children: []
|
|
500
|
+
name: "test",
|
|
501
|
+
type: "struct",
|
|
502
|
+
nullability: "NULLABILITY_UNSPECIFIED",
|
|
503
|
+
children: [],
|
|
497
504
|
};
|
|
498
505
|
// return this.makeSchemaField(outTypes)
|
|
499
506
|
}
|
|
@@ -511,112 +518,114 @@ class SubstraitParser {
|
|
|
511
518
|
// return output
|
|
512
519
|
// }
|
|
513
520
|
/**
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
521
|
+
* Extract the properties from a read relation
|
|
522
|
+
* @param {Object} read a Substrait read relation
|
|
523
|
+
* @return {SimpleProperty[]} properties for the relation
|
|
524
|
+
*/
|
|
518
525
|
readTypeToProps(read) {
|
|
519
|
-
return [
|
|
526
|
+
return [
|
|
527
|
+
{ name: "namedTable.names", value: read.namedTable.names.join(".") },
|
|
528
|
+
];
|
|
520
529
|
}
|
|
521
530
|
/**
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
531
|
+
* Convert a Substrait read relation to a print node
|
|
532
|
+
* @param {Object} read a Substrait read relation
|
|
533
|
+
* @return {PrintNode} a print node
|
|
534
|
+
*/
|
|
526
535
|
readToNode(read) {
|
|
527
536
|
const schema = this.namedStructToSchema(read.baseSchema);
|
|
528
537
|
const props = this.readTypeToProps(read);
|
|
529
|
-
return this.makePrintNode(
|
|
538
|
+
return this.makePrintNode("read", [], props, schema, read.common?.emit);
|
|
530
539
|
}
|
|
531
540
|
/**
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
541
|
+
* True if the object has a field with the given name
|
|
542
|
+
* @param {Object} obj the object to test
|
|
543
|
+
* @param {string} fieldName the field name to look for
|
|
544
|
+
* @return {boolean} the result of the test
|
|
545
|
+
*/
|
|
537
546
|
hasValue(obj, fieldName) {
|
|
538
547
|
return fieldName in obj;
|
|
539
548
|
}
|
|
540
549
|
/**
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
550
|
+
* Convert a Substrait fetch relation to a print node
|
|
551
|
+
* @param {Object} fetch a Substrait fetch relation
|
|
552
|
+
* @return {PrintNode} a print node
|
|
553
|
+
*/
|
|
545
554
|
fetchToNode(fetch) {
|
|
546
555
|
const input = this.relToNode(fetch.input);
|
|
547
556
|
const schema = input.schema;
|
|
548
557
|
const props = [];
|
|
549
|
-
if (this.hasValue(fetch,
|
|
550
|
-
props.push({ name:
|
|
558
|
+
if (this.hasValue(fetch, "offset")) {
|
|
559
|
+
props.push({ name: "offset", value: String(fetch.offset) });
|
|
551
560
|
}
|
|
552
|
-
if (this.hasValue(fetch,
|
|
553
|
-
props.push({ name:
|
|
561
|
+
if (this.hasValue(fetch, "count")) {
|
|
562
|
+
props.push({ name: "count", value: String(fetch.count) });
|
|
554
563
|
}
|
|
555
|
-
return this.makePrintNode(
|
|
564
|
+
return this.makePrintNode("fetch", [input], props, schema, fetch.common.emit);
|
|
556
565
|
}
|
|
557
566
|
/**
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
567
|
+
* Convert a Substrait sort field to a property
|
|
568
|
+
* @param {Object} sortField a Substrait sort field
|
|
569
|
+
* @param {Field} schema the schema of the sort input
|
|
570
|
+
* @return {SimpleProperty} a property describing the sort
|
|
571
|
+
*/
|
|
563
572
|
sortFieldToStr(sortField, schema) {
|
|
564
573
|
const exprStr = this.expressionToStr(sortField.expr, schema).name;
|
|
565
574
|
if (sortField.direction !== null) {
|
|
566
575
|
const dir = sortField.direction;
|
|
567
|
-
let sortDesc =
|
|
576
|
+
let sortDesc = "";
|
|
568
577
|
if (dir === SortDirection.ASCENDING_NULLS_FIRST) {
|
|
569
|
-
sortDesc =
|
|
578
|
+
sortDesc = "ascending (nulls first)";
|
|
570
579
|
}
|
|
571
580
|
else if (dir === SortDirection.ASCENDING_NULLS_LAST) {
|
|
572
|
-
sortDesc =
|
|
581
|
+
sortDesc = "ascending (nulls last)";
|
|
573
582
|
}
|
|
574
583
|
else if (dir === SortDirection.DESCENDING_NULLS_FIRST) {
|
|
575
|
-
sortDesc =
|
|
584
|
+
sortDesc = "descending (nulls first)";
|
|
576
585
|
}
|
|
577
586
|
else if (dir === SortDirection.DESCENDING_NULLS_LAST) {
|
|
578
|
-
sortDesc =
|
|
587
|
+
sortDesc = "descending (nulls last)";
|
|
579
588
|
}
|
|
580
589
|
else if (dir === SortDirection.CLUSTERED) {
|
|
581
|
-
sortDesc =
|
|
590
|
+
sortDesc = "clustered";
|
|
582
591
|
}
|
|
583
592
|
else {
|
|
584
|
-
sortDesc =
|
|
593
|
+
sortDesc = "unspecified";
|
|
585
594
|
}
|
|
586
595
|
return `${exprStr} | ${sortDesc}`;
|
|
587
596
|
}
|
|
588
597
|
}
|
|
589
598
|
/**
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
599
|
+
* Convert a Substrait sort relation to a print node
|
|
600
|
+
* @param {Object} sort a Substrait sort relation
|
|
601
|
+
* @return {PrintNode} a print node
|
|
602
|
+
*/
|
|
594
603
|
sortToNode(sort) {
|
|
595
604
|
const input = this.relToNode(sort.input);
|
|
596
605
|
const schema = input.schema;
|
|
597
606
|
const props = sort.sorts.map((sort, idx) => {
|
|
598
607
|
return {
|
|
599
608
|
name: `sorts[${idx}]`,
|
|
600
|
-
value: this.sortFieldToStr(sort, schema) ??
|
|
609
|
+
value: this.sortFieldToStr(sort, schema) ?? "undefined",
|
|
601
610
|
};
|
|
602
611
|
});
|
|
603
|
-
return this.makePrintNode(
|
|
612
|
+
return this.makePrintNode("sort", [input], props, schema, sort.common.emit);
|
|
604
613
|
}
|
|
605
614
|
/**
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
615
|
+
* Convert a Substrait aggregate function to a field
|
|
616
|
+
* @param {Object} func the Substrait aggregate function
|
|
617
|
+
* @param {Field} inp the input field
|
|
618
|
+
* @return {Field} a field representation of the function
|
|
619
|
+
*/
|
|
611
620
|
aggregateFunctionToField(func, inp) {
|
|
612
621
|
// TODO (weston) sorts, phases, invocation
|
|
613
622
|
return this.scalarFunctionToField(func, inp);
|
|
614
623
|
}
|
|
615
624
|
/**
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
625
|
+
* Convert a Substrait aggregate relation to a print node
|
|
626
|
+
* @param {Object} agg a Substrait aggregate relation
|
|
627
|
+
* @return {PrintNode} a print node
|
|
628
|
+
*/
|
|
620
629
|
aggToNode(agg) {
|
|
621
630
|
const input = this.relToNode(agg.input);
|
|
622
631
|
const props = [];
|
|
@@ -633,64 +642,64 @@ class SubstraitParser {
|
|
|
633
642
|
fields.push(aggregate);
|
|
634
643
|
const innerProps = [
|
|
635
644
|
{
|
|
636
|
-
name:
|
|
637
|
-
value: aggregate.name
|
|
638
|
-
}
|
|
645
|
+
name: "measure",
|
|
646
|
+
value: aggregate.name,
|
|
647
|
+
},
|
|
639
648
|
];
|
|
640
649
|
innerProps.push({
|
|
641
|
-
name:
|
|
642
|
-
value: this.expressionToStr(measure.filter, input.schema).name
|
|
650
|
+
name: "filter",
|
|
651
|
+
value: this.expressionToStr(measure.filter, input.schema).name,
|
|
643
652
|
});
|
|
644
653
|
props.push({
|
|
645
654
|
name: `measures[${idx}]`,
|
|
646
|
-
value: JSON.stringify(innerProps) // TODO Is this right?
|
|
655
|
+
value: JSON.stringify(innerProps), // TODO Is this right?
|
|
647
656
|
});
|
|
648
657
|
});
|
|
649
658
|
const schema = this.makeSchemaField(fields);
|
|
650
|
-
return this.makePrintNode(
|
|
659
|
+
return this.makePrintNode("aggregate", [input], props, schema, agg.common.emit);
|
|
651
660
|
}
|
|
652
661
|
/**
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
662
|
+
* Convert a Substrait filter relation to a print node
|
|
663
|
+
* @param {Object} filt a Substrait filter relation
|
|
664
|
+
* @return {PrintNode} a print node
|
|
665
|
+
*/
|
|
657
666
|
filterToNode(filt) {
|
|
658
667
|
const input = this.relToNode(filt.input);
|
|
659
668
|
const props = [];
|
|
660
669
|
const schema = input.schema;
|
|
661
670
|
props.push({
|
|
662
|
-
name:
|
|
663
|
-
value: this.expressionToStr(filt.condition, schema).name
|
|
671
|
+
name: "condition",
|
|
672
|
+
value: this.expressionToStr(filt.condition, schema).name,
|
|
664
673
|
});
|
|
665
|
-
const printNode = this.makePrintNode(
|
|
674
|
+
const printNode = this.makePrintNode("filter", [input], props, schema, filt.common.emit);
|
|
666
675
|
return printNode;
|
|
667
676
|
}
|
|
668
677
|
/**
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
678
|
+
* Converts a Substrait join relation to a print node
|
|
679
|
+
* @param {Object} join a Substrait join relation
|
|
680
|
+
* @return {PrintNode} a print node
|
|
681
|
+
*/
|
|
673
682
|
joinToNode(join) {
|
|
674
683
|
const left = this.relToNode(join.left);
|
|
675
684
|
const right = this.relToNode(join.right);
|
|
676
685
|
const props = [];
|
|
677
686
|
const schema = this.makeSchemaField(left.schema.children.concat(right.schema.children));
|
|
678
|
-
let joinType =
|
|
687
|
+
let joinType = "unspecified";
|
|
679
688
|
if (join.type !== null) {
|
|
680
689
|
joinType = JoinType[join.type];
|
|
681
690
|
}
|
|
682
|
-
props.push({ name:
|
|
691
|
+
props.push({ name: "type", value: joinType });
|
|
683
692
|
if (join.expression !== null) {
|
|
684
693
|
const exprStr = this.expressionToStr(join.expression, schema).name;
|
|
685
|
-
props.push({ name:
|
|
694
|
+
props.push({ name: "expression", value: exprStr });
|
|
686
695
|
}
|
|
687
|
-
return this.makePrintNode(
|
|
696
|
+
return this.makePrintNode("join", [left, right], props, schema, join.common.emit);
|
|
688
697
|
}
|
|
689
698
|
/**
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
699
|
+
* Converts a Substrait relation to a print node
|
|
700
|
+
* @param {Object} rel the Substrait relation to convert
|
|
701
|
+
* @return {PrintNode} a print node
|
|
702
|
+
*/
|
|
694
703
|
relToNode(rel) {
|
|
695
704
|
if (rel.project != null) {
|
|
696
705
|
return this.projectToNode(rel.project);
|
|
@@ -718,25 +727,25 @@ class SubstraitParser {
|
|
|
718
727
|
}
|
|
719
728
|
}
|
|
720
729
|
/**
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
730
|
+
* Converts a Substrait root relation to a print node
|
|
731
|
+
* @param {Object} rel the Substrait relation to convert
|
|
732
|
+
* @return {PrintNode} a print node
|
|
733
|
+
*/
|
|
725
734
|
rootRelToNode(rel) {
|
|
726
735
|
const props = rel.names.map((val, idx) => ({
|
|
727
736
|
name: `name[${idx}]`,
|
|
728
|
-
value: val
|
|
737
|
+
value: val,
|
|
729
738
|
}));
|
|
730
739
|
this.selectedFields = this.selectedFields.concat(rel.names);
|
|
731
740
|
const inputs = [this.relToNode(rel.input)];
|
|
732
741
|
const schema = this.makeSchemaField([]);
|
|
733
|
-
return this.makePrintNode(
|
|
742
|
+
return this.makePrintNode("sink", inputs, props, schema);
|
|
734
743
|
}
|
|
735
744
|
/**
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
745
|
+
* Converts a Substrait plan to a print node
|
|
746
|
+
* @param {Object} plan the Substrait plan to convert
|
|
747
|
+
* @return {PrintNode} a print node
|
|
748
|
+
*/
|
|
740
749
|
planToNode(plan) {
|
|
741
750
|
const nodes = [];
|
|
742
751
|
for (const relation of plan.relations) {
|
|
@@ -748,7 +757,7 @@ class SubstraitParser {
|
|
|
748
757
|
}
|
|
749
758
|
}
|
|
750
759
|
const schema = this.makeSchemaField([]);
|
|
751
|
-
return this.makePrintNode(
|
|
760
|
+
return this.makePrintNode("plan", nodes, [], schema);
|
|
752
761
|
}
|
|
753
762
|
parse(plan) {
|
|
754
763
|
this.selectedFields = [];
|
|
@@ -756,7 +765,7 @@ class SubstraitParser {
|
|
|
756
765
|
return {
|
|
757
766
|
selection: this.selectedFields,
|
|
758
767
|
filters: this.deduplicate(this.filters),
|
|
759
|
-
logicalOperators: this.deduplicate(this.logicalOperators)
|
|
768
|
+
logicalOperators: this.deduplicate(this.logicalOperators),
|
|
760
769
|
};
|
|
761
770
|
}
|
|
762
771
|
/**
|
|
@@ -774,7 +783,7 @@ class SubstraitParser {
|
|
|
774
783
|
*/
|
|
775
784
|
base64ToDecimal(base64, precision, scale) {
|
|
776
785
|
// Decode base64 string to a Buffer
|
|
777
|
-
const buffer = Buffer.from(base64,
|
|
786
|
+
const buffer = Buffer.from(base64, "base64");
|
|
778
787
|
let value = BigInt(0);
|
|
779
788
|
for (let i = buffer.length - 1; i >= 0; i--) {
|
|
780
789
|
value = (value << BigInt(8)) + BigInt(buffer[i]);
|