@narrative.io/data-collaboration-sdk-ts 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,785 +0,0 @@
1
- /*!
2
- Copyright 2023 Narrative I/O, Inc.
3
-
4
- Licensed under the Apache License, Version 2.0 (the "License");
5
- you may not use this file except in compliance with the License.
6
- You may obtain a copy of the License at
7
-
8
- http://www.apache.org/licenses/LICENSE-2.0
9
-
10
- NOTICE: THIS FILE HAS BEEN MODIFIED BY NARRATIVE I/O UNDER COMPLIANCE WITH THE APACHE 2.0 LICENCE FROM THE ORIGINAL WORK
11
- OF THE COMPANY Substrait (https://github.com/substrait-io/substrait-js).
12
- */
13
- var AggregationInvocation;
14
- (function (AggregationInvocation) {
15
- AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_UNSPECIFIED"] = 0] = "AGGREGATION_INVOCATION_UNSPECIFIED";
16
- AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_ALL"] = 1] = "AGGREGATION_INVOCATION_ALL";
17
- AggregationInvocation[AggregationInvocation["AGGREGATION_INVOCATION_DISTINCT"] = 2] = "AGGREGATION_INVOCATION_DISTINCT";
18
- })(AggregationInvocation || (AggregationInvocation = {}));
19
- var AggregationPhase;
20
- (function (AggregationPhase) {
21
- AggregationPhase[AggregationPhase["AGGREGATION_PHASE_UNSPECIFIED"] = 0] = "AGGREGATION_PHASE_UNSPECIFIED";
22
- AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE"] = 1] = "AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE";
23
- AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE"] = 2] = "AGGREGATION_PHASE_INTERMEDIATE_TO_INTERMEDIATE";
24
- AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INITIAL_TO_RESULT"] = 3] = "AGGREGATION_PHASE_INITIAL_TO_RESULT";
25
- AggregationPhase[AggregationPhase["AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT"] = 4] = "AGGREGATION_PHASE_INTERMEDIATE_TO_RESULT";
26
- })(AggregationPhase || (AggregationPhase = {}));
27
- var JoinType;
28
- (function (JoinType) {
29
- JoinType[JoinType["JOIN_TYPE_UNSPECIFIED"] = 0] = "JOIN_TYPE_UNSPECIFIED";
30
- JoinType[JoinType["JOIN_TYPE_INNER"] = 1] = "JOIN_TYPE_INNER";
31
- JoinType[JoinType["JOIN_TYPE_OUTER"] = 2] = "JOIN_TYPE_OUTER";
32
- JoinType[JoinType["JOIN_TYPE_LEFT"] = 3] = "JOIN_TYPE_LEFT";
33
- JoinType[JoinType["JOIN_TYPE_RIGHT"] = 4] = "JOIN_TYPE_RIGHT";
34
- JoinType[JoinType["JOIN_TYPE_SEMI"] = 5] = "JOIN_TYPE_SEMI";
35
- JoinType[JoinType["JOIN_TYPE_ANTI"] = 6] = "JOIN_TYPE_ANTI";
36
- JoinType[JoinType["JOIN_TYPE_SINGLE"] = 7] = "JOIN_TYPE_SINGLE";
37
- })(JoinType || (JoinType = {}));
38
- var SetOp;
39
- (function (SetOp) {
40
- SetOp[SetOp["SET_OP_UNSPECIFIED"] = 0] = "SET_OP_UNSPECIFIED";
41
- SetOp[SetOp["SET_OP_MINUS_PRIMARY"] = 1] = "SET_OP_MINUS_PRIMARY";
42
- SetOp[SetOp["SET_OP_MINUS_MULTISET"] = 2] = "SET_OP_MINUS_MULTISET";
43
- SetOp[SetOp["SET_OP_INTERSECTION_PRIMARY"] = 3] = "SET_OP_INTERSECTION_PRIMARY";
44
- SetOp[SetOp["SET_OP_INTERSECTION_MULTISET"] = 4] = "SET_OP_INTERSECTION_MULTISET";
45
- SetOp[SetOp["SET_OP_UNION_DISTINCT"] = 5] = "SET_OP_UNION_DISTINCT";
46
- SetOp[SetOp["SET_OP_UNION_ALL"] = 6] = "SET_OP_UNION_ALL";
47
- })(SetOp || (SetOp = {}));
48
- var SortDirection;
49
- (function (SortDirection) {
50
- SortDirection[SortDirection["UNSPECIFIED"] = 0] = "UNSPECIFIED";
51
- SortDirection[SortDirection["ASCENDING_NULLS_FIRST"] = 1] = "ASCENDING_NULLS_FIRST";
52
- SortDirection[SortDirection["ASCENDING_NULLS_LAST"] = 2] = "ASCENDING_NULLS_LAST";
53
- SortDirection[SortDirection["DESCENDING_NULLS_FIRST"] = 3] = "DESCENDING_NULLS_FIRST";
54
- SortDirection[SortDirection["DESCENDING_NULLS_LAST"] = 4] = "DESCENDING_NULLS_LAST";
55
- SortDirection[SortDirection["CLUSTERED"] = 5] = "CLUSTERED";
56
- })(SortDirection || (SortDirection = {}));
57
- class SubstraitParser {
58
- idCounters;
59
- extSet;
60
- fieldNames = [];
61
- selectedFields = [];
62
- filters = [];
63
- logicalOperators = [];
64
- /**
65
- * Constructs a SubstraitParser
66
- * @param {SubstraitPlan} plan the Substrait plan
67
- */
68
- constructor(plan) {
69
- this.idCounters = new Map();
70
- this.extSet = this.buildExtensionSet(plan);
71
- }
72
- /**
73
- * Creates a map from anchor to function name
74
- * @param {SubstraitPlan} plan the Substrait plan
75
- * @return {Map<number, string>} the created mapping
76
- */
77
- buildExtensionSet(plan) {
78
- const extSet = new Map();
79
- for (const extensionDecl of plan.extensions) {
80
- const func = extensionDecl.extensionFunction;
81
- extSet.set(func.functionAnchor, func.name.split(":")[0].toLowerCase());
82
- }
83
- return extSet;
84
- }
85
- /**
86
- * Geneate a unique id for the node
87
- * @param {string} type The type of the node
88
- * @return {string} A unique id
89
- */
90
- nextId(type) {
91
- const value = this.idCounters.get(type) ?? 0;
92
- this.idCounters.set(type, value + 1);
93
- return `${type}_${value}`;
94
- }
95
- /**
96
- * Create a schema field
97
- *
98
- * This is a special field with an empty name and an empty type string
99
- * and uknown nullability
100
- *
101
- * @param {Field[]} fields the fields of the schema
102
- * @return {Field} a schema field
103
- */
104
- makeSchemaField(fields) {
105
- return {
106
- name: "",
107
- children: fields,
108
- type: "",
109
- nullability: "NULLABILITY_UNSPECIFIED",
110
- };
111
- }
112
- /**
113
- * Applies a Substrait emit to an output schema
114
- * @param {Field} schema the original output schema
115
- * @param {Object} emit a potential remapping of fields
116
- * @return {Field} the remapped output schema
117
- */
118
- applyEmit(schema, emit) {
119
- let outputFields = schema.children;
120
- if (emit?.outputMapping != null) {
121
- outputFields = emit.outputMapping.map((idx) => schema.children[idx]);
122
- }
123
- return this.makeSchemaField(outputFields);
124
- }
125
- /**
126
- * Create a print node, generating a new id
127
- *
128
- * @param {string} type the node type
129
- * @param {PrintNode[]} inputs the inputs to the node
130
- * @param {SimpleProperty[]} props properties describing the node
131
- * @param {Field} schema the output schema of the node
132
- * @param {Object} emit the Substrait emit for the node
133
- * @return {PrintNode} a print node
134
- */
135
- makePrintNode(type, inputs, props, schema, emit) {
136
- const emitSchema = this.applyEmit(schema, emit);
137
- return {
138
- id: this.nextId(type),
139
- type,
140
- inputs,
141
- props,
142
- schema: emitSchema,
143
- };
144
- }
145
- /**
146
- * Recursive helper for structFieldToStr
147
- * @param {*} field the Substrait field to convert
148
- * @param {*} currentName the current name
149
- * @param {Field} currentField the current input field
150
- * @return {Field} a stringified represenstation
151
- */
152
- structFieldHelper(field, currentName, currentField) {
153
- const fieldNum = field.field;
154
- const fieldName = currentField.name !== "" ? currentField.name : `$${fieldNum}`;
155
- const nextName = `${currentName}${fieldName}`;
156
- if (field.child != null) {
157
- return this.structFieldHelper(field.child, `${nextName}.`, currentField.children[fieldNum]);
158
- }
159
- return currentField.children[fieldNum];
160
- }
161
- /**
162
- * Convert a struct field to a stringified representation
163
- *
164
- * Struct fields are usually intuitively written with names.
165
- * For example, a.b.c. However, fields in Substrait plans
166
- * are not typically named. So instead we represent them as
167
- * $0, $1, $2, ... where $N means the field at index N.
168
- *
169
- * If the field does happen to be named we will use that.
170
- *
171
- * @param {StructItem} field a Substrait struct field
172
- * @param {Field} exprIn the input to the expression
173
- * @return {Field} a stringified representation
174
- */
175
- structFieldToField(field, exprIn) {
176
- return this.structFieldHelper(field, "", exprIn);
177
- }
178
- /**
179
- * Convert a direct reference to a stringified representation
180
- * @param {Object} ref A Substrait direct reference
181
- * @param {Object} exprIn the input to the expression
182
- * @return {Field} a string representation
183
- */
184
- directReferenceToField(ref, exprIn) {
185
- if (ref.structField != null) {
186
- return this.structFieldToField(ref.structField, exprIn);
187
- }
188
- throw new Error(`Unrecognized direct reference: ${JSON.stringify(ref)}`);
189
- }
190
- /**
191
- * Convert a reference to a stringified representation
192
- * @param {Object} ref A Substrait reference expression
193
- * @param {Object} exprIn the input to the expression
194
- * @return {Field} a string representation
195
- */
196
- referenceToField(ref, exprIn) {
197
- if (ref.directReference != null) {
198
- return this.directReferenceToField(ref.directReference, exprIn);
199
- }
200
- throw new Error(`Unrecognized reference: ${JSON.stringify(ref)}`);
201
- }
202
- /**
203
- * Convert a Substrait function argument to a string
204
- * @param {Object} arg the argument to convert
205
- * @param {Field} inp the input field
206
- * @return {string} a string representation
207
- */
208
- functionArgumentToStr(arg, inp) {
209
- if (arg.enum != null) {
210
- if (arg.enum.specified == null) {
211
- return "unspecified";
212
- }
213
- return arg.enum.specified;
214
- }
215
- if (arg.type != null) {
216
- return this.typeToField(arg.type).type;
217
- }
218
- if (arg.value != null) {
219
- return this.expressionToStr(arg.value, inp).name;
220
- }
221
- throw new Error("A FunctionArgument did not have an enum/type/value");
222
- }
223
- /**
224
- * Converts a Substrait function reference to a string
225
- * @param {number} ref a Substrait function reference
226
- * @return {string} a string version of the reference
227
- */
228
- functionRefToName(ref) {
229
- return this.extSet.get(ref) ?? "unknown";
230
- }
231
- /**
232
- * Convert arguments to a joined string
233
- * @param {FunctionTyp} func A Substrait function call
234
- * @param {Field} schema The input type
235
- * @return {string} A string representation
236
- */
237
- argsToString(func, schema) {
238
- const args = func.arguments.length > 0
239
- ? func.arguments.map((arg) => this.functionArgumentToStr(arg, schema))
240
- : [];
241
- return args.join(",");
242
- }
243
- /**
244
- * Convert a scalar function to a stringified representation
245
- * @param {Object} func the function to convert
246
- * @param {Field} schema the input type info
247
- * @return {Field} a string version of the function
248
- */
249
- scalarFunctionToField(func, schema) {
250
- const argsString = this.argsToString(func, schema);
251
- const outputType = this.typeToField(func.outputType);
252
- const functionName = this.functionRefToName(func.functionReference);
253
- if (!["and", "or", "not", "cast"].includes(functionName)) {
254
- const args = argsString.split(",");
255
- if (args.length === 1) {
256
- this.filters.push(`${args[0]} ${functionName} `);
257
- }
258
- else if (args.length === 2) {
259
- this.filters.push(`${args[0]} ${functionName} ${args[1]}`);
260
- }
261
- else {
262
- this.filters.push(`${functionName}(${args.join(", ")})`);
263
- }
264
- }
265
- else {
266
- this.logicalOperators.push(functionName);
267
- }
268
- const fieldName = `${functionName}(${argsString})`;
269
- outputType.name = fieldName;
270
- return outputType;
271
- }
272
- /**
273
- * Converts a Substrait literal to a field
274
- * @param {Object} lit the Substrait literal
275
- * @return {Field} a field representation
276
- */
277
- literalToField(lit) {
278
- const nullability = lit.nullable
279
- ? "NULLABILITY_NULLABLE"
280
- : "NULLABILITY_REQUIRED";
281
- if (lit.bool != null) {
282
- return {
283
- name: String(lit.bool.value),
284
- type: "boolean",
285
- nullability,
286
- children: [],
287
- };
288
- }
289
- if (lit.date != null) {
290
- return {
291
- name: new Date((lit.date?.value ?? 0) * 8.64e7).toISOString(),
292
- type: "date",
293
- nullability,
294
- children: [],
295
- };
296
- }
297
- if (lit.decimal != null) {
298
- return {
299
- name: String(this.base64ToDecimal(lit.decimal.value ?? "", lit.decimal.precision, lit.decimal.scale)),
300
- type: "decimal",
301
- nullability,
302
- children: [],
303
- };
304
- }
305
- if (lit.i32 != null) {
306
- return {
307
- name: lit.i32?.value?.toString() ?? "unspecified",
308
- type: "i32",
309
- nullability,
310
- children: [],
311
- };
312
- }
313
- if (lit.intervalDayToSecond != null) {
314
- const val = lit.intervalDayToSecond;
315
- return {
316
- name: `${val.days}d${val.seconds}s${val.microseconds}u`,
317
- type: "interval_day",
318
- nullability,
319
- children: [],
320
- };
321
- }
322
- if (lit.fixedChar != null) {
323
- const val = typeof lit.fixedChar === "string"
324
- ? lit.fixedChar
325
- : lit.fixedChar.value ?? "unspecified";
326
- return {
327
- name: `${val}`,
328
- type: "interval_day",
329
- nullability,
330
- children: [],
331
- };
332
- }
333
- throw new Error(`Unrecognized literal: ${JSON.stringify(lit)}`);
334
- }
335
- /**
336
- * Convert a cast expression to a stringified representation
337
- * @param {Object} cast A Substrait cast expression
338
- * @param {Field} schema The input schema
339
- * @return {Field} A field representation
340
- */
341
- castToField(cast, schema) {
342
- const input = this.expressionToStr(cast.input, schema);
343
- const outputType = this.typeToField(cast.type);
344
- let fb = "unspecified";
345
- if (cast.failureBehavior === "FAILURE_BEHAVIOR_RETURN_NULL") {
346
- fb = "or null";
347
- }
348
- else if (cast.failureBehavior === "THROW_EXCEPTION") {
349
- fb = "or die";
350
- }
351
- const name = `cast(${input.name}, ${outputType.type}, ${fb})`;
352
- outputType.name = name;
353
- return outputType;
354
- }
355
- /**
356
- * Convert an expression to a stringified representation
357
- *
358
- * The expression is returned as a field where the type
359
- * of the field is the output type of the expression and the
360
- * name of the field is a string representation of the expression
361
- *
362
- * @param {Object} expr A Substrait expression
363
- * @param {Object} exprIn the input to the expression
364
- * @return {Field} a string representation
365
- */
366
- expressionToStr(expr, exprIn) {
367
- if (expr.selection != null) {
368
- return this.referenceToField(expr.selection, exprIn);
369
- }
370
- if (expr.scalarFunction != null) {
371
- return this.scalarFunctionToField(expr.scalarFunction, exprIn);
372
- }
373
- if (expr.literal != null) {
374
- return this.literalToField(expr.literal);
375
- }
376
- if (expr.cast != null) {
377
- return this.castToField(expr.cast, exprIn);
378
- }
379
- throw new Error(`Unrecognized expression: ${JSON.stringify(expr)}`);
380
- }
381
- /**
382
- * Convert a project relation to a print node
383
- * @param {Object} project A Substrait ProjectRel
384
- * @return {PrintNode} a print node
385
- */
386
- projectToNode(project) {
387
- const input = this.relToNode(project.input);
388
- const expressions = project.expressions.map((val, idx) => {
389
- return {
390
- name: `expressions[${idx}]`,
391
- field: this.expressionToStr(val, input.schema),
392
- };
393
- });
394
- const outFields = expressions.map((expr) => expr.field);
395
- const fields = input.schema.children.concat(outFields);
396
- const schema = this.makeSchemaField(fields);
397
- const props = expressions.map((expr) => ({
398
- name: expr.name,
399
- value: expr.field.name,
400
- }));
401
- const inputs = [this.relToNode(project.input)];
402
- return this.makePrintNode("project", inputs, props, schema, project.common?.emit);
403
- }
404
- /**
405
- * Convert a Substrait data type to an unnamed Field
406
- *
407
- * The field's name will be an empty string
408
- *
409
- * @param {Object} type a Substrait data type
410
- * @return {Field} an unnamed field
411
- */
412
- typeToField(type) {
413
- let typeStr = null;
414
- let nullability = "NULLABILITY_UNSPECIFIED";
415
- if (type.varchar != null) {
416
- typeStr = `VARCHAR<${type.varchar.length}>`;
417
- nullability = type.varchar.nullability;
418
- }
419
- else if (type.fixedChar != null) {
420
- typeStr = `FIXEDCHAR<${type.fixedChar.length}>`;
421
- nullability =
422
- typeof type.fixedChar !== "string"
423
- ? type.fixedChar?.nullability
424
- : "NULLABILITY_REQUIRED";
425
- }
426
- else if (type.date != null) {
427
- typeStr = "date";
428
- nullability = type.date.nullability;
429
- }
430
- else if (type.decimal != null) {
431
- typeStr = `DECIMAL<${type.decimal.precision},${type.decimal.scale}>`;
432
- nullability = type.decimal.nullability ?? "NULLABILITY_UNSPECIFIED";
433
- }
434
- else if (type.i32 != null) {
435
- typeStr = "i32";
436
- nullability = type.i32.nullability;
437
- }
438
- else if (type.i64 != null) {
439
- typeStr = "i64";
440
- nullability = type.i64.nullability;
441
- }
442
- else if (type.bool != null) {
443
- typeStr = "boolean";
444
- nullability = type.bool.nullability;
445
- }
446
- else if (type.string != null) {
447
- typeStr = "string";
448
- nullability = type.string.nullability;
449
- }
450
- else if (type.fp64 != null) {
451
- typeStr = "fp64";
452
- nullability = type.fp64.nullability;
453
- }
454
- else if (type.timestampTz != null) {
455
- typeStr = "timestampTz";
456
- nullability = type.timestampTz.nullability;
457
- }
458
- else if (type.struct != null) {
459
- typeStr = "struct";
460
- nullability = type.struct.nullability;
461
- }
462
- else if (type.fixedBinary != null) {
463
- typeStr = "fixedBinary";
464
- nullability = type.fixedBinary.nullability;
465
- }
466
- else {
467
- throw new Error(`Unrecognized type: ${JSON.stringify(type)}`);
468
- }
469
- return {
470
- name: "",
471
- type: typeStr,
472
- nullability,
473
- children: [],
474
- };
475
- }
476
- /**
477
- * Convert a named struct to an array of Field
478
- * @param {NamedStruct} struct a Substrait named struct
479
- * @return {Field[]} named fields
480
- */
481
- namedStructToSchema(struct) {
482
- // const names = struct.names
483
- // const types = this.flattenTypes(struct.struct)
484
- // console.log('names', names.length)
485
- // console.log('types', types.length)
486
- // if (names.length !== types.length) {
487
- // throw new Error('Mismatched names and types')
488
- // }
489
- return {
490
- name: "test",
491
- type: "struct",
492
- nullability: "NULLABILITY_UNSPECIFIED",
493
- children: [],
494
- };
495
- // return this.makeSchemaField(outTypes)
496
- }
497
- // mapNames (input: NamedStruct, prefix: string = ''): string[] {
498
- // const output: string[] = []
499
- // for (let i = 0; i < input.names.length; i++) {
500
- // const name = input.names[i]
501
- // const type = input.struct.types[i]
502
- // if (type.struct != null) {
503
- // output.push(...this.mapNames({ name, types: type.struct.types }, `${prefix}${name}.`))
504
- // } else {
505
- // output.push(`${prefix}${name}`)
506
- // }
507
- // }
508
- // return output
509
- // }
510
- /**
511
- * Extract the properties from a read relation
512
- * @param {Object} read a Substrait read relation
513
- * @return {SimpleProperty[]} properties for the relation
514
- */
515
- readTypeToProps(read) {
516
- return [
517
- { name: "namedTable.names", value: read.namedTable.names.join(".") },
518
- ];
519
- }
520
- /**
521
- * Convert a Substrait read relation to a print node
522
- * @param {Object} read a Substrait read relation
523
- * @return {PrintNode} a print node
524
- */
525
- readToNode(read) {
526
- const schema = this.namedStructToSchema(read.baseSchema);
527
- const props = this.readTypeToProps(read);
528
- return this.makePrintNode("read", [], props, schema, read.common?.emit);
529
- }
530
- /**
531
- * True if the object has a field with the given name
532
- * @param {Object} obj the object to test
533
- * @param {string} fieldName the field name to look for
534
- * @return {boolean} the result of the test
535
- */
536
- hasValue(obj, fieldName) {
537
- return fieldName in obj;
538
- }
539
- /**
540
- * Convert a Substrait fetch relation to a print node
541
- * @param {Object} fetch a Substrait fetch relation
542
- * @return {PrintNode} a print node
543
- */
544
- fetchToNode(fetch) {
545
- const input = this.relToNode(fetch.input);
546
- const schema = input.schema;
547
- const props = [];
548
- if (this.hasValue(fetch, "offset")) {
549
- props.push({ name: "offset", value: String(fetch.offset) });
550
- }
551
- if (this.hasValue(fetch, "count")) {
552
- props.push({ name: "count", value: String(fetch.count) });
553
- }
554
- return this.makePrintNode("fetch", [input], props, schema, fetch.common.emit);
555
- }
556
- /**
557
- * Convert a Substrait sort field to a property
558
- * @param {Object} sortField a Substrait sort field
559
- * @param {Field} schema the schema of the sort input
560
- * @return {SimpleProperty} a property describing the sort
561
- */
562
- sortFieldToStr(sortField, schema) {
563
- const exprStr = this.expressionToStr(sortField.expr, schema).name;
564
- if (sortField.direction !== null) {
565
- const dir = sortField.direction;
566
- let sortDesc = "";
567
- if (dir === SortDirection.ASCENDING_NULLS_FIRST) {
568
- sortDesc = "ascending (nulls first)";
569
- }
570
- else if (dir === SortDirection.ASCENDING_NULLS_LAST) {
571
- sortDesc = "ascending (nulls last)";
572
- }
573
- else if (dir === SortDirection.DESCENDING_NULLS_FIRST) {
574
- sortDesc = "descending (nulls first)";
575
- }
576
- else if (dir === SortDirection.DESCENDING_NULLS_LAST) {
577
- sortDesc = "descending (nulls last)";
578
- }
579
- else if (dir === SortDirection.CLUSTERED) {
580
- sortDesc = "clustered";
581
- }
582
- else {
583
- sortDesc = "unspecified";
584
- }
585
- return `${exprStr} | ${sortDesc}`;
586
- }
587
- }
588
- /**
589
- * Convert a Substrait sort relation to a print node
590
- * @param {Object} sort a Substrait sort relation
591
- * @return {PrintNode} a print node
592
- */
593
- sortToNode(sort) {
594
- const input = this.relToNode(sort.input);
595
- const schema = input.schema;
596
- const props = sort.sorts.map((sort, idx) => {
597
- return {
598
- name: `sorts[${idx}]`,
599
- value: this.sortFieldToStr(sort, schema) ?? "undefined",
600
- };
601
- });
602
- return this.makePrintNode("sort", [input], props, schema, sort.common.emit);
603
- }
604
- /**
605
- * Convert a Substrait aggregate function to a field
606
- * @param {Object} func the Substrait aggregate function
607
- * @param {Field} inp the input field
608
- * @return {Field} a field representation of the function
609
- */
610
- aggregateFunctionToField(func, inp) {
611
- // TODO (weston) sorts, phases, invocation
612
- return this.scalarFunctionToField(func, inp);
613
- }
614
- /**
615
- * Convert a Substrait aggregate relation to a print node
616
- * @param {Object} agg a Substrait aggregate relation
617
- * @return {PrintNode} a print node
618
- */
619
- aggToNode(agg) {
620
- const input = this.relToNode(agg.input);
621
- const props = [];
622
- const fields = [];
623
- for (const [idx, grouping] of agg.groupings.entries()) {
624
- for (const groupingExpr of grouping.groupingExpressions) {
625
- const groupingField = this.expressionToStr(groupingExpr, input.schema);
626
- props.push({ name: `grouping[${idx}]`, value: groupingField.name });
627
- fields.push(groupingField);
628
- }
629
- }
630
- for (const [idx, measure] of agg.measures.entries()) {
631
- const aggregate = this.aggregateFunctionToField(measure.measure, input.schema);
632
- fields.push(aggregate);
633
- const innerProps = [
634
- {
635
- name: "measure",
636
- value: aggregate.name,
637
- },
638
- ];
639
- innerProps.push({
640
- name: "filter",
641
- value: this.expressionToStr(measure.filter, input.schema).name,
642
- });
643
- props.push({
644
- name: `measures[${idx}]`,
645
- value: JSON.stringify(innerProps), // TODO Is this right?
646
- });
647
- }
648
- const schema = this.makeSchemaField(fields);
649
- return this.makePrintNode("aggregate", [input], props, schema, agg.common.emit);
650
- }
651
- /**
652
- * Convert a Substrait filter relation to a print node
653
- * @param {Object} filt a Substrait filter relation
654
- * @return {PrintNode} a print node
655
- */
656
- filterToNode(filt) {
657
- const input = this.relToNode(filt.input);
658
- const props = [];
659
- const schema = input.schema;
660
- props.push({
661
- name: "condition",
662
- value: this.expressionToStr(filt.condition, schema).name,
663
- });
664
- const printNode = this.makePrintNode("filter", [input], props, schema, filt.common.emit);
665
- return printNode;
666
- }
667
- /**
668
- * Converts a Substrait join relation to a print node
669
- * @param {Object} join a Substrait join relation
670
- * @return {PrintNode} a print node
671
- */
672
- joinToNode(join) {
673
- const left = this.relToNode(join.left);
674
- const right = this.relToNode(join.right);
675
- const props = [];
676
- const schema = this.makeSchemaField(left.schema.children.concat(right.schema.children));
677
- let joinType = "unspecified";
678
- if (join.type !== null) {
679
- joinType = JoinType[join.type];
680
- }
681
- props.push({ name: "type", value: joinType });
682
- if (join.expression !== null) {
683
- const exprStr = this.expressionToStr(join.expression, schema).name;
684
- props.push({ name: "expression", value: exprStr });
685
- }
686
- return this.makePrintNode("join", [left, right], props, schema, join.common.emit);
687
- }
688
- /**
689
- * Converts a Substrait relation to a print node
690
- * @param {Object} rel the Substrait relation to convert
691
- * @return {PrintNode} a print node
692
- */
693
- relToNode(rel) {
694
- if (rel.project != null) {
695
- return this.projectToNode(rel.project);
696
- }
697
- if (rel.read != null) {
698
- return this.readToNode(rel.read);
699
- }
700
- if (rel.fetch != null) {
701
- return this.fetchToNode(rel.fetch);
702
- }
703
- if (rel.sort != null) {
704
- return this.sortToNode(rel.sort);
705
- }
706
- if (rel.aggregate != null) {
707
- return this.aggToNode(rel.aggregate);
708
- }
709
- if (rel.filter != null) {
710
- return this.filterToNode(rel.filter);
711
- }
712
- if (rel.join != null) {
713
- return this.joinToNode(rel.join);
714
- }
715
- throw new Error(`Unrecognized relation: ${JSON.stringify(rel)}`);
716
- }
717
- /**
718
- * Converts a Substrait root relation to a print node
719
- * @param {Object} rel the Substrait relation to convert
720
- * @return {PrintNode} a print node
721
- */
722
- rootRelToNode(rel) {
723
- const props = rel.names.map((val, idx) => ({
724
- name: `name[${idx}]`,
725
- value: val,
726
- }));
727
- this.selectedFields = this.selectedFields.concat(rel.names);
728
- const inputs = [this.relToNode(rel.input)];
729
- const schema = this.makeSchemaField([]);
730
- return this.makePrintNode("sink", inputs, props, schema);
731
- }
732
- /**
733
- * Converts a Substrait plan to a print node
734
- * @param {Object} plan the Substrait plan to convert
735
- * @return {PrintNode} a print node
736
- */
737
- planToNode(plan) {
738
- const nodes = [];
739
- for (const relation of plan.relations) {
740
- if (relation.root != null) {
741
- nodes.push(this.rootRelToNode(relation.root));
742
- }
743
- else {
744
- throw new Error(`Unrecognized plan relation ${JSON.stringify(relation)}`);
745
- }
746
- }
747
- const schema = this.makeSchemaField([]);
748
- return this.makePrintNode("plan", nodes, [], schema);
749
- }
750
- parse(plan) {
751
- this.selectedFields = [];
752
- this.planToNode(plan);
753
- return {
754
- selection: this.selectedFields,
755
- filters: this.deduplicate(this.filters),
756
- logicalOperators: this.deduplicate(this.logicalOperators),
757
- };
758
- }
759
- /**
760
- * Converts a base64 string into a decimal number.
761
- *
762
- * @public
763
- * @param {string} base64 - The base64 string to convert.
764
- * @param {number} precision - The total number of digits in the output number.
765
- * @param {number} scale - The number of digits after the decimal point in the output number.
766
- * @throws {Error} Will throw an error if the conversion fails.
767
- * @returns {number} The converted decimal number.
768
- * @example
769
- * // returns 123.45
770
- * this.base64ToDecimal('MTIzNDU=', 5, 2);
771
- */
772
- base64ToDecimal(base64, precision, scale) {
773
- // Decode base64 string to a Buffer
774
- const buffer = Buffer.from(base64, "base64");
775
- let value = BigInt(0);
776
- for (let i = buffer.length - 1; i >= 0; i--) {
777
- value = (value << BigInt(8)) + BigInt(buffer[i]);
778
- }
779
- return Number(value) / 10 ** scale;
780
- }
781
- deduplicate(array) {
782
- return Array.from(new Set(array));
783
- }
784
- }
785
- export { SubstraitParser };