@elaraai/east 1.0.84 → 1.0.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/analyze.d.ts.map +1 -1
- package/dist/src/analyze.js +48 -0
- package/dist/src/analyze.js.map +1 -1
- package/dist/src/ast_to_ir.coerce_to.spec.js +4 -4
- package/dist/src/ast_to_ir.coerce_to.spec.js.map +1 -1
- package/dist/src/ast_to_ir.d.ts +6 -1
- package/dist/src/ast_to_ir.d.ts.map +1 -1
- package/dist/src/ast_to_ir.js +20 -7
- package/dist/src/ast_to_ir.js.map +1 -1
- package/dist/src/builtins.d.ts +1 -1
- package/dist/src/builtins.d.ts.map +1 -1
- package/dist/src/builtins.js +52 -0
- package/dist/src/builtins.js.map +1 -1
- package/dist/src/codegen/canonical.d.ts +37 -0
- package/dist/src/codegen/canonical.d.ts.map +1 -0
- package/dist/src/codegen/canonical.js +208 -0
- package/dist/src/codegen/canonical.js.map +1 -0
- package/dist/src/codegen/codegen.spec.js +132 -201
- package/dist/src/codegen/codegen.spec.js.map +1 -1
- package/dist/src/codegen/printer.d.ts.map +1 -1
- package/dist/src/codegen/printer.js +142 -24
- package/dist/src/codegen/printer.js.map +1 -1
- package/dist/src/codegen/spellings.d.ts +4 -1
- package/dist/src/codegen/spellings.d.ts.map +1 -1
- package/dist/src/codegen/spellings.js +5 -1
- package/dist/src/codegen/spellings.js.map +1 -1
- package/dist/src/compile/builtins/index.d.ts.map +1 -1
- package/dist/src/compile/builtins/index.js +2 -0
- package/dist/src/compile/builtins/index.js.map +1 -1
- package/dist/src/compile/builtins/primitives.d.ts.map +1 -1
- package/dist/src/compile/builtins/primitives.js +2 -1
- package/dist/src/compile/builtins/primitives.js.map +1 -1
- package/dist/src/compile/builtins/query.d.ts +10 -0
- package/dist/src/compile/builtins/query.d.ts.map +1 -0
- package/dist/src/compile/builtins/query.js +5 -0
- package/dist/src/compile/builtins/query.js.map +1 -0
- package/dist/src/expr/block.d.ts.map +1 -1
- package/dist/src/expr/block.js +2 -1
- package/dist/src/expr/block.js.map +1 -1
- package/dist/src/expr/index.d.ts +75 -47
- package/dist/src/expr/index.d.ts.map +1 -1
- package/dist/src/expr/index.js +31 -0
- package/dist/src/expr/index.js.map +1 -1
- package/dist/src/expr/libs/datetime.d.ts +20 -20
- package/dist/src/expr/libs/datetime.d.ts.map +1 -1
- package/dist/src/expr/libs/datetime.js +208 -167
- package/dist/src/expr/libs/datetime.js.map +1 -1
- package/dist/src/expr/libs/float.d.ts +15 -15
- package/dist/src/expr/libs/float.d.ts.map +1 -1
- package/dist/src/expr/libs/float.js +298 -267
- package/dist/src/expr/libs/float.js.map +1 -1
- package/dist/src/expr/libs/integer.d.ts +12 -12
- package/dist/src/expr/libs/integer.d.ts.map +1 -1
- package/dist/src/expr/libs/integer.js +219 -194
- package/dist/src/expr/libs/integer.js.map +1 -1
- package/dist/src/expr/libs/lazy.d.ts +28 -0
- package/dist/src/expr/libs/lazy.d.ts.map +1 -0
- package/dist/src/expr/libs/lazy.js +55 -0
- package/dist/src/expr/libs/lazy.js.map +1 -0
- package/dist/src/expr/libs/lazy.spec.d.ts +6 -0
- package/dist/src/expr/libs/lazy.spec.d.ts.map +1 -0
- package/dist/src/expr/libs/lazy.spec.js +102 -0
- package/dist/src/expr/libs/lazy.spec.js.map +1 -0
- package/dist/src/expr/query.d.ts +60 -0
- package/dist/src/expr/query.d.ts.map +1 -0
- package/dist/src/expr/query.js +84 -0
- package/dist/src/expr/query.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/internal.d.ts +4 -0
- package/dist/src/internal.d.ts.map +1 -1
- package/dist/src/internal.js +9 -0
- package/dist/src/internal.js.map +1 -1
- package/dist/src/location.d.ts +22 -0
- package/dist/src/location.d.ts.map +1 -1
- package/dist/src/location.js +31 -0
- package/dist/src/location.js.map +1 -1
- package/dist/src/naming.d.ts +5 -3
- package/dist/src/naming.d.ts.map +1 -1
- package/dist/src/naming.js.map +1 -1
- package/dist/src/portable.spec.d.ts +6 -0
- package/dist/src/portable.spec.d.ts.map +1 -0
- package/dist/src/portable.spec.js +181 -0
- package/dist/src/portable.spec.js.map +1 -0
- package/dist/src/query/evaluate.d.ts +98 -0
- package/dist/src/query/evaluate.d.ts.map +1 -0
- package/dist/src/query/evaluate.js +193 -0
- package/dist/src/query/evaluate.js.map +1 -0
- package/dist/src/query/index.d.ts +17 -0
- package/dist/src/query/index.d.ts.map +1 -0
- package/dist/src/query/index.js +17 -0
- package/dist/src/query/index.js.map +1 -0
- package/dist/src/query/jq/catalog.d.ts +174 -0
- package/dist/src/query/jq/catalog.d.ts.map +1 -0
- package/dist/src/query/jq/catalog.js +1740 -0
- package/dist/src/query/jq/catalog.js.map +1 -0
- package/dist/src/query/jq/check.d.ts +210 -0
- package/dist/src/query/jq/check.d.ts.map +1 -0
- package/dist/src/query/jq/check.js +2261 -0
- package/dist/src/query/jq/check.js.map +1 -0
- package/dist/src/query/jq/complete.d.ts +89 -0
- package/dist/src/query/jq/complete.d.ts.map +1 -0
- package/dist/src/query/jq/complete.js +320 -0
- package/dist/src/query/jq/complete.js.map +1 -0
- package/dist/src/query/jq/describe.d.ts +53 -0
- package/dist/src/query/jq/describe.d.ts.map +1 -0
- package/dist/src/query/jq/describe.js +149 -0
- package/dist/src/query/jq/describe.js.map +1 -0
- package/dist/src/query/jq/lex.d.ts +118 -0
- package/dist/src/query/jq/lex.d.ts.map +1 -0
- package/dist/src/query/jq/lex.js +371 -0
- package/dist/src/query/jq/lex.js.map +1 -0
- package/dist/src/query/jq/literals.d.ts +49 -0
- package/dist/src/query/jq/literals.d.ts.map +1 -0
- package/dist/src/query/jq/literals.js +78 -0
- package/dist/src/query/jq/literals.js.map +1 -0
- package/dist/src/query/jq/messages.d.ts +126 -0
- package/dist/src/query/jq/messages.d.ts.map +1 -0
- package/dist/src/query/jq/messages.js +172 -0
- package/dist/src/query/jq/messages.js.map +1 -0
- package/dist/src/query/jq/parse.d.ts +70 -0
- package/dist/src/query/jq/parse.d.ts.map +1 -0
- package/dist/src/query/jq/parse.js +1013 -0
- package/dist/src/query/jq/parse.js.map +1 -0
- package/dist/src/query/jq/print.d.ts +59 -0
- package/dist/src/query/jq/print.d.ts.map +1 -0
- package/dist/src/query/jq/print.js +587 -0
- package/dist/src/query/jq/print.js.map +1 -0
- package/dist/src/query/jq/shapes.d.ts +345 -0
- package/dist/src/query/jq/shapes.d.ts.map +1 -0
- package/dist/src/query/jq/shapes.js +584 -0
- package/dist/src/query/jq/shapes.js.map +1 -0
- package/dist/src/query/jq/spans.d.ts +133 -0
- package/dist/src/query/jq/spans.d.ts.map +1 -0
- package/dist/src/query/jq/spans.js +195 -0
- package/dist/src/query/jq/spans.js.map +1 -0
- package/dist/src/query/jq/split.d.ts +334 -0
- package/dist/src/query/jq/split.d.ts.map +1 -0
- package/dist/src/query/jq/split.js +2108 -0
- package/dist/src/query/jq/split.js.map +1 -0
- package/dist/src/query/jq/strftime.d.ts +20 -0
- package/dist/src/query/jq/strftime.d.ts.map +1 -0
- package/dist/src/query/jq/strftime.js +65 -0
- package/dist/src/query/jq/strftime.js.map +1 -0
- package/dist/src/query/jq/summary.d.ts +139 -0
- package/dist/src/query/jq/summary.d.ts.map +1 -0
- package/dist/src/query/jq/summary.js +222 -0
- package/dist/src/query/jq/summary.js.map +1 -0
- package/dist/src/query/jq/translate-builtins.d.ts +21 -0
- package/dist/src/query/jq/translate-builtins.d.ts.map +1 -0
- package/dist/src/query/jq/translate-builtins.js +1873 -0
- package/dist/src/query/jq/translate-builtins.js.map +1 -0
- package/dist/src/query/jq/translate.d.ts +627 -0
- package/dist/src/query/jq/translate.d.ts.map +1 -0
- package/dist/src/query/jq/translate.js +3364 -0
- package/dist/src/query/jq/translate.js.map +1 -0
- package/dist/src/query/types.d.ts +505 -0
- package/dist/src/query/types.d.ts.map +1 -0
- package/dist/src/query/types.js +231 -0
- package/dist/src/query/types.js.map +1 -0
- package/dist/src/runner_exec.d.ts +134 -0
- package/dist/src/runner_exec.d.ts.map +1 -0
- package/dist/src/runner_exec.js +371 -0
- package/dist/src/runner_exec.js.map +1 -0
- package/dist/src/runner_exec.spec.d.ts +6 -0
- package/dist/src/runner_exec.spec.d.ts.map +1 -0
- package/dist/src/runner_exec.spec.js +198 -0
- package/dist/src/runner_exec.spec.js.map +1 -0
- package/dist/src/runner_io.d.ts +370 -0
- package/dist/src/runner_io.d.ts.map +1 -0
- package/dist/src/runner_io.js +563 -0
- package/dist/src/runner_io.js.map +1 -0
- package/dist/src/runner_io.spec.d.ts +6 -0
- package/dist/src/runner_io.spec.d.ts.map +1 -0
- package/dist/src/runner_io.spec.js +224 -0
- package/dist/src/runner_io.spec.js.map +1 -0
- package/dist/src/runner_merge.d.ts +95 -0
- package/dist/src/runner_merge.d.ts.map +1 -0
- package/dist/src/runner_merge.js +238 -0
- package/dist/src/runner_merge.js.map +1 -0
- package/dist/src/runner_protocol.d.ts +11 -4
- package/dist/src/runner_protocol.d.ts.map +1 -1
- package/dist/src/runner_protocol.js +11 -4
- package/dist/src/runner_protocol.js.map +1 -1
- package/dist/src/serialization/beast2/index.d.ts +1 -1
- package/dist/src/serialization/beast2/index.d.ts.map +1 -1
- package/dist/src/serialization/beast2/index.js.map +1 -1
- package/dist/src/serialization/beast2/v5/frames.d.ts +10 -6
- package/dist/src/serialization/beast2/v5/frames.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/frames.js +10 -6
- package/dist/src/serialization/beast2/v5/frames.js.map +1 -1
- package/dist/src/serialization/beast2/v5/frames.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/frames.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/frames.spec.js +42 -0
- package/dist/src/serialization/beast2/v5/frames.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/index.spec.js +108 -0
- package/dist/src/serialization/beast2/v5/index.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.d.ts +12 -7
- package/dist/src/serialization/beast2/v5/lazy.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.js +16 -6
- package/dist/src/serialization/beast2/v5/lazy.js.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.spec.js +159 -18
- package/dist/src/serialization/beast2/v5/lazy.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/stream.d.ts +110 -21
- package/dist/src/serialization/beast2/v5/stream.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/stream.js +153 -42
- package/dist/src/serialization/beast2/v5/stream.js.map +1 -1
- package/dist/src/serialization/beast2/v5/weight.d.ts +36 -0
- package/dist/src/serialization/beast2/v5/weight.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/weight.js +238 -0
- package/dist/src/serialization/beast2/v5/weight.js.map +1 -0
- package/dist/src/serialization/crc32.d.ts +38 -0
- package/dist/src/serialization/crc32.d.ts.map +1 -0
- package/dist/src/serialization/crc32.js +70 -0
- package/dist/src/serialization/crc32.js.map +1 -0
- package/dist/src/serialization/crc32.spec.d.ts +6 -0
- package/dist/src/serialization/crc32.spec.d.ts.map +1 -0
- package/dist/src/serialization/crc32.spec.js +92 -0
- package/dist/src/serialization/crc32.spec.js.map +1 -0
- package/dist/src/serialization/east-inferred.spec.js +4 -0
- package/dist/src/serialization/east-inferred.spec.js.map +1 -1
- package/dist/src/serialization/east.d.ts.map +1 -1
- package/dist/src/serialization/east.js +134 -110
- package/dist/src/serialization/east.js.map +1 -1
- package/dist/src/serialization/east.spec.js +185 -4
- package/dist/src/serialization/east.spec.js.map +1 -1
- package/dist/src/serialization/east_text.d.ts +25 -0
- package/dist/src/serialization/east_text.d.ts.map +1 -0
- package/dist/src/serialization/east_text.js +29 -0
- package/dist/src/serialization/east_text.js.map +1 -0
- package/dist/src/serialization/index.d.ts +1 -1
- package/dist/src/serialization/index.d.ts.map +1 -1
- package/dist/src/types.d.ts +1 -1
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +7 -6
- package/dist/src/types.js.map +1 -1
- package/package.json +6 -3
|
@@ -0,0 +1,2108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Splitting a checked query over one dataset's pieces (#941).
|
|
7
|
+
*
|
|
8
|
+
* A query that works through one root dataset's rows can run as a split call:
|
|
9
|
+
* each piece of the dataset runs the work done row by row and emits into an
|
|
10
|
+
* output kind, e3 combines the pieces' outputs by that kind, and a final
|
|
11
|
+
* function runs the rest of the query once, over the combined result
|
|
12
|
+
* (`devdocs/QUERY.md` §17.1).
|
|
13
|
+
*
|
|
14
|
+
* Every program here is the translator's code for a part of the checked
|
|
15
|
+
* program, so a piece computes what the whole query computes on its rows, and
|
|
16
|
+
* a runtime error names its place in the query as the one unit's does. Every
|
|
17
|
+
* merge is built, field by field, from associative primitives: adding as
|
|
18
|
+
* `add` adds (numbers, strings, arrays and dicts), counting, the least, the
|
|
19
|
+
* greatest, the first, the last, or, and, union and concatenation.
|
|
20
|
+
*
|
|
21
|
+
* A piece whose output combines by key — a grouping by totals, `unique`,
|
|
22
|
+
* `unique_by`, a `reduce` — folds its rows into a table inside the piece, as
|
|
23
|
+
* the merge folds them, and sends the table's entries on every so many rows
|
|
24
|
+
* and at its end (#1093): the runner's sorter then folds a pair per key per
|
|
25
|
+
* flush, not a pair per row. A table that gathers more keys than its rows
|
|
26
|
+
* repay sends them on, and the piece's rows from there go straight on.
|
|
27
|
+
*
|
|
28
|
+
* @packageDocumentation
|
|
29
|
+
*/
|
|
30
|
+
import { compareFor } from "../../comparison.js";
|
|
31
|
+
import { SortedMap } from "../../containers/sortedmap.js";
|
|
32
|
+
import { SortedSet } from "../../containers/sortedset.js";
|
|
33
|
+
import { none } from "../../containers/variant.js";
|
|
34
|
+
import { func } from "../../expr/block.js";
|
|
35
|
+
import { withLocationCapture } from "../../location.js";
|
|
36
|
+
import { ArrayType, BooleanType, DictType, FunctionType, IntegerType, NullType, OptionType, SetType, StructType, isImmutableType, isTypeEqual, } from "../../types.js";
|
|
37
|
+
import { nullablePayload, unwrap } from "./shapes.js";
|
|
38
|
+
import { childPath, jqChildren } from "./spans.js";
|
|
39
|
+
import { addInto } from "./translate-builtins.js";
|
|
40
|
+
import { TranslationError, Translator, parts } from "./translate.js";
|
|
41
|
+
/** A pipeline's steps, through its pipes and the bodies of its bindings. */
|
|
42
|
+
function pipeline(node, path, out = []) {
|
|
43
|
+
if (node.type === "pipe") {
|
|
44
|
+
pipeline(node.value.left, childPath(path, "pipe.left"), out);
|
|
45
|
+
return pipeline(node.value.right, childPath(path, "pipe.right"), out);
|
|
46
|
+
}
|
|
47
|
+
if (node.type === "bind" && node.value.patterns.length === 1) {
|
|
48
|
+
out.push({ kind: "bind", node, path });
|
|
49
|
+
return pipeline(node.value.body, childPath(path, "bind.body"), out);
|
|
50
|
+
}
|
|
51
|
+
out.push({ kind: "stage", node, path });
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
/** A root field read, `.name`, on the root. */
|
|
55
|
+
function rootField(node, readsRoot) {
|
|
56
|
+
if (!readsRoot)
|
|
57
|
+
return undefined;
|
|
58
|
+
if (node.type === "field" && node.value.target.type === "identity" && !node.value.optional)
|
|
59
|
+
return node.value.name;
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
/** `.name[]`, on the root. */
|
|
63
|
+
function rootStream(node) {
|
|
64
|
+
if (node.type !== "iterate" || node.value.optional)
|
|
65
|
+
return undefined;
|
|
66
|
+
return rootField(node.value.target, true);
|
|
67
|
+
}
|
|
68
|
+
/** Whether a node is a call of a builtin by name and arity, not of a def. */
|
|
69
|
+
function isCall(node, name, arity) {
|
|
70
|
+
return node.type === "call" && node.value.name === name && node.value.args.length === arity;
|
|
71
|
+
}
|
|
72
|
+
/** Whether a node is `.[]` on its input. */
|
|
73
|
+
function isIterate(node) {
|
|
74
|
+
return node.type === "iterate" && !node.value.optional && node.value.target.type === "identity";
|
|
75
|
+
}
|
|
76
|
+
/** Whether the value after a node is a dict. */
|
|
77
|
+
function isDict(checked, path) {
|
|
78
|
+
return unwrap(typeAfter(checked, path)).type === "Dict";
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* A dict's entries at a pipeline's step: `to_entries`, a collection of them;
|
|
82
|
+
* `to_entries[]` or `to_entries | .[]`, a stream of them. The `to_entries`
|
|
83
|
+
* node's path, the stream's, the last step's, and the index of the step after.
|
|
84
|
+
*/
|
|
85
|
+
function entriesAt(steps, i) {
|
|
86
|
+
const step = steps[i];
|
|
87
|
+
if (step?.kind !== "stage")
|
|
88
|
+
return undefined;
|
|
89
|
+
const node = step.node;
|
|
90
|
+
if (node.type === "iterate" && !node.value.optional && isCall(node.value.target, "to_entries", 0)) {
|
|
91
|
+
return { entries: childPath(step.path, "iterate.target"), stream: step.path, end: step.path, next: i + 1 };
|
|
92
|
+
}
|
|
93
|
+
if (!isCall(node, "to_entries", 0))
|
|
94
|
+
return undefined;
|
|
95
|
+
const after = steps[i + 1];
|
|
96
|
+
if (after?.kind === "stage" && isIterate(after.node))
|
|
97
|
+
return { entries: step.path, stream: after.path, end: after.path, next: i + 2 };
|
|
98
|
+
return { entries: step.path, stream: undefined, end: step.path, next: i + 1 };
|
|
99
|
+
}
|
|
100
|
+
// ─── What the query reads, and what it does ─────────────────────────────────
|
|
101
|
+
/** The names of the variables a node and everything inside it read. */
|
|
102
|
+
function variablesIn(node, out = new Set()) {
|
|
103
|
+
if (node.type === "variable")
|
|
104
|
+
out.add(node.value);
|
|
105
|
+
if (node.type === "object") {
|
|
106
|
+
for (const entry of node.value)
|
|
107
|
+
if (entry.key.type === "variable" && entry.value.type === "none")
|
|
108
|
+
out.add(entry.key.value);
|
|
109
|
+
}
|
|
110
|
+
for (const child of jqChildren(node))
|
|
111
|
+
if (child.node !== undefined)
|
|
112
|
+
variablesIn(child.node, out);
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
/** The names a destructuring pattern binds. */
|
|
116
|
+
function patternNames(pattern) {
|
|
117
|
+
switch (pattern.type) {
|
|
118
|
+
case "variable": return [pattern.value];
|
|
119
|
+
case "array": return pattern.value.flatMap(patternNames);
|
|
120
|
+
case "object":
|
|
121
|
+
return pattern.value
|
|
122
|
+
.flatMap(entry => entry.value.type === "none" ? [entry.key] : patternNames(entry.value.value));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
/** Whether a node, given an input, reads it: `.` reaches it. Conservative: a builtin call reads its input. */
|
|
126
|
+
function readsInput(node) {
|
|
127
|
+
switch (node.type) {
|
|
128
|
+
case "identity":
|
|
129
|
+
case "descend":
|
|
130
|
+
case "call":
|
|
131
|
+
case "update":
|
|
132
|
+
case "format":
|
|
133
|
+
case "def":
|
|
134
|
+
return true;
|
|
135
|
+
case "literal":
|
|
136
|
+
case "variable":
|
|
137
|
+
case "break":
|
|
138
|
+
return false;
|
|
139
|
+
case "pipe":
|
|
140
|
+
return readsInput(node.value.left);
|
|
141
|
+
case "field":
|
|
142
|
+
case "iterate":
|
|
143
|
+
return readsInput(node.value.target);
|
|
144
|
+
case "index":
|
|
145
|
+
return readsInput(node.value.target) || readsInput(node.value.index);
|
|
146
|
+
case "slice":
|
|
147
|
+
return readsInput(node.value.target) || jqChildren(node).some(c => c.node !== undefined && c.step !== "slice.target" && readsInput(c.node));
|
|
148
|
+
case "object":
|
|
149
|
+
return node.value.some(entry => (entry.key.type === "name" && entry.value.type === "none")
|
|
150
|
+
|| (entry.key.type === "computed" && readsInput(entry.key.value))
|
|
151
|
+
|| (entry.value.type === "some" && readsInput(entry.value.value)));
|
|
152
|
+
case "if":
|
|
153
|
+
return node.value.otherwise.type === "none" || jqChildren(node).some(c => c.node !== undefined && readsInput(c.node));
|
|
154
|
+
case "reduce":
|
|
155
|
+
case "foreach":
|
|
156
|
+
return readsInput(node.value.source) || readsInput(node.value.init);
|
|
157
|
+
default:
|
|
158
|
+
return jqChildren(node).some(c => c.node !== undefined && readsInput(c.node));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/** The first node in a subtree for which `test` holds, and its path. */
|
|
162
|
+
function findNode(node, path, test) {
|
|
163
|
+
if (test(node))
|
|
164
|
+
return { node, path };
|
|
165
|
+
for (const child of jqChildren(node)) {
|
|
166
|
+
if (child.node === undefined)
|
|
167
|
+
continue;
|
|
168
|
+
const found = findNode(child.node, childPath(path, child.step), test);
|
|
169
|
+
if (found !== undefined)
|
|
170
|
+
return found;
|
|
171
|
+
}
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
/** Builtins that carry state from one row to the next: they read the program's further inputs. */
|
|
175
|
+
const STATEFUL = new Set(["input", "inputs", "input_line_number"]);
|
|
176
|
+
/** Builtins that need every row of the collection they are given at once. */
|
|
177
|
+
const EVERY_ROW = new Set([
|
|
178
|
+
"sort", "sort_by", "group_by", "unique", "unique_by", "reverse", "min", "max", "min_by", "max_by", "add", "any", "all",
|
|
179
|
+
"to_entries", "from_entries", "with_entries", "keys", "keys_unsorted", "transpose", "combinations", "join", "flatten",
|
|
180
|
+
"tojson", "tostring", "walk", "paths", "leaf_paths", "length", "indices", "index", "rindex", "inside", "contains",
|
|
181
|
+
]);
|
|
182
|
+
/** Builtins that take rows by their position. */
|
|
183
|
+
const POSITIONAL = new Set(["first", "last", "nth", "limit", "skip", "until"]);
|
|
184
|
+
/** A reason, with the range of the node it is about. */
|
|
185
|
+
function reason(checked, code, path, name = null) {
|
|
186
|
+
return { code, range: path === null ? null : rangeOf(checked, path), name };
|
|
187
|
+
}
|
|
188
|
+
/** A node's range of text. */
|
|
189
|
+
function rangeOf(checked, path) {
|
|
190
|
+
const range = checked.source.spans.get(path);
|
|
191
|
+
if (range === undefined)
|
|
192
|
+
throw new TranslationError(`no text for the node at ${path || "the root"}`);
|
|
193
|
+
return range;
|
|
194
|
+
}
|
|
195
|
+
/** The range from one node's start to another's end. */
|
|
196
|
+
function spanning(checked, from, to) {
|
|
197
|
+
return { from: rangeOf(checked, from).from, to: rangeOf(checked, to).to };
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Finds a query's row work: the bindings before it, the dataset it streams,
|
|
201
|
+
* the steps done to each row, and the steps after.
|
|
202
|
+
*/
|
|
203
|
+
function findRows(checked, program) {
|
|
204
|
+
const steps = pipeline(program, "");
|
|
205
|
+
let i = 0;
|
|
206
|
+
const binds = [];
|
|
207
|
+
while (i < steps.length && steps[i].kind === "bind")
|
|
208
|
+
binds.push(steps[i++]);
|
|
209
|
+
const first = steps[i];
|
|
210
|
+
if (first === undefined || first.kind !== "stage")
|
|
211
|
+
return { whole: reason(checked, "shape", null) };
|
|
212
|
+
// A def before the stream would have to be in scope where every piece runs.
|
|
213
|
+
if (first.node.type === "def")
|
|
214
|
+
return { whole: reason(checked, "shape", first.path) };
|
|
215
|
+
let source;
|
|
216
|
+
let stream = false;
|
|
217
|
+
let end = first.path;
|
|
218
|
+
const rowSteps = [];
|
|
219
|
+
const field = rootField(first.node, true);
|
|
220
|
+
const streamed = rootStream(first.node);
|
|
221
|
+
if (field !== undefined) {
|
|
222
|
+
source = { over: field, path: first.path };
|
|
223
|
+
// `.D | to_entries`, `.D | to_entries[]`: a dict's entries are the rows (#942).
|
|
224
|
+
const entries = isDict(checked, first.path) ? entriesAt(steps, i + 1) : undefined;
|
|
225
|
+
if (entries !== undefined) {
|
|
226
|
+
source = { ...source, entries: entries.entries };
|
|
227
|
+
if (entries.stream !== undefined) {
|
|
228
|
+
rowSteps.push({ kind: "as", path: entries.stream, element: false });
|
|
229
|
+
stream = true;
|
|
230
|
+
}
|
|
231
|
+
end = entries.end;
|
|
232
|
+
i = entries.next - 1;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
else if (streamed !== undefined) {
|
|
236
|
+
source = { over: streamed, path: first.path };
|
|
237
|
+
stream = true;
|
|
238
|
+
rowSteps.push({ kind: "as", path: first.path, element: false });
|
|
239
|
+
}
|
|
240
|
+
else if (first.node.type === "array" && first.node.value.type === "some") {
|
|
241
|
+
// `[.D[] | f]`, `[.D | to_entries[] | f]`: the stream's outputs, collected.
|
|
242
|
+
const inner = pipeline(first.node.value.value, childPath(first.path, "array.some"));
|
|
243
|
+
const head = inner[0];
|
|
244
|
+
if (head?.kind !== "stage")
|
|
245
|
+
return { whole: nestedOrNone(checked, first) };
|
|
246
|
+
const over = rootStream(head.node);
|
|
247
|
+
const dict = rootField(head.node, true);
|
|
248
|
+
const entries = dict !== undefined && isDict(checked, head.path) ? entriesAt(inner, 1) : undefined;
|
|
249
|
+
if (over !== undefined) {
|
|
250
|
+
source = { over, path: head.path };
|
|
251
|
+
rowSteps.push({ kind: "as", path: head.path, element: false }, ...streamSteps(inner.slice(1)), { kind: "as", path: first.path, element: true });
|
|
252
|
+
}
|
|
253
|
+
else if (dict !== undefined && entries?.stream !== undefined) {
|
|
254
|
+
source = { over: dict, path: head.path, entries: entries.entries };
|
|
255
|
+
rowSteps.push({ kind: "as", path: entries.stream, element: false }, ...streamSteps(inner.slice(entries.next)), { kind: "as", path: first.path, element: true });
|
|
256
|
+
}
|
|
257
|
+
else {
|
|
258
|
+
return { whole: nestedOrNone(checked, first) };
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
else if (first.node.type === "reduce" && rootStream(first.node.value.source) !== undefined) {
|
|
262
|
+
// `reduce .D[] as $x (…)`: the reduce combines the dataset's rows.
|
|
263
|
+
const sourcePath = childPath(first.path, "reduce.source");
|
|
264
|
+
const row = typeAfter(checked, sourcePath);
|
|
265
|
+
return checkBinds(checked, {
|
|
266
|
+
binds, source: { over: rootStream(first.node.value.source), path: sourcePath },
|
|
267
|
+
steps: [{ kind: "as", path: sourcePath, element: false }], stream: false, start: first.path, end: sourcePath, work: false,
|
|
268
|
+
elements: { type: ArrayType(row), element: row }, after: steps.slice(i),
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
else {
|
|
272
|
+
return { whole: earlyOrNested(checked, first) };
|
|
273
|
+
}
|
|
274
|
+
i += 1;
|
|
275
|
+
// The collection's row work: `map`, `flatten`, `[.[] | f]`; or `.[]` making a stream of the dataset.
|
|
276
|
+
while (!stream && i < steps.length) {
|
|
277
|
+
const step = steps[i];
|
|
278
|
+
if (step.kind !== "stage")
|
|
279
|
+
break;
|
|
280
|
+
const node = step.node;
|
|
281
|
+
const working = rowSteps.some(s => s.kind !== "as");
|
|
282
|
+
if (isCall(node, "map", 1))
|
|
283
|
+
rowSteps.push({ kind: "map", node, path: step.path });
|
|
284
|
+
else if (isCall(node, "flatten", 0))
|
|
285
|
+
rowSteps.push({ kind: "flatten", node, path: step.path });
|
|
286
|
+
else if (node.type === "array" && node.value.type === "some") {
|
|
287
|
+
const inner = pipeline(node.value.value, childPath(step.path, "array.some"));
|
|
288
|
+
const head = inner[0];
|
|
289
|
+
if (head?.kind !== "stage" || !isIterate(head.node))
|
|
290
|
+
break;
|
|
291
|
+
rowSteps.push({ kind: "as", path: head.path, element: false }, ...streamSteps(inner.slice(1)), { kind: "as", path: step.path, element: true });
|
|
292
|
+
}
|
|
293
|
+
else if (isIterate(node) && !working) {
|
|
294
|
+
// `.D | .[]`: the dataset's rows as a stream, which one unit stops early.
|
|
295
|
+
rowSteps.push({ kind: "as", path: step.path, element: false });
|
|
296
|
+
stream = true;
|
|
297
|
+
}
|
|
298
|
+
else
|
|
299
|
+
break;
|
|
300
|
+
end = step.path;
|
|
301
|
+
i += 1;
|
|
302
|
+
}
|
|
303
|
+
if (stream) {
|
|
304
|
+
// A stream's later steps all run row by row: it is the query's output.
|
|
305
|
+
rowSteps.push(...streamSteps(steps.slice(i)));
|
|
306
|
+
if (steps.length > i)
|
|
307
|
+
end = steps[steps.length - 1].path;
|
|
308
|
+
i = steps.length;
|
|
309
|
+
}
|
|
310
|
+
for (const s of rowSteps) {
|
|
311
|
+
if (!("node" in s))
|
|
312
|
+
continue;
|
|
313
|
+
const call = findNode(s.node, s.path, n => n.type === "call" && n.value.name === "call");
|
|
314
|
+
if (call !== undefined)
|
|
315
|
+
return { whole: reason(checked, "calls", call.path) };
|
|
316
|
+
const stateful = findNode(s.node, s.path, n => n.type === "call" && STATEFUL.has(n.value.name));
|
|
317
|
+
if (stateful !== undefined)
|
|
318
|
+
return { whole: reason(checked, "state", stateful.path, nodeName(stateful.node)) };
|
|
319
|
+
}
|
|
320
|
+
return checkBinds(checked, {
|
|
321
|
+
binds, source, steps: rowSteps, stream, start: first.path, end, work: rowSteps.some(s => s.kind !== "as"),
|
|
322
|
+
elements: stream ? undefined : elementsOf(typeAfter(checked, end)), after: steps.slice(i),
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
/** Row work whose bindings before the stream every piece can read: none reads the dataset again, and each gives one value. */
|
|
326
|
+
function checkBinds(checked, rows) {
|
|
327
|
+
for (const bind of rows.binds) {
|
|
328
|
+
if (readsRootField(bind.node.value.source, rows.source.over))
|
|
329
|
+
return { whole: reason(checked, "reads_again", bind.path, rows.source.over) };
|
|
330
|
+
const result = checked.resultAt(childPath(bind.path, "bind.source"));
|
|
331
|
+
if (result === null || result.mult.lo !== 1 || result.mult.hi !== 1)
|
|
332
|
+
return { whole: reason(checked, "shape", bind.path) };
|
|
333
|
+
}
|
|
334
|
+
return { rows };
|
|
335
|
+
}
|
|
336
|
+
/** The steps of a stream after its source, each run on each row. */
|
|
337
|
+
function streamSteps(steps) {
|
|
338
|
+
return steps.map((step) => step.kind === "bind" ? { kind: "bind", node: step.node, path: step.path } : { kind: "each", node: step.node, path: step.path });
|
|
339
|
+
}
|
|
340
|
+
/** The name a node is known by in a reason: a builtin's, or its kind. */
|
|
341
|
+
function nodeName(node) {
|
|
342
|
+
if (node.type === "call")
|
|
343
|
+
return node.value.name;
|
|
344
|
+
return node.type;
|
|
345
|
+
}
|
|
346
|
+
/** Whether a node reads a root field anywhere in it, with the root as its input. */
|
|
347
|
+
function readsRootField(node, name) {
|
|
348
|
+
if (rootField(node, true) === name)
|
|
349
|
+
return true;
|
|
350
|
+
if (node.type === "pipe")
|
|
351
|
+
return readsRootField(node.value.left, name);
|
|
352
|
+
return jqChildren(node).some(child => child.node !== undefined && child.step !== "pipe.right" && readsRootField(child.node, name));
|
|
353
|
+
}
|
|
354
|
+
/** Why a query whose first stage is not a stream runs as one unit: it stops early, or streams inside an expression, or streams nothing. */
|
|
355
|
+
function earlyOrNested(checked, first) {
|
|
356
|
+
const node = first.node;
|
|
357
|
+
if (node.type === "call" && (POSITIONAL.has(node.value.name) || node.value.name === "isempty")) {
|
|
358
|
+
if (findNode(node, first.path, n => rootStream(n) !== undefined) !== undefined)
|
|
359
|
+
return reason(checked, "stops_early", first.path, node.value.name);
|
|
360
|
+
}
|
|
361
|
+
if (node.type === "reduce" || node.type === "foreach")
|
|
362
|
+
return reason(checked, node.type === "reduce" ? "shape" : "state", first.path, node.type);
|
|
363
|
+
return nestedOrNone(checked, first);
|
|
364
|
+
}
|
|
365
|
+
/** `nested` when a stream over a root field is somewhere inside the node; `no_stream` otherwise. */
|
|
366
|
+
function nestedOrNone(checked, first) {
|
|
367
|
+
const inner = findNode(first.node, first.path, n => rootStream(n) !== undefined || (n.type === "pipe" && rootField(n.value.left, true) !== undefined));
|
|
368
|
+
if (inner !== undefined && inner.path !== first.path)
|
|
369
|
+
return reason(checked, "nested", inner.path);
|
|
370
|
+
return reason(checked, "no_stream", null);
|
|
371
|
+
}
|
|
372
|
+
/** The element type of a collection type that totals work through: an array's, a set's or a dict's values. */
|
|
373
|
+
function elementsOf(type) {
|
|
374
|
+
const t = unwrap(type);
|
|
375
|
+
if (t.type === "Array")
|
|
376
|
+
return { type: t, element: t.value };
|
|
377
|
+
if (t.type === "Set")
|
|
378
|
+
return { type: t, element: t.key };
|
|
379
|
+
if (t.type === "Dict")
|
|
380
|
+
return { type: t, element: t.value };
|
|
381
|
+
return undefined;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Decomposes an expression over a collection into totals.
|
|
385
|
+
*
|
|
386
|
+
* @param node - the expression
|
|
387
|
+
* @param path - its path
|
|
388
|
+
* @param input - the collection it reads: its type and its elements' type
|
|
389
|
+
* @param maps - the maps whose outputs the collection is, outermost first
|
|
390
|
+
* @returns the totals, or `undefined` when the expression reads the collection other than through totals
|
|
391
|
+
*/
|
|
392
|
+
function decompose(checked, node, path, input, maps, out) {
|
|
393
|
+
const at = (step) => childPath(path, step);
|
|
394
|
+
const leaf = leafOf(checked, node, path, input, maps);
|
|
395
|
+
if (leaf !== undefined) {
|
|
396
|
+
out.leaves.push(leaf);
|
|
397
|
+
return true;
|
|
398
|
+
}
|
|
399
|
+
switch (node.type) {
|
|
400
|
+
case "literal":
|
|
401
|
+
case "variable":
|
|
402
|
+
return true;
|
|
403
|
+
case "pipe": {
|
|
404
|
+
const left = node.value.left;
|
|
405
|
+
const leftPath = at("pipe.left");
|
|
406
|
+
if (isCall(left, "map", 1)) {
|
|
407
|
+
const mapped = checked.typeAt(leftPath)?.type;
|
|
408
|
+
const elements = mapped === undefined ? undefined : elementsOf(mapped);
|
|
409
|
+
if (elements === undefined)
|
|
410
|
+
return false;
|
|
411
|
+
const context = { node: left, path: leftPath };
|
|
412
|
+
out.maps.push(context);
|
|
413
|
+
return decompose(checked, node.value.right, at("pipe.right"), elements, [...maps, context], out);
|
|
414
|
+
}
|
|
415
|
+
// The right side reads the left's value, never the collection.
|
|
416
|
+
return decompose(checked, left, leftPath, input, maps, out);
|
|
417
|
+
}
|
|
418
|
+
case "field":
|
|
419
|
+
return node.value.target.type !== "identity" && decompose(checked, node.value.target, at("field.target"), input, maps, out);
|
|
420
|
+
case "index":
|
|
421
|
+
return node.value.target.type !== "identity"
|
|
422
|
+
&& decompose(checked, node.value.target, at("index.target"), input, maps, out)
|
|
423
|
+
&& decompose(checked, node.value.index, at("index.index"), input, maps, out);
|
|
424
|
+
case "object":
|
|
425
|
+
return node.value.every((entry, i) => {
|
|
426
|
+
if (entry.key.type === "computed")
|
|
427
|
+
return false;
|
|
428
|
+
if (entry.value.type === "none")
|
|
429
|
+
return entry.key.type === "variable";
|
|
430
|
+
return decompose(checked, entry.value.value, at(`object[${i}].value.some`), input, maps, out);
|
|
431
|
+
});
|
|
432
|
+
case "array":
|
|
433
|
+
return node.value.type === "none" || decompose(checked, node.value.value, at("array.some"), input, maps, out);
|
|
434
|
+
case "binary":
|
|
435
|
+
case "alternative":
|
|
436
|
+
case "comma":
|
|
437
|
+
return decompose(checked, node.value.left, at(`${node.type}.left`), input, maps, out)
|
|
438
|
+
&& decompose(checked, node.value.right, at(`${node.type}.right`), input, maps, out);
|
|
439
|
+
case "negate":
|
|
440
|
+
return decompose(checked, node.value, at("negate"), input, maps, out);
|
|
441
|
+
case "if":
|
|
442
|
+
return node.value.otherwise.type === "some" && jqChildren(node).every(child => child.node === undefined || decompose(checked, child.node, at(child.step), input, maps, out));
|
|
443
|
+
case "string":
|
|
444
|
+
return jqChildren(node).every(child => child.node === undefined || decompose(checked, child.node, at(child.step), input, maps, out));
|
|
445
|
+
default:
|
|
446
|
+
return false;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/** The algebra of a total, or `undefined` when a node is not one. */
|
|
450
|
+
function leafOf(checked, node, path, input, maps) {
|
|
451
|
+
// A call by name and arity; not narrowing `node`, which several tests below read.
|
|
452
|
+
const named = (name, arity) => node.type === "call" && node.value.name === name && node.value.args.length === arity;
|
|
453
|
+
const call = node;
|
|
454
|
+
const E = input.element;
|
|
455
|
+
const kind = unwrap(input.type).type;
|
|
456
|
+
const range = maps.length === 0 ? rangeOf(checked, path) : spanning(checked, maps[0].path, path);
|
|
457
|
+
const base = { path, node, range, maps, input };
|
|
458
|
+
const option = OptionType(E);
|
|
459
|
+
const representative = (t, $, partial, env, value) => {
|
|
460
|
+
// The leaf on a collection of the one element its partial holds: what it gives on every element.
|
|
461
|
+
const type = t.typeAt(path, env);
|
|
462
|
+
const one = t.matchValue(partial, {
|
|
463
|
+
none: () => kind === "Set" ? t.value(new SortedSet([], compareFor(E)), input.type) : t.emptyArray(E),
|
|
464
|
+
some: ($2, p) => {
|
|
465
|
+
if (kind === "Set") {
|
|
466
|
+
const set = t.declare($2, t.value(new SortedSet([], compareFor(E)), input.type), "one");
|
|
467
|
+
t.stmt($2, t.b("SetTryInsert", [E], [set, value(p)], BooleanType, path));
|
|
468
|
+
return set;
|
|
469
|
+
}
|
|
470
|
+
const array = t.declare($2, t.emptyArray(E), "one");
|
|
471
|
+
t.push($2, array, value(p), path);
|
|
472
|
+
return array;
|
|
473
|
+
},
|
|
474
|
+
}, input.type, path);
|
|
475
|
+
return t.one(node, path, $, t.bind($, one, "elements"), env, type);
|
|
476
|
+
};
|
|
477
|
+
const pickBetter = (t, $, a, b, better, type) =>
|
|
478
|
+
// b replaces a when a has none, or b is better.
|
|
479
|
+
t.matchValue(b, {
|
|
480
|
+
none: () => a,
|
|
481
|
+
some: (_$2, y) => t.matchValue(a, {
|
|
482
|
+
none: () => b,
|
|
483
|
+
some: (_$3, x) => t.ifValue(better(y, x), () => b, () => a, type, path),
|
|
484
|
+
}, type, path),
|
|
485
|
+
}, type, path);
|
|
486
|
+
if (named("length", 0)) {
|
|
487
|
+
return {
|
|
488
|
+
...base, rule: "count", type: IntegerType, zero: 0n,
|
|
489
|
+
add: (t, $, acc) => t.assign($, acc, t.add(acc, t.int(1), path)),
|
|
490
|
+
combine: (t, _$, a, b) => t.add(a, b, path),
|
|
491
|
+
finish: (_t, _$, partial) => partial,
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
if (named("add", 0)) {
|
|
495
|
+
const O = checked.typeAt(path)?.type;
|
|
496
|
+
if (O === undefined)
|
|
497
|
+
return undefined;
|
|
498
|
+
const zero = addZero(O);
|
|
499
|
+
if (zero === undefined)
|
|
500
|
+
return undefined;
|
|
501
|
+
const site = (t, $) => ({ name: "add", path, args: [], argPaths: [], $, x: t.null(path), env: emptyEnv(), emit: () => { } });
|
|
502
|
+
return {
|
|
503
|
+
...base, rule: "add", type: O, zero: zero.value,
|
|
504
|
+
add: (t, $, acc, element) => addInto(t, site(t, $), $, acc, element, O),
|
|
505
|
+
combine: (t, $, a, b) => {
|
|
506
|
+
const sum = t.declare($, t.value(zero.value, O), "sum");
|
|
507
|
+
addInto(t, site(t, $), $, sum, a, O);
|
|
508
|
+
addInto(t, site(t, $), $, sum, b, O);
|
|
509
|
+
return sum;
|
|
510
|
+
},
|
|
511
|
+
finish: (_t, _$, partial) => partial,
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
if ((named("min", 0) || named("max", 0)) && kind !== "Dict") {
|
|
515
|
+
const min = call.value.name === "min";
|
|
516
|
+
const better = (t) => (x, y) => min ? t.lt(x, y, path) : t.b("GreaterEqual", [E], [x, y], BooleanType, path);
|
|
517
|
+
return {
|
|
518
|
+
...base, rule: min ? "min" : "max", type: option, zero: none,
|
|
519
|
+
add: (t, $, acc, element) => {
|
|
520
|
+
const value = t.bind($, element, "element");
|
|
521
|
+
const take = t.matchValue(acc, { none: () => t.bool(true), some: (_$2, b) => better(t)(value, b) }, BooleanType, path);
|
|
522
|
+
t.ifElse($, take, $2 => t.assign($2, acc, t.some(value, option)), undefined, path);
|
|
523
|
+
},
|
|
524
|
+
combine: (t, $, a, b) => pickBetter(t, $, a, b, better(t), option),
|
|
525
|
+
finish: (t, $, partial, env) => representative(t, $, partial, env, p => p),
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
if ((named("min_by", 1) || named("max_by", 1)) && kind !== "Dict") {
|
|
529
|
+
const min = call.value.name === "min_by";
|
|
530
|
+
const argPath = childPath(path, "call.args[0]");
|
|
531
|
+
const sample = checked.resultAt(argPath);
|
|
532
|
+
if (sample === null)
|
|
533
|
+
return undefined;
|
|
534
|
+
const keyOne = sample.mult.lo === 1 && sample.mult.hi === 1;
|
|
535
|
+
const outputKey = checked.typeAt(argPath)?.type;
|
|
536
|
+
if (outputKey === undefined)
|
|
537
|
+
return undefined;
|
|
538
|
+
const K = keyOne ? outputKey : ArrayType(outputKey);
|
|
539
|
+
const Pair = StructType({ key: K, value: E });
|
|
540
|
+
const pair = OptionType(Pair);
|
|
541
|
+
const better = (t) => (x, y) => min ? t.lt(t.field(x, "key"), t.field(y, "key"), path) : t.b("GreaterEqual", [K], [t.field(x, "key"), t.field(y, "key")], BooleanType, path);
|
|
542
|
+
return {
|
|
543
|
+
...base, rule: min ? "min" : "max", type: pair, zero: none,
|
|
544
|
+
add: (t, $, acc, element, env) => {
|
|
545
|
+
const value = t.bind($, element, "element");
|
|
546
|
+
let key;
|
|
547
|
+
if (keyOne)
|
|
548
|
+
key = t.one(call.value.args[0], argPath, $, value, env, K);
|
|
549
|
+
else {
|
|
550
|
+
const keys = t.declare($, t.emptyArray(outputKey), "keys");
|
|
551
|
+
t.gen(call.value.args[0], argPath, $, value, env, ($2, k) => t.push($2, keys, k, path));
|
|
552
|
+
key = keys;
|
|
553
|
+
}
|
|
554
|
+
const candidate = t.bind($, t.struct(Pair, { key, value }), "pair");
|
|
555
|
+
const take = t.matchValue(acc, { none: () => t.bool(true), some: (_$2, b) => better(t)(candidate, b) }, BooleanType, path);
|
|
556
|
+
t.ifElse($, take, $2 => t.assign($2, acc, t.some(candidate, pair)), undefined, path);
|
|
557
|
+
},
|
|
558
|
+
combine: (t, $, a, b) => pickBetter(t, $, a, b, better(t), pair),
|
|
559
|
+
finish: (t, $, partial, env) => representative(t, $, partial, env, p => t.field(p, "value")),
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
const firstOrLast = named("first", 0) ? "first" : named("last", 0) ? "last"
|
|
563
|
+
: node.type === "index" && node.value.target.type === "identity" ? indexLiteral(node) : undefined;
|
|
564
|
+
if (firstOrLast !== undefined && kind !== "Dict") {
|
|
565
|
+
const first = firstOrLast === "first";
|
|
566
|
+
return {
|
|
567
|
+
...base, rule: firstOrLast, type: option, zero: none,
|
|
568
|
+
add: (t, $, acc, element) => {
|
|
569
|
+
if (first)
|
|
570
|
+
t.match($, acc, { none: $2 => t.assign($2, acc, t.some(element, option)) }, path);
|
|
571
|
+
else
|
|
572
|
+
t.assign($, acc, t.some(element, option));
|
|
573
|
+
},
|
|
574
|
+
combine: (t, _$, a, b) => first
|
|
575
|
+
? t.matchValue(a, { none: () => b, some: () => a }, option, path)
|
|
576
|
+
: t.matchValue(b, { none: () => a, some: () => b }, option, path),
|
|
577
|
+
finish: (t, $, partial, env) => representative(t, $, partial, env, p => p),
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
if ((named("any", 0) || named("all", 0) || named("any", 1) || named("all", 1))) {
|
|
581
|
+
const any = call.value.name === "any";
|
|
582
|
+
const argPath = childPath(path, "call.args[0]");
|
|
583
|
+
const test = (t, $, acc, v) => {
|
|
584
|
+
const truth = t.truthy(v, path);
|
|
585
|
+
if (any)
|
|
586
|
+
t.branch($, truth, $2 => t.assign($2, acc, t.bool(true)), () => { }, path);
|
|
587
|
+
else
|
|
588
|
+
t.branch($, truth, () => { }, $2 => t.assign($2, acc, t.bool(false)), path);
|
|
589
|
+
};
|
|
590
|
+
return {
|
|
591
|
+
...base, rule: any ? "any" : "all", type: BooleanType, zero: !any,
|
|
592
|
+
add: (t, $, acc, element, env) => {
|
|
593
|
+
if (call.value.args.length === 0)
|
|
594
|
+
test(t, $, acc, element);
|
|
595
|
+
else
|
|
596
|
+
t.gen(call.value.args[0], argPath, $, element, env, ($2, v) => test(t, $2, acc, v));
|
|
597
|
+
},
|
|
598
|
+
combine: (t, _$, a, b) => t.b(any ? "BooleanOr" : "BooleanAnd", [], [a, b], BooleanType, path),
|
|
599
|
+
finish: (_t, _$, partial) => partial,
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
if (named("unique", 0) && kind !== "Dict" && isImmutableType(E)) {
|
|
603
|
+
const S = SetType(E);
|
|
604
|
+
return {
|
|
605
|
+
...base, rule: "union", type: S, zero: new SortedSet([], compareFor(E)),
|
|
606
|
+
add: (t, $, acc, element) => t.stmt($, t.b("SetTryInsert", [E], [acc, element], BooleanType, path)),
|
|
607
|
+
combine: (t, _$, a, b) => t.b("SetUnion", [E], [a, b], S, path),
|
|
608
|
+
finish: (t, $, partial) => t.asArray($, partial, path),
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
return undefined;
|
|
612
|
+
}
|
|
613
|
+
/** `first` or `last` for `.[0]` or `.[-1]`. */
|
|
614
|
+
function indexLiteral(node) {
|
|
615
|
+
if (node.value.optional)
|
|
616
|
+
return undefined;
|
|
617
|
+
const index = node.value.index;
|
|
618
|
+
// `.[-1]` is a negated literal.
|
|
619
|
+
const literal = index.type === "literal" ? integerOf(index.value)
|
|
620
|
+
: index.type === "negate" && index.value.type === "literal" ? negated(integerOf(index.value.value)) : undefined;
|
|
621
|
+
return literal === 0n ? "first" : literal === -1n ? "last" : undefined;
|
|
622
|
+
}
|
|
623
|
+
/** A whole number negated, or `undefined`. */
|
|
624
|
+
function negated(n) {
|
|
625
|
+
return n === undefined ? undefined : -n;
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* How many of an array's first elements a step reads, when it reads no others
|
|
629
|
+
* (#942): `.[:n]` and `.[0:n]` its first `n`, `.[k]` its first `k + 1`,
|
|
630
|
+
* `first` its first, and a read into one of those (`.[0].id`, `.[:3][1]`) as
|
|
631
|
+
* many; `undefined` for any other step.
|
|
632
|
+
*/
|
|
633
|
+
function leadingRows(node) {
|
|
634
|
+
const count = (n) => n === undefined || n < 0n || n > BigInt(Number.MAX_SAFE_INTEGER) ? undefined : Number(n);
|
|
635
|
+
const literal = (option) => option.type === "none" ? null : option.value.type === "literal" ? integerOf(option.value.value) : undefined;
|
|
636
|
+
if (isCall(node, "first", 0))
|
|
637
|
+
return 1;
|
|
638
|
+
switch (node.type) {
|
|
639
|
+
case "field":
|
|
640
|
+
return node.value.target.type === "identity" ? undefined : leadingRows(node.value.target);
|
|
641
|
+
case "index": {
|
|
642
|
+
const { index, target } = node.value;
|
|
643
|
+
if (target.type !== "identity")
|
|
644
|
+
return readsInput(index) ? undefined : leadingRows(target);
|
|
645
|
+
const k = index.type === "literal" ? count(integerOf(index.value)) : undefined;
|
|
646
|
+
return k === undefined ? undefined : k + 1;
|
|
647
|
+
}
|
|
648
|
+
case "slice": {
|
|
649
|
+
const { from, target, to } = node.value;
|
|
650
|
+
if (target.type !== "identity") {
|
|
651
|
+
const bounds = [from, to].every(b => b.type === "none" || !readsInput(b.value));
|
|
652
|
+
return bounds ? leadingRows(target) : undefined;
|
|
653
|
+
}
|
|
654
|
+
// From the start, or a literal index of it; to a literal index.
|
|
655
|
+
const a = literal(from);
|
|
656
|
+
const b = literal(to);
|
|
657
|
+
if (a === undefined || (a !== null && count(a) === undefined) || b === null || b === undefined)
|
|
658
|
+
return undefined;
|
|
659
|
+
return count(b);
|
|
660
|
+
}
|
|
661
|
+
default:
|
|
662
|
+
return undefined;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
// ─── Values and zeros ───────────────────────────────────────────────────────
|
|
666
|
+
/** An Integer literal's value, or `undefined` for a literal of another type. */
|
|
667
|
+
function integerOf(literal) {
|
|
668
|
+
return literal.type === "integer" ? literal.value : undefined;
|
|
669
|
+
}
|
|
670
|
+
/** `add`'s start, as the `add` rule starts: its identity, for its output type. */
|
|
671
|
+
function addZero(type) {
|
|
672
|
+
const t = unwrap(type);
|
|
673
|
+
switch (t.type) {
|
|
674
|
+
case "Null": return { value: null };
|
|
675
|
+
case "Integer": return { value: 0n };
|
|
676
|
+
case "Float": return { value: 0 };
|
|
677
|
+
case "String": return { value: "" };
|
|
678
|
+
case "Array": return { value: [] };
|
|
679
|
+
case "Dict": return { value: new SortedMap([], compareFor(t.key)) };
|
|
680
|
+
case "Variant": return t.cases !== undefined && "none" in t.cases ? { value: none } : undefined;
|
|
681
|
+
default: return undefined;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
/** An environment with nothing in scope. */
|
|
685
|
+
function emptyEnv() {
|
|
686
|
+
return { vars: new Map(), defs: new Map(), labels: new Map(), instance: "", recursion: new Map() };
|
|
687
|
+
}
|
|
688
|
+
// ─── The translator, with holes ─────────────────────────────────────────────
|
|
689
|
+
/**
|
|
690
|
+
* The translator for a part of a checked program: a node whose value is known
|
|
691
|
+
* — a total the pieces computed, a map the totals read through — is a hole,
|
|
692
|
+
* and gives that value where the program reaches it.
|
|
693
|
+
*/
|
|
694
|
+
class SplitTranslator extends Translator {
|
|
695
|
+
holes = new Map();
|
|
696
|
+
constructor(checked, options) {
|
|
697
|
+
super(checked, options);
|
|
698
|
+
}
|
|
699
|
+
gen(node, path, $, x, env, emit) {
|
|
700
|
+
const hole = this.holes.get(path);
|
|
701
|
+
if (hole === undefined) {
|
|
702
|
+
super.gen(node, path, $, x, env, emit);
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
if (!this.ended($))
|
|
706
|
+
emit($, hole);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
/** The type of the value after a path, as the checker gave it. */
|
|
710
|
+
function typeAfter(checked, path) {
|
|
711
|
+
const type = checked.typeAt(path)?.type;
|
|
712
|
+
if (type === undefined)
|
|
713
|
+
throw new TranslationError(`no type for the node at ${path || "the root"}`);
|
|
714
|
+
return type;
|
|
715
|
+
}
|
|
716
|
+
/** The type of each row as the row work leaves it: a binding passes its row on. */
|
|
717
|
+
function rowTypeOf(checked, rows) {
|
|
718
|
+
const last = [...rows.steps].reverse().find(s => s.kind !== "bind");
|
|
719
|
+
if (last === undefined) {
|
|
720
|
+
// A dict read as its entries: the rows are `to_entries`' elements.
|
|
721
|
+
const elements = elementsOf(typeAfter(checked, rows.source.entries ?? rows.source.path));
|
|
722
|
+
if (elements === undefined)
|
|
723
|
+
throw new TranslationError(`.${rows.source.over} is not a collection`);
|
|
724
|
+
return elements.element;
|
|
725
|
+
}
|
|
726
|
+
switch (last.kind) {
|
|
727
|
+
case "map":
|
|
728
|
+
case "flatten": return parts(typeAfter(checked, last.path)).value;
|
|
729
|
+
case "as": return last.element ? parts(typeAfter(checked, last.path)).value : typeAfter(checked, last.path);
|
|
730
|
+
default: return typeAfter(checked, last.path);
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
/** A key a node gives on each row, when it is one value a dict can hold. */
|
|
734
|
+
function keyOf(checked, path) {
|
|
735
|
+
const result = checked.resultAt(path);
|
|
736
|
+
if (result === null || result.mult.lo !== 1 || result.mult.hi !== 1)
|
|
737
|
+
return undefined;
|
|
738
|
+
const type = checked.typeAt(path)?.type;
|
|
739
|
+
return type !== undefined && isImmutableType(type) ? type : undefined;
|
|
740
|
+
}
|
|
741
|
+
/**
|
|
742
|
+
* How the rows combine: the step after the row work, when it is one the
|
|
743
|
+
* pieces can combine for.
|
|
744
|
+
*/
|
|
745
|
+
function findCombine(checked, rows) {
|
|
746
|
+
const [next, after] = [rows.after[0], rows.after[1]];
|
|
747
|
+
const elements = rows.elements;
|
|
748
|
+
if (next === undefined || next.kind !== "stage" || elements === undefined)
|
|
749
|
+
return undefined;
|
|
750
|
+
const node = next.node;
|
|
751
|
+
const at = (step) => childPath(next.path, step);
|
|
752
|
+
const kind = unwrap(elements.type).type;
|
|
753
|
+
const exactlyOne = (path) => {
|
|
754
|
+
const r = checked.resultAt(path);
|
|
755
|
+
return r !== null && r.mult.lo === 1 && r.mult.hi === 1;
|
|
756
|
+
};
|
|
757
|
+
if (isCall(node, "group_by", 1) && kind !== "Dict" && after?.kind === "stage" && isCall(after.node, "map", 1)) {
|
|
758
|
+
const keyPath = at("call.args[0]");
|
|
759
|
+
const keyType = keyOf(checked, keyPath);
|
|
760
|
+
if (keyType === undefined)
|
|
761
|
+
return { whole: reason(checked, "key", keyPath) };
|
|
762
|
+
const bodyPath = childPath(after.path, "call.args[0]");
|
|
763
|
+
const body = after.node.value.args[0];
|
|
764
|
+
// Each group is an array of the rows, as `group_by` gives it.
|
|
765
|
+
const group = elementsOf(parts(typeAfter(checked, next.path)).value);
|
|
766
|
+
const out = { leaves: [], maps: [] };
|
|
767
|
+
const decomposed = group !== undefined && exactlyOne(bodyPath) && decompose(checked, body, bodyPath, group, [], out);
|
|
768
|
+
return {
|
|
769
|
+
combine: {
|
|
770
|
+
kind: "group", path: next.path, mapPath: after.path,
|
|
771
|
+
key: { node: node.value.args[0], path: keyPath, type: keyType },
|
|
772
|
+
body: { node: body, path: bodyPath },
|
|
773
|
+
totals: decomposed ? out : null,
|
|
774
|
+
},
|
|
775
|
+
rest: rows.after.slice(2),
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
if (isCall(node, "unique", 0) && kind !== "Dict" && isImmutableType(elements.element)) {
|
|
779
|
+
return { combine: { kind: "distinct", path: next.path }, rest: rows.after.slice(1) };
|
|
780
|
+
}
|
|
781
|
+
if (isCall(node, "unique_by", 1) && kind !== "Dict") {
|
|
782
|
+
const keyPath = at("call.args[0]");
|
|
783
|
+
const keyType = keyOf(checked, keyPath);
|
|
784
|
+
if (keyType === undefined)
|
|
785
|
+
return { whole: reason(checked, "key", keyPath) };
|
|
786
|
+
return { combine: { kind: "distinct_by", path: next.path, key: { node: node.value.args[0], path: keyPath, type: keyType } }, rest: rows.after.slice(1) };
|
|
787
|
+
}
|
|
788
|
+
if (node.type === "reduce") {
|
|
789
|
+
const reduce = reduceOf(checked, node, next.path);
|
|
790
|
+
return reduce === undefined ? undefined : { combine: reduce, rest: rows.after.slice(1) };
|
|
791
|
+
}
|
|
792
|
+
// `sort_by(g) | .[:n]`, `sort | first`: the first rows of a sort, each piece's kept (#942).
|
|
793
|
+
if ((isCall(node, "sort_by", 1) || isCall(node, "sort", 0)) && kind !== "Dict" && after?.kind === "stage") {
|
|
794
|
+
const kept = leadingRows(after.node);
|
|
795
|
+
if (kept !== undefined) {
|
|
796
|
+
const element = parts(typeAfter(checked, next.path)).value;
|
|
797
|
+
let key = null;
|
|
798
|
+
if (node.value.args.length === 1) {
|
|
799
|
+
const argPath = at("call.args[0]");
|
|
800
|
+
const sample = checked.resultAt(argPath);
|
|
801
|
+
const output = checked.typeAt(argPath)?.type;
|
|
802
|
+
if (sample === null || output === undefined)
|
|
803
|
+
return undefined;
|
|
804
|
+
const one = sample.mult.lo === 1 && sample.mult.hi === 1;
|
|
805
|
+
key = { node: node.value.args[0], path: argPath, type: one ? output : ArrayType(output), one };
|
|
806
|
+
}
|
|
807
|
+
return { combine: { kind: "top", path: next.path, keepPath: after.path, key, rows: kept, element }, rest: rows.after.slice(1) };
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
const out = { leaves: [], maps: [] };
|
|
811
|
+
if (exactlyOne(next.path) && decompose(checked, node, next.path, elements, [], out) && out.leaves.length > 0) {
|
|
812
|
+
return { combine: { kind: "totals", path: next.path, node, totals: out }, rest: rows.after.slice(1) };
|
|
813
|
+
}
|
|
814
|
+
return undefined;
|
|
815
|
+
}
|
|
816
|
+
/** `reduce .[] as $x ({}; .[k] += v)`, or `= v`: a dict its updates combine into by key. */
|
|
817
|
+
function reduceOf(checked, node, path) {
|
|
818
|
+
const { init, source, update } = node.value;
|
|
819
|
+
// Over the collection's rows, `.[]`, or the dataset's own, `.D[]`.
|
|
820
|
+
if (!isIterate(source) && rootStream(source) === undefined)
|
|
821
|
+
return undefined;
|
|
822
|
+
if (init.type !== "object" || init.value.length !== 0)
|
|
823
|
+
return undefined;
|
|
824
|
+
if (update.type !== "update" || (update.value.op !== "+=" && update.value.op !== "="))
|
|
825
|
+
return undefined;
|
|
826
|
+
const target = update.value.path;
|
|
827
|
+
if (target.type !== "index" || target.value.optional || target.value.target.type !== "identity")
|
|
828
|
+
return undefined;
|
|
829
|
+
const updatePath = childPath(path, "reduce.update");
|
|
830
|
+
const keyPath = childPath(updatePath, "update.path.index.index");
|
|
831
|
+
const valuePath = childPath(updatePath, "update.value");
|
|
832
|
+
if (readsInput(target.value.index) || readsInput(update.value.value))
|
|
833
|
+
return undefined;
|
|
834
|
+
const one = (at) => {
|
|
835
|
+
const r = checked.resultAt(at);
|
|
836
|
+
return r !== null && r.mult.lo === 1 && r.mult.hi === 1;
|
|
837
|
+
};
|
|
838
|
+
if (!one(keyPath) || !one(valuePath))
|
|
839
|
+
return undefined;
|
|
840
|
+
const accType = checked.typeAt(`${path}#acc`)?.type;
|
|
841
|
+
if (accType === undefined)
|
|
842
|
+
return undefined;
|
|
843
|
+
const acc = unwrap(accType);
|
|
844
|
+
if (acc.type !== "Dict" || !isImmutableType(acc.key))
|
|
845
|
+
return undefined;
|
|
846
|
+
return {
|
|
847
|
+
kind: "reduce", path, node, accType,
|
|
848
|
+
keyType: acc.key, valueType: acc.value,
|
|
849
|
+
update: update.value.op === "+=" ? "add" : "replace",
|
|
850
|
+
};
|
|
851
|
+
}
|
|
852
|
+
// ─── Splitting ──────────────────────────────────────────────────────────────
|
|
853
|
+
/**
|
|
854
|
+
* Splits a checked query over one dataset's pieces, or says why it runs as one
|
|
855
|
+
* unit.
|
|
856
|
+
*
|
|
857
|
+
* @param checked - what `checkJq` made of the query, checked as an e3 root; it
|
|
858
|
+
* must have no error
|
|
859
|
+
* @param options - `maxOutputs` for a `many` query's limit
|
|
860
|
+
* @returns the split call's parts, or the reason it runs as one unit, with the
|
|
861
|
+
* reads that skip what they don't need either way
|
|
862
|
+
* @throws {TranslationError} When the query does not check, or was not checked
|
|
863
|
+
* as an e3 root.
|
|
864
|
+
*
|
|
865
|
+
* @remarks
|
|
866
|
+
* A query splits when its pipeline reads one root dataset, `.D`, `.D[]` or
|
|
867
|
+
* `[.D[] | f]`, and works through its rows: `map(f)`, `[.[] | f]`, `flatten`,
|
|
868
|
+
* and every step of a stream. Then:
|
|
869
|
+
* - **totals** over the rows (`length`, `add`, `min` / `max` and `_by`,
|
|
870
|
+
* `.[0]` / `first`, `.[-1]` / `last`, `any`, `all`, `unique`, each perhaps
|
|
871
|
+
* after `map(g)`, inside any expression of them) fold;
|
|
872
|
+
* - `group_by(k) | map(E)` combines by key, `E`'s totals field by field, or
|
|
873
|
+
* each group's rows collected when `E` is not made of totals;
|
|
874
|
+
* - `unique` is a set, and `unique_by(g)` a dict keeping each key's first row;
|
|
875
|
+
* - `reduce .[] as $x ({}; .[k] += v)`, or `= v`, combines by key;
|
|
876
|
+
* - anything else after the row work runs once, over the rows concatenated.
|
|
877
|
+
*
|
|
878
|
+
* Bindings before the stream, `.C as $c | …`, reach every piece: each piece
|
|
879
|
+
* reads their datasets whole. A query whose row work stops early (a stream's
|
|
880
|
+
* own outputs, `first`, `limit`) runs as one unit, which reads only as far as
|
|
881
|
+
* it needs.
|
|
882
|
+
*
|
|
883
|
+
* @example
|
|
884
|
+
* ```ts
|
|
885
|
+
* const Order = StructType({ region: StringType, total: FloatType });
|
|
886
|
+
* const root = StructType({ orders: ArrayType(Order) });
|
|
887
|
+
* const checked = checkJq(".orders | group_by(.region) | map({region: .[0].region, revenue: map(.total) | add})", root, { root: true });
|
|
888
|
+
* const split = splitJq(checked);
|
|
889
|
+
* split.kind; // "split"
|
|
890
|
+
* split.output.kind; // "dict": revenue by region, added
|
|
891
|
+
* ```
|
|
892
|
+
*/
|
|
893
|
+
export function splitJq(checked, options = {}) {
|
|
894
|
+
if (checked.program === null || checked.elementType === null || checked.multiplicity === null) {
|
|
895
|
+
const errors = checked.diagnostics.filter(d => d.severity.type === "error").map(d => d.message);
|
|
896
|
+
throw new TranslationError(`the program does not check: ${errors.join(" ")}`);
|
|
897
|
+
}
|
|
898
|
+
if (!checked.source.root)
|
|
899
|
+
throw new TranslationError("splitJq splits a query checked as an e3 root");
|
|
900
|
+
const program = checked.program;
|
|
901
|
+
const pruning = pruningOf(checked, program);
|
|
902
|
+
const whole = (r) => ({ kind: "whole", reason: r, pruning });
|
|
903
|
+
const found = findRows(checked, program);
|
|
904
|
+
if ("whole" in found)
|
|
905
|
+
return whole(found.whole);
|
|
906
|
+
const rows = found.rows;
|
|
907
|
+
if (rows.stream && rows.after.length === 0) {
|
|
908
|
+
// A stream is the query's outputs, which its caller caps: one unit stops as soon as it has them.
|
|
909
|
+
return { kind: "whole", reason: reason(checked, "stops_early", null, rows.source.over), pruning: [...pruning, { kind: "stop", name: rows.source.over, range: rangeOf(checked, rows.source.path) }] };
|
|
910
|
+
}
|
|
911
|
+
const combined = findCombine(checked, rows);
|
|
912
|
+
if (combined !== undefined && "whole" in combined)
|
|
913
|
+
return whole(combined.whole);
|
|
914
|
+
let combine;
|
|
915
|
+
let rest;
|
|
916
|
+
if (combined !== undefined) {
|
|
917
|
+
({ combine, rest } = combined);
|
|
918
|
+
// Totals with no work done to the rows, each read from the index or an end
|
|
919
|
+
// of the dataset, are cheaper as one unit, which reads only those.
|
|
920
|
+
if (combine.kind === "totals" && !rows.work && combine.totals.leaves.every(l => l.maps.length === 0 && (l.rule === "count" || l.rule === "first" || l.rule === "last"))) {
|
|
921
|
+
const ends = combine.totals.leaves.find(l => l.rule !== "count");
|
|
922
|
+
return whole(ends === undefined ? reason(checked, "no_stream", null, rows.source.over) : reason(checked, "position", ends.path, nodeName(ends.node)));
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
else {
|
|
926
|
+
const next = rows.after[0];
|
|
927
|
+
if (!rows.work) {
|
|
928
|
+
if (next === undefined)
|
|
929
|
+
return whole(reason(checked, "no_stream", null, rows.source.over));
|
|
930
|
+
return whole(whyNot(checked, next, rows.source.over));
|
|
931
|
+
}
|
|
932
|
+
// The rows are concatenated, and everything after them runs once.
|
|
933
|
+
if (unwrap(typeAfter(checked, rows.end)).type !== "Array")
|
|
934
|
+
return whole(reason(checked, "shape", rows.end));
|
|
935
|
+
combine = { kind: "concat" };
|
|
936
|
+
rest = rows.after;
|
|
937
|
+
}
|
|
938
|
+
const plan = { rows, combine, rest, rowType: rowTypeOf(checked, rows), cutType: typeAfter(checked, rows.end) };
|
|
939
|
+
return splitCall(checked, plan, options, pruning);
|
|
940
|
+
}
|
|
941
|
+
/** Why the step after a query's row work keeps it one unit, when no work is done row by row. */
|
|
942
|
+
function whyNot(checked, step, over) {
|
|
943
|
+
if (step.kind === "bind")
|
|
944
|
+
return reason(checked, "every_row", step.path, "as");
|
|
945
|
+
const node = step.node;
|
|
946
|
+
// A key seek: one segment, not the rows.
|
|
947
|
+
if (node.type === "call" && node.value.name === "has" && node.value.args.length === 1)
|
|
948
|
+
return reason(checked, "no_stream", null, over);
|
|
949
|
+
if (node.type === "slice" || (node.type === "index" && node.value.target.type === "identity"))
|
|
950
|
+
return reason(checked, "position", step.path, node.type);
|
|
951
|
+
if (node.type === "call" && POSITIONAL.has(node.value.name))
|
|
952
|
+
return reason(checked, "position", step.path, node.value.name);
|
|
953
|
+
if (node.type === "foreach")
|
|
954
|
+
return reason(checked, "state", step.path, "foreach");
|
|
955
|
+
if (node.type === "call" && EVERY_ROW.has(node.value.name))
|
|
956
|
+
return reason(checked, "every_row", step.path, node.value.name);
|
|
957
|
+
if (node.type === "reduce")
|
|
958
|
+
return reason(checked, "state", step.path, "reduce");
|
|
959
|
+
return reason(checked, "every_row", step.path, nodeName(node));
|
|
960
|
+
}
|
|
961
|
+
/** The reads a query makes that skip what they don't need: counts from the index, key seeks. */
|
|
962
|
+
function pruningOf(checked, program) {
|
|
963
|
+
const out = [];
|
|
964
|
+
const steps = pipeline(program, "");
|
|
965
|
+
const visitPipeline = (list) => {
|
|
966
|
+
list.forEach((step, i) => {
|
|
967
|
+
if (step.kind === "bind") {
|
|
968
|
+
visitPipeline(pipeline(step.node.value.source, childPath(step.path, "bind.source")));
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
const field = rootField(step.node, true);
|
|
972
|
+
const next = list[i + 1];
|
|
973
|
+
if (field !== undefined && next?.kind === "stage") {
|
|
974
|
+
if (isCall(next.node, "length", 0))
|
|
975
|
+
out.push({ kind: "count", name: field, range: spanning(checked, step.path, next.path) });
|
|
976
|
+
if (isCall(next.node, "has", 1) && unwrap(typeAfter(checked, step.path)).type === "Dict")
|
|
977
|
+
out.push({ kind: "seek", name: field, range: spanning(checked, step.path, next.path) });
|
|
978
|
+
}
|
|
979
|
+
if (step.node.type === "index") {
|
|
980
|
+
const target = rootField(step.node.value.target, true);
|
|
981
|
+
if (target !== undefined && unwrap(typeAfter(checked, childPath(step.path, "index.target"))).type === "Dict") {
|
|
982
|
+
out.push({ kind: "seek", name: target, range: rangeOf(checked, step.path) });
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
});
|
|
986
|
+
};
|
|
987
|
+
visitPipeline(steps);
|
|
988
|
+
return out;
|
|
989
|
+
}
|
|
990
|
+
/** Builtins whose arguments run on the call's own input, so `.` in them is what it is at the call. */
|
|
991
|
+
const SAME_INPUT = new Set([
|
|
992
|
+
"select", "has", "in", "contains", "inside", "startswith", "endswith", "ltrimstr", "rtrimstr", "trimstr", "test", "split",
|
|
993
|
+
"join", "error", "isempty", "first", "last", "nth", "limit", "skip", "index", "rindex", "indices", "range", "IN",
|
|
994
|
+
]);
|
|
995
|
+
/** Whether a filter gives its input, or nothing: `.`, `select(f)`, and pipes of them. */
|
|
996
|
+
function keepsInput(node) {
|
|
997
|
+
if (node.type === "identity")
|
|
998
|
+
return true;
|
|
999
|
+
if (isCall(node, "select", 1))
|
|
1000
|
+
return true;
|
|
1001
|
+
return node.type === "pipe" && keepsInput(node.value.left) && keepsInput(node.value.right);
|
|
1002
|
+
}
|
|
1003
|
+
/** The field names from the row to a node's value, when it reads a field of the row: `.a.b` where `.` is the row, `$r.a` where `$r` holds it, `$k` where `$k` holds one. */
|
|
1004
|
+
function rowField(node, cx) {
|
|
1005
|
+
if (node.type === "variable")
|
|
1006
|
+
return cx.fields.get(node.value);
|
|
1007
|
+
if (node.type !== "field" || node.value.optional)
|
|
1008
|
+
return undefined;
|
|
1009
|
+
const target = node.value.target;
|
|
1010
|
+
if (target.type === "identity")
|
|
1011
|
+
return cx.row ? [node.value.name] : undefined;
|
|
1012
|
+
if (target.type === "variable" && cx.rows.has(target.value))
|
|
1013
|
+
return [node.value.name];
|
|
1014
|
+
const inner = rowField(target, cx);
|
|
1015
|
+
return inner === undefined ? undefined : [...inner, node.value.name];
|
|
1016
|
+
}
|
|
1017
|
+
/** A context without some variables, which a pattern binds anew. */
|
|
1018
|
+
function forgetting(cx, names) {
|
|
1019
|
+
if (names.length === 0)
|
|
1020
|
+
return cx;
|
|
1021
|
+
return {
|
|
1022
|
+
...cx,
|
|
1023
|
+
rows: new Set([...cx.rows].filter(v => !names.includes(v))),
|
|
1024
|
+
fields: new Map([...cx.fields].filter(([v]) => !names.includes(v))),
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
/** The context in a binding's body: the pattern's variables bound, one holding the row or a field of it when its source does. */
|
|
1028
|
+
function binding(cx, pattern, source) {
|
|
1029
|
+
const next = forgetting(cx, patternNames(pattern));
|
|
1030
|
+
if (pattern.type !== "variable")
|
|
1031
|
+
return next;
|
|
1032
|
+
if (source.type === "identity" && cx.row)
|
|
1033
|
+
return { ...next, rows: new Set([...next.rows, pattern.value]) };
|
|
1034
|
+
const fields = rowField(source, cx);
|
|
1035
|
+
return fields === undefined || fields.length === 0 ? next : { ...next, fields: new Map([...next.fields, [pattern.value, fields]]) };
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* Whether every read of `$name` in a node is a lookup at a field of the row:
|
|
1039
|
+
* `$name[.k]`, `$name[$r.k]`, `$name[$k]`, or `$name | has($k)`. Each lookup
|
|
1040
|
+
* is added to `found`. Conservative: what it cannot follow, a def's body or
|
|
1041
|
+
* a builtin whose arguments run on other values, reads `$name` otherwise.
|
|
1042
|
+
*/
|
|
1043
|
+
function lookupsOnly(node, path, cx, name, found) {
|
|
1044
|
+
const at = (step) => childPath(path, step);
|
|
1045
|
+
const visit = (child, step, inner = cx) => lookupsOnly(child, at(step), inner, name, found);
|
|
1046
|
+
const away = { ...cx, row: false };
|
|
1047
|
+
switch (node.type) {
|
|
1048
|
+
case "variable":
|
|
1049
|
+
return node.value !== name;
|
|
1050
|
+
case "index": {
|
|
1051
|
+
const { index, target } = node.value;
|
|
1052
|
+
if (target.type === "variable" && target.value === name) {
|
|
1053
|
+
const fields = rowField(index, cx);
|
|
1054
|
+
if (fields === undefined || fields.length === 0)
|
|
1055
|
+
return false;
|
|
1056
|
+
found.push({ fields, path: at("index.index") });
|
|
1057
|
+
return true;
|
|
1058
|
+
}
|
|
1059
|
+
return visit(target, "index.target") && visit(index, "index.index");
|
|
1060
|
+
}
|
|
1061
|
+
case "pipe": {
|
|
1062
|
+
const { left, right } = node.value;
|
|
1063
|
+
if (left.type === "variable" && left.value === name) {
|
|
1064
|
+
// `$name | has($k)`, perhaps piped on (`| not`): has's key runs on `$name`, so only a variable holding a field of the row is one.
|
|
1065
|
+
let has = right;
|
|
1066
|
+
let hasPath = at("pipe.right");
|
|
1067
|
+
while (has.type === "pipe") {
|
|
1068
|
+
has = has.value.left;
|
|
1069
|
+
hasPath = childPath(hasPath, "pipe.left");
|
|
1070
|
+
}
|
|
1071
|
+
if (!isCall(has, "has", 1) || variablesIn(right).has(name))
|
|
1072
|
+
return false;
|
|
1073
|
+
const fields = rowField(has.value.args[0], away);
|
|
1074
|
+
if (fields === undefined || fields.length === 0)
|
|
1075
|
+
return false;
|
|
1076
|
+
found.push({ fields, path: childPath(hasPath, "call.args[0]") });
|
|
1077
|
+
return true;
|
|
1078
|
+
}
|
|
1079
|
+
return visit(left, "pipe.left") && visit(right, "pipe.right", { ...cx, row: cx.row && keepsInput(left) });
|
|
1080
|
+
}
|
|
1081
|
+
case "bind": {
|
|
1082
|
+
const { body, patterns, source } = node.value;
|
|
1083
|
+
if (!visit(source, "bind.source"))
|
|
1084
|
+
return false;
|
|
1085
|
+
if (patterns.length !== 1)
|
|
1086
|
+
return !variablesIn(body).has(name);
|
|
1087
|
+
// A body where the pattern binds `$name` again reads another value.
|
|
1088
|
+
if (patternNames(patterns[0]).includes(name))
|
|
1089
|
+
return true;
|
|
1090
|
+
return visit(body, "bind.body", binding(cx, patterns[0], source));
|
|
1091
|
+
}
|
|
1092
|
+
case "reduce":
|
|
1093
|
+
case "foreach": {
|
|
1094
|
+
const v = node.value;
|
|
1095
|
+
if (!visit(v.source, `${node.type}.source`) || !visit(v.init, `${node.type}.init`))
|
|
1096
|
+
return false;
|
|
1097
|
+
const names = patternNames(v.pattern);
|
|
1098
|
+
if (names.includes(name))
|
|
1099
|
+
return true;
|
|
1100
|
+
// The update, and an extract, run on the state.
|
|
1101
|
+
const state = { ...forgetting(cx, names), row: false };
|
|
1102
|
+
if (!visit(v.update, `${node.type}.update`, state))
|
|
1103
|
+
return false;
|
|
1104
|
+
return node.type === "reduce" || node.value.extract.type === "none" || visit(node.value.extract.value, "foreach.extract.some", state);
|
|
1105
|
+
}
|
|
1106
|
+
case "call": {
|
|
1107
|
+
const { args } = node.value;
|
|
1108
|
+
if (cx.defs.has(`${node.value.name}/${args.length}`))
|
|
1109
|
+
return args.every(arg => !variablesIn(arg).has(name));
|
|
1110
|
+
const same = SAME_INPUT.has(node.value.name);
|
|
1111
|
+
return args.every((arg, i) => visit(arg, `call.args[${i}]`, same ? cx : away));
|
|
1112
|
+
}
|
|
1113
|
+
case "object":
|
|
1114
|
+
return node.value.every((entry, i) => {
|
|
1115
|
+
if (entry.key.type === "variable" && entry.value.type === "none")
|
|
1116
|
+
return entry.key.value !== name;
|
|
1117
|
+
if (entry.key.type === "computed" && !visit(entry.key.value, `object[${i}].key.computed`))
|
|
1118
|
+
return false;
|
|
1119
|
+
return entry.value.type === "none" || visit(entry.value.value, `object[${i}].value.some`);
|
|
1120
|
+
});
|
|
1121
|
+
case "try":
|
|
1122
|
+
// The handler runs on the error's message.
|
|
1123
|
+
return visit(node.value.body, "try.body") && (node.value.catch.type === "none" || visit(node.value.catch.value, "try.catch.some", away));
|
|
1124
|
+
case "update":
|
|
1125
|
+
// `|=`'s value runs on each position, the others' on `.`.
|
|
1126
|
+
return visit(node.value.path, "update.path") && visit(node.value.value, "update.value", node.value.op === "|=" ? away : cx);
|
|
1127
|
+
case "def": {
|
|
1128
|
+
if (variablesIn(node.value.body).has(name))
|
|
1129
|
+
return false;
|
|
1130
|
+
return visit(node.value.rest, "def.rest", { ...cx, defs: new Set([...cx.defs, `${node.value.name}/${node.value.params.length}`]) });
|
|
1131
|
+
}
|
|
1132
|
+
default:
|
|
1133
|
+
return jqChildren(node).every(child => child.node === undefined || visit(child.node, child.step));
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
/**
|
|
1137
|
+
* Every lookup the pieces make of a dict bound before the stream to `$name`,
|
|
1138
|
+
* each at a field of the row as the dataset gives it; `undefined` when a
|
|
1139
|
+
* piece reads it otherwise — whole, or at another key — or a later binding
|
|
1140
|
+
* every piece makes reads it.
|
|
1141
|
+
*/
|
|
1142
|
+
function rowLookups(plan, name) {
|
|
1143
|
+
const found = [];
|
|
1144
|
+
const binds = plan.rows.binds;
|
|
1145
|
+
const bound = binds.findIndex(b => patternNames(b.node.value.patterns[0]).includes(name));
|
|
1146
|
+
if (binds.slice(bound + 1).some(b => variablesIn(b.node.value.source).has(name)))
|
|
1147
|
+
return undefined;
|
|
1148
|
+
let cx = { row: true, rows: new Set(), fields: new Map(), defs: new Set() };
|
|
1149
|
+
// The rows are the dataset's own until a step makes others.
|
|
1150
|
+
let intact = true;
|
|
1151
|
+
for (const step of plan.rows.steps) {
|
|
1152
|
+
const here = { ...cx, row: intact };
|
|
1153
|
+
switch (step.kind) {
|
|
1154
|
+
case "as":
|
|
1155
|
+
break;
|
|
1156
|
+
case "flatten":
|
|
1157
|
+
intact = false;
|
|
1158
|
+
break;
|
|
1159
|
+
case "map": {
|
|
1160
|
+
const arg = step.node.value.args[0];
|
|
1161
|
+
if (!lookupsOnly(arg, childPath(step.path, "call.args[0]"), here, name, found))
|
|
1162
|
+
return undefined;
|
|
1163
|
+
intact = intact && keepsInput(arg);
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
case "each":
|
|
1167
|
+
if (!lookupsOnly(step.node, step.path, here, name, found))
|
|
1168
|
+
return undefined;
|
|
1169
|
+
intact = intact && keepsInput(step.node);
|
|
1170
|
+
break;
|
|
1171
|
+
case "bind": {
|
|
1172
|
+
const { patterns, source } = step.node.value;
|
|
1173
|
+
if (!lookupsOnly(source, childPath(step.path, "bind.source"), here, name, found))
|
|
1174
|
+
return undefined;
|
|
1175
|
+
// From here on `$name` is another value.
|
|
1176
|
+
if (patternNames(patterns[0]).includes(name))
|
|
1177
|
+
return found;
|
|
1178
|
+
cx = binding(here, patterns[0], source);
|
|
1179
|
+
break;
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
const end = { ...cx, row: intact };
|
|
1184
|
+
const c = plan.combine;
|
|
1185
|
+
const visit = (node, path, inner) => lookupsOnly(node, path, inner, name, found);
|
|
1186
|
+
// A total runs on the rows together, never on one row.
|
|
1187
|
+
const totalsRead = (totals) => totals !== null && [...totals.maps.map(m => m.node), ...totals.leaves.map(l => l.node)].some(n => variablesIn(n).has(name));
|
|
1188
|
+
switch (c.kind) {
|
|
1189
|
+
case "totals":
|
|
1190
|
+
return totalsRead(c.totals) ? undefined : found;
|
|
1191
|
+
case "group":
|
|
1192
|
+
return visit(c.key.node, c.key.path, end) && !totalsRead(c.totals) ? found : undefined;
|
|
1193
|
+
case "distinct_by":
|
|
1194
|
+
return visit(c.key.node, c.key.path, end) ? found : undefined;
|
|
1195
|
+
case "top":
|
|
1196
|
+
return c.key === null || visit(c.key.node, c.key.path, end) ? found : undefined;
|
|
1197
|
+
case "reduce": {
|
|
1198
|
+
const v = c.node.value;
|
|
1199
|
+
const names = patternNames(v.pattern);
|
|
1200
|
+
if (names.includes(name))
|
|
1201
|
+
return found;
|
|
1202
|
+
let inner = forgetting(cx, names);
|
|
1203
|
+
if (intact && v.pattern.type === "variable")
|
|
1204
|
+
inner = { ...inner, rows: new Set([...inner.rows, v.pattern.value]) };
|
|
1205
|
+
const state = { ...inner, row: false };
|
|
1206
|
+
return visit(v.init, childPath(c.path, "reduce.init"), { ...cx, row: false }) && visit(v.update, childPath(c.path, "reduce.update"), state) ? found : undefined;
|
|
1207
|
+
}
|
|
1208
|
+
default:
|
|
1209
|
+
return found;
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Whether every binding before the stream that reads a dataset binds it whole
|
|
1214
|
+
* to one variable, `.B as $b`, which the pieces read only at fields of the
|
|
1215
|
+
* row: every lookup, by the field names from the row.
|
|
1216
|
+
*/
|
|
1217
|
+
function joinedAt(plan, dataset) {
|
|
1218
|
+
const binds = plan.rows.binds.filter(b => rootFieldsRead(b.node.value.source).includes(dataset));
|
|
1219
|
+
const found = [];
|
|
1220
|
+
for (const b of binds) {
|
|
1221
|
+
const pattern = b.node.value.patterns[0];
|
|
1222
|
+
if (pattern.type !== "variable" || rootField(b.node.value.source, true) !== dataset)
|
|
1223
|
+
return undefined;
|
|
1224
|
+
const lookups = rowLookups(plan, pattern.value);
|
|
1225
|
+
if (lookups === undefined)
|
|
1226
|
+
return undefined;
|
|
1227
|
+
found.push(...lookups);
|
|
1228
|
+
}
|
|
1229
|
+
return found;
|
|
1230
|
+
}
|
|
1231
|
+
/** Whether two lists of field names are the same. */
|
|
1232
|
+
function sameFields(a, b) {
|
|
1233
|
+
return a.length === b.length && a.every((name, i) => name === b[i]);
|
|
1234
|
+
}
|
|
1235
|
+
/** The type of a field a row reads through struct fields alone, or `undefined`. */
|
|
1236
|
+
function fieldType(row, fields) {
|
|
1237
|
+
let t = row;
|
|
1238
|
+
for (const name of fields) {
|
|
1239
|
+
const s = unwrap(t);
|
|
1240
|
+
if (s.type !== "Struct" || !(name in s.fields))
|
|
1241
|
+
return undefined;
|
|
1242
|
+
t = s.fields[name];
|
|
1243
|
+
}
|
|
1244
|
+
return t;
|
|
1245
|
+
}
|
|
1246
|
+
/**
|
|
1247
|
+
* The datasets a split call partitions with the one it is cut from (#942):
|
|
1248
|
+
* dicts keyed as it is, bound before the stream and read by the pieces only at
|
|
1249
|
+
* the row's key — the rows being the dataset's entries, `{key, value}`
|
|
1250
|
+
* (`.A | to_entries | map(… $b[.key] …)`). Cut at the same keys, a piece of
|
|
1251
|
+
* each holds the same keys, so a row's lookup finds in its piece what it
|
|
1252
|
+
* would find in the whole.
|
|
1253
|
+
*/
|
|
1254
|
+
function copartitionedOf(plan, inputs) {
|
|
1255
|
+
const source = plan.rows.source;
|
|
1256
|
+
if (source.entries === undefined)
|
|
1257
|
+
return [];
|
|
1258
|
+
const A = unwrap(inputs.find(i => i.name === source.over).type);
|
|
1259
|
+
if (A.type !== "Dict")
|
|
1260
|
+
return [];
|
|
1261
|
+
const out = [];
|
|
1262
|
+
for (const dataset of new Set(plan.rows.binds.flatMap(b => rootFieldsRead(b.node.value.source)))) {
|
|
1263
|
+
const B = inputs.find(i => i.name === dataset);
|
|
1264
|
+
if (dataset === source.over || B === undefined)
|
|
1265
|
+
continue;
|
|
1266
|
+
const t = unwrap(B.type);
|
|
1267
|
+
if (t.type !== "Dict" || !isTypeEqual(t.key, A.key))
|
|
1268
|
+
continue;
|
|
1269
|
+
const lookups = joinedAt(plan, dataset);
|
|
1270
|
+
if (lookups === undefined || lookups.length === 0 || !lookups.every(l => sameFields(l.fields, ["key"])))
|
|
1271
|
+
continue;
|
|
1272
|
+
out.push(dataset);
|
|
1273
|
+
}
|
|
1274
|
+
return out;
|
|
1275
|
+
}
|
|
1276
|
+
/** The field names a node reads from `.`, `.a.b`, or `undefined` for another node. */
|
|
1277
|
+
function fieldsRead(node) {
|
|
1278
|
+
if (node.type !== "field" || node.value.optional)
|
|
1279
|
+
return undefined;
|
|
1280
|
+
if (node.value.target.type === "identity")
|
|
1281
|
+
return [node.value.name];
|
|
1282
|
+
const inner = fieldsRead(node.value.target);
|
|
1283
|
+
return inner === undefined ? undefined : [...inner, node.value.name];
|
|
1284
|
+
}
|
|
1285
|
+
/** The node at a path inside a node at another. */
|
|
1286
|
+
function nodeAt(node, path, wanted) {
|
|
1287
|
+
if (path === wanted)
|
|
1288
|
+
return node;
|
|
1289
|
+
for (const child of jqChildren(node)) {
|
|
1290
|
+
if (child.node === undefined)
|
|
1291
|
+
continue;
|
|
1292
|
+
const at = childPath(path, child.step);
|
|
1293
|
+
if (wanted === at || wanted.startsWith(`${at}.`))
|
|
1294
|
+
return nodeAt(child.node, at, wanted);
|
|
1295
|
+
}
|
|
1296
|
+
return undefined;
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Whether a group's total is order-blind: it counts, adds numbers, takes the
|
|
1300
|
+
* least or the greatest value, tests, or unites — or it is the group's first
|
|
1301
|
+
* or last row read at the group's own key (`.[0].region`, grouped by
|
|
1302
|
+
* `.region`), which every row of the group shares.
|
|
1303
|
+
*/
|
|
1304
|
+
function leafOrderBlind(leaf, group) {
|
|
1305
|
+
switch (leaf.rule) {
|
|
1306
|
+
case "count":
|
|
1307
|
+
case "any":
|
|
1308
|
+
case "all":
|
|
1309
|
+
case "union":
|
|
1310
|
+
return true;
|
|
1311
|
+
case "min":
|
|
1312
|
+
case "max":
|
|
1313
|
+
// `min_by` and `max_by` keep the first least and the last greatest: on a tie, the order decides.
|
|
1314
|
+
return !(leaf.node.type === "call" && (leaf.node.value.name === "min_by" || leaf.node.value.name === "max_by"));
|
|
1315
|
+
case "add": {
|
|
1316
|
+
const t = unwrap(nullablePayload(unwrap(leaf.type)) ?? leaf.type);
|
|
1317
|
+
return t.type === "Integer" || t.type === "Float" || t.type === "Null";
|
|
1318
|
+
}
|
|
1319
|
+
case "first":
|
|
1320
|
+
case "last": {
|
|
1321
|
+
if (group === undefined || group.key === undefined || leaf.maps.length > 0)
|
|
1322
|
+
return false;
|
|
1323
|
+
// The field reads around the total, from it outward.
|
|
1324
|
+
const read = [];
|
|
1325
|
+
let path = leaf.path;
|
|
1326
|
+
while (read.length < group.key.length && path.endsWith(".field.target")) {
|
|
1327
|
+
path = path.slice(0, -".field.target".length);
|
|
1328
|
+
const node = nodeAt(group.body.node, group.body.path, path);
|
|
1329
|
+
if (node?.type !== "field" || node.value.optional)
|
|
1330
|
+
return false;
|
|
1331
|
+
read.push(node.value.name);
|
|
1332
|
+
}
|
|
1333
|
+
return sameFields(read, group.key);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
/**
|
|
1338
|
+
* Whether a combine gives the same answer whatever order its rows come in, as
|
|
1339
|
+
* re-keyed rows come, grouped by their key (#942): totals that are all
|
|
1340
|
+
* order-blind, a grouping by them, `unique`, and a `reduce` that adds numbers.
|
|
1341
|
+
* A Float sum is added in another grouping, as any split's is.
|
|
1342
|
+
*/
|
|
1343
|
+
function orderBlind(plan) {
|
|
1344
|
+
const c = plan.combine;
|
|
1345
|
+
switch (c.kind) {
|
|
1346
|
+
case "totals":
|
|
1347
|
+
return c.totals.leaves.every(leaf => leafOrderBlind(leaf, undefined));
|
|
1348
|
+
case "group":
|
|
1349
|
+
return c.totals !== null && c.totals.leaves.every(leaf => leafOrderBlind(leaf, { body: c.body, key: fieldsRead(c.key.node) }));
|
|
1350
|
+
case "distinct":
|
|
1351
|
+
return true;
|
|
1352
|
+
case "reduce": {
|
|
1353
|
+
const t = unwrap(c.valueType);
|
|
1354
|
+
return c.update === "add" && (t.type === "Integer" || t.type === "Float");
|
|
1355
|
+
}
|
|
1356
|
+
default:
|
|
1357
|
+
return false;
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
// ─── Building the call ──────────────────────────────────────────────────────
|
|
1361
|
+
/** The input names and types of a split's programs: the root fields the query reads, as `translateJq` takes them. */
|
|
1362
|
+
function inputsOf(checked) {
|
|
1363
|
+
const fields = parts(unwrap(checked.inputType)).fields;
|
|
1364
|
+
return checked.reads.map(name => ({ name, type: fields[name] }));
|
|
1365
|
+
}
|
|
1366
|
+
/** The parts of the split call for a plan. */
|
|
1367
|
+
function splitCall(checked, plan, options, pruning) {
|
|
1368
|
+
const inputs = inputsOf(checked);
|
|
1369
|
+
if (!inputs.some(i => i.name === plan.rows.source.over))
|
|
1370
|
+
throw new TranslationError(`.${plan.rows.source.over} is not one of the query's inputs`);
|
|
1371
|
+
const element = checked.elementType;
|
|
1372
|
+
const multiplicity = checked.multiplicity;
|
|
1373
|
+
const resultType = multiplicity === "one" ? element : multiplicity === "maybe" ? OptionType(element) : ArrayType(element);
|
|
1374
|
+
const output = outputOf(checked, plan);
|
|
1375
|
+
const rangeEnd = plan.rest.length === 0 ? null : spanning(checked, plan.rest[0].path, plan.rest[plan.rest.length - 1].path);
|
|
1376
|
+
// The assembled rows are the result when nothing follows them and they are its type.
|
|
1377
|
+
const thenNeeded = !(plan.combine.kind === "concat" && plan.rest.length === 0 && multiplicity === "one" && isTypeEqual(output.type, resultType));
|
|
1378
|
+
const copartitioned = copartitionedOf(plan, inputs);
|
|
1379
|
+
return {
|
|
1380
|
+
kind: "split",
|
|
1381
|
+
over: plan.rows.source.over,
|
|
1382
|
+
inputs,
|
|
1383
|
+
resultType,
|
|
1384
|
+
output,
|
|
1385
|
+
piece: () => pieceProgram(checked, plan, inputs, output, options),
|
|
1386
|
+
then: () => thenNeeded ? thenProgram(checked, plan, inputs, output, resultType, options) : null,
|
|
1387
|
+
stages: {
|
|
1388
|
+
piece: spanning(checked, plan.rows.start, plan.rows.end),
|
|
1389
|
+
combine: combineOf(checked, plan),
|
|
1390
|
+
then: rangeEnd,
|
|
1391
|
+
},
|
|
1392
|
+
broadcast: [...new Set(plan.rows.binds.flatMap(bind => rootFieldsRead(bind.node.value.source)))].filter(name => !copartitioned.includes(name)),
|
|
1393
|
+
copartitioned,
|
|
1394
|
+
rekey: copartitioned.length > 0 ? null : rekeyOf(checked, plan, inputs, output, resultType, thenNeeded, options),
|
|
1395
|
+
pruning,
|
|
1396
|
+
};
|
|
1397
|
+
}
|
|
1398
|
+
/**
|
|
1399
|
+
* The re-keyed join a split can make instead of reading a dict whole in every
|
|
1400
|
+
* piece (#942): the one dict, bound before the stream, that the pieces read
|
|
1401
|
+
* only at one field of the row as the dataset gives it — of the dict's key
|
|
1402
|
+
* type — when the combine is order-blind; `null` when there is none, or more
|
|
1403
|
+
* than one.
|
|
1404
|
+
*/
|
|
1405
|
+
function rekeyOf(checked, plan, inputs, output, resultType, thenNeeded, options) {
|
|
1406
|
+
const source = plan.rows.source;
|
|
1407
|
+
if (source.entries !== undefined || !orderBlind(plan))
|
|
1408
|
+
return null;
|
|
1409
|
+
const A = inputs.find(i => i.name === source.over);
|
|
1410
|
+
const elements = elementsOf(A.type);
|
|
1411
|
+
if (elements === undefined)
|
|
1412
|
+
return null;
|
|
1413
|
+
const R = elements.element;
|
|
1414
|
+
const joins = [];
|
|
1415
|
+
for (const dataset of new Set(plan.rows.binds.flatMap(b => rootFieldsRead(b.node.value.source)))) {
|
|
1416
|
+
const B = inputs.find(i => i.name === dataset);
|
|
1417
|
+
if (dataset === source.over || B === undefined)
|
|
1418
|
+
continue;
|
|
1419
|
+
const t = unwrap(B.type);
|
|
1420
|
+
if (t.type !== "Dict")
|
|
1421
|
+
continue;
|
|
1422
|
+
const lookups = joinedAt(plan, dataset);
|
|
1423
|
+
if (lookups === undefined || lookups.length === 0)
|
|
1424
|
+
continue;
|
|
1425
|
+
const fields = lookups[0].fields;
|
|
1426
|
+
const key = fieldType(R, fields);
|
|
1427
|
+
if (!lookups.every(l => sameFields(l.fields, fields)) || key === undefined || !isTypeEqual(key, t.key))
|
|
1428
|
+
continue;
|
|
1429
|
+
joins.push({ name: dataset, fields, key, range: rangeOf(checked, lookups[0].path) });
|
|
1430
|
+
}
|
|
1431
|
+
if (joins.length !== 1)
|
|
1432
|
+
return null;
|
|
1433
|
+
const join = joins[0];
|
|
1434
|
+
const Rows = ArrayType(R);
|
|
1435
|
+
const Keyed = DictType(join.key, Rows);
|
|
1436
|
+
const joinInputs = inputs.map(i => i.name === source.over ? { name: i.name, type: Keyed } : i);
|
|
1437
|
+
// Each key's rows, in input order: a re-keyed piece's rows.
|
|
1438
|
+
const rekeyed = (t, $, piece, each) => t.forEach($, piece, ($2, rows) => t.forEach($2, rows, ($3, item) => each($3, item), source.path, "row"), source.path, "rows");
|
|
1439
|
+
return {
|
|
1440
|
+
name: join.name,
|
|
1441
|
+
key: join.range,
|
|
1442
|
+
keyType: join.key,
|
|
1443
|
+
output: { kind: "dict", type: Keyed, merge: () => mergeFunction(checked, join.key, Rows, (t, _$, earlier, later) => t.b("ArrayConcat", [R], [earlier, later], Rows, source.path)) },
|
|
1444
|
+
piece: () => rekeyPiece(checked, A.type, R, join.fields, join.key, source, options),
|
|
1445
|
+
inputs: joinInputs,
|
|
1446
|
+
joinPiece: () => pieceProgram(checked, plan, joinInputs, output, options, rekeyed),
|
|
1447
|
+
joinThen: () => thenNeeded ? thenProgram(checked, plan, joinInputs, output, resultType, options) : null,
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
/**
|
|
1451
|
+
* The re-key call's piece program (#942): each row of the piece sent on under
|
|
1452
|
+
* its join key, a key's rows gathered first in the piece's table, appended in
|
|
1453
|
+
* place, since the table alone holds them until it sends them on.
|
|
1454
|
+
*/
|
|
1455
|
+
function rekeyPiece(checked, A, R, fields, K, source, options) {
|
|
1456
|
+
const Rows = ArrayType(R);
|
|
1457
|
+
return withLocationCapture(false, () => func([A, FunctionType([K, Rows], NullType)], NullType, ($, piece, emit) => {
|
|
1458
|
+
const t = new SplitTranslator(checked, {});
|
|
1459
|
+
const send = ($2, ...args) => t.stmt($2, t.callFn(emit, args, source.path));
|
|
1460
|
+
const append = (t2, $2, earlier, later) => {
|
|
1461
|
+
t2.stmt($2, t2.b("ArrayAppend", [R], [earlier, later], NullType, source.path));
|
|
1462
|
+
return earlier;
|
|
1463
|
+
};
|
|
1464
|
+
const table = pieceTable(t, $, K, { type: Rows, fold: append }, source.path, options, send);
|
|
1465
|
+
t.forEach($, piece, ($2, item) => {
|
|
1466
|
+
const row = t.bind($2, t.widenTo($2, item, R, source.path), "row");
|
|
1467
|
+
let key = row;
|
|
1468
|
+
for (const name of fields)
|
|
1469
|
+
key = t.field(key, name);
|
|
1470
|
+
const one = t.declare($2, t.emptyArray(R), "rows");
|
|
1471
|
+
t.push($2, one, row, source.path);
|
|
1472
|
+
table.add($2, key, one);
|
|
1473
|
+
}, source.path);
|
|
1474
|
+
table.flush($);
|
|
1475
|
+
return t.null();
|
|
1476
|
+
}));
|
|
1477
|
+
}
|
|
1478
|
+
/** The root fields a node reads with the root as its input. */
|
|
1479
|
+
function rootFieldsRead(node, out = []) {
|
|
1480
|
+
const field = rootField(node, true);
|
|
1481
|
+
if (field !== undefined) {
|
|
1482
|
+
out.push(field);
|
|
1483
|
+
return out;
|
|
1484
|
+
}
|
|
1485
|
+
if (node.type === "pipe")
|
|
1486
|
+
return rootFieldsRead(node.value.left, out);
|
|
1487
|
+
for (const child of jqChildren(node))
|
|
1488
|
+
if (child.node !== undefined && child.step !== "pipe.right")
|
|
1489
|
+
rootFieldsRead(child.node, out);
|
|
1490
|
+
return out;
|
|
1491
|
+
}
|
|
1492
|
+
/** What the explanation says of a plan's combine. */
|
|
1493
|
+
function combineOf(checked, plan) {
|
|
1494
|
+
const c = plan.combine;
|
|
1495
|
+
const totals = (t) => t.leaves.map(l => ({ range: l.range, rule: l.rule }));
|
|
1496
|
+
switch (c.kind) {
|
|
1497
|
+
case "concat": return { kind: "concat" };
|
|
1498
|
+
case "totals": return { kind: "totals", range: rangeOf(checked, c.path), totals: totals(c.totals) };
|
|
1499
|
+
case "group": return { kind: "group", range: spanning(checked, c.path, c.mapPath), key: rangeOf(checked, c.key.path), totals: c.totals === null ? null : totals(c.totals) };
|
|
1500
|
+
case "distinct": return { kind: "distinct", range: rangeOf(checked, c.path) };
|
|
1501
|
+
case "distinct_by": return { kind: "distinct_by", range: rangeOf(checked, c.path), key: rangeOf(checked, c.key.path) };
|
|
1502
|
+
case "reduce": return { kind: "reduce", range: rangeOf(checked, c.path), key: rangeOf(checked, childPath(c.path, "reduce.update.update.path.index.index")), update: c.update };
|
|
1503
|
+
case "top": return { kind: "top", range: spanning(checked, c.path, c.keepPath), key: c.key === null ? null : rangeOf(checked, c.key.path), rows: c.rows };
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
/** The struct of a set of totals' partials: a field each, `p0`, `p1`, … */
|
|
1507
|
+
function partialsType(leaves) {
|
|
1508
|
+
return StructType(Object.fromEntries(leaves.map((leaf, i) => [`p${i}`, leaf.type])));
|
|
1509
|
+
}
|
|
1510
|
+
/** A group's partial: its totals', or its rows collected. */
|
|
1511
|
+
function groupValueType(plan, c) {
|
|
1512
|
+
return c.totals === null ? ArrayType(plan.rowType) : partialsType(c.totals.leaves);
|
|
1513
|
+
}
|
|
1514
|
+
/** The split call's output kind for a plan. */
|
|
1515
|
+
function outputOf(checked, plan) {
|
|
1516
|
+
const c = plan.combine;
|
|
1517
|
+
const R = plan.rowType;
|
|
1518
|
+
switch (c.kind) {
|
|
1519
|
+
case "concat":
|
|
1520
|
+
return { kind: "array", type: ArrayType(R) };
|
|
1521
|
+
case "distinct":
|
|
1522
|
+
return { kind: "set", type: SetType(R) };
|
|
1523
|
+
case "distinct_by": {
|
|
1524
|
+
const K = c.key.type;
|
|
1525
|
+
return { kind: "dict", type: DictType(K, R), merge: () => mergeFunction(checked, K, R, foldOf(plan, c, R)) };
|
|
1526
|
+
}
|
|
1527
|
+
case "group": {
|
|
1528
|
+
const K = c.key.type;
|
|
1529
|
+
const V = groupValueType(plan, c);
|
|
1530
|
+
return { kind: "dict", type: DictType(K, V), merge: () => mergeFunction(checked, K, V, foldOf(plan, c, V)) };
|
|
1531
|
+
}
|
|
1532
|
+
case "totals": {
|
|
1533
|
+
const leaves = c.totals.leaves;
|
|
1534
|
+
const T = partialsType(leaves);
|
|
1535
|
+
const zero = Object.fromEntries(leaves.map((leaf, i) => [`p${i}`, leaf.zero]));
|
|
1536
|
+
return { kind: "fold", type: T, zero, combine: () => combineFunction(checked, T, (t, $, a, b) => combineStruct(t, $, leaves, a, b, T)) };
|
|
1537
|
+
}
|
|
1538
|
+
case "reduce": {
|
|
1539
|
+
const K = c.keyType;
|
|
1540
|
+
const V = c.valueType;
|
|
1541
|
+
return { kind: "dict", type: DictType(K, V), merge: () => mergeFunction(checked, K, V, foldOf(plan, c, V)) };
|
|
1542
|
+
}
|
|
1543
|
+
case "top": {
|
|
1544
|
+
// Each piece's first rows with their keys, in order; two pieces' merged, the earlier's first on a tie.
|
|
1545
|
+
const T = ArrayType(topPair(c));
|
|
1546
|
+
return { kind: "fold", type: T, zero: [], combine: () => combineFunction(checked, T, (t, $, a, b) => firstSorted(t, $, t.b("ArrayConcat", [topPair(c)], [a, b], T, c.path), c)) };
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
/** A row the first rows of a sort keep, with its key: the pairs `sort_by` sorts. */
|
|
1551
|
+
function topPair(c) {
|
|
1552
|
+
return StructType({ key: c.key?.type ?? c.element, value: c.element });
|
|
1553
|
+
}
|
|
1554
|
+
/**
|
|
1555
|
+
* Pairs sorted by their keys, stably, as `sort_by` sorts them, and the first
|
|
1556
|
+
* of them a top keeps.
|
|
1557
|
+
*
|
|
1558
|
+
* @param pairs - an array of {@link topPair}s
|
|
1559
|
+
* @param c - the top
|
|
1560
|
+
* @returns the first `c.rows` pairs in order
|
|
1561
|
+
*/
|
|
1562
|
+
function firstSorted(t, $, pairs, c) {
|
|
1563
|
+
const Pair = topPair(c);
|
|
1564
|
+
const K = c.key?.type ?? c.element;
|
|
1565
|
+
const A = ArrayType(Pair);
|
|
1566
|
+
const byKey = t.lambda([Pair], K, ["pair"], (_$f, pair) => t.field(pair, "key"), c.path);
|
|
1567
|
+
const sorted = t.bind($, t.b("ArraySort", [Pair, K], [pairs, byKey], A, c.path), "sorted");
|
|
1568
|
+
const size = t.bind($, t.size(sorted, c.path), "size");
|
|
1569
|
+
const end = t.ifValue(t.lt(size, t.int(c.rows), c.path), () => size, () => t.int(c.rows), IntegerType, c.path);
|
|
1570
|
+
return t.b("ArraySlice", [Pair], [sorted, t.int(0), end], A, c.path);
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1573
|
+
* How a dict output's values of one key fold, the earlier first: the merge e3
|
|
1574
|
+
* runs, and a piece's table. `unique_by` keeps the earlier row, a grouping
|
|
1575
|
+
* combines its totals field by field or concatenates its rows, and a `reduce`
|
|
1576
|
+
* adds as `+=` adds or keeps the later value.
|
|
1577
|
+
*/
|
|
1578
|
+
function foldOf(plan, c, V) {
|
|
1579
|
+
switch (c.kind) {
|
|
1580
|
+
case "distinct_by":
|
|
1581
|
+
return (_t, _$, earlier) => earlier;
|
|
1582
|
+
case "group": {
|
|
1583
|
+
const totals = c.totals;
|
|
1584
|
+
return (t, $, earlier, later) => totals === null
|
|
1585
|
+
? t.b("ArrayConcat", [plan.rowType], [earlier, later], V, c.path)
|
|
1586
|
+
: combineStruct(t, $, totals.leaves, earlier, later, V);
|
|
1587
|
+
}
|
|
1588
|
+
case "reduce": {
|
|
1589
|
+
const path = childPath(c.path, "reduce.update");
|
|
1590
|
+
return (t, $, earlier, later) => c.update === "replace" ? later : t.widenTo($, t.arith($, "+", earlier, later, path), V, path);
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
/** Two structs of partials combined field by field, each by its total's rule. */
|
|
1595
|
+
function combineStruct(t, $, leaves, a, b, type) {
|
|
1596
|
+
const fields = {};
|
|
1597
|
+
leaves.forEach((leaf, i) => {
|
|
1598
|
+
fields[`p${i}`] = t.bind($, leaf.combine(t, $, t.field(a, `p${i}`), t.field(b, `p${i}`)), `p${i}`);
|
|
1599
|
+
});
|
|
1600
|
+
return t.struct(type, fields);
|
|
1601
|
+
}
|
|
1602
|
+
/** A dict output's merge: a function of the key and two values. */
|
|
1603
|
+
function mergeFunction(checked, K, V, body) {
|
|
1604
|
+
return withLocationCapture(false, () => func([K, V, V], V, ($, _key, a, b) => {
|
|
1605
|
+
const t = new SplitTranslator(checked, {});
|
|
1606
|
+
return t.as(body(t, $, a, b), V);
|
|
1607
|
+
}));
|
|
1608
|
+
}
|
|
1609
|
+
/** A fold output's combine: a function of two values. */
|
|
1610
|
+
function combineFunction(checked, T, body) {
|
|
1611
|
+
return withLocationCapture(false, () => func([T, T], T, ($, a, b) => {
|
|
1612
|
+
const t = new SplitTranslator(checked, {});
|
|
1613
|
+
return t.as(body(t, $, a, b), T);
|
|
1614
|
+
}));
|
|
1615
|
+
}
|
|
1616
|
+
// ─── The piece program ──────────────────────────────────────────────────────
|
|
1617
|
+
/** The variables a plan's bindings must give a part of the query that reads some of them. */
|
|
1618
|
+
function bindsFor(binds, reads) {
|
|
1619
|
+
// A binding is kept when what reads its names, or a later binding kept, needs it.
|
|
1620
|
+
const kept = [];
|
|
1621
|
+
const needed = new Set(reads);
|
|
1622
|
+
for (let i = binds.length - 1; i >= 0; i--) {
|
|
1623
|
+
const bind = binds[i];
|
|
1624
|
+
if (!patternNames(bind.node.value.patterns[0]).some(name => needed.has(name)))
|
|
1625
|
+
continue;
|
|
1626
|
+
kept.unshift(bind);
|
|
1627
|
+
for (const name of variablesIn(bind.node.value.source))
|
|
1628
|
+
needed.add(name);
|
|
1629
|
+
}
|
|
1630
|
+
return kept;
|
|
1631
|
+
}
|
|
1632
|
+
/** Generates bindings from the root, then the rest with them in scope. */
|
|
1633
|
+
function genBinds(t, $, root, binds, env, then) {
|
|
1634
|
+
const step = ($2, i, scope) => {
|
|
1635
|
+
if (i === binds.length) {
|
|
1636
|
+
then($2, scope);
|
|
1637
|
+
return;
|
|
1638
|
+
}
|
|
1639
|
+
const bind = binds[i];
|
|
1640
|
+
const sourcePath = childPath(bind.path, "bind.source");
|
|
1641
|
+
const bodyPath = childPath(bind.path, "bind.body");
|
|
1642
|
+
t.gen(bind.node.value.source, sourcePath, $2, root, scope, ($3, value) => {
|
|
1643
|
+
t.destructure($3, bind.node.value.patterns[0], childPath(bind.path, "bind.patterns[0]"), value, scope, bodyPath, ($4, vars) => step($4, i + 1, { ...scope, vars }));
|
|
1644
|
+
});
|
|
1645
|
+
};
|
|
1646
|
+
step($, 0, env);
|
|
1647
|
+
}
|
|
1648
|
+
/** Runs the row work on one row: each row it gives, with the bindings in scope then. */
|
|
1649
|
+
function genRow(t, $, row, steps, i, env, done) {
|
|
1650
|
+
if (t.ended($))
|
|
1651
|
+
return;
|
|
1652
|
+
if (i === steps.length) {
|
|
1653
|
+
done($, row, env);
|
|
1654
|
+
return;
|
|
1655
|
+
}
|
|
1656
|
+
const step = steps[i];
|
|
1657
|
+
const next = ($2, value, scope = env) => genRow(t, $2, value, steps, i + 1, scope, done);
|
|
1658
|
+
switch (step.kind) {
|
|
1659
|
+
case "map": {
|
|
1660
|
+
const element = t.typeAt(step.path, env).value;
|
|
1661
|
+
t.gen(step.node.value.args[0], childPath(step.path, "call.args[0]"), $, row, env, ($2, v) => next($2, t.bind($2, t.widenTo($2, v, element, step.path), "row")));
|
|
1662
|
+
return;
|
|
1663
|
+
}
|
|
1664
|
+
case "flatten": {
|
|
1665
|
+
// The row flattened as the one row of an array is: `flatten` is the same on every row.
|
|
1666
|
+
const one = t.declare($, t.emptyArray(t.type(row)), "row");
|
|
1667
|
+
t.push($, one, row, step.path);
|
|
1668
|
+
t.genBuiltin("flatten", [], [], step.path, $, one, env, ($2, flat) => t.forEach($2, flat, ($3, item) => next($3, item), step.path));
|
|
1669
|
+
return;
|
|
1670
|
+
}
|
|
1671
|
+
case "each":
|
|
1672
|
+
t.gen(step.node, step.path, $, row, env, ($2, v) => next($2, v));
|
|
1673
|
+
return;
|
|
1674
|
+
case "bind": {
|
|
1675
|
+
const source = childPath(step.path, "bind.source");
|
|
1676
|
+
const body = childPath(step.path, "bind.body");
|
|
1677
|
+
t.gen(step.node.value.source, source, $, row, env, ($2, value) => {
|
|
1678
|
+
t.destructure($2, step.node.value.patterns[0], childPath(step.path, "bind.patterns[0]"), value, env, body, ($3, vars) => next($3, row, { ...env, vars }));
|
|
1679
|
+
});
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1682
|
+
case "as": {
|
|
1683
|
+
const type = step.element ? t.typeAt(step.path, env).value : t.typeAt(step.path, env);
|
|
1684
|
+
next($, t.bind($, t.widenTo($, row, type, step.path), "row"));
|
|
1685
|
+
return;
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
/** The nodes the piece program generates: the row work, and what the combine computes on each row. */
|
|
1690
|
+
function pieceNodes(plan) {
|
|
1691
|
+
const nodes = [];
|
|
1692
|
+
for (const step of plan.rows.steps)
|
|
1693
|
+
if ("node" in step)
|
|
1694
|
+
nodes.push(step.node);
|
|
1695
|
+
const c = plan.combine;
|
|
1696
|
+
const leafNodes = (totals) => totals === null ? [] : [...totals.maps.map(m => m.node), ...totals.leaves.map(l => l.node)];
|
|
1697
|
+
switch (c.kind) {
|
|
1698
|
+
case "totals":
|
|
1699
|
+
nodes.push(...leafNodes(c.totals));
|
|
1700
|
+
break;
|
|
1701
|
+
case "group":
|
|
1702
|
+
nodes.push(c.key.node, ...leafNodes(c.totals));
|
|
1703
|
+
break;
|
|
1704
|
+
case "distinct_by":
|
|
1705
|
+
nodes.push(c.key.node);
|
|
1706
|
+
break;
|
|
1707
|
+
case "reduce":
|
|
1708
|
+
nodes.push(c.node);
|
|
1709
|
+
break;
|
|
1710
|
+
case "top":
|
|
1711
|
+
if (c.key !== null)
|
|
1712
|
+
nodes.push(c.key.node);
|
|
1713
|
+
break;
|
|
1714
|
+
default: break;
|
|
1715
|
+
}
|
|
1716
|
+
return nodes;
|
|
1717
|
+
}
|
|
1718
|
+
/** A piece's rows as its dataset gives them: its elements or a dict's values, or a dict's entries, `{key, value}`. */
|
|
1719
|
+
function datasetRows(checked, source) {
|
|
1720
|
+
const entries = source.entries;
|
|
1721
|
+
if (entries === undefined)
|
|
1722
|
+
return (t, $, piece, each) => t.forEach($, piece, ($2, item) => each($2, item), source.path);
|
|
1723
|
+
const Entry = parts(typeAfter(checked, entries)).value;
|
|
1724
|
+
const fields = parts(Entry).fields;
|
|
1725
|
+
return (t, $, piece, each) => t.forEach($, piece, ($2, value, key) => each($2, t.bind($2, t.struct(Entry, {
|
|
1726
|
+
key: t.widenTo($2, key, fields["key"], entries),
|
|
1727
|
+
value: t.widenTo($2, value, fields["value"], entries),
|
|
1728
|
+
}), "entry")), entries);
|
|
1729
|
+
}
|
|
1730
|
+
/**
|
|
1731
|
+
* A piece's first rows of a sort (#942): each row with its sort key, the
|
|
1732
|
+
* pairs sorted and cut back to the first `rows` once they are twice that, so
|
|
1733
|
+
* the piece holds a bounded number of them, and at the end.
|
|
1734
|
+
*/
|
|
1735
|
+
function topRows(t, $, c) {
|
|
1736
|
+
const Pair = topPair(c);
|
|
1737
|
+
const kept = t.declare($, t.emptyArray(Pair), "top");
|
|
1738
|
+
const most = 2n * BigInt(c.rows) + 64n;
|
|
1739
|
+
return {
|
|
1740
|
+
add: ($2, row, env) => {
|
|
1741
|
+
const value = t.bind($2, t.widenTo($2, row, c.element, c.path), "row");
|
|
1742
|
+
t.push($2, kept, t.struct(Pair, { key: sortKey(t, $2, c, value, env), value }), c.path);
|
|
1743
|
+
t.ifElse($2, t.b("GreaterEqual", [IntegerType], [t.size(kept, c.path), t.int(most)], BooleanType, c.path), $3 => t.assign($3, kept, firstSorted(t, $3, kept, c)), undefined, c.path);
|
|
1744
|
+
},
|
|
1745
|
+
result: $2 => firstSorted(t, $2, kept, c),
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
/** A row's sort key, as `sort_by` keys it: the key filter's one output, or the array of its outputs; for `sort`, the row. */
|
|
1749
|
+
function sortKey(t, $, c, row, env) {
|
|
1750
|
+
const key = c.key;
|
|
1751
|
+
if (key === null)
|
|
1752
|
+
return row;
|
|
1753
|
+
if (key.one)
|
|
1754
|
+
return t.one(key.node, key.path, $, row, env, key.type);
|
|
1755
|
+
const keys = t.declare($, t.emptyArray(parts(key.type).value), "keys");
|
|
1756
|
+
t.gen(key.node, key.path, $, row, env, ($2, k) => t.push($2, keys, k, c.path));
|
|
1757
|
+
return keys;
|
|
1758
|
+
}
|
|
1759
|
+
/** Generates the elements a total works through, from one row: the row, or the outputs of its maps. */
|
|
1760
|
+
function genElements(t, $, row, maps, env, each) {
|
|
1761
|
+
const step = ($2, value, i) => {
|
|
1762
|
+
if (i === maps.length) {
|
|
1763
|
+
each($2, value);
|
|
1764
|
+
return;
|
|
1765
|
+
}
|
|
1766
|
+
const map = maps[i];
|
|
1767
|
+
const element = t.typeAt(map.path, env).value;
|
|
1768
|
+
t.gen(map.node.value.args[0], childPath(map.path, "call.args[0]"), $2, value, env, ($3, v) => step($3, t.bind($3, t.widenTo($3, v, element, map.path), "element"), i + 1));
|
|
1769
|
+
};
|
|
1770
|
+
step($, row, 0);
|
|
1771
|
+
}
|
|
1772
|
+
/** Adds one row to totals' partials, each held in a variable. */
|
|
1773
|
+
function addRow(t, $, totals, accs, row, env) {
|
|
1774
|
+
totals.leaves.forEach((leaf, i) => genElements(t, $, row, leaf.maps, env, ($2, element) => leaf.add(t, $2, accs[i], element, env)));
|
|
1775
|
+
}
|
|
1776
|
+
/** The partials of totals over one row, as their struct. */
|
|
1777
|
+
function rowPartials(t, $, totals, row, env, type) {
|
|
1778
|
+
const accs = totals.leaves.map((leaf, i) => t.declare($, t.value(leaf.zero, leaf.type), `p${i}`));
|
|
1779
|
+
addRow(t, $, totals, accs, row, env);
|
|
1780
|
+
return t.struct(type, Object.fromEntries(accs.map((acc, i) => [`p${i}`, acc])));
|
|
1781
|
+
}
|
|
1782
|
+
/**
|
|
1783
|
+
* The program each piece runs: the bindings, the row work over the piece, and
|
|
1784
|
+
* what it emits. `rowsOf` reads the piece's rows: by default as the dataset
|
|
1785
|
+
* gives them; a re-keyed join's, each key's rows in turn.
|
|
1786
|
+
*/
|
|
1787
|
+
function pieceProgram(checked, plan, inputs, output, options, rowsOf = datasetRows(checked, plan.rows.source)) {
|
|
1788
|
+
const emitType = output.kind === "dict" ? FunctionType([parts(output.type).key, parts(output.type).value], NullType)
|
|
1789
|
+
: output.kind === "fold" ? FunctionType([output.type], NullType)
|
|
1790
|
+
: FunctionType([output.kind === "set" ? parts(output.type).key : parts(output.type).value], NullType);
|
|
1791
|
+
const reads = new Set();
|
|
1792
|
+
for (const node of pieceNodes(plan))
|
|
1793
|
+
variablesIn(node, reads);
|
|
1794
|
+
const binds = bindsFor(plan.rows.binds, reads);
|
|
1795
|
+
return withLocationCapture(false, () => func([...inputs.map(i => i.type), emitType], NullType, ($, ...values) => {
|
|
1796
|
+
const t = new SplitTranslator(checked, {});
|
|
1797
|
+
const emit = values[values.length - 1];
|
|
1798
|
+
const root = { root: true, fields: new Map(inputs.map((input, i) => [input.name, values[i]])) };
|
|
1799
|
+
const c = plan.combine;
|
|
1800
|
+
const send = ($2, ...args) => t.stmt($2, t.callFn(emit, args, plan.rows.source.path));
|
|
1801
|
+
// A fold's partials are added up in the piece, and emitted once.
|
|
1802
|
+
const folded = c.kind === "totals" ? c.totals.leaves.map((leaf, i) => t.declare($, t.value(leaf.zero, leaf.type), `p${i}`)) : [];
|
|
1803
|
+
// An output that combines by key is folded by key in the piece first; a grouping that keeps its rows sends each on.
|
|
1804
|
+
const table = c.kind === "distinct" ? pieceTable(t, $, plan.rowType, undefined, c.path, options, send)
|
|
1805
|
+
: c.kind === "distinct_by" || c.kind === "reduce" || (c.kind === "group" && c.totals !== null)
|
|
1806
|
+
? pieceTable(t, $, parts(output.type).key, { type: parts(output.type).value, fold: foldOf(plan, c, parts(output.type).value) }, c.path, options, send)
|
|
1807
|
+
: undefined;
|
|
1808
|
+
// The first rows of a sort are kept in the piece, and emitted once.
|
|
1809
|
+
const top = c.kind === "top" ? topRows(t, $, c) : undefined;
|
|
1810
|
+
genBinds(t, $, root, binds, emptyEnv(), ($2, env) => {
|
|
1811
|
+
const piece = root.fields.get(plan.rows.source.over);
|
|
1812
|
+
rowsOf(t, $2, piece, ($3, item) => genRow(t, $3, item, plan.rows.steps, 0, env, ($4, row, scope) => {
|
|
1813
|
+
switch (c.kind) {
|
|
1814
|
+
case "concat":
|
|
1815
|
+
send($4, t.widenTo($4, row, plan.rowType, plan.rows.end));
|
|
1816
|
+
return;
|
|
1817
|
+
case "distinct":
|
|
1818
|
+
table.add($4, t.widenTo($4, row, plan.rowType, plan.rows.end));
|
|
1819
|
+
return;
|
|
1820
|
+
case "totals":
|
|
1821
|
+
addRow(t, $4, c.totals, folded, row, scope);
|
|
1822
|
+
return;
|
|
1823
|
+
case "group": {
|
|
1824
|
+
const key = t.one(c.key.node, c.key.path, $4, row, scope, c.key.type);
|
|
1825
|
+
if (c.totals === null) {
|
|
1826
|
+
const rows = t.declare($4, t.emptyArray(plan.rowType), "rows");
|
|
1827
|
+
t.push($4, rows, row, c.path);
|
|
1828
|
+
send($4, key, rows);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
table.add($4, key, rowPartials(t, $4, c.totals, row, scope, groupValueType(plan, c)));
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1834
|
+
case "distinct_by":
|
|
1835
|
+
table.add($4, t.one(c.key.node, c.key.path, $4, row, scope, c.key.type), t.widenTo($4, row, plan.rowType, plan.rows.end));
|
|
1836
|
+
return;
|
|
1837
|
+
case "reduce": {
|
|
1838
|
+
const { node, path } = c;
|
|
1839
|
+
const updatePath = childPath(path, "reduce.update");
|
|
1840
|
+
const update = node.value.update;
|
|
1841
|
+
const target = update.value.path;
|
|
1842
|
+
t.destructure($4, node.value.pattern, childPath(path, "reduce.pattern"), row, scope, updatePath, ($5, vars) => {
|
|
1843
|
+
const inner = { ...scope, vars };
|
|
1844
|
+
const accEnv = t.placeholder(c.accType);
|
|
1845
|
+
const key = t.one(target.value.index, childPath(updatePath, "update.path.index.index"), $5, accEnv, inner, c.keyType);
|
|
1846
|
+
const value = t.one(update.value.value, childPath(updatePath, "update.value"), $5, accEnv, inner, t.typeAt(childPath(updatePath, "update.value"), inner));
|
|
1847
|
+
table.add($5, t.widenTo($5, key, c.keyType, path), t.widenTo($5, value, c.valueType, path));
|
|
1848
|
+
});
|
|
1849
|
+
return;
|
|
1850
|
+
}
|
|
1851
|
+
case "top":
|
|
1852
|
+
top.add($4, row, scope);
|
|
1853
|
+
return;
|
|
1854
|
+
}
|
|
1855
|
+
}));
|
|
1856
|
+
});
|
|
1857
|
+
if (c.kind === "totals")
|
|
1858
|
+
send($, t.struct(output.type, Object.fromEntries(folded.map((acc, i) => [`p${i}`, acc]))));
|
|
1859
|
+
if (top !== undefined)
|
|
1860
|
+
send($, top.result($));
|
|
1861
|
+
table?.flush($);
|
|
1862
|
+
return t.null();
|
|
1863
|
+
}));
|
|
1864
|
+
}
|
|
1865
|
+
// ─── The piece's table ──────────────────────────────────────────────────────
|
|
1866
|
+
/** The rows a piece folds into its table before it sends the table's entries on: what bounds the table, whatever the keys. */
|
|
1867
|
+
const FLUSH_ROWS = 32_768;
|
|
1868
|
+
/**
|
|
1869
|
+
* The share of a flush's rows its table's keys may be: 1,024 keys in 32,768
|
|
1870
|
+
* rows, each key standing for 32 rows on average. A table costs more a row
|
|
1871
|
+
* than the runner's sorter it spares, unless each key stands for many rows:
|
|
1872
|
+
* on east-c, Release, over a piece's orders, a table of 8 to 1,024 keys made
|
|
1873
|
+
* a grouping by totals 9 to 23% cheaper and `unique_by` about a third, and
|
|
1874
|
+
* cost `unique` and a `reduce` adding a Float at most 9% more, while one of
|
|
1875
|
+
* 16,384 keys cost them 30 to 43% more (#1093). Trino's adaptive partial
|
|
1876
|
+
* aggregation stops at 0.8, but its table spares a shuffle over the network,
|
|
1877
|
+
* not a sort in the same process.
|
|
1878
|
+
*/
|
|
1879
|
+
const BYPASS_RATIO = 1 / 32;
|
|
1880
|
+
/**
|
|
1881
|
+
* Declares a piece's table (#1093): a set of the rows, or a dict whose value
|
|
1882
|
+
* for a key is its rows' values folded by the output's merge, in input order.
|
|
1883
|
+
*
|
|
1884
|
+
* @param key - the type of the set's rows, or the dict's keys
|
|
1885
|
+
* @param value - the dict's value type and the merge's fold; `undefined` for a set
|
|
1886
|
+
* @param path - the combine's node, where the table's work is placed
|
|
1887
|
+
* @param options - `flushRows` and `bypassRatio`, for a test
|
|
1888
|
+
* @param send - sends a row, or a key and its value, on to the output
|
|
1889
|
+
* @returns the table
|
|
1890
|
+
*
|
|
1891
|
+
* @remarks
|
|
1892
|
+
* Every `flushRows` rows, and at the piece's end, the table sends each of its
|
|
1893
|
+
* entries on and empties, so it never holds more than that many rows'
|
|
1894
|
+
* contributions. A table may hold `bypassRatio` of a flush's rows as keys: the
|
|
1895
|
+
* key one past that sends the table on at once, and from there the piece
|
|
1896
|
+
* sends each row on as it comes, as a piece with no table does. A flush's keys
|
|
1897
|
+
* only grow, so the table is bypassed at its first key too many rather than at
|
|
1898
|
+
* the flush it would be found at, and never grows past that many keys. Either
|
|
1899
|
+
* way the runner's sorter folds each key's values in the order the piece sends
|
|
1900
|
+
* them, which is input order, so the output is the same values; a Float sum is
|
|
1901
|
+
* only added in another grouping.
|
|
1902
|
+
*/
|
|
1903
|
+
function pieceTable(t, $, key, value, path, options, send) {
|
|
1904
|
+
const flushRows = options.flushRows ?? FLUSH_ROWS;
|
|
1905
|
+
const ratio = options.bypassRatio ?? BYPASS_RATIO;
|
|
1906
|
+
if (!Number.isInteger(flushRows) || flushRows < 1)
|
|
1907
|
+
throw new RangeError(`splitJq: flushRows is ${flushRows}, not a whole number of rows from 1`);
|
|
1908
|
+
if (!(ratio >= 0 && ratio <= 1))
|
|
1909
|
+
throw new RangeError(`splitJq: bypassRatio is ${ratio}, not a share from 0 to 1`);
|
|
1910
|
+
// The keys a table may hold: the share of a flush's rows, as a count.
|
|
1911
|
+
const most = Math.floor(ratio * flushRows);
|
|
1912
|
+
const table = value === undefined
|
|
1913
|
+
? t.declare($, t.value(new SortedSet([], compareFor(key)), SetType(key)), "table")
|
|
1914
|
+
: t.declare($, t.value(new SortedMap([], compareFor(key)), DictType(key, value.type)), "table");
|
|
1915
|
+
const held = t.declare($, t.int(0), "held");
|
|
1916
|
+
const direct = t.declare($, t.bool(false), "direct");
|
|
1917
|
+
const size = () => value === undefined
|
|
1918
|
+
? t.b("SetSize", [key], [table], IntegerType, path)
|
|
1919
|
+
: t.b("DictSize", [key, value.type], [table], IntegerType, path);
|
|
1920
|
+
const flush = ($2) => {
|
|
1921
|
+
if (value === undefined) {
|
|
1922
|
+
t.forEach($2, table, ($3, row) => send($3, row), path, "row");
|
|
1923
|
+
t.stmt($2, t.b("SetClear", [key], [table], NullType, path));
|
|
1924
|
+
return;
|
|
1925
|
+
}
|
|
1926
|
+
t.forEach($2, table, ($3, v, k) => send($3, k, v), path, "value");
|
|
1927
|
+
t.stmt($2, t.b("DictClear", [key, value.type], [table], NullType, path));
|
|
1928
|
+
};
|
|
1929
|
+
// A key new to the table: one more than a flush may hold sends the table on, and every row after it.
|
|
1930
|
+
const added = ($2) => {
|
|
1931
|
+
t.ifElse($2, t.lt(t.int(most), size(), path), $3 => {
|
|
1932
|
+
flush($3);
|
|
1933
|
+
t.assign($3, direct, t.bool(true));
|
|
1934
|
+
}, undefined, path);
|
|
1935
|
+
};
|
|
1936
|
+
// A row into a set; a key's value into a dict, folded into the value it holds.
|
|
1937
|
+
const fold = ($2, k, v) => {
|
|
1938
|
+
if (value === undefined) {
|
|
1939
|
+
t.ifElse($2, t.b("SetTryInsert", [key], [table, k], BooleanType, path), $3 => added($3), undefined, path);
|
|
1940
|
+
return;
|
|
1941
|
+
}
|
|
1942
|
+
if (v === undefined)
|
|
1943
|
+
throw new TranslationError("a key with no value for a dict's table");
|
|
1944
|
+
const V = value.type;
|
|
1945
|
+
t.match($2, t.b("DictTryGet", [key, V], [table, k], OptionType(V), path), {
|
|
1946
|
+
none: $3 => {
|
|
1947
|
+
t.stmt($3, t.b("DictInsert", [key, V], [table, k, v], NullType, path));
|
|
1948
|
+
added($3);
|
|
1949
|
+
},
|
|
1950
|
+
some: ($3, earlier) => {
|
|
1951
|
+
const folded = value.fold(t, $3, earlier, v);
|
|
1952
|
+
// A fold that keeps the earlier value changes nothing.
|
|
1953
|
+
if (folded !== earlier)
|
|
1954
|
+
t.stmt($3, t.b("DictUpdate", [key, V], [table, k, t.as(folded, V)], NullType, path));
|
|
1955
|
+
},
|
|
1956
|
+
}, path);
|
|
1957
|
+
};
|
|
1958
|
+
return {
|
|
1959
|
+
add: ($2, k, v) => {
|
|
1960
|
+
// Each computed once, whichever way it goes.
|
|
1961
|
+
const bound = t.bind($2, k, "key");
|
|
1962
|
+
const boundValue = v === undefined ? undefined : t.bind($2, v, "value");
|
|
1963
|
+
t.ifElse($2, direct, $3 => send($3, ...(boundValue === undefined ? [bound] : [bound, boundValue])), $3 => {
|
|
1964
|
+
fold($3, bound, boundValue);
|
|
1965
|
+
t.assign($3, held, t.add(held, t.int(1), path));
|
|
1966
|
+
t.ifElse($3, t.b("GreaterEqual", [IntegerType], [held, t.int(flushRows)], BooleanType, path), $4 => {
|
|
1967
|
+
flush($4);
|
|
1968
|
+
t.assign($4, held, t.int(0));
|
|
1969
|
+
}, undefined, path);
|
|
1970
|
+
}, path);
|
|
1971
|
+
},
|
|
1972
|
+
flush,
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
// ─── The final function ─────────────────────────────────────────────────────
|
|
1976
|
+
/** The final function: the value the combine gives, the rest of the query on it, and the query's sink. */
|
|
1977
|
+
function thenProgram(checked, plan, inputs, output, resultType, options) {
|
|
1978
|
+
const c = plan.combine;
|
|
1979
|
+
const reads = new Set();
|
|
1980
|
+
for (const step of plan.rest)
|
|
1981
|
+
variablesIn(step.node, reads);
|
|
1982
|
+
if (c.kind === "totals")
|
|
1983
|
+
variablesIn(c.node, reads);
|
|
1984
|
+
if (c.kind === "group")
|
|
1985
|
+
variablesIn(c.body.node, reads);
|
|
1986
|
+
const binds = bindsFor(plan.rows.binds, reads);
|
|
1987
|
+
const element = checked.elementType;
|
|
1988
|
+
const multiplicity = checked.multiplicity;
|
|
1989
|
+
return withLocationCapture(false, () => func([output.type, ...inputs.map(i => i.type)], resultType, ($, assembled, ...values) => {
|
|
1990
|
+
const t = new SplitTranslator(checked, options);
|
|
1991
|
+
const root = { root: true, fields: new Map(inputs.map((input, i) => [input.name, values[i]])) };
|
|
1992
|
+
const all = assembled;
|
|
1993
|
+
// The value where the combine ends, and the rest of the query on it, each output to `sink`.
|
|
1994
|
+
const run = ($2, env, sink) => {
|
|
1995
|
+
const rest = ($3, value) => genSteps(t, $3, value, plan.rest, 0, env, sink);
|
|
1996
|
+
switch (c.kind) {
|
|
1997
|
+
case "concat":
|
|
1998
|
+
rest($2, t.bind($2, t.widenTo($2, all, plan.cutType, plan.rows.end), "rows"));
|
|
1999
|
+
return;
|
|
2000
|
+
case "distinct":
|
|
2001
|
+
rest($2, t.bind($2, t.widenTo($2, t.asArray($2, all, c.path), typeAfter(checked, c.path), c.path), "rows"));
|
|
2002
|
+
return;
|
|
2003
|
+
case "distinct_by": {
|
|
2004
|
+
const out = t.declare($2, t.emptyArray(plan.rowType), "rows");
|
|
2005
|
+
t.forEach($2, all, ($3, value) => t.push($3, out, value, c.path), c.path);
|
|
2006
|
+
rest($2, t.bind($2, t.widenTo($2, out, typeAfter(checked, c.path), c.path), "rows"));
|
|
2007
|
+
return;
|
|
2008
|
+
}
|
|
2009
|
+
case "reduce":
|
|
2010
|
+
rest($2, t.bind($2, t.widenTo($2, all, typeAfter(checked, c.path), c.path), "reduced"));
|
|
2011
|
+
return;
|
|
2012
|
+
case "top": {
|
|
2013
|
+
// The first rows, in order: the sort's own first rows, which are all the step after it reads.
|
|
2014
|
+
const out = t.declare($2, t.emptyArray(c.element), "sorted");
|
|
2015
|
+
t.forEach($2, all, ($3, pair) => t.push($3, out, t.field(pair, "value"), c.path), c.path, "pair");
|
|
2016
|
+
rest($2, t.bind($2, t.widenTo($2, out, typeAfter(checked, c.path), c.path), "rows"));
|
|
2017
|
+
return;
|
|
2018
|
+
}
|
|
2019
|
+
case "totals": {
|
|
2020
|
+
const value = withTotals(t, $2, c.totals, all, env, c.node, c.path, plan.cutType);
|
|
2021
|
+
rest($2, value);
|
|
2022
|
+
return;
|
|
2023
|
+
}
|
|
2024
|
+
case "group": {
|
|
2025
|
+
const mapped = typeAfter(checked, c.mapPath);
|
|
2026
|
+
const out = t.declare($2, t.emptyArray(parts(mapped).value), "groups");
|
|
2027
|
+
t.forEach($2, all, ($3, partial) => {
|
|
2028
|
+
const groupType = parts(typeAfter(checked, c.path)).value;
|
|
2029
|
+
if (c.totals === null) {
|
|
2030
|
+
t.gen(c.body.node, c.body.path, $3, t.bind($3, t.widenTo($3, partial, groupType, c.path), "group"), env, ($4, v) => t.push($4, out, v, c.mapPath));
|
|
2031
|
+
return;
|
|
2032
|
+
}
|
|
2033
|
+
t.push($3, out, withTotals(t, $3, c.totals, partial, env, c.body.node, c.body.path, groupType), c.mapPath);
|
|
2034
|
+
}, c.path, "partial");
|
|
2035
|
+
rest($2, out);
|
|
2036
|
+
return;
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
};
|
|
2040
|
+
return sink(t, $, element, multiplicity, options, ($2, emit) => genBinds(t, $2, root, binds, emptyEnv(), ($3, env) => run($3, env, emit)));
|
|
2041
|
+
}));
|
|
2042
|
+
}
|
|
2043
|
+
/**
|
|
2044
|
+
* The value of an expression made of totals, given their partials: each total
|
|
2045
|
+
* finished, its node a hole that gives it, then the expression generated on a
|
|
2046
|
+
* collection it never reads.
|
|
2047
|
+
*/
|
|
2048
|
+
function withTotals(t, $, totals, partials, env, node, path, inputType) {
|
|
2049
|
+
const finished = totals.leaves.map((leaf, i) => t.bind($, leaf.finish(t, $, t.field(partials, `p${i}`), env), `total${i}`));
|
|
2050
|
+
totals.leaves.forEach((leaf, i) => t.holes.set(leaf.path, finished[i]));
|
|
2051
|
+
for (const map of totals.maps)
|
|
2052
|
+
t.holes.set(map.path, t.placeholder(t.typeAt(map.path, env)));
|
|
2053
|
+
try {
|
|
2054
|
+
return t.one(node, path, $, t.placeholder(inputType), env, t.typeAt(path, env));
|
|
2055
|
+
}
|
|
2056
|
+
finally {
|
|
2057
|
+
totals.leaves.forEach(leaf => t.holes.delete(leaf.path));
|
|
2058
|
+
for (const map of totals.maps)
|
|
2059
|
+
t.holes.delete(map.path);
|
|
2060
|
+
}
|
|
2061
|
+
}
|
|
2062
|
+
/** Generates pipeline steps on a value, each output to the next, the last's to `emit`. */
|
|
2063
|
+
function genSteps(t, $, value, steps, i, env, emit) {
|
|
2064
|
+
if (t.ended($))
|
|
2065
|
+
return;
|
|
2066
|
+
if (i === steps.length) {
|
|
2067
|
+
emit($, value);
|
|
2068
|
+
return;
|
|
2069
|
+
}
|
|
2070
|
+
const step = steps[i];
|
|
2071
|
+
if (step.kind === "stage") {
|
|
2072
|
+
t.gen(step.node, step.path, $, value, env, ($2, v) => genSteps(t, $2, v, steps, i + 1, env, emit));
|
|
2073
|
+
return;
|
|
2074
|
+
}
|
|
2075
|
+
const bodyPath = childPath(step.path, "bind.body");
|
|
2076
|
+
t.gen(step.node.value.source, childPath(step.path, "bind.source"), $, value, env, ($2, bound) => {
|
|
2077
|
+
t.destructure($2, step.node.value.patterns[0], childPath(step.path, "bind.patterns[0]"), bound, env, bodyPath, ($3, vars) => genSteps(t, $3, value, steps, i + 1, { ...env, vars }, emit));
|
|
2078
|
+
});
|
|
2079
|
+
}
|
|
2080
|
+
/** The query's sink, as `translateJq` gives its result: the value, an option, or the outputs up to the limit and one more. */
|
|
2081
|
+
function sink(t, $, element, multiplicity, options, gen) {
|
|
2082
|
+
if (multiplicity === "one") {
|
|
2083
|
+
const out = t.declare($, t.placeholder(element), "result");
|
|
2084
|
+
gen($, ($2, v) => t.assign($2, out, t.widenTo($2, v, element, "")));
|
|
2085
|
+
return out;
|
|
2086
|
+
}
|
|
2087
|
+
if (multiplicity === "maybe") {
|
|
2088
|
+
const option = OptionType(element);
|
|
2089
|
+
const out = t.declare($, t.none(option), "result");
|
|
2090
|
+
t.once($, ($2, label) => gen($2, ($3, v) => {
|
|
2091
|
+
t.assign($3, out, t.some(t.widenTo($3, v, element, ""), option));
|
|
2092
|
+
t.brk($3, label);
|
|
2093
|
+
}), "");
|
|
2094
|
+
return out;
|
|
2095
|
+
}
|
|
2096
|
+
const out = t.declare($, t.emptyArray(element), "results");
|
|
2097
|
+
const limit = options.maxOutputs;
|
|
2098
|
+
if (limit === undefined) {
|
|
2099
|
+
gen($, ($2, v) => t.push($2, out, v, ""));
|
|
2100
|
+
return out;
|
|
2101
|
+
}
|
|
2102
|
+
t.once($, ($2, label) => gen($2, ($3, v) => {
|
|
2103
|
+
t.push($3, out, v, "");
|
|
2104
|
+
t.ifElse($3, t.b("GreaterEqual", [IntegerType], [t.size(out, ""), t.int(limit + 1)], BooleanType, ""), $4 => t.brk($4, label), undefined, "");
|
|
2105
|
+
}), "");
|
|
2106
|
+
return out;
|
|
2107
|
+
}
|
|
2108
|
+
//# sourceMappingURL=split.js.map
|