@math.gl/sql 0.0.0-stage → 5.0.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +249 -0
- package/README.md +26 -2
- package/THIRD-PARTY-NOTICES.md +5 -0
- package/dist/arrow/aggregates.d.ts +46 -0
- package/dist/arrow/aggregates.d.ts.map +1 -0
- package/dist/arrow/aggregates.js +289 -0
- package/dist/arrow/aggregates.js.map +1 -0
- package/dist/arrow/buffers.d.ts +30 -0
- package/dist/arrow/buffers.d.ts.map +1 -0
- package/dist/arrow/buffers.js +177 -0
- package/dist/arrow/buffers.js.map +1 -0
- package/dist/arrow/cast.d.ts +19 -0
- package/dist/arrow/cast.d.ts.map +1 -0
- package/dist/arrow/cast.js +138 -0
- package/dist/arrow/cast.js.map +1 -0
- package/dist/arrow/compiler.d.ts +10 -0
- package/dist/arrow/compiler.d.ts.map +1 -0
- package/dist/arrow/compiler.js +1538 -0
- package/dist/arrow/compiler.js.map +1 -0
- package/dist/arrow/float16.d.ts +7 -0
- package/dist/arrow/float16.d.ts.map +1 -0
- package/dist/arrow/float16.js +26 -0
- package/dist/arrow/float16.js.map +1 -0
- package/dist/arrow/index.cjs +3321 -0
- package/dist/arrow/index.cjs.map +6 -0
- package/dist/arrow/index.d.ts +5 -0
- package/dist/arrow/index.d.ts.map +1 -0
- package/dist/arrow/index.js +7 -0
- package/dist/arrow/index.js.map +1 -0
- package/dist/arrow/like.d.ts +3 -0
- package/dist/arrow/like.d.ts.map +1 -0
- package/dist/arrow/like.js +75 -0
- package/dist/arrow/like.js.map +1 -0
- package/dist/arrow/list-values.d.ts +14 -0
- package/dist/arrow/list-values.d.ts.map +1 -0
- package/dist/arrow/list-values.js +201 -0
- package/dist/arrow/list-values.js.map +1 -0
- package/dist/arrow/predicate.d.ts +7 -0
- package/dist/arrow/predicate.d.ts.map +1 -0
- package/dist/arrow/predicate.js +10 -0
- package/dist/arrow/predicate.js.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts +24 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js +64 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js.map +1 -0
- package/dist/arrow/query-arrow-sql.d.ts +8 -0
- package/dist/arrow/query-arrow-sql.d.ts.map +1 -0
- package/dist/arrow/query-arrow-sql.js +223 -0
- package/dist/arrow/query-arrow-sql.js.map +1 -0
- package/dist/arrow/query-arrow-table.d.ts +36 -0
- package/dist/arrow/query-arrow-table.d.ts.map +1 -0
- package/dist/arrow/query-arrow-table.js +208 -0
- package/dist/arrow/query-arrow-table.js.map +1 -0
- package/dist/arrow/query-worker-entry.d.ts +2 -0
- package/dist/arrow/query-worker-entry.d.ts.map +1 -0
- package/dist/arrow/query-worker-entry.js +7 -0
- package/dist/arrow/query-worker-entry.js.map +1 -0
- package/dist/arrow/query-worker.d.ts +65 -0
- package/dist/arrow/query-worker.d.ts.map +1 -0
- package/dist/arrow/query-worker.js +135 -0
- package/dist/arrow/query-worker.js.map +1 -0
- package/dist/arrow/registry.d.ts +4 -0
- package/dist/arrow/registry.d.ts.map +1 -0
- package/dist/arrow/registry.js +66 -0
- package/dist/arrow/registry.js.map +1 -0
- package/dist/arrow/relational-query.d.ts +36 -0
- package/dist/arrow/relational-query.d.ts.map +1 -0
- package/dist/arrow/relational-query.js +174 -0
- package/dist/arrow/relational-query.js.map +1 -0
- package/dist/arrow/replace-arrow-columns.d.ts +8 -0
- package/dist/arrow/replace-arrow-columns.d.ts.map +1 -0
- package/dist/arrow/replace-arrow-columns.js +63 -0
- package/dist/arrow/replace-arrow-columns.js.map +1 -0
- package/dist/arrow/signature.d.ts +6 -0
- package/dist/arrow/signature.d.ts.map +1 -0
- package/dist/arrow/signature.js +51 -0
- package/dist/arrow/signature.js.map +1 -0
- package/dist/arrow/temporal-cast.d.ts +9 -0
- package/dist/arrow/temporal-cast.d.ts.map +1 -0
- package/dist/arrow/temporal-cast.js +60 -0
- package/dist/arrow/temporal-cast.js.map +1 -0
- package/dist/arrow/types.d.ts +62 -0
- package/dist/arrow/types.d.ts.map +1 -0
- package/dist/arrow/types.js +5 -0
- package/dist/arrow/types.js.map +1 -0
- package/dist/arrow/unnest-arrow-column.d.ts +22 -0
- package/dist/arrow/unnest-arrow-column.d.ts.map +1 -0
- package/dist/arrow/unnest-arrow-column.js +112 -0
- package/dist/arrow/unnest-arrow-column.js.map +1 -0
- package/dist/arrow/worker-entry.d.ts +2 -0
- package/dist/arrow/worker-entry.d.ts.map +1 -0
- package/dist/arrow/worker-entry.js +9 -0
- package/dist/arrow/worker-entry.js.map +1 -0
- package/dist/arrow/worker.cjs +4074 -0
- package/dist/arrow/worker.cjs.map +6 -0
- package/dist/arrow/worker.d.ts +58 -0
- package/dist/arrow/worker.d.ts.map +1 -0
- package/dist/arrow/worker.js +84 -0
- package/dist/arrow/worker.js.map +1 -0
- package/dist/color-functions/index.cjs +382 -0
- package/dist/color-functions/index.cjs.map +6 -0
- package/dist/color-functions/index.d.ts +26 -0
- package/dist/color-functions/index.d.ts.map +1 -0
- package/dist/color-functions/index.js +306 -0
- package/dist/color-functions/index.js.map +1 -0
- package/dist/diagnostic-log.d.ts +8 -0
- package/dist/diagnostic-log.d.ts.map +1 -0
- package/dist/diagnostic-log.js +15 -0
- package/dist/diagnostic-log.js.map +1 -0
- package/dist/expressions/ast.d.ts +12 -0
- package/dist/expressions/ast.d.ts.map +1 -0
- package/dist/expressions/ast.js +114 -0
- package/dist/expressions/ast.js.map +1 -0
- package/dist/expressions/columnar-predicate.d.ts +56 -0
- package/dist/expressions/columnar-predicate.d.ts.map +1 -0
- package/dist/expressions/columnar-predicate.js +176 -0
- package/dist/expressions/columnar-predicate.js.map +1 -0
- package/dist/expressions/evaluation-error.d.ts +14 -0
- package/dist/expressions/evaluation-error.d.ts.map +1 -0
- package/dist/expressions/evaluation-error.js +26 -0
- package/dist/expressions/evaluation-error.js.map +1 -0
- package/dist/expressions/expression-eval.d.ts +105 -0
- package/dist/expressions/expression-eval.d.ts.map +1 -0
- package/dist/expressions/expression-eval.js +330 -0
- package/dist/expressions/expression-eval.js.map +1 -0
- package/dist/expressions/function-libraries.d.ts +63 -0
- package/dist/expressions/function-libraries.d.ts.map +1 -0
- package/dist/expressions/function-libraries.js +69 -0
- package/dist/expressions/function-libraries.js.map +1 -0
- package/dist/expressions/function-registry.d.ts +93 -0
- package/dist/expressions/function-registry.d.ts.map +1 -0
- package/dist/expressions/function-registry.js +139 -0
- package/dist/expressions/function-registry.js.map +1 -0
- package/dist/expressions/get.d.ts +9 -0
- package/dist/expressions/get.d.ts.map +1 -0
- package/dist/expressions/get.js +32 -0
- package/dist/expressions/get.js.map +1 -0
- package/dist/expressions/index.cjs +1888 -0
- package/dist/expressions/index.cjs.map +6 -0
- package/dist/expressions/index.d.ts +13 -0
- package/dist/expressions/index.d.ts.map +1 -0
- package/dist/expressions/index.js +14 -0
- package/dist/expressions/index.js.map +1 -0
- package/dist/expressions/numeric-functions.d.ts +277 -0
- package/dist/expressions/numeric-functions.d.ts.map +1 -0
- package/dist/expressions/numeric-functions.js +52 -0
- package/dist/expressions/numeric-functions.js.map +1 -0
- package/dist/expressions/parse-expression-string.d.ts +21 -0
- package/dist/expressions/parse-expression-string.d.ts.map +1 -0
- package/dist/expressions/parse-expression-string.js +61 -0
- package/dist/expressions/parse-expression-string.js.map +1 -0
- package/dist/expressions/predicate-expression.d.ts +7 -0
- package/dist/expressions/predicate-expression.d.ts.map +1 -0
- package/dist/expressions/predicate-expression.js +112 -0
- package/dist/expressions/predicate-expression.js.map +1 -0
- package/dist/expressions/scalar-type.d.ts +6 -0
- package/dist/expressions/scalar-type.d.ts.map +1 -0
- package/dist/expressions/scalar-type.js +23 -0
- package/dist/expressions/scalar-type.js.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts +3 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.js +972 -0
- package/dist/expressions/vendor/jsep/runtime.js.map +1 -0
- package/dist/expressions/vendor/jsep/types.d.ts +148 -0
- package/dist/expressions/vendor/jsep/types.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/types.js +6 -0
- package/dist/expressions/vendor/jsep/types.js.map +1 -0
- package/dist/index.cjs +5294 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/math-functions/index.cjs +444 -0
- package/dist/math-functions/index.cjs.map +6 -0
- package/dist/math-functions/index.d.ts +44 -0
- package/dist/math-functions/index.d.ts.map +1 -0
- package/dist/math-functions/index.js +182 -0
- package/dist/math-functions/index.js.map +1 -0
- package/dist/parser/bind-sql-predicate.d.ts +4 -0
- package/dist/parser/bind-sql-predicate.d.ts.map +1 -0
- package/dist/parser/bind-sql-predicate.js +11 -0
- package/dist/parser/bind-sql-predicate.js.map +1 -0
- package/dist/parser/index.cjs +1807 -0
- package/dist/parser/index.cjs.map +6 -0
- package/dist/parser/index.d.ts +12 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +11 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/parameter-snapshots.d.ts +21 -0
- package/dist/parser/parameter-snapshots.d.ts.map +1 -0
- package/dist/parser/parameter-snapshots.js +85 -0
- package/dist/parser/parameter-snapshots.js.map +1 -0
- package/dist/parser/parse-sql-expression.d.ts +50 -0
- package/dist/parser/parse-sql-expression.d.ts.map +1 -0
- package/dist/parser/parse-sql-expression.js +501 -0
- package/dist/parser/parse-sql-expression.js.map +1 -0
- package/dist/parser/parse-sql-predicate.d.ts +13 -0
- package/dist/parser/parse-sql-predicate.d.ts.map +1 -0
- package/dist/parser/parse-sql-predicate.js +365 -0
- package/dist/parser/parse-sql-predicate.js.map +1 -0
- package/dist/parser/parse-sql-query.d.ts +9 -0
- package/dist/parser/parse-sql-query.d.ts.map +1 -0
- package/dist/parser/parse-sql-query.js +81 -0
- package/dist/parser/parse-sql-query.js.map +1 -0
- package/dist/parser/parse-sql-select.d.ts +57 -0
- package/dist/parser/parse-sql-select.d.ts.map +1 -0
- package/dist/parser/parse-sql-select.js +247 -0
- package/dist/parser/parse-sql-select.js.map +1 -0
- package/dist/parser/parse-sql-statement.d.ts +54 -0
- package/dist/parser/parse-sql-statement.d.ts.map +1 -0
- package/dist/parser/parse-sql-statement.js +207 -0
- package/dist/parser/parse-sql-statement.js.map +1 -0
- package/dist/parser/query-types.d.ts +56 -0
- package/dist/parser/query-types.d.ts.map +1 -0
- package/dist/parser/query-types.js +5 -0
- package/dist/parser/query-types.js.map +1 -0
- package/dist/parser/sql-parameters.d.ts +29 -0
- package/dist/parser/sql-parameters.d.ts.map +1 -0
- package/dist/parser/sql-parameters.js +110 -0
- package/dist/parser/sql-parameters.js.map +1 -0
- package/dist/parser/sql-parse-error.d.ts +13 -0
- package/dist/parser/sql-parse-error.d.ts.map +1 -0
- package/dist/parser/sql-parse-error.js +20 -0
- package/dist/parser/sql-parse-error.js.map +1 -0
- package/dist/parser/sql-predicate-schema.d.ts +164 -0
- package/dist/parser/sql-predicate-schema.d.ts.map +1 -0
- package/dist/parser/sql-predicate-schema.js +249 -0
- package/dist/parser/sql-predicate-schema.js.map +1 -0
- package/dist/parser/sql-predicate-types.d.ts +48 -0
- package/dist/parser/sql-predicate-types.d.ts.map +1 -0
- package/dist/parser/sql-predicate-types.js +9 -0
- package/dist/parser/sql-predicate-types.js.map +1 -0
- package/dist/parser/sql-predicate.d.ts +6 -0
- package/dist/parser/sql-predicate.d.ts.map +1 -0
- package/dist/parser/sql-predicate.js +8 -0
- package/dist/parser/sql-predicate.js.map +1 -0
- package/dist/scales/arrow-domains.d.ts +22 -0
- package/dist/scales/arrow-domains.d.ts.map +1 -0
- package/dist/scales/arrow-domains.js +166 -0
- package/dist/scales/arrow-domains.js.map +1 -0
- package/dist/scales/arrow-scales.d.ts +44 -0
- package/dist/scales/arrow-scales.d.ts.map +1 -0
- package/dist/scales/arrow-scales.js +255 -0
- package/dist/scales/arrow-scales.js.map +1 -0
- package/dist/scales/compile-scale.d.ts +2 -0
- package/dist/scales/compile-scale.d.ts.map +1 -0
- package/dist/scales/compile-scale.js +5 -0
- package/dist/scales/compile-scale.js.map +1 -0
- package/dist/scales/conformance.cjs +180 -0
- package/dist/scales/conformance.cjs.map +6 -0
- package/dist/scales/conformance.d.ts +26 -0
- package/dist/scales/conformance.d.ts.map +1 -0
- package/dist/scales/conformance.js +164 -0
- package/dist/scales/conformance.js.map +1 -0
- package/dist/scales/hdr-colors.d.ts +2 -0
- package/dist/scales/hdr-colors.d.ts.map +1 -0
- package/dist/scales/hdr-colors.js +5 -0
- package/dist/scales/hdr-colors.js.map +1 -0
- package/dist/scales/index.cjs +4334 -0
- package/dist/scales/index.cjs.map +6 -0
- package/dist/scales/index.d.ts +10 -0
- package/dist/scales/index.d.ts.map +1 -0
- package/dist/scales/index.js +11 -0
- package/dist/scales/index.js.map +1 -0
- package/dist/scales/palettes.d.ts +2 -0
- package/dist/scales/palettes.d.ts.map +1 -0
- package/dist/scales/palettes.js +5 -0
- package/dist/scales/palettes.js.map +1 -0
- package/dist/scales/parse-sql-scale.d.ts +27 -0
- package/dist/scales/parse-sql-scale.d.ts.map +1 -0
- package/dist/scales/parse-sql-scale.js +257 -0
- package/dist/scales/parse-sql-scale.js.map +1 -0
- package/dist/scales/quantile-sketch.d.ts +2 -0
- package/dist/scales/quantile-sketch.d.ts.map +1 -0
- package/dist/scales/quantile-sketch.js +5 -0
- package/dist/scales/quantile-sketch.js.map +1 -0
- package/dist/scales/scale-types.d.ts +2 -0
- package/dist/scales/scale-types.d.ts.map +1 -0
- package/dist/scales/scale-types.js +5 -0
- package/dist/scales/scale-types.js.map +1 -0
- package/dist/scales/streaming-domains.d.ts +18 -0
- package/dist/scales/streaming-domains.d.ts.map +1 -0
- package/dist/scales/streaming-domains.js +115 -0
- package/dist/scales/streaming-domains.js.map +1 -0
- package/package.json +92 -4
- package/src/arrow/aggregates.ts +386 -0
- package/src/arrow/buffers.ts +177 -0
- package/src/arrow/cast.ts +160 -0
- package/src/arrow/compiler.ts +1778 -0
- package/src/arrow/float16.ts +25 -0
- package/src/arrow/index.ts +22 -0
- package/src/arrow/like.ts +72 -0
- package/src/arrow/list-values.ts +221 -0
- package/src/arrow/predicate.ts +19 -0
- package/src/arrow/query-arrow-aggregate-snapshots.ts +107 -0
- package/src/arrow/query-arrow-sql.ts +255 -0
- package/src/arrow/query-arrow-table.ts +298 -0
- package/src/arrow/query-worker-entry.ts +8 -0
- package/src/arrow/query-worker.ts +194 -0
- package/src/arrow/registry.ts +87 -0
- package/src/arrow/relational-query.ts +247 -0
- package/src/arrow/replace-arrow-columns.ts +71 -0
- package/src/arrow/signature.ts +50 -0
- package/src/arrow/temporal-cast.ts +66 -0
- package/src/arrow/types.ts +76 -0
- package/src/arrow/unnest-arrow-column.ts +145 -0
- package/src/arrow/worker-entry.ts +10 -0
- package/src/arrow/worker.ts +143 -0
- package/src/color-functions/index.ts +405 -0
- package/src/diagnostic-log.ts +20 -0
- package/src/expressions/ast.ts +143 -0
- package/src/expressions/columnar-predicate.ts +293 -0
- package/src/expressions/evaluation-error.ts +29 -0
- package/src/expressions/expression-eval.ts +489 -0
- package/src/expressions/function-libraries.ts +126 -0
- package/src/expressions/function-registry.ts +171 -0
- package/src/expressions/get.ts +37 -0
- package/src/expressions/index.ts +40 -0
- package/src/expressions/numeric-functions.ts +99 -0
- package/src/expressions/parse-expression-string.ts +80 -0
- package/src/expressions/predicate-expression.ts +128 -0
- package/src/expressions/scalar-type.ts +39 -0
- package/src/expressions/vendor/jsep/LICENSE.txt +23 -0
- package/src/expressions/vendor/jsep/runtime.ts +949 -0
- package/src/expressions/vendor/jsep/types.ts +191 -0
- package/src/index.ts +54 -0
- package/src/math-functions/index.ts +235 -0
- package/src/parser/bind-sql-predicate.ts +21 -0
- package/src/parser/index.ts +43 -0
- package/src/parser/parameter-snapshots.ts +103 -0
- package/src/parser/parse-sql-expression.ts +500 -0
- package/src/parser/parse-sql-predicate.ts +454 -0
- package/src/parser/parse-sql-query.ts +90 -0
- package/src/parser/parse-sql-select.ts +272 -0
- package/src/parser/parse-sql-statement.ts +260 -0
- package/src/parser/query-types.ts +71 -0
- package/src/parser/sql-parameters.ts +174 -0
- package/src/parser/sql-parse-error.ts +33 -0
- package/src/parser/sql-predicate-schema.ts +279 -0
- package/src/parser/sql-predicate-types.ts +91 -0
- package/src/parser/sql-predicate.ts +28 -0
- package/src/scales/arrow-domains.ts +190 -0
- package/src/scales/arrow-scales.ts +363 -0
- package/src/scales/compile-scale.ts +5 -0
- package/src/scales/conformance.ts +186 -0
- package/src/scales/hdr-colors.ts +5 -0
- package/src/scales/index.ts +37 -0
- package/src/scales/palettes.ts +5 -0
- package/src/scales/parse-sql-scale.ts +300 -0
- package/src/scales/quantile-sketch.ts +5 -0
- package/src/scales/scale-types.ts +5 -0
- package/src/scales/streaming-domains.ts +139 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DataType, type Table} from 'apache-arrow';
|
|
6
|
+
import {unnestArrowColumn} from './unnest-arrow-column';
|
|
7
|
+
import type {Expression} from '../expressions';
|
|
8
|
+
import {
|
|
9
|
+
parseSQLSelectCompound,
|
|
10
|
+
type SQLSelectStatement,
|
|
11
|
+
type SQLSelectStatementOptions,
|
|
12
|
+
type SQLSort
|
|
13
|
+
} from '../parser/parse-sql-statement';
|
|
14
|
+
import {queryArrowTable, type ArrowTableQuery} from './query-arrow-table';
|
|
15
|
+
import type {ArrowAggregate} from './aggregates';
|
|
16
|
+
|
|
17
|
+
type Execution = Pick<ArrowTableQuery, 'signal' | 'functions'>;
|
|
18
|
+
|
|
19
|
+
/** Executes relational SQL over a caller-supplied catalog of in-memory Arrow tables. */
|
|
20
|
+
export function queryArrowSQL(
|
|
21
|
+
sql: string,
|
|
22
|
+
tables: ReadonlyMap<string, Table>,
|
|
23
|
+
options: Execution & SQLSelectStatementOptions = {}
|
|
24
|
+
): Table {
|
|
25
|
+
const {inputs, orderBy, offset, limit} = parseSQLSelectCompound(sql, options);
|
|
26
|
+
const execution = {signal: options.signal, functions: options.functions};
|
|
27
|
+
const [first, ...rest] = inputs.map(input => executeSelect(input, tables, execution)) as [
|
|
28
|
+
Table,
|
|
29
|
+
...Table[]
|
|
30
|
+
];
|
|
31
|
+
// Plan the final order against the projected names, including SELECT aliases.
|
|
32
|
+
return queryArrowTable(first, {
|
|
33
|
+
...execution,
|
|
34
|
+
union: rest.map(table => {
|
|
35
|
+
if (table.numCols !== first.numCols)
|
|
36
|
+
throw new Error('Arrow SQL UNION ALL requires the same number of columns in each input');
|
|
37
|
+
return queryArrowTable(table, {
|
|
38
|
+
select: table.schema.fields.map((field, index) => ({
|
|
39
|
+
column: field.name,
|
|
40
|
+
as: first.schema.fields[index]!.name
|
|
41
|
+
})),
|
|
42
|
+
signal: execution.signal
|
|
43
|
+
});
|
|
44
|
+
}),
|
|
45
|
+
orderBy: planOrder(orderBy),
|
|
46
|
+
offset,
|
|
47
|
+
limit
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function planOrder(orderBy?: readonly SQLSort[]) {
|
|
52
|
+
return orderBy?.map(({expression, direction, nulls}) => {
|
|
53
|
+
/** Qualified projected output name, including UNNEST aliases. */
|
|
54
|
+
const column = qualifiedColumn(expression);
|
|
55
|
+
return {
|
|
56
|
+
column: column?.qualifier ? column.qualifier + '.' + column.name : requireColumn(expression),
|
|
57
|
+
direction,
|
|
58
|
+
nulls
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function executeSelect(
|
|
64
|
+
statement: SQLSelectStatement,
|
|
65
|
+
tables: ReadonlyMap<string, Table>,
|
|
66
|
+
execution: Execution
|
|
67
|
+
): Table {
|
|
68
|
+
let source = getTable(tables, statement.from.name);
|
|
69
|
+
const leftAlias = statement.from.alias ?? statement.from.name;
|
|
70
|
+
const [right] = statement.joins;
|
|
71
|
+
let join: ArrowTableQuery['join'];
|
|
72
|
+
let rightAlias: string | undefined;
|
|
73
|
+
let rightTable: Table | undefined;
|
|
74
|
+
if (right?.unnest) {
|
|
75
|
+
if (statement.joins.length !== 1)
|
|
76
|
+
throw new Error('UNNEST currently supports one expansion per SELECT');
|
|
77
|
+
const {expression, columns, ordinality} = right.unnest;
|
|
78
|
+
const column = qualifiedColumn(expression);
|
|
79
|
+
if (!column || (column.qualifier && column.qualifier !== leftAlias))
|
|
80
|
+
throw new Error('UNNEST requires a column of its parent table');
|
|
81
|
+
const field = source.schema.fields.find(field => field.name === column.name);
|
|
82
|
+
if (!field) throw new Error(`Unknown UNNEST column: ${column.name}`);
|
|
83
|
+
const map = DataType.isMap(field.type);
|
|
84
|
+
if (!map && !DataType.isList(field.type) && !DataType.isFixedSizeList(field.type))
|
|
85
|
+
throw new Error('SQL UNNEST requires List, FixedSizeList or Map');
|
|
86
|
+
if (columns.length !== (map ? 2 : 1) + (ordinality ? 1 : 0))
|
|
87
|
+
throw new Error('UNNEST column alias count does not match its output');
|
|
88
|
+
rightAlias = right.table.alias!;
|
|
89
|
+
if (rightAlias === leftAlias) throw new Error('Duplicate Arrow SQL table alias');
|
|
90
|
+
source = unnestArrowColumn(source, column.name, {
|
|
91
|
+
valueColumn: rightAlias + '.' + columns[map ? 1 : 0],
|
|
92
|
+
keyColumn: map ? rightAlias + '.' + columns[0] : undefined,
|
|
93
|
+
ordinalColumn: ordinality ? rightAlias + '.' + columns[map ? 2 : 1] : undefined,
|
|
94
|
+
outer: right.type === 'left',
|
|
95
|
+
signal: execution.signal
|
|
96
|
+
});
|
|
97
|
+
} else if (right) {
|
|
98
|
+
if (statement.joins.length > 1 || right.type !== 'inner')
|
|
99
|
+
throw new Error('Arrow SQL supports one INNER JOIN per SELECT');
|
|
100
|
+
rightAlias = right.table.alias ?? right.table.name;
|
|
101
|
+
rightTable = getTable(tables, right.table.name);
|
|
102
|
+
if (leftAlias === rightAlias) throw new Error('Duplicate Arrow SQL table alias');
|
|
103
|
+
const {on} = right;
|
|
104
|
+
if (on.type !== 'BinaryExpression' || on.operator !== '==')
|
|
105
|
+
throw new Error('Arrow SQL JOIN requires equality between two columns');
|
|
106
|
+
const sides = [on.left, on.right].map(node => qualifiedColumn(node as Expression));
|
|
107
|
+
const left = sides.find(column => column?.qualifier === leftAlias);
|
|
108
|
+
const rightColumn = sides.find(column => column?.qualifier === rightAlias);
|
|
109
|
+
if (!left || !rightColumn) throw new Error('Qualify both Arrow SQL JOIN keys');
|
|
110
|
+
join = {
|
|
111
|
+
table: rightTable,
|
|
112
|
+
left: left.name,
|
|
113
|
+
right: rightColumn.name,
|
|
114
|
+
rightPrefix: rightAlias + '.'
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const bind = (node: Expression): Expression =>
|
|
119
|
+
bindColumns(node, column => {
|
|
120
|
+
if (column.qualifier === leftAlias) return column.name;
|
|
121
|
+
if (column.qualifier && column.qualifier === rightAlias)
|
|
122
|
+
return column.qualifier + '.' + column.name;
|
|
123
|
+
if (column.qualifier) {
|
|
124
|
+
if (
|
|
125
|
+
[source, rightTable].some(table =>
|
|
126
|
+
table?.schema.fields.some(field => field.name === column.qualifier)
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
return undefined;
|
|
130
|
+
throw new Error(`Unknown Arrow SQL table alias: ${column.qualifier}`);
|
|
131
|
+
}
|
|
132
|
+
const onRight = rightTable?.schema.fields.some(field => field.name === column.name);
|
|
133
|
+
if (!onRight) return column.name;
|
|
134
|
+
if (source.schema.fields.some(field => field.name === column.name))
|
|
135
|
+
throw new Error(`Ambiguous Arrow SQL column: ${column.name}`);
|
|
136
|
+
return rightAlias + '.' + column.name;
|
|
137
|
+
});
|
|
138
|
+
const groupBy = statement.groupBy?.map(expression => requireColumn(bind(expression)));
|
|
139
|
+
const aggregates = new Map<string, ArrowAggregate>();
|
|
140
|
+
const select = statement.select?.map(({expression, alias}, index) => {
|
|
141
|
+
const aggregate = planAggregate(expression, bind);
|
|
142
|
+
if (aggregate) {
|
|
143
|
+
const name = alias ?? `aggregate_${index + 1}`;
|
|
144
|
+
if (aggregates.has(name) || groupBy?.includes(name))
|
|
145
|
+
throw new Error(`Duplicate Arrow SQL aggregate name: ${name}`);
|
|
146
|
+
aggregates.set(name, aggregate);
|
|
147
|
+
return name;
|
|
148
|
+
}
|
|
149
|
+
const bound = bind(expression);
|
|
150
|
+
if (bound.type === 'Identifier') return {column: bound.name, as: alias};
|
|
151
|
+
return {expression: bound, as: alias};
|
|
152
|
+
});
|
|
153
|
+
return queryArrowTable(source, {
|
|
154
|
+
...execution,
|
|
155
|
+
join,
|
|
156
|
+
select,
|
|
157
|
+
where: statement.where && {expression: bind(statement.where)},
|
|
158
|
+
groupBy,
|
|
159
|
+
aggregates: Object.fromEntries(aggregates)
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function getTable(tables: ReadonlyMap<string, Table>, name: string): Table {
|
|
164
|
+
const table = tables.get(name);
|
|
165
|
+
if (!table) throw new Error(`Unknown Arrow SQL table: ${name}`);
|
|
166
|
+
return table;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
type Column = {name: string; qualifier?: string};
|
|
170
|
+
|
|
171
|
+
function qualifiedColumn(node: Expression): Column | undefined {
|
|
172
|
+
if (node.type === 'Identifier') return {name: node.name};
|
|
173
|
+
if (
|
|
174
|
+
node.type === 'MemberExpression' &&
|
|
175
|
+
!node.computed &&
|
|
176
|
+
node.object.type === 'Identifier' &&
|
|
177
|
+
node.property.type === 'Identifier'
|
|
178
|
+
)
|
|
179
|
+
return {name: String(node.property['name']), qualifier: String(node.object['name'])};
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function requireColumn(node: Expression): string {
|
|
184
|
+
if (node.type !== 'Identifier')
|
|
185
|
+
throw new Error('Arrow SQL requires a column name in this clause');
|
|
186
|
+
return node.name;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Resolves SQL column references while leaving function names and literals intact. */
|
|
190
|
+
function bindColumns(
|
|
191
|
+
node: Expression,
|
|
192
|
+
resolve: (column: Column) => string | undefined
|
|
193
|
+
): Expression {
|
|
194
|
+
const column = qualifiedColumn(node);
|
|
195
|
+
const name = column && resolve(column);
|
|
196
|
+
if (name !== undefined) return {type: 'Identifier', name};
|
|
197
|
+
const bind = (child: unknown) => bindColumns(child as Expression, resolve);
|
|
198
|
+
switch (node.type) {
|
|
199
|
+
case 'BinaryExpression':
|
|
200
|
+
case 'LogicalExpression':
|
|
201
|
+
return {...node, left: bind(node.left), right: bind(node.right)};
|
|
202
|
+
case 'UnaryExpression':
|
|
203
|
+
return {...node, argument: bind(node.argument)};
|
|
204
|
+
case 'CallExpression':
|
|
205
|
+
return {...node, arguments: node.arguments.map(bind)};
|
|
206
|
+
case 'MemberExpression': {
|
|
207
|
+
if (node.computed || node.property.type !== 'Identifier')
|
|
208
|
+
throw new Error('Arrow SQL struct access requires a field identifier');
|
|
209
|
+
const name = String(node.property['name']);
|
|
210
|
+
return {
|
|
211
|
+
type: 'CallExpression',
|
|
212
|
+
callee: {type: 'Identifier', name: 'STRUCT_EXTRACT'},
|
|
213
|
+
arguments: [bind(node.object), {type: 'Literal', value: name, raw: JSON.stringify(name)}]
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
case 'ConditionalExpression':
|
|
217
|
+
return {
|
|
218
|
+
...node,
|
|
219
|
+
test: bind(node.test),
|
|
220
|
+
consequent: bind(node.consequent),
|
|
221
|
+
alternate: bind(node.alternate)
|
|
222
|
+
};
|
|
223
|
+
default:
|
|
224
|
+
return node;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const aggregateOperations = new Set<ArrowAggregate['operation']>([
|
|
229
|
+
'COUNT',
|
|
230
|
+
'SUM',
|
|
231
|
+
'MIN',
|
|
232
|
+
'MAX',
|
|
233
|
+
'AVG',
|
|
234
|
+
'EVERY',
|
|
235
|
+
'BOOL_AND',
|
|
236
|
+
'BOOL_OR',
|
|
237
|
+
'BIT_AND',
|
|
238
|
+
'BIT_OR',
|
|
239
|
+
'BIT_XOR'
|
|
240
|
+
]);
|
|
241
|
+
|
|
242
|
+
function planAggregate(
|
|
243
|
+
node: Expression,
|
|
244
|
+
bind: (expression: Expression) => Expression
|
|
245
|
+
): ArrowAggregate | undefined {
|
|
246
|
+
if (node.type !== 'CallExpression' || node.callee.type !== 'Identifier' || !node.sqlFunction)
|
|
247
|
+
return undefined;
|
|
248
|
+
const operation = String(node.callee['name']).toUpperCase() as ArrowAggregate['operation'];
|
|
249
|
+
if (!aggregateOperations.has(operation)) return undefined;
|
|
250
|
+
if (node.arguments.length !== 1) throw new Error(`Arrow SQL ${operation} requires one argument`);
|
|
251
|
+
const argument = node.arguments[0] as Expression & {sqlWildcard?: boolean};
|
|
252
|
+
return operation === 'COUNT' && argument.sqlWildcard
|
|
253
|
+
? {operation}
|
|
254
|
+
: {operation, expression: bind(argument)};
|
|
255
|
+
}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
Data,
|
|
7
|
+
DataType,
|
|
8
|
+
makeData,
|
|
9
|
+
type RecordBatch,
|
|
10
|
+
Schema,
|
|
11
|
+
Struct,
|
|
12
|
+
Table,
|
|
13
|
+
type Field
|
|
14
|
+
} from 'apache-arrow';
|
|
15
|
+
import {
|
|
16
|
+
getRecordBatchMetadata,
|
|
17
|
+
createRecordBatchWithMetadata,
|
|
18
|
+
validateArrowTableSchema,
|
|
19
|
+
takeArrowTableRows
|
|
20
|
+
} from '@math.gl/core/arrow';
|
|
21
|
+
import type {
|
|
22
|
+
ColumnarPredicateParameterValues,
|
|
23
|
+
ParameterizedColumnarPredicate
|
|
24
|
+
} from '../expressions';
|
|
25
|
+
import type {TableQuery} from '../parser/query-types';
|
|
26
|
+
import {compileArrowArray, compileArrowExpression} from './compiler';
|
|
27
|
+
import {compileArrowPredicate} from './predicate';
|
|
28
|
+
import type {ArrowAggregate} from './aggregates';
|
|
29
|
+
import {
|
|
30
|
+
groupArrowTable,
|
|
31
|
+
joinArrowTables,
|
|
32
|
+
orderArrowTable,
|
|
33
|
+
unionArrowTables,
|
|
34
|
+
type ArrowQueryJoin,
|
|
35
|
+
type ArrowQueryOrder
|
|
36
|
+
} from './relational-query';
|
|
37
|
+
import type {ArrowExpressionOptions, CompiledArrowExpression} from './types';
|
|
38
|
+
|
|
39
|
+
/** A linear Arrow query that can execute incrementally over batches. */
|
|
40
|
+
export type ArrowBatchQuery = TableQuery<ParameterizedColumnarPredicate> &
|
|
41
|
+
Omit<ArrowExpressionOptions, 'inputSchema'> & {
|
|
42
|
+
/** Named values for a parameterized predicate. */
|
|
43
|
+
parameters?: ColumnarPredicateParameterValues;
|
|
44
|
+
/** Checked between batches and during expression evaluation. */
|
|
45
|
+
signal?: AbortSignal;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/** Options specific to streaming Arrow queries. */
|
|
49
|
+
export type ArrowStreamingQuery = ArrowBatchQuery & {
|
|
50
|
+
/** Emit a schema-bearing empty batch when an input batch produces no rows. */
|
|
51
|
+
emitEmptyBatches?: boolean;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/** An in-memory SELECT with optional relational operators. */
|
|
55
|
+
export type ArrowTableQuery = ArrowBatchQuery & {
|
|
56
|
+
/** Append schema-compatible tables before filtering. */
|
|
57
|
+
union?: readonly Table[];
|
|
58
|
+
/** Join a second table before filtering. */
|
|
59
|
+
join?: ArrowQueryJoin;
|
|
60
|
+
/** Scalar source columns that define groups. */
|
|
61
|
+
groupBy?: readonly string[];
|
|
62
|
+
/** Aggregate reductions indexed by output name. */
|
|
63
|
+
aggregates?: Readonly<Record<string, ArrowAggregate>>;
|
|
64
|
+
/** Stable ordering applied after grouping and before pagination. */
|
|
65
|
+
orderBy?: readonly ArrowQueryOrder[];
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
/** A bound output field and either a source column index or a computed expression. */
|
|
69
|
+
type Projection = {field: Field} & ({index: number} | {expression: CompiledArrowExpression});
|
|
70
|
+
|
|
71
|
+
/** Applies WHERE, projection, and pagination directly to a native Apache Arrow table. */
|
|
72
|
+
export function queryArrowTable(source: Table, query: ArrowTableQuery = {}): Table {
|
|
73
|
+
const {select, where, offset = 0, limit, functions, parameters, signal} = query;
|
|
74
|
+
signal?.throwIfAborted();
|
|
75
|
+
validatePagination(offset, limit);
|
|
76
|
+
|
|
77
|
+
const {union, join, groupBy, aggregates, orderBy} = query;
|
|
78
|
+
if (union?.length) source = unionArrowTables(source, union);
|
|
79
|
+
if (join) source = joinArrowTables(source, join, signal);
|
|
80
|
+
|
|
81
|
+
if (groupBy?.length || Object.keys(aggregates ?? {}).length || orderBy?.length) {
|
|
82
|
+
source = queryArrowTable(source, {where, functions, parameters, signal});
|
|
83
|
+
if (groupBy?.length || Object.keys(aggregates ?? {}).length) {
|
|
84
|
+
source = groupArrowTable(source, groupBy ?? [], aggregates ?? {}, {
|
|
85
|
+
inputSchema: source.schema,
|
|
86
|
+
functions,
|
|
87
|
+
signal
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
if (orderBy?.length) {
|
|
91
|
+
source = orderArrowTable(source, orderBy, offset, limit, signal);
|
|
92
|
+
return queryArrowTable(source, {select, functions, signal});
|
|
93
|
+
}
|
|
94
|
+
return queryArrowTable(source, {select, offset, limit, functions, signal});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const plan = compileArrowQuery(source.schema, query);
|
|
98
|
+
const result: RecordBatch[] = [];
|
|
99
|
+
for (const batch of source.batches) {
|
|
100
|
+
if (plan.done()) break;
|
|
101
|
+
for (const output of plan.execute(batch)) result.push(output);
|
|
102
|
+
}
|
|
103
|
+
return new Table(plan.schema, result);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Streams a linear Arrow query with global pagination and source backpressure. */
|
|
107
|
+
export async function* queryArrowTableInBatches(
|
|
108
|
+
input: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
|
|
109
|
+
inputSchema: Schema,
|
|
110
|
+
query: ArrowStreamingQuery = {}
|
|
111
|
+
): AsyncIterable<RecordBatch> {
|
|
112
|
+
query.signal?.throwIfAborted();
|
|
113
|
+
validatePagination(query.offset ?? 0, query.limit);
|
|
114
|
+
const plan = compileArrowQuery(inputSchema, query);
|
|
115
|
+
if (plan.done()) return;
|
|
116
|
+
for await (const batch of input) {
|
|
117
|
+
query.signal?.throwIfAborted();
|
|
118
|
+
if (batch.schema !== inputSchema) {
|
|
119
|
+
validateArrowTableSchema(new Table(batch.schema, []), inputSchema, {
|
|
120
|
+
rejectExtraFields: true,
|
|
121
|
+
schemaName: 'query input schema'
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
yield* plan.execute(batch, query.emitEmptyBatches);
|
|
125
|
+
if (plan.done()) return;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Validates pagination before the query reads or transforms any rows. */
|
|
130
|
+
function validatePagination(offset: number, limit?: number): void {
|
|
131
|
+
if (!Number.isSafeInteger(offset) || offset < 0) throw new Error('Invalid Arrow query offset');
|
|
132
|
+
if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0))
|
|
133
|
+
throw new Error('Invalid Arrow query limit');
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Binds predicates and projection to a schema once per execution. */
|
|
137
|
+
function compileArrowQuery(inputSchema: Schema, query: ArrowBatchQuery) {
|
|
138
|
+
const {select, where, offset = 0, limit, functions, parameters, signal} = query;
|
|
139
|
+
const options = {inputSchema, functions};
|
|
140
|
+
const predicate = where
|
|
141
|
+
? 'expression' in where
|
|
142
|
+
? compileArrowExpression(where.expression, options)
|
|
143
|
+
: compileArrowPredicate(where, {...options, parameters})
|
|
144
|
+
: undefined;
|
|
145
|
+
if (predicate && !DataType.isBool(predicate.outputField.type))
|
|
146
|
+
throw new Error('Arrow query WHERE requires a Boolean expression');
|
|
147
|
+
|
|
148
|
+
const names = new Set<string>();
|
|
149
|
+
for (const selection of select ?? []) {
|
|
150
|
+
const name =
|
|
151
|
+
typeof selection === 'string'
|
|
152
|
+
? selection
|
|
153
|
+
: (selection.as ?? ('column' in selection ? selection.column : undefined));
|
|
154
|
+
if (name === undefined) continue;
|
|
155
|
+
if (names.has(name) || (typeof selection !== 'string' && selection.as === ''))
|
|
156
|
+
throw new Error(`Invalid or duplicate Arrow query output name: ${name}`);
|
|
157
|
+
names.add(name);
|
|
158
|
+
}
|
|
159
|
+
const projection: Projection[] = select
|
|
160
|
+
? select.map((selection, ordinal) => {
|
|
161
|
+
if (typeof selection !== 'string' && ('array' in selection || 'expression' in selection)) {
|
|
162
|
+
const expression =
|
|
163
|
+
'array' in selection
|
|
164
|
+
? compileArrowArray(selection.array, {...options, type: selection.type})
|
|
165
|
+
: compileArrowExpression(selection.expression, options);
|
|
166
|
+
let name = selection.as;
|
|
167
|
+
if (name === undefined) {
|
|
168
|
+
name = `expression_${ordinal + 1}`;
|
|
169
|
+
while (names.has(name)) name += '_';
|
|
170
|
+
names.add(name);
|
|
171
|
+
}
|
|
172
|
+
return {expression, field: expression.outputField.clone({name})};
|
|
173
|
+
}
|
|
174
|
+
const column = typeof selection === 'string' ? selection : selection.column;
|
|
175
|
+
const index = inputSchema.fields.findIndex(field => field.name === column);
|
|
176
|
+
if (index < 0) throw new Error(`Arrow query column not found: ${column}`);
|
|
177
|
+
const field = inputSchema.fields[index]!;
|
|
178
|
+
return {
|
|
179
|
+
index,
|
|
180
|
+
field: typeof selection === 'string' ? field : field.clone({name: selection.as ?? column})
|
|
181
|
+
};
|
|
182
|
+
})
|
|
183
|
+
: inputSchema.fields.map((field, index) => ({field, index}));
|
|
184
|
+
|
|
185
|
+
const schema = select
|
|
186
|
+
? new Schema(
|
|
187
|
+
projection.map(({field}) => field),
|
|
188
|
+
inputSchema.metadata,
|
|
189
|
+
undefined,
|
|
190
|
+
inputSchema.metadataVersion
|
|
191
|
+
)
|
|
192
|
+
: inputSchema;
|
|
193
|
+
let skip = offset;
|
|
194
|
+
let remaining = limit ?? Infinity;
|
|
195
|
+
/** Evaluates one batch using the remaining pagination budget. */
|
|
196
|
+
function* execute(batch: RecordBatch, emitEmpty = false): Iterable<RecordBatch> {
|
|
197
|
+
signal?.throwIfAborted();
|
|
198
|
+
const metadata = getRecordBatchMetadata(batch);
|
|
199
|
+
const mask = predicate?.evaluate(batch, {signal});
|
|
200
|
+
let selected: RecordBatch | undefined;
|
|
201
|
+
if (mask) {
|
|
202
|
+
const indexes: number[] = [];
|
|
203
|
+
for (let row = 0; row < batch.numRows && indexes.length < remaining; row++) {
|
|
204
|
+
if ((row & 1023) === 0) signal?.throwIfAborted();
|
|
205
|
+
if (mask.get(row) !== true) continue;
|
|
206
|
+
if (skip > 0) skip--;
|
|
207
|
+
else indexes.push(row);
|
|
208
|
+
}
|
|
209
|
+
if (indexes.length === batch.numRows) selected = batch;
|
|
210
|
+
else if (
|
|
211
|
+
indexes.length &&
|
|
212
|
+
indexes[indexes.length - 1]! - indexes[0]! + 1 === indexes.length
|
|
213
|
+
) {
|
|
214
|
+
selected = createRecordBatchWithMetadata(
|
|
215
|
+
batch.schema,
|
|
216
|
+
batch.data.slice(indexes[0]!, indexes.length),
|
|
217
|
+
metadata
|
|
218
|
+
);
|
|
219
|
+
} else if (indexes.length) {
|
|
220
|
+
const gathered = takeArrowTableRows(
|
|
221
|
+
new Table(inputSchema, [batch]),
|
|
222
|
+
Int32Array.from(indexes)
|
|
223
|
+
);
|
|
224
|
+
selected = makeQueryBatch(
|
|
225
|
+
inputSchema,
|
|
226
|
+
gathered.batches[0]!.data.children,
|
|
227
|
+
indexes.length,
|
|
228
|
+
metadata
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
} else {
|
|
232
|
+
const start = Math.min(skip, batch.numRows);
|
|
233
|
+
skip -= start;
|
|
234
|
+
const end = start + Math.min(batch.numRows - start, remaining);
|
|
235
|
+
if (end > start)
|
|
236
|
+
selected =
|
|
237
|
+
start === 0 && end === batch.numRows
|
|
238
|
+
? batch
|
|
239
|
+
: createRecordBatchWithMetadata(
|
|
240
|
+
batch.schema,
|
|
241
|
+
batch.data.slice(start, end - start),
|
|
242
|
+
metadata
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
if (!selected?.numRows) {
|
|
246
|
+
if (emitEmpty) {
|
|
247
|
+
const children = schema.fields.map(field => makeData({type: field.type, nullCount: 0}));
|
|
248
|
+
yield makeQueryBatch(schema, children, 0, metadata);
|
|
249
|
+
}
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
remaining -= selected.numRows;
|
|
253
|
+
if (!select) {
|
|
254
|
+
yield selected;
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const columns = projection.map(item =>
|
|
259
|
+
'index' in item
|
|
260
|
+
? selected.data.children[item.index]!
|
|
261
|
+
: item.expression.evaluate(selected, {signal})
|
|
262
|
+
);
|
|
263
|
+
// Computed vectors can introduce chunk boundaries inside the selected source batch.
|
|
264
|
+
const boundaries = new Set([selected.numRows]);
|
|
265
|
+
for (const column of columns) {
|
|
266
|
+
if ('children' in column) continue;
|
|
267
|
+
let end = 0;
|
|
268
|
+
for (const chunk of column.data) {
|
|
269
|
+
end += chunk.length;
|
|
270
|
+
if (end > 0) boundaries.add(end);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
let start = 0;
|
|
274
|
+
for (const end of [...boundaries].sort((left, right) => left - right)) {
|
|
275
|
+
const children = columns.map(column =>
|
|
276
|
+
'children' in column
|
|
277
|
+
? start === 0 && end === column.length
|
|
278
|
+
? column
|
|
279
|
+
: column.slice(start, end - start)
|
|
280
|
+
: column.slice(start, end).data[0]!
|
|
281
|
+
);
|
|
282
|
+
yield makeQueryBatch(schema, children, end - start, metadata);
|
|
283
|
+
start = end;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return {schema, done: () => remaining === 0, execute};
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Creates a batch while preserving its row count even for an empty projection. */
|
|
290
|
+
function makeQueryBatch(
|
|
291
|
+
schema: Schema,
|
|
292
|
+
children: Data[],
|
|
293
|
+
length: number,
|
|
294
|
+
metadata: Map<string, string> | undefined
|
|
295
|
+
): RecordBatch {
|
|
296
|
+
const data = new Data(new Struct(schema.fields), 0, length, 0, undefined, children);
|
|
297
|
+
return createRecordBatchWithMetadata(schema, data, metadata);
|
|
298
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {initializeTaskWorker} from '@math.gl/tasks';
|
|
6
|
+
import {createArrowQueryTask} from './query-worker';
|
|
7
|
+
|
|
8
|
+
void initializeTaskWorker(createArrowQueryTask());
|