@math.gl/sql 0.0.0-stage → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +249 -0
- package/README.md +26 -2
- package/THIRD-PARTY-NOTICES.md +5 -0
- package/dist/arrow/aggregates.d.ts +46 -0
- package/dist/arrow/aggregates.d.ts.map +1 -0
- package/dist/arrow/aggregates.js +289 -0
- package/dist/arrow/aggregates.js.map +1 -0
- package/dist/arrow/buffers.d.ts +30 -0
- package/dist/arrow/buffers.d.ts.map +1 -0
- package/dist/arrow/buffers.js +177 -0
- package/dist/arrow/buffers.js.map +1 -0
- package/dist/arrow/cast.d.ts +19 -0
- package/dist/arrow/cast.d.ts.map +1 -0
- package/dist/arrow/cast.js +138 -0
- package/dist/arrow/cast.js.map +1 -0
- package/dist/arrow/compiler.d.ts +10 -0
- package/dist/arrow/compiler.d.ts.map +1 -0
- package/dist/arrow/compiler.js +1421 -0
- package/dist/arrow/compiler.js.map +1 -0
- package/dist/arrow/float16.d.ts +7 -0
- package/dist/arrow/float16.d.ts.map +1 -0
- package/dist/arrow/float16.js +26 -0
- package/dist/arrow/float16.js.map +1 -0
- package/dist/arrow/index.cjs +3220 -0
- package/dist/arrow/index.cjs.map +6 -0
- package/dist/arrow/index.d.ts +5 -0
- package/dist/arrow/index.d.ts.map +1 -0
- package/dist/arrow/index.js +7 -0
- package/dist/arrow/index.js.map +1 -0
- package/dist/arrow/like.d.ts +3 -0
- package/dist/arrow/like.d.ts.map +1 -0
- package/dist/arrow/like.js +75 -0
- package/dist/arrow/like.js.map +1 -0
- package/dist/arrow/list-values.d.ts +14 -0
- package/dist/arrow/list-values.d.ts.map +1 -0
- package/dist/arrow/list-values.js +180 -0
- package/dist/arrow/list-values.js.map +1 -0
- package/dist/arrow/predicate.d.ts +7 -0
- package/dist/arrow/predicate.d.ts.map +1 -0
- package/dist/arrow/predicate.js +10 -0
- package/dist/arrow/predicate.js.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts +24 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js +64 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js.map +1 -0
- package/dist/arrow/query-arrow-sql.d.ts +8 -0
- package/dist/arrow/query-arrow-sql.d.ts.map +1 -0
- package/dist/arrow/query-arrow-sql.js +191 -0
- package/dist/arrow/query-arrow-sql.js.map +1 -0
- package/dist/arrow/query-arrow-table.d.ts +36 -0
- package/dist/arrow/query-arrow-table.d.ts.map +1 -0
- package/dist/arrow/query-arrow-table.js +208 -0
- package/dist/arrow/query-arrow-table.js.map +1 -0
- package/dist/arrow/query-worker-entry.d.ts +2 -0
- package/dist/arrow/query-worker-entry.d.ts.map +1 -0
- package/dist/arrow/query-worker-entry.js +7 -0
- package/dist/arrow/query-worker-entry.js.map +1 -0
- package/dist/arrow/query-worker.d.ts +65 -0
- package/dist/arrow/query-worker.d.ts.map +1 -0
- package/dist/arrow/query-worker.js +135 -0
- package/dist/arrow/query-worker.js.map +1 -0
- package/dist/arrow/registry.d.ts +4 -0
- package/dist/arrow/registry.d.ts.map +1 -0
- package/dist/arrow/registry.js +65 -0
- package/dist/arrow/registry.js.map +1 -0
- package/dist/arrow/relational-query.d.ts +36 -0
- package/dist/arrow/relational-query.d.ts.map +1 -0
- package/dist/arrow/relational-query.js +174 -0
- package/dist/arrow/relational-query.js.map +1 -0
- package/dist/arrow/replace-arrow-columns.d.ts +8 -0
- package/dist/arrow/replace-arrow-columns.d.ts.map +1 -0
- package/dist/arrow/replace-arrow-columns.js +63 -0
- package/dist/arrow/replace-arrow-columns.js.map +1 -0
- package/dist/arrow/signature.d.ts +6 -0
- package/dist/arrow/signature.d.ts.map +1 -0
- package/dist/arrow/signature.js +51 -0
- package/dist/arrow/signature.js.map +1 -0
- package/dist/arrow/temporal-cast.d.ts +9 -0
- package/dist/arrow/temporal-cast.d.ts.map +1 -0
- package/dist/arrow/temporal-cast.js +60 -0
- package/dist/arrow/temporal-cast.js.map +1 -0
- package/dist/arrow/types.d.ts +62 -0
- package/dist/arrow/types.d.ts.map +1 -0
- package/dist/arrow/types.js +5 -0
- package/dist/arrow/types.js.map +1 -0
- package/dist/arrow/worker-entry.d.ts +2 -0
- package/dist/arrow/worker-entry.d.ts.map +1 -0
- package/dist/arrow/worker-entry.js +9 -0
- package/dist/arrow/worker-entry.js.map +1 -0
- package/dist/arrow/worker.cjs +3973 -0
- package/dist/arrow/worker.cjs.map +6 -0
- package/dist/arrow/worker.d.ts +58 -0
- package/dist/arrow/worker.d.ts.map +1 -0
- package/dist/arrow/worker.js +84 -0
- package/dist/arrow/worker.js.map +1 -0
- package/dist/color/index.cjs +382 -0
- package/dist/color/index.cjs.map +6 -0
- package/dist/color/index.d.ts +26 -0
- package/dist/color/index.d.ts.map +1 -0
- package/dist/color/index.js +306 -0
- package/dist/color/index.js.map +1 -0
- package/dist/diagnostic-log.d.ts +8 -0
- package/dist/diagnostic-log.d.ts.map +1 -0
- package/dist/diagnostic-log.js +15 -0
- package/dist/diagnostic-log.js.map +1 -0
- package/dist/expressions/ast.d.ts +12 -0
- package/dist/expressions/ast.d.ts.map +1 -0
- package/dist/expressions/ast.js +112 -0
- package/dist/expressions/ast.js.map +1 -0
- package/dist/expressions/columnar-predicate.d.ts +56 -0
- package/dist/expressions/columnar-predicate.d.ts.map +1 -0
- package/dist/expressions/columnar-predicate.js +176 -0
- package/dist/expressions/columnar-predicate.js.map +1 -0
- package/dist/expressions/evaluation-error.d.ts +14 -0
- package/dist/expressions/evaluation-error.d.ts.map +1 -0
- package/dist/expressions/evaluation-error.js +26 -0
- package/dist/expressions/evaluation-error.js.map +1 -0
- package/dist/expressions/expression-eval.d.ts +96 -0
- package/dist/expressions/expression-eval.d.ts.map +1 -0
- package/dist/expressions/expression-eval.js +330 -0
- package/dist/expressions/expression-eval.js.map +1 -0
- package/dist/expressions/function-libraries.d.ts +63 -0
- package/dist/expressions/function-libraries.d.ts.map +1 -0
- package/dist/expressions/function-libraries.js +69 -0
- package/dist/expressions/function-libraries.js.map +1 -0
- package/dist/expressions/function-registry.d.ts +93 -0
- package/dist/expressions/function-registry.d.ts.map +1 -0
- package/dist/expressions/function-registry.js +139 -0
- package/dist/expressions/function-registry.js.map +1 -0
- package/dist/expressions/get.d.ts +9 -0
- package/dist/expressions/get.d.ts.map +1 -0
- package/dist/expressions/get.js +32 -0
- package/dist/expressions/get.js.map +1 -0
- package/dist/expressions/index.cjs +1888 -0
- package/dist/expressions/index.cjs.map +6 -0
- package/dist/expressions/index.d.ts +13 -0
- package/dist/expressions/index.d.ts.map +1 -0
- package/dist/expressions/index.js +14 -0
- package/dist/expressions/index.js.map +1 -0
- package/dist/expressions/numeric-functions.d.ts +277 -0
- package/dist/expressions/numeric-functions.d.ts.map +1 -0
- package/dist/expressions/numeric-functions.js +52 -0
- package/dist/expressions/numeric-functions.js.map +1 -0
- package/dist/expressions/parse-expression-string.d.ts +21 -0
- package/dist/expressions/parse-expression-string.d.ts.map +1 -0
- package/dist/expressions/parse-expression-string.js +61 -0
- package/dist/expressions/parse-expression-string.js.map +1 -0
- package/dist/expressions/predicate-expression.d.ts +7 -0
- package/dist/expressions/predicate-expression.d.ts.map +1 -0
- package/dist/expressions/predicate-expression.js +112 -0
- package/dist/expressions/predicate-expression.js.map +1 -0
- package/dist/expressions/scalar-type.d.ts +6 -0
- package/dist/expressions/scalar-type.d.ts.map +1 -0
- package/dist/expressions/scalar-type.js +23 -0
- package/dist/expressions/scalar-type.js.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts +3 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.js +972 -0
- package/dist/expressions/vendor/jsep/runtime.js.map +1 -0
- package/dist/expressions/vendor/jsep/types.d.ts +148 -0
- package/dist/expressions/vendor/jsep/types.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/types.js +6 -0
- package/dist/expressions/vendor/jsep/types.js.map +1 -0
- package/dist/index.cjs +4865 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/math/index.cjs +444 -0
- package/dist/math/index.cjs.map +6 -0
- package/dist/math/index.d.ts +44 -0
- package/dist/math/index.d.ts.map +1 -0
- package/dist/math/index.js +182 -0
- package/dist/math/index.js.map +1 -0
- package/dist/scales/arrow-domains.d.ts +15 -0
- package/dist/scales/arrow-domains.d.ts.map +1 -0
- package/dist/scales/arrow-domains.js +124 -0
- package/dist/scales/arrow-domains.js.map +1 -0
- package/dist/scales/arrow-scales.cjs +3837 -0
- package/dist/scales/arrow-scales.cjs.map +6 -0
- package/dist/scales/arrow-scales.d.ts +20 -0
- package/dist/scales/arrow-scales.d.ts.map +1 -0
- package/dist/scales/arrow-scales.js +109 -0
- package/dist/scales/arrow-scales.js.map +1 -0
- package/dist/scales/compile-scale.d.ts +7 -0
- package/dist/scales/compile-scale.d.ts.map +1 -0
- package/dist/scales/compile-scale.js +195 -0
- package/dist/scales/compile-scale.js.map +1 -0
- package/dist/scales/hdr-colors.d.ts +13 -0
- package/dist/scales/hdr-colors.d.ts.map +1 -0
- package/dist/scales/hdr-colors.js +115 -0
- package/dist/scales/hdr-colors.js.map +1 -0
- package/dist/scales/index.cjs +1214 -0
- package/dist/scales/index.cjs.map +6 -0
- package/dist/scales/index.d.ts +9 -0
- package/dist/scales/index.d.ts.map +1 -0
- package/dist/scales/index.js +10 -0
- package/dist/scales/index.js.map +1 -0
- package/dist/scales/palettes.d.ts +5 -0
- package/dist/scales/palettes.d.ts.map +1 -0
- package/dist/scales/palettes.js +85 -0
- package/dist/scales/palettes.js.map +1 -0
- package/dist/scales/parse-sql-scale.d.ts +14 -0
- package/dist/scales/parse-sql-scale.d.ts.map +1 -0
- package/dist/scales/parse-sql-scale.js +165 -0
- package/dist/scales/parse-sql-scale.js.map +1 -0
- package/dist/scales/quantile-sketch.d.ts +22 -0
- package/dist/scales/quantile-sketch.d.ts.map +1 -0
- package/dist/scales/quantile-sketch.js +108 -0
- package/dist/scales/quantile-sketch.js.map +1 -0
- package/dist/scales/scale-types.d.ts +92 -0
- package/dist/scales/scale-types.d.ts.map +1 -0
- package/dist/scales/scale-types.js +119 -0
- package/dist/scales/scale-types.js.map +1 -0
- package/dist/scales/streaming-domains.d.ts +18 -0
- package/dist/scales/streaming-domains.d.ts.map +1 -0
- package/dist/scales/streaming-domains.js +109 -0
- package/dist/scales/streaming-domains.js.map +1 -0
- package/dist/sql/bind-sql-predicate.d.ts +4 -0
- package/dist/sql/bind-sql-predicate.d.ts.map +1 -0
- package/dist/sql/bind-sql-predicate.js +11 -0
- package/dist/sql/bind-sql-predicate.js.map +1 -0
- package/dist/sql/index.cjs +1588 -0
- package/dist/sql/index.cjs.map +6 -0
- package/dist/sql/index.d.ts +10 -0
- package/dist/sql/index.d.ts.map +1 -0
- package/dist/sql/index.js +10 -0
- package/dist/sql/index.js.map +1 -0
- package/dist/sql/parse-sql-expression.d.ts +44 -0
- package/dist/sql/parse-sql-expression.d.ts.map +1 -0
- package/dist/sql/parse-sql-expression.js +455 -0
- package/dist/sql/parse-sql-expression.js.map +1 -0
- package/dist/sql/parse-sql-predicate.d.ts +13 -0
- package/dist/sql/parse-sql-predicate.d.ts.map +1 -0
- package/dist/sql/parse-sql-predicate.js +365 -0
- package/dist/sql/parse-sql-predicate.js.map +1 -0
- package/dist/sql/parse-sql-query.d.ts +8 -0
- package/dist/sql/parse-sql-query.d.ts.map +1 -0
- package/dist/sql/parse-sql-query.js +66 -0
- package/dist/sql/parse-sql-query.js.map +1 -0
- package/dist/sql/parse-sql-select.d.ts +55 -0
- package/dist/sql/parse-sql-select.d.ts.map +1 -0
- package/dist/sql/parse-sql-select.js +244 -0
- package/dist/sql/parse-sql-select.js.map +1 -0
- package/dist/sql/parse-sql-statement.d.ts +50 -0
- package/dist/sql/parse-sql-statement.d.ts.map +1 -0
- package/dist/sql/parse-sql-statement.js +182 -0
- package/dist/sql/parse-sql-statement.js.map +1 -0
- package/dist/sql/query-types.d.ts +56 -0
- package/dist/sql/query-types.d.ts.map +1 -0
- package/dist/sql/query-types.js +5 -0
- package/dist/sql/query-types.js.map +1 -0
- package/dist/sql/sql-parse-error.d.ts +13 -0
- package/dist/sql/sql-parse-error.d.ts.map +1 -0
- package/dist/sql/sql-parse-error.js +20 -0
- package/dist/sql/sql-parse-error.js.map +1 -0
- package/dist/sql/sql-predicate-schema.d.ts +164 -0
- package/dist/sql/sql-predicate-schema.d.ts.map +1 -0
- package/dist/sql/sql-predicate-schema.js +249 -0
- package/dist/sql/sql-predicate-schema.js.map +1 -0
- package/dist/sql/sql-predicate-types.d.ts +48 -0
- package/dist/sql/sql-predicate-types.d.ts.map +1 -0
- package/dist/sql/sql-predicate-types.js +9 -0
- package/dist/sql/sql-predicate-types.js.map +1 -0
- package/dist/sql/sql-predicate.d.ts +6 -0
- package/dist/sql/sql-predicate.d.ts.map +1 -0
- package/dist/sql/sql-predicate.js +8 -0
- package/dist/sql/sql-predicate.js.map +1 -0
- package/package.json +92 -4
- package/src/arrow/aggregates.ts +386 -0
- package/src/arrow/buffers.ts +177 -0
- package/src/arrow/cast.ts +160 -0
- package/src/arrow/compiler.ts +1659 -0
- package/src/arrow/float16.ts +25 -0
- package/src/arrow/index.ts +22 -0
- package/src/arrow/like.ts +72 -0
- package/src/arrow/list-values.ts +200 -0
- package/src/arrow/predicate.ts +19 -0
- package/src/arrow/query-arrow-aggregate-snapshots.ts +107 -0
- package/src/arrow/query-arrow-sql.ts +227 -0
- package/src/arrow/query-arrow-table.ts +298 -0
- package/src/arrow/query-worker-entry.ts +8 -0
- package/src/arrow/query-worker.ts +190 -0
- package/src/arrow/registry.ts +86 -0
- package/src/arrow/relational-query.ts +247 -0
- package/src/arrow/replace-arrow-columns.ts +71 -0
- package/src/arrow/signature.ts +50 -0
- package/src/arrow/temporal-cast.ts +66 -0
- package/src/arrow/types.ts +76 -0
- package/src/arrow/worker-entry.ts +10 -0
- package/src/arrow/worker.ts +139 -0
- package/src/color/index.ts +405 -0
- package/src/diagnostic-log.ts +20 -0
- package/src/expressions/ast.ts +141 -0
- package/src/expressions/columnar-predicate.ts +293 -0
- package/src/expressions/evaluation-error.ts +29 -0
- package/src/expressions/expression-eval.ts +480 -0
- package/src/expressions/function-libraries.ts +126 -0
- package/src/expressions/function-registry.ts +171 -0
- package/src/expressions/get.ts +37 -0
- package/src/expressions/index.ts +40 -0
- package/src/expressions/numeric-functions.ts +99 -0
- package/src/expressions/parse-expression-string.ts +80 -0
- package/src/expressions/predicate-expression.ts +128 -0
- package/src/expressions/scalar-type.ts +39 -0
- package/src/expressions/vendor/jsep/LICENSE.txt +23 -0
- package/src/expressions/vendor/jsep/README.md +22 -0
- package/src/expressions/vendor/jsep/runtime.ts +949 -0
- package/src/expressions/vendor/jsep/types.ts +191 -0
- package/src/index.ts +40 -0
- package/src/math/index.ts +235 -0
- package/src/scales/arrow-domains.ts +131 -0
- package/src/scales/arrow-scales.ts +156 -0
- package/src/scales/compile-scale.ts +217 -0
- package/src/scales/hdr-colors.ts +126 -0
- package/src/scales/index.ts +35 -0
- package/src/scales/palettes.ts +86 -0
- package/src/scales/parse-sql-scale.ts +195 -0
- package/src/scales/quantile-sketch.ts +106 -0
- package/src/scales/scale-types.ts +224 -0
- package/src/scales/streaming-domains.ts +133 -0
- package/src/sql/bind-sql-predicate.ts +21 -0
- package/src/sql/index.ts +28 -0
- package/src/sql/parse-sql-expression.ts +457 -0
- package/src/sql/parse-sql-predicate.ts +454 -0
- package/src/sql/parse-sql-query.ts +72 -0
- package/src/sql/parse-sql-select.ts +266 -0
- package/src/sql/parse-sql-statement.ts +242 -0
- package/src/sql/query-types.ts +71 -0
- package/src/sql/sql-parse-error.ts +33 -0
- package/src/sql/sql-predicate-schema.ts +279 -0
- package/src/sql/sql-predicate-types.ts +91 -0
- package/src/sql/sql-predicate.ts +28 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
DataType,
|
|
7
|
+
Field,
|
|
8
|
+
Float64,
|
|
9
|
+
Int64,
|
|
10
|
+
Schema,
|
|
11
|
+
Table,
|
|
12
|
+
vectorFromArray,
|
|
13
|
+
type Vector
|
|
14
|
+
} from 'apache-arrow';
|
|
15
|
+
import {takeArrowTableRows, validateArrowTableSchema} from '@math.gl/arrow';
|
|
16
|
+
import {compileArrowAggregates, type ArrowAggregate} from './aggregates';
|
|
17
|
+
import {compareUTF8} from '@math.gl/arrow/utf8';
|
|
18
|
+
import {ColumnReader, valueType} from './buffers';
|
|
19
|
+
import {compileArrowExpression} from './compiler';
|
|
20
|
+
import type {ArrowExpressionOptions, ArrowExpressionValue} from './types';
|
|
21
|
+
|
|
22
|
+
/** A stable ordering key resolved against the source or grouped result. */
|
|
23
|
+
export type ArrowQueryOrder = {
|
|
24
|
+
/** Column name before SELECT aliases are applied. */
|
|
25
|
+
column: string;
|
|
26
|
+
/** Sort direction, ascending by default. */
|
|
27
|
+
direction?: 'asc' | 'desc';
|
|
28
|
+
/** Explicit null placement, last by default. */
|
|
29
|
+
nulls?: 'first' | 'last';
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** An inner equi-join with a prefix for right-side output names. */
|
|
33
|
+
export type ArrowQueryJoin = {
|
|
34
|
+
/** Native Arrow right input. */
|
|
35
|
+
table: Table;
|
|
36
|
+
/** Left key column. */
|
|
37
|
+
left: string;
|
|
38
|
+
/** Right key column. */
|
|
39
|
+
right: string;
|
|
40
|
+
/** Prefix including any separator, for example `products.`. */
|
|
41
|
+
rightPrefix: string;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** Appends inputs with matching schemas in input order. */
|
|
45
|
+
export function unionArrowTables(source: Table, inputs: readonly Table[]): Table {
|
|
46
|
+
for (const table of inputs) {
|
|
47
|
+
validateArrowTableSchema(table, source.schema, {
|
|
48
|
+
rejectExtraFields: true,
|
|
49
|
+
schemaName: 'UNION ALL'
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
return new Table(
|
|
53
|
+
source.schema,
|
|
54
|
+
[source, ...inputs].flatMap(table => table.batches)
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Gathers matching row pairs in left row order and then right row order. */
|
|
59
|
+
export function joinArrowTables(left: Table, join: ArrowQueryJoin, signal?: AbortSignal): Table {
|
|
60
|
+
const right = join.table;
|
|
61
|
+
const {vector: leftKey} = getScalarColumn(left, join.left);
|
|
62
|
+
const {vector: rightKey} = getScalarColumn(right, join.right);
|
|
63
|
+
const fields = [
|
|
64
|
+
...left.schema.fields,
|
|
65
|
+
...right.schema.fields.map(field => field.clone({name: join.rightPrefix + field.name}))
|
|
66
|
+
];
|
|
67
|
+
if (new Set(fields.map(field => field.name)).size !== fields.length)
|
|
68
|
+
throw new Error('Duplicate Arrow join output name');
|
|
69
|
+
const schema = new Schema(fields, left.schema.metadata, undefined, left.schema.metadataVersion);
|
|
70
|
+
|
|
71
|
+
const rightRows = new Map<ArrowExpressionValue, number[]>();
|
|
72
|
+
for (let row = 0; row < right.numRows; row++) {
|
|
73
|
+
signal?.throwIfAborted();
|
|
74
|
+
const key = normalizeJoinKey(rightKey!.get(row));
|
|
75
|
+
if (key === null || (typeof key === 'number' && Number.isNaN(key))) continue;
|
|
76
|
+
const matches = rightRows.get(key);
|
|
77
|
+
if (matches) matches.push(row);
|
|
78
|
+
else rightRows.set(key, [row]);
|
|
79
|
+
}
|
|
80
|
+
const leftIndexes: number[] = [];
|
|
81
|
+
const rightIndexes: number[] = [];
|
|
82
|
+
for (let row = 0; row < left.numRows; row++) {
|
|
83
|
+
signal?.throwIfAborted();
|
|
84
|
+
const key = normalizeJoinKey(leftKey!.get(row));
|
|
85
|
+
for (const match of rightRows.get(key) ?? []) {
|
|
86
|
+
leftIndexes.push(row);
|
|
87
|
+
rightIndexes.push(match);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (!leftIndexes.length) return new Table(schema, []);
|
|
91
|
+
|
|
92
|
+
const tables = [
|
|
93
|
+
takeArrowTableRows(left, Int32Array.from(leftIndexes)),
|
|
94
|
+
takeArrowTableRows(right, Int32Array.from(rightIndexes))
|
|
95
|
+
];
|
|
96
|
+
const columns = tables.flatMap(table =>
|
|
97
|
+
table.schema.fields.map((_, index) => table.getChildAt(index)!)
|
|
98
|
+
);
|
|
99
|
+
return new Table(
|
|
100
|
+
schema,
|
|
101
|
+
Object.fromEntries(fields.map((field, index) => [field.name, columns[index]!]))
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Canonicalizes numeric keys without narrowing 64-bit integers to Number. */
|
|
106
|
+
function normalizeJoinKey(value: ArrowExpressionValue): ArrowExpressionValue {
|
|
107
|
+
return typeof value === 'number' && Number.isInteger(value) ? BigInt(value) : value;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Sorts stably by scalar keys and gathers only the requested rows. */
|
|
111
|
+
export function orderArrowTable(
|
|
112
|
+
table: Table,
|
|
113
|
+
orderBy: readonly ArrowQueryOrder[],
|
|
114
|
+
offset: number,
|
|
115
|
+
limit: number | undefined,
|
|
116
|
+
signal?: AbortSignal
|
|
117
|
+
): Table {
|
|
118
|
+
const keys = orderBy.map(key => {
|
|
119
|
+
const {field, vector} = getScalarColumn(table, key.column);
|
|
120
|
+
return {
|
|
121
|
+
...key,
|
|
122
|
+
reader: vector && new ColumnReader(vector),
|
|
123
|
+
utf8: DataType.isUtf8(valueType(field.type))
|
|
124
|
+
};
|
|
125
|
+
});
|
|
126
|
+
const indexes = Array.from({length: table.numRows}, (_, index) => index);
|
|
127
|
+
indexes.sort((left, right) => {
|
|
128
|
+
signal?.throwIfAborted();
|
|
129
|
+
for (const {reader, utf8, direction, nulls} of keys) {
|
|
130
|
+
const leftValid = reader!.isValid(left);
|
|
131
|
+
const rightValid = reader!.isValid(right);
|
|
132
|
+
if (!leftValid || !rightValid) {
|
|
133
|
+
if (leftValid === rightValid) continue;
|
|
134
|
+
return (leftValid ? 1 : -1) * (nulls === 'first' ? 1 : -1);
|
|
135
|
+
}
|
|
136
|
+
let order: number;
|
|
137
|
+
if (utf8) {
|
|
138
|
+
const [a, start, end] = reader!.utf8(left);
|
|
139
|
+
order = compareUTF8(a, start, end, ...reader!.utf8(right));
|
|
140
|
+
} else {
|
|
141
|
+
const a = reader!.get(left)!;
|
|
142
|
+
const b = reader!.get(right)!;
|
|
143
|
+
order = a < b ? -1 : a > b ? 1 : Number(Number.isNaN(a)) - Number(Number.isNaN(b));
|
|
144
|
+
}
|
|
145
|
+
if (order) return direction === 'desc' ? -order : order;
|
|
146
|
+
}
|
|
147
|
+
return 0;
|
|
148
|
+
});
|
|
149
|
+
return takeArrowTableRows(
|
|
150
|
+
table,
|
|
151
|
+
Int32Array.from(indexes.slice(offset, limit === undefined ? undefined : offset + limit))
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Groups by scalar source columns and evaluates the shared Arrow aggregate kernels. */
|
|
156
|
+
export function groupArrowTable(
|
|
157
|
+
table: Table,
|
|
158
|
+
groupBy: readonly string[],
|
|
159
|
+
aggregates: Readonly<Record<string, ArrowAggregate>>,
|
|
160
|
+
options: ArrowExpressionOptions & {signal?: AbortSignal}
|
|
161
|
+
): Table {
|
|
162
|
+
const keys = groupBy.map(name => getScalarColumn(table, name));
|
|
163
|
+
const fields = keys.map(({field}) => field);
|
|
164
|
+
const definitions = Object.entries(aggregates);
|
|
165
|
+
const names = [...groupBy, ...definitions.map(([name]) => name)];
|
|
166
|
+
if (new Set(names).size !== names.length) throw new Error('Duplicate Arrow grouping output name');
|
|
167
|
+
|
|
168
|
+
const compiled = compileArrowAggregates(aggregates, options);
|
|
169
|
+
const groups = new Map<string, number[]>();
|
|
170
|
+
if (!groupBy.length) groups.set('[]', []);
|
|
171
|
+
for (let row = 0; groupBy.length && row < table.numRows; row++) {
|
|
172
|
+
options.signal?.throwIfAborted();
|
|
173
|
+
const key = JSON.stringify(
|
|
174
|
+
keys.map(({vector}) => {
|
|
175
|
+
const value = vector!.get(row);
|
|
176
|
+
return [typeof value, String(value)];
|
|
177
|
+
})
|
|
178
|
+
);
|
|
179
|
+
const rows = groups.get(key);
|
|
180
|
+
if (rows) rows.push(row);
|
|
181
|
+
else groups.set(key, [row]);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const rows = [...groups.values()];
|
|
185
|
+
const representatives = rows.flatMap(indexes => (indexes.length ? [indexes[0]!] : []));
|
|
186
|
+
const selected = takeArrowTableRows(table, Int32Array.from(representatives));
|
|
187
|
+
const results = rows.map(indexes =>
|
|
188
|
+
compiled.evaluate(
|
|
189
|
+
groupBy.length ? takeArrowTableRows(table, Int32Array.from(indexes)) : table,
|
|
190
|
+
options
|
|
191
|
+
)
|
|
192
|
+
);
|
|
193
|
+
const aggregateFields = getArrowAggregateFields(aggregates, options);
|
|
194
|
+
const schema = new Schema([...fields, ...aggregateFields], table.schema.metadata);
|
|
195
|
+
if (!results.length) return new Table(schema, []);
|
|
196
|
+
const columns = [
|
|
197
|
+
...fields.map(field => selected.getChild(field.name)!),
|
|
198
|
+
...aggregateFields.map(field =>
|
|
199
|
+
vectorFromArray(
|
|
200
|
+
results.map(result => result[field.name]),
|
|
201
|
+
field.type
|
|
202
|
+
)
|
|
203
|
+
)
|
|
204
|
+
];
|
|
205
|
+
return new Table(schema, Object.fromEntries(names.map((name, index) => [name, columns[index]!])));
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Derives output fields consistently for materialized and incremental aggregates. */
|
|
209
|
+
export function getArrowAggregateFields(
|
|
210
|
+
aggregates: Readonly<Record<string, ArrowAggregate>>,
|
|
211
|
+
options: ArrowExpressionOptions
|
|
212
|
+
): Field[] {
|
|
213
|
+
return Object.entries(aggregates).map(([name, aggregate]) => {
|
|
214
|
+
const expression =
|
|
215
|
+
aggregate.expression === undefined
|
|
216
|
+
? undefined
|
|
217
|
+
: compileArrowExpression(aggregate.expression, options);
|
|
218
|
+
const inputType = expression && valueType(expression.outputField.type);
|
|
219
|
+
const type =
|
|
220
|
+
aggregate.operation === 'COUNT' ||
|
|
221
|
+
(aggregate.operation === 'SUM' && inputType && DataType.isInt(inputType))
|
|
222
|
+
? new Int64()
|
|
223
|
+
: aggregate.operation === 'AVG' || !inputType
|
|
224
|
+
? new Float64()
|
|
225
|
+
: inputType;
|
|
226
|
+
return new Field(name, type, aggregate.operation !== 'COUNT');
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Validates scalar key types consistently even when there are no rows. */
|
|
231
|
+
function getScalarColumn(table: Table, name: string): {field: Field; vector: Vector | null} {
|
|
232
|
+
const field = table.schema.fields.find(field => field.name === name);
|
|
233
|
+
if (!field) throw new Error(`Arrow query column not found: ${name}`);
|
|
234
|
+
const type = valueType(field.type);
|
|
235
|
+
if (
|
|
236
|
+
!(
|
|
237
|
+
DataType.isNull(type) ||
|
|
238
|
+
DataType.isBool(type) ||
|
|
239
|
+
DataType.isInt(type) ||
|
|
240
|
+
DataType.isFloat(type) ||
|
|
241
|
+
DataType.isUtf8(type)
|
|
242
|
+
)
|
|
243
|
+
) {
|
|
244
|
+
throw new Error(`Unsupported Arrow relational key type: ${type}`);
|
|
245
|
+
}
|
|
246
|
+
return {field, vector: table.getChild(name)};
|
|
247
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
import {RecordBatch, Schema, Struct, Table} from 'apache-arrow';
|
|
5
|
+
import {compileArrowExpression} from './compiler';
|
|
6
|
+
import type {ArrowExpression, ArrowExpressionOptions} from './types';
|
|
7
|
+
|
|
8
|
+
/** Replace named columns in their original slots. Each expression reads the same input table. */
|
|
9
|
+
export function replaceArrowColumns(
|
|
10
|
+
table: Table,
|
|
11
|
+
replacements: readonly Readonly<{as: string; expression: ArrowExpression}>[],
|
|
12
|
+
options: Omit<ArrowExpressionOptions, 'inputSchema'> = {}
|
|
13
|
+
): Table {
|
|
14
|
+
if (!replacements.length) return table;
|
|
15
|
+
const fields = [...table.schema.fields];
|
|
16
|
+
const indices = new Map(fields.map((field, index) => [field.name, index]));
|
|
17
|
+
if (indices.size !== fields.length) throw new Error('Arrow input column names must be unique');
|
|
18
|
+
const seen = new Set<string>();
|
|
19
|
+
const plan = replacements.map(({as, expression}) => {
|
|
20
|
+
const index = indices.get(as);
|
|
21
|
+
if (index === undefined) throw new Error(`Unknown Arrow replacement column: ${as}`);
|
|
22
|
+
if (seen.has(as)) throw new Error(`Duplicate Arrow replacement column: ${as}`);
|
|
23
|
+
seen.add(as);
|
|
24
|
+
return {
|
|
25
|
+
as,
|
|
26
|
+
index,
|
|
27
|
+
compiled: compileArrowExpression(expression, {...options, inputSchema: table.schema})
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
for (const {as, index, compiled} of plan) {
|
|
32
|
+
fields[index] = compiled.outputField.clone({name: as});
|
|
33
|
+
}
|
|
34
|
+
const schema = new Schema(fields, table.schema.metadata, undefined, table.schema.metadataVersion);
|
|
35
|
+
const struct = new Struct(fields);
|
|
36
|
+
const batches = table.batches.flatMap(batch => {
|
|
37
|
+
const columns = plan.map(({index, compiled}) => ({index, vector: compiled.evaluate(batch)}));
|
|
38
|
+
// Vector kernels can return multiple chunks. Align the output batches at each boundary.
|
|
39
|
+
const boundaries = new Set([batch.numRows]);
|
|
40
|
+
for (const {vector} of columns) {
|
|
41
|
+
let end = 0;
|
|
42
|
+
for (const chunk of vector.data) {
|
|
43
|
+
end += chunk.length;
|
|
44
|
+
if (end > 0) boundaries.add(end);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
let start = 0;
|
|
48
|
+
return [...boundaries]
|
|
49
|
+
.sort((a, b) => a - b)
|
|
50
|
+
.map(end => {
|
|
51
|
+
const wholeBatch = start === 0 && end === batch.numRows;
|
|
52
|
+
const source = wholeBatch ? batch.data : batch.data.slice(start, end - start);
|
|
53
|
+
const children = [...source.children];
|
|
54
|
+
for (const {index, vector} of columns) {
|
|
55
|
+
children[index] = (wholeBatch ? vector : vector.slice(start, end)).data[0]!;
|
|
56
|
+
}
|
|
57
|
+
start = end;
|
|
58
|
+
const data = source.clone(
|
|
59
|
+
struct,
|
|
60
|
+
source.offset,
|
|
61
|
+
source.length,
|
|
62
|
+
source.nullCount,
|
|
63
|
+
source.buffers,
|
|
64
|
+
children
|
|
65
|
+
);
|
|
66
|
+
// RecordBatch normalizes child lengths and recreates the parent struct.
|
|
67
|
+
return Object.assign(new RecordBatch(schema, data), {data});
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
return new Table(schema, batches);
|
|
71
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
import {DataType, type Field} from 'apache-arrow';
|
|
5
|
+
import {typeKey} from './buffers';
|
|
6
|
+
/** Compares physical encoding as well as logical type when reusing a bound evaluator. */
|
|
7
|
+
export function physicalKey(type: DataType): string {
|
|
8
|
+
try {
|
|
9
|
+
return DataType.isDictionary(type)
|
|
10
|
+
? `dictionary:${typeKey(type.indices)}:${typeKey(type.dictionary)}:${type.isOrdered}`
|
|
11
|
+
: typeKey(type);
|
|
12
|
+
} catch {
|
|
13
|
+
// Unreferenced columns may have nested or extension layouts outside this evaluator's subset.
|
|
14
|
+
const properties = [
|
|
15
|
+
'bitWidth',
|
|
16
|
+
'isSigned',
|
|
17
|
+
'precision',
|
|
18
|
+
'scale',
|
|
19
|
+
'unit',
|
|
20
|
+
'timezone',
|
|
21
|
+
'byteWidth',
|
|
22
|
+
'listSize',
|
|
23
|
+
'keysSorted',
|
|
24
|
+
'isOrdered',
|
|
25
|
+
'mode'
|
|
26
|
+
].map(key => (type as unknown as Record<string, unknown>)[key]);
|
|
27
|
+
return JSON.stringify([
|
|
28
|
+
type.typeId,
|
|
29
|
+
properties,
|
|
30
|
+
DataType.isUnion(type) ? Array.from(type.typeIds) : null,
|
|
31
|
+
DataType.isDictionary(type)
|
|
32
|
+
? [physicalKey(type.indices), physicalKey(type.dictionary), type.isOrdered]
|
|
33
|
+
: null,
|
|
34
|
+
type.children?.map(field => [
|
|
35
|
+
field.name,
|
|
36
|
+
physicalKey(field.type),
|
|
37
|
+
field.nullable,
|
|
38
|
+
extensionKey(field)
|
|
39
|
+
])
|
|
40
|
+
]);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Geometry encoding/CRS metadata is part of a schema-bound vector function contract. */
|
|
45
|
+
export function extensionKey(field: Field): string {
|
|
46
|
+
return JSON.stringify([
|
|
47
|
+
field.metadata.get('ARROW:extension:name'),
|
|
48
|
+
field.metadata.get('ARROW:extension:metadata')
|
|
49
|
+
]);
|
|
50
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DateDay, TimestampMicrosecond, type DataType} from 'apache-arrow';
|
|
6
|
+
import {ExpressionEvaluationError} from '../expressions/index';
|
|
7
|
+
import {typeKey} from './buffers';
|
|
8
|
+
import type {ArrowExpressionValue} from './types';
|
|
9
|
+
|
|
10
|
+
const DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
11
|
+
const TIMESTAMP =
|
|
12
|
+
/^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,6}))?(Z|[+-]\d{2}:\d{2})?$/;
|
|
13
|
+
|
|
14
|
+
/** Binds ISO text or an already matching Arrow temporal value to a temporal output. */
|
|
15
|
+
export function bindTemporalCast(source: DataType, target: 'date' | 'timestamp' | 'timestamp_utc') {
|
|
16
|
+
const dateOnly = target === 'date';
|
|
17
|
+
const zoned = target === 'timestamp_utc';
|
|
18
|
+
const type = dateOnly ? new DateDay() : new TimestampMicrosecond(zoned ? 'UTC' : undefined);
|
|
19
|
+
const sourceKey = typeKey(source);
|
|
20
|
+
if (sourceKey !== 'utf8' && sourceKey !== 'null' && sourceKey !== typeKey(type))
|
|
21
|
+
throw new Error('Temporal CAST requires UTF8 or an input of the requested Arrow type');
|
|
22
|
+
return {
|
|
23
|
+
type,
|
|
24
|
+
textInput: false,
|
|
25
|
+
evaluate(value: ArrowExpressionValue): ArrowExpressionValue {
|
|
26
|
+
return typeof value === 'string' ? parseTemporalValue(value, dateOnly, zoned) : value;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Converts a validated calendar value to Arrow epoch days or exact epoch microseconds. */
|
|
32
|
+
function parseTemporalValue(text: string, dateOnly: boolean, zoned: boolean): number | bigint {
|
|
33
|
+
const match = (dateOnly ? DATE : TIMESTAMP).exec(text.trim());
|
|
34
|
+
const invalid = () =>
|
|
35
|
+
new ExpressionEvaluationError('conversion', 'Invalid text for temporal CAST');
|
|
36
|
+
if (!match) throw invalid();
|
|
37
|
+
const [, yearText, monthText, dayText, hourText, minuteText, secondText, fraction, zone] = match;
|
|
38
|
+
const year = Number(yearText);
|
|
39
|
+
const month = Number(monthText);
|
|
40
|
+
const day = Number(dayText);
|
|
41
|
+
const calendar = new Date(0);
|
|
42
|
+
// setUTCFullYear also handles four-digit years below 0100 without a 1900 offset.
|
|
43
|
+
calendar.setUTCFullYear(year, month - 1, day);
|
|
44
|
+
if (
|
|
45
|
+
calendar.getUTCFullYear() !== year ||
|
|
46
|
+
calendar.getUTCMonth() !== month - 1 ||
|
|
47
|
+
calendar.getUTCDate() !== day
|
|
48
|
+
)
|
|
49
|
+
throw invalid();
|
|
50
|
+
if (dateOnly) return calendar.getTime() / 86_400_000;
|
|
51
|
+
const hour = Number(hourText);
|
|
52
|
+
const minute = Number(minuteText);
|
|
53
|
+
const second = Number(secondText);
|
|
54
|
+
if (hour > 23 || minute > 59 || second > 59 || zoned !== Boolean(zone)) throw invalid();
|
|
55
|
+
let offsetMinutes = 0;
|
|
56
|
+
if (zone && zone !== 'Z') {
|
|
57
|
+
const offsetHour = Number(zone.slice(1, 3));
|
|
58
|
+
const offsetMinute = Number(zone.slice(4));
|
|
59
|
+
if (offsetHour > 23 || offsetMinute > 59) throw invalid();
|
|
60
|
+
offsetMinutes = (zone.startsWith('-') ? -1 : 1) * (offsetHour * 60 + offsetMinute);
|
|
61
|
+
}
|
|
62
|
+
const seconds = hour * 3600 + minute * 60 + second - offsetMinutes * 60;
|
|
63
|
+
return (
|
|
64
|
+
BigInt(calendar.getTime() + seconds * 1000) * 1000n + BigInt((fraction ?? '').padEnd(6, '0'))
|
|
65
|
+
);
|
|
66
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Field, Schema, RecordBatch, Vector, Table} from 'apache-arrow';
|
|
6
|
+
import type {Expression} from '../expressions/index';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Scalar values exposed to user callbacks. Date32 uses epoch days and microsecond
|
|
10
|
+
* timestamps use bigint microseconds, matching their Arrow storage buffers.
|
|
11
|
+
*/
|
|
12
|
+
export type ArrowExpressionValue = number | bigint | boolean | string | Uint8Array | null;
|
|
13
|
+
|
|
14
|
+
/** A vector kernel receives full-length inputs and must evaluate only active rows. */
|
|
15
|
+
export type ArrowTableFunctionType = (
|
|
16
|
+
arguments_: readonly Vector[],
|
|
17
|
+
context: {
|
|
18
|
+
/** One byte per row; zero means the row must not be evaluated. */
|
|
19
|
+
activeRows: Uint8Array;
|
|
20
|
+
/** Number of rows in every argument and the returned vector. */
|
|
21
|
+
length: number;
|
|
22
|
+
/** Report a row failure; TRY recovers classified errors, otherwise the call throws. */
|
|
23
|
+
reportError?(row: number, error: unknown): void;
|
|
24
|
+
}
|
|
25
|
+
) => Vector;
|
|
26
|
+
|
|
27
|
+
/** One schema-bound implementation of a named table function. */
|
|
28
|
+
export type ArrowTableFunctionPrototype = Readonly<{
|
|
29
|
+
/** Ordered argument fields, with exact types, nullability and extension metadata. */
|
|
30
|
+
arguments: readonly Field[];
|
|
31
|
+
/** Declared result field; its name is retained for direct calls. */
|
|
32
|
+
result: Field;
|
|
33
|
+
/** Propagate null arguments, or pass them to the implementation. Defaults to propagate. */
|
|
34
|
+
nullHandling?: 'propagate' | 'call';
|
|
35
|
+
/** Batch implementation, also independently callable with the declared arguments and mask. */
|
|
36
|
+
evaluate: ArrowTableFunctionType;
|
|
37
|
+
/** Optional equivalent primitive implementation used for fused expression execution. */
|
|
38
|
+
evaluateScalar?(...arguments_: ArrowExpressionValue[]): ArrowExpressionValue;
|
|
39
|
+
}>;
|
|
40
|
+
|
|
41
|
+
/** Named overload group; each argument signature must be unique. */
|
|
42
|
+
export type ArrowTableFunction = Readonly<{
|
|
43
|
+
/** Unique expression identifier; built-in names are reserved. */
|
|
44
|
+
name: string;
|
|
45
|
+
/** Typed implementations selected once when compiling a call. */
|
|
46
|
+
prototypes: readonly ArrowTableFunctionPrototype[];
|
|
47
|
+
}>;
|
|
48
|
+
|
|
49
|
+
/** Schema binding shared by predicate and expression compilation. */
|
|
50
|
+
export type ArrowExpressionOptions = Readonly<{
|
|
51
|
+
/** Expected batch schema, validated before evaluation. */
|
|
52
|
+
inputSchema: Schema;
|
|
53
|
+
/** Named function declarations, snapshotted into a private lookup during compilation. */
|
|
54
|
+
functions?: readonly ArrowTableFunction[];
|
|
55
|
+
}>;
|
|
56
|
+
|
|
57
|
+
/** Controls shared by batch and whole-table evaluation. */
|
|
58
|
+
export type ArrowExpressionEvaluationOptions = {
|
|
59
|
+
/** One byte per logical row, including all batches when evaluating a table. */
|
|
60
|
+
activeRows?: Uint8Array;
|
|
61
|
+
/** Checked before and after each batch; synchronous execution does not yield. */
|
|
62
|
+
signal?: AbortSignal;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** A compiled expression preserves row count and returns a non-dictionary vector. */
|
|
66
|
+
export interface CompiledArrowExpression {
|
|
67
|
+
/** Bound result type and nullability. */
|
|
68
|
+
readonly outputField: Field;
|
|
69
|
+
/** Evaluates a batch; optional masks skip rows and produce null at those positions. */
|
|
70
|
+
evaluate(batch: RecordBatch, options?: ArrowExpressionEvaluationOptions): Vector;
|
|
71
|
+
/** Evaluates ordered batches and combines output chunks without copying their buffers. */
|
|
72
|
+
evaluateTable(arrowTable: Table, options?: ArrowExpressionEvaluationOptions): Vector;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Text or the existing core expression AST. */
|
|
76
|
+
export type ArrowExpression = string | Expression;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {initializeTaskWorker} from '@math.gl/tasks';
|
|
6
|
+
import {createArrowExpressionTask} from './worker';
|
|
7
|
+
|
|
8
|
+
// Default worker for primitive expressions. Applications can provide their own module worker
|
|
9
|
+
// with local declarations by initializing createArrowExpressionTask({functions}) instead.
|
|
10
|
+
void initializeTaskWorker(createArrowExpressionTask());
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {Table, RecordBatch, Schema, Struct, Vector, makeData, type Field} from 'apache-arrow';
|
|
6
|
+
import type {Task, TaskWorker} from '@math.gl/tasks';
|
|
7
|
+
import type {
|
|
8
|
+
ParameterizedColumnarPredicate,
|
|
9
|
+
ColumnarPredicateParameterValues
|
|
10
|
+
} from '../expressions/index';
|
|
11
|
+
import {dehydrateArrowTable, hydrateArrowTable, type DehydratedArrowTable} from '@math.gl/arrow';
|
|
12
|
+
import {compileArrowExpression} from './compiler';
|
|
13
|
+
import {compileArrowPredicate} from './predicate';
|
|
14
|
+
import type {ArrowExpression, CompiledArrowExpression} from './types';
|
|
15
|
+
import type {ArrowTableFunction} from './types';
|
|
16
|
+
|
|
17
|
+
/** Serializable program, compiled once in each initialized task instance. */
|
|
18
|
+
export type ArrowExpressionProgram =
|
|
19
|
+
| {expression: ArrowExpression; predicate?: never; parameters?: never}
|
|
20
|
+
| {
|
|
21
|
+
predicate: ParameterizedColumnarPredicate;
|
|
22
|
+
parameters?: ColumnarPredicateParameterValues;
|
|
23
|
+
expression?: never;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/** Borrowed column-buffer payload; use TaskRunner ownership controls when sending. */
|
|
27
|
+
export type ArrowExpressionTaskInput = {table: DehydratedArrowTable; activeRows?: Uint8Array};
|
|
28
|
+
/** One-column table retains the output field, validity and ordered chunks. */
|
|
29
|
+
export type ArrowExpressionTaskOutput = {table: DehydratedArrowTable};
|
|
30
|
+
|
|
31
|
+
type ExpressionTaskTypes = {
|
|
32
|
+
initialization: ArrowExpressionProgram;
|
|
33
|
+
input: ArrowExpressionTaskInput;
|
|
34
|
+
output: ArrowExpressionTaskOutput;
|
|
35
|
+
state: {program: ArrowExpressionProgram; compiled?: CompiledArrowExpression};
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Creates a reusable task. Construct custom function arrays inside the worker; callbacks never cross
|
|
40
|
+
* the transport. Supply a module-worker URL or factory on the host and initialize the same task
|
|
41
|
+
* with initializeTaskWorker in the worker. A retained task accepts one bound input schema.
|
|
42
|
+
*/
|
|
43
|
+
export function createArrowExpressionTask(
|
|
44
|
+
options: {functions?: readonly ArrowTableFunction[]; worker?: TaskWorker} = {}
|
|
45
|
+
): Task<ExpressionTaskTypes> {
|
|
46
|
+
return {
|
|
47
|
+
id: 'math.gl-arrow-expression',
|
|
48
|
+
worker: options.worker,
|
|
49
|
+
initialize(program) {
|
|
50
|
+
return {program};
|
|
51
|
+
},
|
|
52
|
+
async process(input, _options, context) {
|
|
53
|
+
context.throwIfAborted();
|
|
54
|
+
const table = hydrateArrowTable(input.table);
|
|
55
|
+
const state = context.state;
|
|
56
|
+
state.compiled ||=
|
|
57
|
+
state.program.expression !== undefined
|
|
58
|
+
? compileArrowExpression(state.program.expression, {
|
|
59
|
+
inputSchema: table.schema,
|
|
60
|
+
functions: options.functions
|
|
61
|
+
})
|
|
62
|
+
: compileArrowPredicate(state.program.predicate!, {
|
|
63
|
+
inputSchema: table.schema,
|
|
64
|
+
functions: options.functions,
|
|
65
|
+
parameters: state.program.parameters
|
|
66
|
+
});
|
|
67
|
+
const compiled = state.compiled;
|
|
68
|
+
const mask = input.activeRows;
|
|
69
|
+
if (mask && (mask.length !== table.numRows || mask.some(value => value !== 0 && value !== 1)))
|
|
70
|
+
throw new Error('activeRows must contain one 0 or 1 per input row');
|
|
71
|
+
// Validate even an empty table against the previously compiled schema.
|
|
72
|
+
compiled.evaluateTable(new Table(table.schema, []), {signal: context.signal});
|
|
73
|
+
const chunks = [];
|
|
74
|
+
let offset = 0;
|
|
75
|
+
for (const batch of table.batches) {
|
|
76
|
+
// Yield to cancellation messages between batches, including before the first batch.
|
|
77
|
+
await new Promise<void>(resolve => setTimeout(resolve, 0));
|
|
78
|
+
context.throwIfAborted();
|
|
79
|
+
chunks.push(
|
|
80
|
+
...compiled.evaluate(batch, {
|
|
81
|
+
activeRows: mask?.subarray(offset, offset + batch.numRows),
|
|
82
|
+
signal: context.signal
|
|
83
|
+
}).data
|
|
84
|
+
);
|
|
85
|
+
offset += batch.numRows;
|
|
86
|
+
context.reportProgress({completed: offset, total: table.numRows, unit: 'rows'});
|
|
87
|
+
}
|
|
88
|
+
context.throwIfAborted();
|
|
89
|
+
const vector = chunks.length ? new Vector(chunks) : compiled.evaluateTable(table);
|
|
90
|
+
// A mask may introduce nulls even when the logical expression is non-nullable.
|
|
91
|
+
const field = mask ? compiled.outputField.clone({nullable: true}) : compiled.outputField;
|
|
92
|
+
const schema = new Schema([field]);
|
|
93
|
+
const batches = vector.data.map(
|
|
94
|
+
data =>
|
|
95
|
+
new RecordBatch(
|
|
96
|
+
schema,
|
|
97
|
+
makeData({
|
|
98
|
+
type: new Struct([field]),
|
|
99
|
+
length: data.length,
|
|
100
|
+
nullCount: 0,
|
|
101
|
+
children: [data]
|
|
102
|
+
})
|
|
103
|
+
)
|
|
104
|
+
);
|
|
105
|
+
return {table: dehydrateArrowTable(new Table(schema, batches), {copy: 'none'})};
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Prepares borrowed buffers without copying values or detaching the input. TaskRunner defaults
|
|
112
|
+
* to copying them. inputOwnership:'transfer' relinquishes every referenced ArrayBuffer, including
|
|
113
|
+
* buffers shared with slices or other columns; do not read the original table afterward.
|
|
114
|
+
*/
|
|
115
|
+
export function prepareArrowExpressionInput(
|
|
116
|
+
arrowTable: Table,
|
|
117
|
+
options: {activeRows?: Uint8Array} = {}
|
|
118
|
+
): ArrowExpressionTaskInput {
|
|
119
|
+
return {table: dehydrateArrowTable(arrowTable, {copy: 'none'}), activeRows: options.activeRows};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Hydrates worker results without copying data buffers. */
|
|
123
|
+
export function hydrateArrowExpressionResult(output: ArrowExpressionTaskOutput): {
|
|
124
|
+
field: Field;
|
|
125
|
+
vector: Vector;
|
|
126
|
+
} {
|
|
127
|
+
const table = hydrateArrowTable(output.table);
|
|
128
|
+
if (table.numCols !== 1) throw new Error('Arrow expression task must return exactly one column');
|
|
129
|
+
return {field: table.schema.fields[0]!, vector: table.getChildAt(0)!};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export {
|
|
133
|
+
createArrowQueryTask,
|
|
134
|
+
createArrowQueryWorker,
|
|
135
|
+
queryArrowTableInWorker,
|
|
136
|
+
type ArrowQueryWorkerOptions,
|
|
137
|
+
type ArrowQueryWorkerSession,
|
|
138
|
+
type ArrowWorkerQuery
|
|
139
|
+
} from './query-worker';
|