@math.gl/sql 0.0.0-stage → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +249 -0
- package/README.md +26 -2
- package/THIRD-PARTY-NOTICES.md +5 -0
- package/dist/arrow/aggregates.d.ts +46 -0
- package/dist/arrow/aggregates.d.ts.map +1 -0
- package/dist/arrow/aggregates.js +289 -0
- package/dist/arrow/aggregates.js.map +1 -0
- package/dist/arrow/buffers.d.ts +30 -0
- package/dist/arrow/buffers.d.ts.map +1 -0
- package/dist/arrow/buffers.js +177 -0
- package/dist/arrow/buffers.js.map +1 -0
- package/dist/arrow/cast.d.ts +19 -0
- package/dist/arrow/cast.d.ts.map +1 -0
- package/dist/arrow/cast.js +138 -0
- package/dist/arrow/cast.js.map +1 -0
- package/dist/arrow/compiler.d.ts +10 -0
- package/dist/arrow/compiler.d.ts.map +1 -0
- package/dist/arrow/compiler.js +1421 -0
- package/dist/arrow/compiler.js.map +1 -0
- package/dist/arrow/float16.d.ts +7 -0
- package/dist/arrow/float16.d.ts.map +1 -0
- package/dist/arrow/float16.js +26 -0
- package/dist/arrow/float16.js.map +1 -0
- package/dist/arrow/index.cjs +3220 -0
- package/dist/arrow/index.cjs.map +6 -0
- package/dist/arrow/index.d.ts +5 -0
- package/dist/arrow/index.d.ts.map +1 -0
- package/dist/arrow/index.js +7 -0
- package/dist/arrow/index.js.map +1 -0
- package/dist/arrow/like.d.ts +3 -0
- package/dist/arrow/like.d.ts.map +1 -0
- package/dist/arrow/like.js +75 -0
- package/dist/arrow/like.js.map +1 -0
- package/dist/arrow/list-values.d.ts +14 -0
- package/dist/arrow/list-values.d.ts.map +1 -0
- package/dist/arrow/list-values.js +180 -0
- package/dist/arrow/list-values.js.map +1 -0
- package/dist/arrow/predicate.d.ts +7 -0
- package/dist/arrow/predicate.d.ts.map +1 -0
- package/dist/arrow/predicate.js +10 -0
- package/dist/arrow/predicate.js.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts +24 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js +64 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js.map +1 -0
- package/dist/arrow/query-arrow-sql.d.ts +8 -0
- package/dist/arrow/query-arrow-sql.d.ts.map +1 -0
- package/dist/arrow/query-arrow-sql.js +191 -0
- package/dist/arrow/query-arrow-sql.js.map +1 -0
- package/dist/arrow/query-arrow-table.d.ts +36 -0
- package/dist/arrow/query-arrow-table.d.ts.map +1 -0
- package/dist/arrow/query-arrow-table.js +208 -0
- package/dist/arrow/query-arrow-table.js.map +1 -0
- package/dist/arrow/query-worker-entry.d.ts +2 -0
- package/dist/arrow/query-worker-entry.d.ts.map +1 -0
- package/dist/arrow/query-worker-entry.js +7 -0
- package/dist/arrow/query-worker-entry.js.map +1 -0
- package/dist/arrow/query-worker.d.ts +65 -0
- package/dist/arrow/query-worker.d.ts.map +1 -0
- package/dist/arrow/query-worker.js +135 -0
- package/dist/arrow/query-worker.js.map +1 -0
- package/dist/arrow/registry.d.ts +4 -0
- package/dist/arrow/registry.d.ts.map +1 -0
- package/dist/arrow/registry.js +65 -0
- package/dist/arrow/registry.js.map +1 -0
- package/dist/arrow/relational-query.d.ts +36 -0
- package/dist/arrow/relational-query.d.ts.map +1 -0
- package/dist/arrow/relational-query.js +174 -0
- package/dist/arrow/relational-query.js.map +1 -0
- package/dist/arrow/replace-arrow-columns.d.ts +8 -0
- package/dist/arrow/replace-arrow-columns.d.ts.map +1 -0
- package/dist/arrow/replace-arrow-columns.js +63 -0
- package/dist/arrow/replace-arrow-columns.js.map +1 -0
- package/dist/arrow/signature.d.ts +6 -0
- package/dist/arrow/signature.d.ts.map +1 -0
- package/dist/arrow/signature.js +51 -0
- package/dist/arrow/signature.js.map +1 -0
- package/dist/arrow/temporal-cast.d.ts +9 -0
- package/dist/arrow/temporal-cast.d.ts.map +1 -0
- package/dist/arrow/temporal-cast.js +60 -0
- package/dist/arrow/temporal-cast.js.map +1 -0
- package/dist/arrow/types.d.ts +62 -0
- package/dist/arrow/types.d.ts.map +1 -0
- package/dist/arrow/types.js +5 -0
- package/dist/arrow/types.js.map +1 -0
- package/dist/arrow/worker-entry.d.ts +2 -0
- package/dist/arrow/worker-entry.d.ts.map +1 -0
- package/dist/arrow/worker-entry.js +9 -0
- package/dist/arrow/worker-entry.js.map +1 -0
- package/dist/arrow/worker.cjs +3973 -0
- package/dist/arrow/worker.cjs.map +6 -0
- package/dist/arrow/worker.d.ts +58 -0
- package/dist/arrow/worker.d.ts.map +1 -0
- package/dist/arrow/worker.js +84 -0
- package/dist/arrow/worker.js.map +1 -0
- package/dist/color/index.cjs +382 -0
- package/dist/color/index.cjs.map +6 -0
- package/dist/color/index.d.ts +26 -0
- package/dist/color/index.d.ts.map +1 -0
- package/dist/color/index.js +306 -0
- package/dist/color/index.js.map +1 -0
- package/dist/diagnostic-log.d.ts +8 -0
- package/dist/diagnostic-log.d.ts.map +1 -0
- package/dist/diagnostic-log.js +15 -0
- package/dist/diagnostic-log.js.map +1 -0
- package/dist/expressions/ast.d.ts +12 -0
- package/dist/expressions/ast.d.ts.map +1 -0
- package/dist/expressions/ast.js +112 -0
- package/dist/expressions/ast.js.map +1 -0
- package/dist/expressions/columnar-predicate.d.ts +56 -0
- package/dist/expressions/columnar-predicate.d.ts.map +1 -0
- package/dist/expressions/columnar-predicate.js +176 -0
- package/dist/expressions/columnar-predicate.js.map +1 -0
- package/dist/expressions/evaluation-error.d.ts +14 -0
- package/dist/expressions/evaluation-error.d.ts.map +1 -0
- package/dist/expressions/evaluation-error.js +26 -0
- package/dist/expressions/evaluation-error.js.map +1 -0
- package/dist/expressions/expression-eval.d.ts +96 -0
- package/dist/expressions/expression-eval.d.ts.map +1 -0
- package/dist/expressions/expression-eval.js +330 -0
- package/dist/expressions/expression-eval.js.map +1 -0
- package/dist/expressions/function-libraries.d.ts +63 -0
- package/dist/expressions/function-libraries.d.ts.map +1 -0
- package/dist/expressions/function-libraries.js +69 -0
- package/dist/expressions/function-libraries.js.map +1 -0
- package/dist/expressions/function-registry.d.ts +93 -0
- package/dist/expressions/function-registry.d.ts.map +1 -0
- package/dist/expressions/function-registry.js +139 -0
- package/dist/expressions/function-registry.js.map +1 -0
- package/dist/expressions/get.d.ts +9 -0
- package/dist/expressions/get.d.ts.map +1 -0
- package/dist/expressions/get.js +32 -0
- package/dist/expressions/get.js.map +1 -0
- package/dist/expressions/index.cjs +1888 -0
- package/dist/expressions/index.cjs.map +6 -0
- package/dist/expressions/index.d.ts +13 -0
- package/dist/expressions/index.d.ts.map +1 -0
- package/dist/expressions/index.js +14 -0
- package/dist/expressions/index.js.map +1 -0
- package/dist/expressions/numeric-functions.d.ts +277 -0
- package/dist/expressions/numeric-functions.d.ts.map +1 -0
- package/dist/expressions/numeric-functions.js +52 -0
- package/dist/expressions/numeric-functions.js.map +1 -0
- package/dist/expressions/parse-expression-string.d.ts +21 -0
- package/dist/expressions/parse-expression-string.d.ts.map +1 -0
- package/dist/expressions/parse-expression-string.js +61 -0
- package/dist/expressions/parse-expression-string.js.map +1 -0
- package/dist/expressions/predicate-expression.d.ts +7 -0
- package/dist/expressions/predicate-expression.d.ts.map +1 -0
- package/dist/expressions/predicate-expression.js +112 -0
- package/dist/expressions/predicate-expression.js.map +1 -0
- package/dist/expressions/scalar-type.d.ts +6 -0
- package/dist/expressions/scalar-type.d.ts.map +1 -0
- package/dist/expressions/scalar-type.js +23 -0
- package/dist/expressions/scalar-type.js.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts +3 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.js +972 -0
- package/dist/expressions/vendor/jsep/runtime.js.map +1 -0
- package/dist/expressions/vendor/jsep/types.d.ts +148 -0
- package/dist/expressions/vendor/jsep/types.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/types.js +6 -0
- package/dist/expressions/vendor/jsep/types.js.map +1 -0
- package/dist/index.cjs +4865 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/math/index.cjs +444 -0
- package/dist/math/index.cjs.map +6 -0
- package/dist/math/index.d.ts +44 -0
- package/dist/math/index.d.ts.map +1 -0
- package/dist/math/index.js +182 -0
- package/dist/math/index.js.map +1 -0
- package/dist/scales/arrow-domains.d.ts +15 -0
- package/dist/scales/arrow-domains.d.ts.map +1 -0
- package/dist/scales/arrow-domains.js +124 -0
- package/dist/scales/arrow-domains.js.map +1 -0
- package/dist/scales/arrow-scales.cjs +3837 -0
- package/dist/scales/arrow-scales.cjs.map +6 -0
- package/dist/scales/arrow-scales.d.ts +20 -0
- package/dist/scales/arrow-scales.d.ts.map +1 -0
- package/dist/scales/arrow-scales.js +109 -0
- package/dist/scales/arrow-scales.js.map +1 -0
- package/dist/scales/compile-scale.d.ts +7 -0
- package/dist/scales/compile-scale.d.ts.map +1 -0
- package/dist/scales/compile-scale.js +195 -0
- package/dist/scales/compile-scale.js.map +1 -0
- package/dist/scales/hdr-colors.d.ts +13 -0
- package/dist/scales/hdr-colors.d.ts.map +1 -0
- package/dist/scales/hdr-colors.js +115 -0
- package/dist/scales/hdr-colors.js.map +1 -0
- package/dist/scales/index.cjs +1214 -0
- package/dist/scales/index.cjs.map +6 -0
- package/dist/scales/index.d.ts +9 -0
- package/dist/scales/index.d.ts.map +1 -0
- package/dist/scales/index.js +10 -0
- package/dist/scales/index.js.map +1 -0
- package/dist/scales/palettes.d.ts +5 -0
- package/dist/scales/palettes.d.ts.map +1 -0
- package/dist/scales/palettes.js +85 -0
- package/dist/scales/palettes.js.map +1 -0
- package/dist/scales/parse-sql-scale.d.ts +14 -0
- package/dist/scales/parse-sql-scale.d.ts.map +1 -0
- package/dist/scales/parse-sql-scale.js +165 -0
- package/dist/scales/parse-sql-scale.js.map +1 -0
- package/dist/scales/quantile-sketch.d.ts +22 -0
- package/dist/scales/quantile-sketch.d.ts.map +1 -0
- package/dist/scales/quantile-sketch.js +108 -0
- package/dist/scales/quantile-sketch.js.map +1 -0
- package/dist/scales/scale-types.d.ts +92 -0
- package/dist/scales/scale-types.d.ts.map +1 -0
- package/dist/scales/scale-types.js +119 -0
- package/dist/scales/scale-types.js.map +1 -0
- package/dist/scales/streaming-domains.d.ts +18 -0
- package/dist/scales/streaming-domains.d.ts.map +1 -0
- package/dist/scales/streaming-domains.js +109 -0
- package/dist/scales/streaming-domains.js.map +1 -0
- package/dist/sql/bind-sql-predicate.d.ts +4 -0
- package/dist/sql/bind-sql-predicate.d.ts.map +1 -0
- package/dist/sql/bind-sql-predicate.js +11 -0
- package/dist/sql/bind-sql-predicate.js.map +1 -0
- package/dist/sql/index.cjs +1588 -0
- package/dist/sql/index.cjs.map +6 -0
- package/dist/sql/index.d.ts +10 -0
- package/dist/sql/index.d.ts.map +1 -0
- package/dist/sql/index.js +10 -0
- package/dist/sql/index.js.map +1 -0
- package/dist/sql/parse-sql-expression.d.ts +44 -0
- package/dist/sql/parse-sql-expression.d.ts.map +1 -0
- package/dist/sql/parse-sql-expression.js +455 -0
- package/dist/sql/parse-sql-expression.js.map +1 -0
- package/dist/sql/parse-sql-predicate.d.ts +13 -0
- package/dist/sql/parse-sql-predicate.d.ts.map +1 -0
- package/dist/sql/parse-sql-predicate.js +365 -0
- package/dist/sql/parse-sql-predicate.js.map +1 -0
- package/dist/sql/parse-sql-query.d.ts +8 -0
- package/dist/sql/parse-sql-query.d.ts.map +1 -0
- package/dist/sql/parse-sql-query.js +66 -0
- package/dist/sql/parse-sql-query.js.map +1 -0
- package/dist/sql/parse-sql-select.d.ts +55 -0
- package/dist/sql/parse-sql-select.d.ts.map +1 -0
- package/dist/sql/parse-sql-select.js +244 -0
- package/dist/sql/parse-sql-select.js.map +1 -0
- package/dist/sql/parse-sql-statement.d.ts +50 -0
- package/dist/sql/parse-sql-statement.d.ts.map +1 -0
- package/dist/sql/parse-sql-statement.js +182 -0
- package/dist/sql/parse-sql-statement.js.map +1 -0
- package/dist/sql/query-types.d.ts +56 -0
- package/dist/sql/query-types.d.ts.map +1 -0
- package/dist/sql/query-types.js +5 -0
- package/dist/sql/query-types.js.map +1 -0
- package/dist/sql/sql-parse-error.d.ts +13 -0
- package/dist/sql/sql-parse-error.d.ts.map +1 -0
- package/dist/sql/sql-parse-error.js +20 -0
- package/dist/sql/sql-parse-error.js.map +1 -0
- package/dist/sql/sql-predicate-schema.d.ts +164 -0
- package/dist/sql/sql-predicate-schema.d.ts.map +1 -0
- package/dist/sql/sql-predicate-schema.js +249 -0
- package/dist/sql/sql-predicate-schema.js.map +1 -0
- package/dist/sql/sql-predicate-types.d.ts +48 -0
- package/dist/sql/sql-predicate-types.d.ts.map +1 -0
- package/dist/sql/sql-predicate-types.js +9 -0
- package/dist/sql/sql-predicate-types.js.map +1 -0
- package/dist/sql/sql-predicate.d.ts +6 -0
- package/dist/sql/sql-predicate.d.ts.map +1 -0
- package/dist/sql/sql-predicate.js +8 -0
- package/dist/sql/sql-predicate.js.map +1 -0
- package/package.json +92 -4
- package/src/arrow/aggregates.ts +386 -0
- package/src/arrow/buffers.ts +177 -0
- package/src/arrow/cast.ts +160 -0
- package/src/arrow/compiler.ts +1659 -0
- package/src/arrow/float16.ts +25 -0
- package/src/arrow/index.ts +22 -0
- package/src/arrow/like.ts +72 -0
- package/src/arrow/list-values.ts +200 -0
- package/src/arrow/predicate.ts +19 -0
- package/src/arrow/query-arrow-aggregate-snapshots.ts +107 -0
- package/src/arrow/query-arrow-sql.ts +227 -0
- package/src/arrow/query-arrow-table.ts +298 -0
- package/src/arrow/query-worker-entry.ts +8 -0
- package/src/arrow/query-worker.ts +190 -0
- package/src/arrow/registry.ts +86 -0
- package/src/arrow/relational-query.ts +247 -0
- package/src/arrow/replace-arrow-columns.ts +71 -0
- package/src/arrow/signature.ts +50 -0
- package/src/arrow/temporal-cast.ts +66 -0
- package/src/arrow/types.ts +76 -0
- package/src/arrow/worker-entry.ts +10 -0
- package/src/arrow/worker.ts +139 -0
- package/src/color/index.ts +405 -0
- package/src/diagnostic-log.ts +20 -0
- package/src/expressions/ast.ts +141 -0
- package/src/expressions/columnar-predicate.ts +293 -0
- package/src/expressions/evaluation-error.ts +29 -0
- package/src/expressions/expression-eval.ts +480 -0
- package/src/expressions/function-libraries.ts +126 -0
- package/src/expressions/function-registry.ts +171 -0
- package/src/expressions/get.ts +37 -0
- package/src/expressions/index.ts +40 -0
- package/src/expressions/numeric-functions.ts +99 -0
- package/src/expressions/parse-expression-string.ts +80 -0
- package/src/expressions/predicate-expression.ts +128 -0
- package/src/expressions/scalar-type.ts +39 -0
- package/src/expressions/vendor/jsep/LICENSE.txt +23 -0
- package/src/expressions/vendor/jsep/README.md +22 -0
- package/src/expressions/vendor/jsep/runtime.ts +949 -0
- package/src/expressions/vendor/jsep/types.ts +191 -0
- package/src/index.ts +40 -0
- package/src/math/index.ts +235 -0
- package/src/scales/arrow-domains.ts +131 -0
- package/src/scales/arrow-scales.ts +156 -0
- package/src/scales/compile-scale.ts +217 -0
- package/src/scales/hdr-colors.ts +126 -0
- package/src/scales/index.ts +35 -0
- package/src/scales/palettes.ts +86 -0
- package/src/scales/parse-sql-scale.ts +195 -0
- package/src/scales/quantile-sketch.ts +106 -0
- package/src/scales/scale-types.ts +224 -0
- package/src/scales/streaming-domains.ts +133 -0
- package/src/sql/bind-sql-predicate.ts +21 -0
- package/src/sql/index.ts +28 -0
- package/src/sql/parse-sql-expression.ts +457 -0
- package/src/sql/parse-sql-predicate.ts +454 -0
- package/src/sql/parse-sql-query.ts +72 -0
- package/src/sql/parse-sql-select.ts +266 -0
- package/src/sql/parse-sql-statement.ts +242 -0
- package/src/sql/query-types.ts +71 -0
- package/src/sql/sql-parse-error.ts +33 -0
- package/src/sql/sql-predicate-schema.ts +279 -0
- package/src/sql/sql-predicate-types.ts +91 -0
- package/src/sql/sql-predicate.ts +28 -0
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DataType, type Table, type RecordBatch} from 'apache-arrow';
|
|
6
|
+
import {compareUTF8} from '@math.gl/arrow/utf8';
|
|
7
|
+
import {ColumnReader, valueType} from './buffers';
|
|
8
|
+
import {compileArrowExpression} from './compiler';
|
|
9
|
+
import type {
|
|
10
|
+
ArrowExpression,
|
|
11
|
+
ArrowExpressionOptions,
|
|
12
|
+
ArrowExpressionValue,
|
|
13
|
+
CompiledArrowExpression
|
|
14
|
+
} from './types';
|
|
15
|
+
|
|
16
|
+
type Operation =
|
|
17
|
+
| 'COUNT'
|
|
18
|
+
| 'SUM'
|
|
19
|
+
| 'MIN'
|
|
20
|
+
| 'MAX'
|
|
21
|
+
| 'AVG'
|
|
22
|
+
| 'EVERY'
|
|
23
|
+
| 'BOOL_AND'
|
|
24
|
+
| 'BOOL_OR'
|
|
25
|
+
| 'BIT_AND'
|
|
26
|
+
| 'BIT_OR'
|
|
27
|
+
| 'BIT_XOR';
|
|
28
|
+
|
|
29
|
+
/** One scalar reduction. Omit the COUNT expression for COUNT(*). */
|
|
30
|
+
export type ArrowAggregate =
|
|
31
|
+
| {readonly operation: 'COUNT'; readonly expression?: ArrowExpression}
|
|
32
|
+
| {readonly operation: Exclude<Operation, 'COUNT'>; readonly expression: ArrowExpression};
|
|
33
|
+
|
|
34
|
+
export interface ArrowAggregateAccumulator<K extends string> {
|
|
35
|
+
/** Adds a batch. A failed expression leaves the previous state intact. */
|
|
36
|
+
update(batch: RecordBatch, options?: {signal?: AbortSignal}): void;
|
|
37
|
+
/** Adds the partial state of another accumulator from this compiled plan. */
|
|
38
|
+
merge(other: ArrowAggregateAccumulator<K>): void;
|
|
39
|
+
/** Returns a snapshot. Further batches may change the next result. */
|
|
40
|
+
result(): Record<K, ArrowExpressionValue>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Incrementally reduces rows into groups without storing the input rows. */
|
|
44
|
+
export interface ArrowGroupedAggregateAccumulator<K extends string> {
|
|
45
|
+
/** Adds one batch atomically. An expression error or cancellation leaves the previous state intact. */
|
|
46
|
+
update(batch: RecordBatch, options?: {signal?: AbortSignal}): void;
|
|
47
|
+
/** Returns a detached snapshot in first-seen key order. Null keys form one group. */
|
|
48
|
+
result(): ReadonlyMap<ArrowExpressionValue, Record<K, ArrowExpressionValue>>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A schema-bound plan for scalar or grouped reductions. */
|
|
52
|
+
export interface CompiledArrowAggregates<K extends string> {
|
|
53
|
+
/** Starts a grouped reduction using one scalar key expression. */
|
|
54
|
+
createGroupedAccumulator(groupBy: ArrowExpression): ArrowGroupedAggregateAccumulator<K>;
|
|
55
|
+
createAccumulator(): ArrowAggregateAccumulator<K>;
|
|
56
|
+
evaluate(
|
|
57
|
+
input: Table | Iterable<RecordBatch>,
|
|
58
|
+
options?: {signal?: AbortSignal}
|
|
59
|
+
): Record<K, ArrowExpressionValue>;
|
|
60
|
+
evaluateInBatches(
|
|
61
|
+
input: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
|
|
62
|
+
options?: {signal?: AbortSignal}
|
|
63
|
+
): Promise<Record<K, ArrowExpressionValue>>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
type Value = Exclude<ArrowExpressionValue, null | string> | Uint8Array;
|
|
67
|
+
type State = {
|
|
68
|
+
count: bigint;
|
|
69
|
+
sum: number | bigint;
|
|
70
|
+
values: Map<Operation, Value>;
|
|
71
|
+
};
|
|
72
|
+
type Group = {
|
|
73
|
+
expression?: CompiledArrowExpression;
|
|
74
|
+
operations: Set<Operation>;
|
|
75
|
+
integer: boolean;
|
|
76
|
+
wide: boolean;
|
|
77
|
+
utf8: boolean;
|
|
78
|
+
sums: boolean;
|
|
79
|
+
reductions: Operation[];
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const reductions = new Set<Operation>([
|
|
83
|
+
'MIN',
|
|
84
|
+
'MAX',
|
|
85
|
+
'EVERY',
|
|
86
|
+
'BOOL_AND',
|
|
87
|
+
'BOOL_OR',
|
|
88
|
+
'BIT_AND',
|
|
89
|
+
'BIT_OR',
|
|
90
|
+
'BIT_XOR'
|
|
91
|
+
]);
|
|
92
|
+
|
|
93
|
+
/** Compiles named scalar aggregates. Identical expression strings or shared ASTs run once per batch. */
|
|
94
|
+
export function compileArrowAggregates<K extends string>(
|
|
95
|
+
aggregates: Readonly<Record<K, ArrowAggregate>>,
|
|
96
|
+
options: ArrowExpressionOptions
|
|
97
|
+
): CompiledArrowAggregates<K> {
|
|
98
|
+
const groups: Group[] = [];
|
|
99
|
+
const byExpression = new Map<ArrowExpression | undefined, Group>();
|
|
100
|
+
const outputs = (Object.entries(aggregates) as [K, ArrowAggregate][]).map(([name, aggregate]) => {
|
|
101
|
+
const {operation, expression} = aggregate;
|
|
102
|
+
if (!reductions.has(operation) && !['COUNT', 'SUM', 'AVG'].includes(operation))
|
|
103
|
+
throw new Error(`Unsupported Arrow aggregate: ${operation}`);
|
|
104
|
+
if (expression === undefined && operation !== 'COUNT')
|
|
105
|
+
throw new Error(`${operation} requires an expression`);
|
|
106
|
+
let group = byExpression.get(expression);
|
|
107
|
+
if (!group) {
|
|
108
|
+
const compiled =
|
|
109
|
+
expression === undefined ? undefined : compileArrowExpression(expression, options);
|
|
110
|
+
const type = compiled && valueType(compiled.outputField.type);
|
|
111
|
+
group = {
|
|
112
|
+
expression: compiled,
|
|
113
|
+
operations: new Set(),
|
|
114
|
+
integer: Boolean(type && DataType.isInt(type)),
|
|
115
|
+
wide: Boolean(type && DataType.isInt(type) && type.bitWidth === 64),
|
|
116
|
+
utf8: Boolean(type && DataType.isUtf8(type)),
|
|
117
|
+
sums: false,
|
|
118
|
+
reductions: []
|
|
119
|
+
};
|
|
120
|
+
byExpression.set(expression, group);
|
|
121
|
+
groups.push(group);
|
|
122
|
+
}
|
|
123
|
+
const type = group.expression && valueType(group.expression.outputField.type);
|
|
124
|
+
if (type && !DataType.isNull(type) && operation !== 'COUNT') {
|
|
125
|
+
const numeric = group.integer || DataType.isFloat(type);
|
|
126
|
+
const supported = operation.startsWith('BIT_')
|
|
127
|
+
? group.integer
|
|
128
|
+
: operation === 'EVERY' || operation.startsWith('BOOL_')
|
|
129
|
+
? DataType.isBool(type)
|
|
130
|
+
: numeric || (group.utf8 && (operation === 'MIN' || operation === 'MAX'));
|
|
131
|
+
if (!supported) throw new Error(`${operation} does not support Arrow type ${type}`);
|
|
132
|
+
}
|
|
133
|
+
group.operations.add(operation);
|
|
134
|
+
return {name, operation, group};
|
|
135
|
+
});
|
|
136
|
+
for (const group of groups) {
|
|
137
|
+
group.sums = group.operations.has('SUM') || group.operations.has('AVG');
|
|
138
|
+
group.reductions = [...group.operations].filter(operation => reductions.has(operation));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Materializes one aggregate result without exposing accumulator state. */
|
|
142
|
+
const resultFor = (states: Map<Group, State>): Record<K, ArrowExpressionValue> =>
|
|
143
|
+
Object.fromEntries(
|
|
144
|
+
outputs.map(({name, operation, group}) => [
|
|
145
|
+
name,
|
|
146
|
+
finish(operation, group, states.get(group)!)
|
|
147
|
+
])
|
|
148
|
+
) as Record<K, ArrowExpressionValue>;
|
|
149
|
+
|
|
150
|
+
const accumulators = new WeakMap<ArrowAggregateAccumulator<K>, Map<Group, State>>();
|
|
151
|
+
const createAccumulator = (): ArrowAggregateAccumulator<K> => {
|
|
152
|
+
const states = new Map(groups.map(group => [group, emptyState(group)]));
|
|
153
|
+
const accumulator: ArrowAggregateAccumulator<K> = {
|
|
154
|
+
update(batch, {signal} = {}) {
|
|
155
|
+
signal?.throwIfAborted();
|
|
156
|
+
const partials = groups.map(group => summarize(group, batch, signal)[0]!);
|
|
157
|
+
signal?.throwIfAborted();
|
|
158
|
+
groups.forEach((group, index) => mergeState(group, states.get(group)!, partials[index]!));
|
|
159
|
+
},
|
|
160
|
+
merge(other) {
|
|
161
|
+
const source = accumulators.get(other);
|
|
162
|
+
if (!source || other === accumulator)
|
|
163
|
+
throw new Error('Merge requires another accumulator from the same Arrow aggregate plan');
|
|
164
|
+
for (const group of groups) mergeState(group, states.get(group)!, source.get(group)!);
|
|
165
|
+
},
|
|
166
|
+
result() {
|
|
167
|
+
return resultFor(states);
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
accumulators.set(accumulator, states);
|
|
171
|
+
return accumulator;
|
|
172
|
+
};
|
|
173
|
+
return {
|
|
174
|
+
createAccumulator,
|
|
175
|
+
createGroupedAccumulator(groupBy) {
|
|
176
|
+
const keyExpression = compileArrowExpression(groupBy, options);
|
|
177
|
+
const type = valueType(keyExpression.outputField.type);
|
|
178
|
+
if (
|
|
179
|
+
!(
|
|
180
|
+
DataType.isInt(type) ||
|
|
181
|
+
DataType.isFloat(type) ||
|
|
182
|
+
DataType.isBool(type) ||
|
|
183
|
+
DataType.isUtf8(type) ||
|
|
184
|
+
DataType.isNull(type)
|
|
185
|
+
)
|
|
186
|
+
) {
|
|
187
|
+
throw new Error(`Unsupported Arrow aggregate group key type: ${type}`);
|
|
188
|
+
}
|
|
189
|
+
const statesByKey = new Map<ArrowExpressionValue, Map<Group, State>>();
|
|
190
|
+
return {
|
|
191
|
+
update(batch, {signal} = {}) {
|
|
192
|
+
const partition = partitionRows(keyExpression, batch, signal);
|
|
193
|
+
const partials = groups.map(group => summarize(group, batch, signal, partition));
|
|
194
|
+
signal?.throwIfAborted();
|
|
195
|
+
for (const [key, index] of partition.indexByKey) {
|
|
196
|
+
let states = statesByKey.get(key);
|
|
197
|
+
if (!states) {
|
|
198
|
+
states = new Map(groups.map(group => [group, emptyState(group)]));
|
|
199
|
+
statesByKey.set(key, states);
|
|
200
|
+
}
|
|
201
|
+
groups.forEach((group, groupIndex) =>
|
|
202
|
+
mergeState(group, states.get(group)!, partials[groupIndex]![index]!)
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
result() {
|
|
207
|
+
return new Map(Array.from(statesByKey, ([key, states]) => [key, resultFor(states)]));
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
},
|
|
211
|
+
evaluate(input, evaluation) {
|
|
212
|
+
evaluation?.signal?.throwIfAborted();
|
|
213
|
+
const accumulator = createAccumulator();
|
|
214
|
+
for (const batch of 'batches' in input ? input.batches : input)
|
|
215
|
+
accumulator.update(batch, evaluation);
|
|
216
|
+
return accumulator.result();
|
|
217
|
+
},
|
|
218
|
+
async evaluateInBatches(input, evaluation) {
|
|
219
|
+
evaluation?.signal?.throwIfAborted();
|
|
220
|
+
const accumulator = createAccumulator();
|
|
221
|
+
for await (const batch of input) accumulator.update(batch, evaluation);
|
|
222
|
+
evaluation?.signal?.throwIfAborted();
|
|
223
|
+
return accumulator.result();
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function emptyState(group: Group): State {
|
|
229
|
+
return {count: 0n, sum: group.integer ? 0n : 0, values: new Map()};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Dense batch-local group indices keep expression evaluation independent of group count. */
|
|
233
|
+
type RowPartition = {
|
|
234
|
+
indexByKey: ReadonlyMap<ArrowExpressionValue, number>;
|
|
235
|
+
indexByRow: Uint32Array;
|
|
236
|
+
rowCounts: readonly number[];
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
/** Resolves one group key per row and records first-seen order. */
|
|
240
|
+
function partitionRows(
|
|
241
|
+
expression: CompiledArrowExpression,
|
|
242
|
+
batch: RecordBatch,
|
|
243
|
+
signal?: AbortSignal
|
|
244
|
+
): RowPartition {
|
|
245
|
+
const reader = new ColumnReader(expression.evaluate(batch, {signal}));
|
|
246
|
+
const indexByKey = new Map<ArrowExpressionValue, number>();
|
|
247
|
+
const indexByRow = new Uint32Array(batch.numRows);
|
|
248
|
+
const rowCounts: number[] = [];
|
|
249
|
+
for (let row = 0; row < batch.numRows; row++) {
|
|
250
|
+
if (row % 1024 === 0) signal?.throwIfAborted();
|
|
251
|
+
const key = reader.get(row);
|
|
252
|
+
if (typeof key === 'number' && !Number.isFinite(key))
|
|
253
|
+
throw new Error('Arrow aggregate group keys must be finite');
|
|
254
|
+
let index = indexByKey.get(key);
|
|
255
|
+
if (index === undefined) {
|
|
256
|
+
index = indexByKey.size;
|
|
257
|
+
indexByKey.set(key, index);
|
|
258
|
+
rowCounts.push(0);
|
|
259
|
+
}
|
|
260
|
+
indexByRow[row] = index;
|
|
261
|
+
rowCounts[index]!++;
|
|
262
|
+
}
|
|
263
|
+
return {indexByKey, indexByRow, rowCounts};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Reduces one expression across the batch, optionally routing rows to groups. */
|
|
267
|
+
function summarize(
|
|
268
|
+
group: Group,
|
|
269
|
+
batch: RecordBatch,
|
|
270
|
+
signal?: AbortSignal,
|
|
271
|
+
partition?: RowPartition
|
|
272
|
+
): State[] {
|
|
273
|
+
const states = Array.from({length: partition?.indexByKey.size ?? 1}, () => emptyState(group));
|
|
274
|
+
if (!group.expression) {
|
|
275
|
+
for (let index = 0; index < states.length; index++)
|
|
276
|
+
states[index]!.count = BigInt(partition?.rowCounts[index] ?? batch.numRows);
|
|
277
|
+
return states;
|
|
278
|
+
}
|
|
279
|
+
const vector = group.expression.evaluate(batch, {signal});
|
|
280
|
+
const countOnly = !group.sums && !group.reductions.length;
|
|
281
|
+
if (countOnly && !partition) {
|
|
282
|
+
states[0]!.count = BigInt(vector.length - vector.nullCount);
|
|
283
|
+
return states;
|
|
284
|
+
}
|
|
285
|
+
const reader = new ColumnReader(vector);
|
|
286
|
+
const counts = Array.from({length: states.length}, () => 0);
|
|
287
|
+
for (let row = 0; row < vector.length; row++) {
|
|
288
|
+
if (row % 1024 === 0) signal?.throwIfAborted();
|
|
289
|
+
const index = partition?.indexByRow[row] ?? 0;
|
|
290
|
+
const state = states[index]!;
|
|
291
|
+
if (countOnly || group.utf8) {
|
|
292
|
+
if (!reader.isValid(row)) continue;
|
|
293
|
+
counts[index]!++;
|
|
294
|
+
if (countOnly) continue;
|
|
295
|
+
const bytes = reader.utf8(row);
|
|
296
|
+
for (const operation of group.reductions) {
|
|
297
|
+
const previous = state.values.get(operation) as Uint8Array | undefined;
|
|
298
|
+
if (winsUTF8(operation, bytes, previous))
|
|
299
|
+
state.values.set(operation, bytes[0].slice(bytes[1], bytes[2]));
|
|
300
|
+
}
|
|
301
|
+
} else {
|
|
302
|
+
const value = reader.get(row) as number | bigint | boolean | null;
|
|
303
|
+
if (value === null) continue;
|
|
304
|
+
counts[index]!++;
|
|
305
|
+
if (group.sums) state.sum = add(group, state.sum, value as number | bigint);
|
|
306
|
+
for (const operation of group.reductions) {
|
|
307
|
+
const operand = operation.startsWith('BIT_') ? BigInt(value) : value;
|
|
308
|
+
state.values.set(operation, reduce(operation, state.values.get(operation), operand));
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
states.forEach((state, index) => {
|
|
313
|
+
state.count = BigInt(counts[index]!);
|
|
314
|
+
});
|
|
315
|
+
return states;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function add(group: Group, left: number | bigint, right: number | bigint): number | bigint {
|
|
319
|
+
return group.integer ? BigInt(left) + BigInt(right) : Number(left) + Number(right);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function winsUTF8(
|
|
323
|
+
operation: Operation,
|
|
324
|
+
[bytes, start, end]: [Uint8Array, number, number],
|
|
325
|
+
previous: Uint8Array | undefined
|
|
326
|
+
): boolean {
|
|
327
|
+
if (!previous) return true;
|
|
328
|
+
const order = compareUTF8(bytes, start, end, previous, 0, previous.length);
|
|
329
|
+
return operation === 'MIN' ? order < 0 : order > 0;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function reduce(operation: Operation, left: Value | undefined, right: Value): Value {
|
|
333
|
+
if (left === undefined) return right;
|
|
334
|
+
switch (operation) {
|
|
335
|
+
case 'MIN':
|
|
336
|
+
return typeof left === 'number'
|
|
337
|
+
? Math.min(left, right as number)
|
|
338
|
+
: left < right
|
|
339
|
+
? left
|
|
340
|
+
: right;
|
|
341
|
+
case 'MAX':
|
|
342
|
+
return typeof left === 'number'
|
|
343
|
+
? Math.max(left, right as number)
|
|
344
|
+
: left > right
|
|
345
|
+
? left
|
|
346
|
+
: right;
|
|
347
|
+
case 'EVERY':
|
|
348
|
+
case 'BOOL_AND':
|
|
349
|
+
return Boolean(left && right);
|
|
350
|
+
case 'BOOL_OR':
|
|
351
|
+
return Boolean(left || right);
|
|
352
|
+
case 'BIT_AND':
|
|
353
|
+
return (left as bigint) & (right as bigint);
|
|
354
|
+
case 'BIT_OR':
|
|
355
|
+
return (left as bigint) | (right as bigint);
|
|
356
|
+
case 'BIT_XOR':
|
|
357
|
+
return (left as bigint) ^ (right as bigint);
|
|
358
|
+
default:
|
|
359
|
+
throw new Error(`Invalid Arrow reduction: ${operation}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function mergeState(group: Group, target: State, source: State): void {
|
|
364
|
+
if (!source.count) return;
|
|
365
|
+
target.count += source.count;
|
|
366
|
+
if (group.sums) target.sum = add(group, target.sum, source.sum);
|
|
367
|
+
for (const [operation, value] of source.values) {
|
|
368
|
+
const previous = target.values.get(operation);
|
|
369
|
+
if (value instanceof Uint8Array) {
|
|
370
|
+
if (winsUTF8(operation, [value, 0, value.length], previous as Uint8Array | undefined))
|
|
371
|
+
target.values.set(operation, value);
|
|
372
|
+
} else {
|
|
373
|
+
target.values.set(operation, reduce(operation, previous, value));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function finish(operation: Operation, group: Group, state: State): ArrowExpressionValue {
|
|
379
|
+
if (operation === 'COUNT') return state.count;
|
|
380
|
+
if (!state.count) return null;
|
|
381
|
+
if (operation === 'SUM') return state.sum;
|
|
382
|
+
if (operation === 'AVG') return Number(state.sum) / Number(state.count);
|
|
383
|
+
const value = state.values.get(operation)!;
|
|
384
|
+
if (value instanceof Uint8Array) return new TextDecoder().decode(value);
|
|
385
|
+
return operation.startsWith('BIT_') && !group.wide ? Number(value) : value;
|
|
386
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DataType, DateUnit, Precision, TimeUnit, type Vector, type Data} from 'apache-arrow';
|
|
6
|
+
import {float16Values, requireFloat16} from './float16';
|
|
7
|
+
import {ExpressionEvaluationError} from '../expressions/index';
|
|
8
|
+
import type {ArrowExpressionValue} from './types';
|
|
9
|
+
|
|
10
|
+
/** Dictionary encodings do not change expression value types. */
|
|
11
|
+
export function valueType(type: DataType): DataType {
|
|
12
|
+
return DataType.isDictionary(type) ? valueType(type.dictionary) : type;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Stable primitive type identity, independent of concrete Arrow subclasses. */
|
|
16
|
+
export function typeKey(type: DataType): string {
|
|
17
|
+
type = valueType(type);
|
|
18
|
+
if (DataType.isInt(type)) return `${type.isSigned ? 'i' : 'u'}${type.bitWidth}`;
|
|
19
|
+
if (DataType.isFloat(type)) {
|
|
20
|
+
if (type.precision === Precision.HALF) requireFloat16();
|
|
21
|
+
return `f${type.precision}`;
|
|
22
|
+
}
|
|
23
|
+
if (DataType.isBool(type)) return 'bool';
|
|
24
|
+
if (DataType.isUtf8(type)) return 'utf8';
|
|
25
|
+
if (DataType.isDate(type) && type.unit === DateUnit.DAY) return 'date32';
|
|
26
|
+
if (DataType.isTimestamp(type) && type.unit === TimeUnit.MICROSECOND)
|
|
27
|
+
return `timestamp_us:${type.timezone ?? ''}`;
|
|
28
|
+
if (DataType.isBinary(type)) return 'binary';
|
|
29
|
+
if (DataType.isFixedSizeBinary(type)) return `fixedsizebinary:${type.byteWidth}`;
|
|
30
|
+
if (DataType.isNull(type)) return 'null';
|
|
31
|
+
throw new Error(`Unsupported Arrow expression type: ${type}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Reader over Arrow chunks; slices already adjust numeric buffers and string offsets. */
|
|
35
|
+
export class ColumnReader {
|
|
36
|
+
private readonly ends: number[] = [];
|
|
37
|
+
private readonly location: [Data, number] = [null!, 0];
|
|
38
|
+
private readonly range: [Uint8Array, number, number] = [null!, 0, 0];
|
|
39
|
+
private readonly half = new Map<Data, Float16Array>();
|
|
40
|
+
private readonly dictionaries = new Map<Data, ColumnReader>();
|
|
41
|
+
private readonly decoder = new TextDecoder();
|
|
42
|
+
constructor(readonly vector: Vector) {
|
|
43
|
+
let end = 0;
|
|
44
|
+
for (const data of vector.data) {
|
|
45
|
+
end += data.length;
|
|
46
|
+
this.ends.push(end);
|
|
47
|
+
if (DataType.isFloat(data.type) && data.type.precision === Precision.HALF)
|
|
48
|
+
this.half.set(data, float16Values(data.values as Uint16Array));
|
|
49
|
+
if (DataType.isDictionary(data.type))
|
|
50
|
+
this.dictionaries.set(data, new ColumnReader(data.dictionary!));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Locates a chunk without constructing row objects. */
|
|
54
|
+
private chunk(row: number): [Data, number] {
|
|
55
|
+
let index = 0;
|
|
56
|
+
while (row >= this.ends[index]!) index++;
|
|
57
|
+
this.location[0] = this.vector.data[index]!;
|
|
58
|
+
this.location[1] = row - (index ? this.ends[index - 1]! : 0);
|
|
59
|
+
return this.location;
|
|
60
|
+
}
|
|
61
|
+
/** Tests validity with the physical bit offset. */
|
|
62
|
+
isValid(row: number): boolean {
|
|
63
|
+
const [data, index] = this.chunk(row);
|
|
64
|
+
if (DataType.isNull(data.type)) return false;
|
|
65
|
+
const bit = data.offset + index;
|
|
66
|
+
if (data.nullCount && !(data.nullBitmap[bit >> 3]! & (1 << (bit & 7)))) return false;
|
|
67
|
+
return (
|
|
68
|
+
!DataType.isDictionary(data.type) ||
|
|
69
|
+
this.dictionaries.get(data)!.isValid(Number(data.values[index]))
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
/** Reads a primitive; UTF-8 is decoded only for scalar callbacks or output materialization. */
|
|
73
|
+
get(row: number): ArrowExpressionValue {
|
|
74
|
+
if (!this.isValid(row)) return null;
|
|
75
|
+
const [data, index] = this.chunk(row);
|
|
76
|
+
if (DataType.isDictionary(data.type))
|
|
77
|
+
return this.dictionaries.get(data)!.get(Number(data.values[index]));
|
|
78
|
+
if (DataType.isUtf8(data.type))
|
|
79
|
+
return this.decoder.decode(
|
|
80
|
+
data.values.subarray(data.valueOffsets[index], data.valueOffsets[index + 1])
|
|
81
|
+
);
|
|
82
|
+
if (DataType.isBinary(data.type))
|
|
83
|
+
return data.values.subarray(data.valueOffsets[index], data.valueOffsets[index + 1]);
|
|
84
|
+
if (DataType.isFixedSizeBinary(data.type))
|
|
85
|
+
return data.values.subarray(data.stride * index, data.stride * (index + 1));
|
|
86
|
+
if (DataType.isBool(data.type)) {
|
|
87
|
+
const bit = data.offset + index;
|
|
88
|
+
return Boolean(data.values[bit >> 3]! & (1 << (bit & 7)));
|
|
89
|
+
}
|
|
90
|
+
// Temporal values use their physical units so microseconds never pass through Number.
|
|
91
|
+
return this.half.has(data)
|
|
92
|
+
? this.half.get(data)![index]!
|
|
93
|
+
: (data.values[index] as number | bigint);
|
|
94
|
+
}
|
|
95
|
+
/** Resolves UTF-8 buffers, including dictionary values, without decoding. */
|
|
96
|
+
utf8(row: number): [Uint8Array, number, number] {
|
|
97
|
+
const [data, index] = this.chunk(row);
|
|
98
|
+
if (DataType.isDictionary(data.type))
|
|
99
|
+
return this.dictionaries.get(data)!.utf8(Number(data.values[index]));
|
|
100
|
+
this.range[0] = data.values as Uint8Array;
|
|
101
|
+
this.range[1] = Number(data.valueOffsets[index]);
|
|
102
|
+
this.range[2] = Number(data.valueOffsets[index + 1]);
|
|
103
|
+
return this.range;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Creates a type-specialized scalar validator once per bound operation. */
|
|
108
|
+
export function createValueValidator(inputType: DataType): (value: ArrowExpressionValue) => void {
|
|
109
|
+
const type = valueType(inputType),
|
|
110
|
+
key = typeKey(type);
|
|
111
|
+
if (DataType.isInt(type)) {
|
|
112
|
+
if (type.bitWidth === 64) {
|
|
113
|
+
const bits = BigInt(type.bitWidth),
|
|
114
|
+
min = type.isSigned ? -(1n << (bits - 1n)) : 0n;
|
|
115
|
+
const max = type.isSigned ? (1n << (bits - 1n)) - 1n : (1n << bits) - 1n;
|
|
116
|
+
return value => {
|
|
117
|
+
if (value === null) return;
|
|
118
|
+
if (typeof value !== 'bigint') throw new Error('64-bit integers require bigint values');
|
|
119
|
+
if (value < min || value > max)
|
|
120
|
+
throw new ExpressionEvaluationError('overflow', 'Integer expression overflow');
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
const min = type.isSigned ? -(2 ** (type.bitWidth - 1)) : 0;
|
|
124
|
+
const max = type.isSigned ? 2 ** (type.bitWidth - 1) - 1 : 2 ** type.bitWidth - 1;
|
|
125
|
+
return value => {
|
|
126
|
+
if (value === null) return;
|
|
127
|
+
if (typeof value !== 'number')
|
|
128
|
+
throw new TypeError('Integer expressions require numeric values');
|
|
129
|
+
if (!Number.isInteger(value))
|
|
130
|
+
throw new ExpressionEvaluationError(
|
|
131
|
+
'conversion',
|
|
132
|
+
'Integer expression requires integral numbers'
|
|
133
|
+
);
|
|
134
|
+
if (value < min || value > max)
|
|
135
|
+
throw new ExpressionEvaluationError('overflow', 'Integer expression overflow');
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
if (key === 'date32') {
|
|
139
|
+
return value => {
|
|
140
|
+
if (value === null) return;
|
|
141
|
+
if (
|
|
142
|
+
typeof value !== 'number' ||
|
|
143
|
+
!Number.isInteger(value) ||
|
|
144
|
+
value < -0x80000000 ||
|
|
145
|
+
value > 0x7fffffff
|
|
146
|
+
)
|
|
147
|
+
throw new ExpressionEvaluationError('overflow', 'Invalid Date32 epoch day');
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
if (key.startsWith('timestamp_us:')) {
|
|
151
|
+
return value => {
|
|
152
|
+
if (value === null) return;
|
|
153
|
+
if (typeof value !== 'bigint' || value !== BigInt.asIntN(64, value))
|
|
154
|
+
throw new ExpressionEvaluationError('overflow', 'Invalid timestamp epoch microseconds');
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
if (DataType.isBinary(type) || DataType.isFixedSizeBinary(type)) {
|
|
158
|
+
return value => {
|
|
159
|
+
if (value === null) return;
|
|
160
|
+
if (
|
|
161
|
+
!(value instanceof Uint8Array) ||
|
|
162
|
+
(DataType.isFixedSizeBinary(type) && value.byteLength !== type.byteWidth)
|
|
163
|
+
)
|
|
164
|
+
throw new Error(`Invalid value for ${type}`);
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
const expected = DataType.isFloat(type) ? 'number' : key === 'bool' ? 'boolean' : 'string';
|
|
168
|
+
return value => {
|
|
169
|
+
if (value !== null && (key === 'null' || typeof value !== expected))
|
|
170
|
+
throw new Error(`Invalid value for ${type}`);
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Checks a literal against a declared primitive type during binding. */
|
|
175
|
+
export function validateValue(value: ArrowExpressionValue, type: DataType): void {
|
|
176
|
+
createValueValidator(type)(value);
|
|
177
|
+
}
|