@math.gl/sql 0.0.0-stage → 5.0.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +249 -0
- package/README.md +26 -2
- package/THIRD-PARTY-NOTICES.md +5 -0
- package/dist/arrow/aggregates.d.ts +46 -0
- package/dist/arrow/aggregates.d.ts.map +1 -0
- package/dist/arrow/aggregates.js +289 -0
- package/dist/arrow/aggregates.js.map +1 -0
- package/dist/arrow/buffers.d.ts +30 -0
- package/dist/arrow/buffers.d.ts.map +1 -0
- package/dist/arrow/buffers.js +177 -0
- package/dist/arrow/buffers.js.map +1 -0
- package/dist/arrow/cast.d.ts +19 -0
- package/dist/arrow/cast.d.ts.map +1 -0
- package/dist/arrow/cast.js +138 -0
- package/dist/arrow/cast.js.map +1 -0
- package/dist/arrow/compiler.d.ts +10 -0
- package/dist/arrow/compiler.d.ts.map +1 -0
- package/dist/arrow/compiler.js +1538 -0
- package/dist/arrow/compiler.js.map +1 -0
- package/dist/arrow/float16.d.ts +7 -0
- package/dist/arrow/float16.d.ts.map +1 -0
- package/dist/arrow/float16.js +26 -0
- package/dist/arrow/float16.js.map +1 -0
- package/dist/arrow/index.cjs +3321 -0
- package/dist/arrow/index.cjs.map +6 -0
- package/dist/arrow/index.d.ts +5 -0
- package/dist/arrow/index.d.ts.map +1 -0
- package/dist/arrow/index.js +7 -0
- package/dist/arrow/index.js.map +1 -0
- package/dist/arrow/like.d.ts +3 -0
- package/dist/arrow/like.d.ts.map +1 -0
- package/dist/arrow/like.js +75 -0
- package/dist/arrow/like.js.map +1 -0
- package/dist/arrow/list-values.d.ts +14 -0
- package/dist/arrow/list-values.d.ts.map +1 -0
- package/dist/arrow/list-values.js +201 -0
- package/dist/arrow/list-values.js.map +1 -0
- package/dist/arrow/predicate.d.ts +7 -0
- package/dist/arrow/predicate.d.ts.map +1 -0
- package/dist/arrow/predicate.js +10 -0
- package/dist/arrow/predicate.js.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts +24 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.d.ts.map +1 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js +64 -0
- package/dist/arrow/query-arrow-aggregate-snapshots.js.map +1 -0
- package/dist/arrow/query-arrow-sql.d.ts +8 -0
- package/dist/arrow/query-arrow-sql.d.ts.map +1 -0
- package/dist/arrow/query-arrow-sql.js +223 -0
- package/dist/arrow/query-arrow-sql.js.map +1 -0
- package/dist/arrow/query-arrow-table.d.ts +36 -0
- package/dist/arrow/query-arrow-table.d.ts.map +1 -0
- package/dist/arrow/query-arrow-table.js +208 -0
- package/dist/arrow/query-arrow-table.js.map +1 -0
- package/dist/arrow/query-worker-entry.d.ts +2 -0
- package/dist/arrow/query-worker-entry.d.ts.map +1 -0
- package/dist/arrow/query-worker-entry.js +7 -0
- package/dist/arrow/query-worker-entry.js.map +1 -0
- package/dist/arrow/query-worker.d.ts +65 -0
- package/dist/arrow/query-worker.d.ts.map +1 -0
- package/dist/arrow/query-worker.js +135 -0
- package/dist/arrow/query-worker.js.map +1 -0
- package/dist/arrow/registry.d.ts +4 -0
- package/dist/arrow/registry.d.ts.map +1 -0
- package/dist/arrow/registry.js +66 -0
- package/dist/arrow/registry.js.map +1 -0
- package/dist/arrow/relational-query.d.ts +36 -0
- package/dist/arrow/relational-query.d.ts.map +1 -0
- package/dist/arrow/relational-query.js +174 -0
- package/dist/arrow/relational-query.js.map +1 -0
- package/dist/arrow/replace-arrow-columns.d.ts +8 -0
- package/dist/arrow/replace-arrow-columns.d.ts.map +1 -0
- package/dist/arrow/replace-arrow-columns.js +63 -0
- package/dist/arrow/replace-arrow-columns.js.map +1 -0
- package/dist/arrow/signature.d.ts +6 -0
- package/dist/arrow/signature.d.ts.map +1 -0
- package/dist/arrow/signature.js +51 -0
- package/dist/arrow/signature.js.map +1 -0
- package/dist/arrow/temporal-cast.d.ts +9 -0
- package/dist/arrow/temporal-cast.d.ts.map +1 -0
- package/dist/arrow/temporal-cast.js +60 -0
- package/dist/arrow/temporal-cast.js.map +1 -0
- package/dist/arrow/types.d.ts +62 -0
- package/dist/arrow/types.d.ts.map +1 -0
- package/dist/arrow/types.js +5 -0
- package/dist/arrow/types.js.map +1 -0
- package/dist/arrow/unnest-arrow-column.d.ts +22 -0
- package/dist/arrow/unnest-arrow-column.d.ts.map +1 -0
- package/dist/arrow/unnest-arrow-column.js +112 -0
- package/dist/arrow/unnest-arrow-column.js.map +1 -0
- package/dist/arrow/worker-entry.d.ts +2 -0
- package/dist/arrow/worker-entry.d.ts.map +1 -0
- package/dist/arrow/worker-entry.js +9 -0
- package/dist/arrow/worker-entry.js.map +1 -0
- package/dist/arrow/worker.cjs +4074 -0
- package/dist/arrow/worker.cjs.map +6 -0
- package/dist/arrow/worker.d.ts +58 -0
- package/dist/arrow/worker.d.ts.map +1 -0
- package/dist/arrow/worker.js +84 -0
- package/dist/arrow/worker.js.map +1 -0
- package/dist/color-functions/index.cjs +382 -0
- package/dist/color-functions/index.cjs.map +6 -0
- package/dist/color-functions/index.d.ts +26 -0
- package/dist/color-functions/index.d.ts.map +1 -0
- package/dist/color-functions/index.js +306 -0
- package/dist/color-functions/index.js.map +1 -0
- package/dist/diagnostic-log.d.ts +8 -0
- package/dist/diagnostic-log.d.ts.map +1 -0
- package/dist/diagnostic-log.js +15 -0
- package/dist/diagnostic-log.js.map +1 -0
- package/dist/expressions/ast.d.ts +12 -0
- package/dist/expressions/ast.d.ts.map +1 -0
- package/dist/expressions/ast.js +114 -0
- package/dist/expressions/ast.js.map +1 -0
- package/dist/expressions/columnar-predicate.d.ts +56 -0
- package/dist/expressions/columnar-predicate.d.ts.map +1 -0
- package/dist/expressions/columnar-predicate.js +176 -0
- package/dist/expressions/columnar-predicate.js.map +1 -0
- package/dist/expressions/evaluation-error.d.ts +14 -0
- package/dist/expressions/evaluation-error.d.ts.map +1 -0
- package/dist/expressions/evaluation-error.js +26 -0
- package/dist/expressions/evaluation-error.js.map +1 -0
- package/dist/expressions/expression-eval.d.ts +105 -0
- package/dist/expressions/expression-eval.d.ts.map +1 -0
- package/dist/expressions/expression-eval.js +330 -0
- package/dist/expressions/expression-eval.js.map +1 -0
- package/dist/expressions/function-libraries.d.ts +63 -0
- package/dist/expressions/function-libraries.d.ts.map +1 -0
- package/dist/expressions/function-libraries.js +69 -0
- package/dist/expressions/function-libraries.js.map +1 -0
- package/dist/expressions/function-registry.d.ts +93 -0
- package/dist/expressions/function-registry.d.ts.map +1 -0
- package/dist/expressions/function-registry.js +139 -0
- package/dist/expressions/function-registry.js.map +1 -0
- package/dist/expressions/get.d.ts +9 -0
- package/dist/expressions/get.d.ts.map +1 -0
- package/dist/expressions/get.js +32 -0
- package/dist/expressions/get.js.map +1 -0
- package/dist/expressions/index.cjs +1888 -0
- package/dist/expressions/index.cjs.map +6 -0
- package/dist/expressions/index.d.ts +13 -0
- package/dist/expressions/index.d.ts.map +1 -0
- package/dist/expressions/index.js +14 -0
- package/dist/expressions/index.js.map +1 -0
- package/dist/expressions/numeric-functions.d.ts +277 -0
- package/dist/expressions/numeric-functions.d.ts.map +1 -0
- package/dist/expressions/numeric-functions.js +52 -0
- package/dist/expressions/numeric-functions.js.map +1 -0
- package/dist/expressions/parse-expression-string.d.ts +21 -0
- package/dist/expressions/parse-expression-string.d.ts.map +1 -0
- package/dist/expressions/parse-expression-string.js +61 -0
- package/dist/expressions/parse-expression-string.js.map +1 -0
- package/dist/expressions/predicate-expression.d.ts +7 -0
- package/dist/expressions/predicate-expression.d.ts.map +1 -0
- package/dist/expressions/predicate-expression.js +112 -0
- package/dist/expressions/predicate-expression.js.map +1 -0
- package/dist/expressions/scalar-type.d.ts +6 -0
- package/dist/expressions/scalar-type.d.ts.map +1 -0
- package/dist/expressions/scalar-type.js +23 -0
- package/dist/expressions/scalar-type.js.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts +3 -0
- package/dist/expressions/vendor/jsep/runtime.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/runtime.js +972 -0
- package/dist/expressions/vendor/jsep/runtime.js.map +1 -0
- package/dist/expressions/vendor/jsep/types.d.ts +148 -0
- package/dist/expressions/vendor/jsep/types.d.ts.map +1 -0
- package/dist/expressions/vendor/jsep/types.js +6 -0
- package/dist/expressions/vendor/jsep/types.js.map +1 -0
- package/dist/index.cjs +5294 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/math-functions/index.cjs +444 -0
- package/dist/math-functions/index.cjs.map +6 -0
- package/dist/math-functions/index.d.ts +44 -0
- package/dist/math-functions/index.d.ts.map +1 -0
- package/dist/math-functions/index.js +182 -0
- package/dist/math-functions/index.js.map +1 -0
- package/dist/parser/bind-sql-predicate.d.ts +4 -0
- package/dist/parser/bind-sql-predicate.d.ts.map +1 -0
- package/dist/parser/bind-sql-predicate.js +11 -0
- package/dist/parser/bind-sql-predicate.js.map +1 -0
- package/dist/parser/index.cjs +1807 -0
- package/dist/parser/index.cjs.map +6 -0
- package/dist/parser/index.d.ts +12 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +11 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/parameter-snapshots.d.ts +21 -0
- package/dist/parser/parameter-snapshots.d.ts.map +1 -0
- package/dist/parser/parameter-snapshots.js +85 -0
- package/dist/parser/parameter-snapshots.js.map +1 -0
- package/dist/parser/parse-sql-expression.d.ts +50 -0
- package/dist/parser/parse-sql-expression.d.ts.map +1 -0
- package/dist/parser/parse-sql-expression.js +501 -0
- package/dist/parser/parse-sql-expression.js.map +1 -0
- package/dist/parser/parse-sql-predicate.d.ts +13 -0
- package/dist/parser/parse-sql-predicate.d.ts.map +1 -0
- package/dist/parser/parse-sql-predicate.js +365 -0
- package/dist/parser/parse-sql-predicate.js.map +1 -0
- package/dist/parser/parse-sql-query.d.ts +9 -0
- package/dist/parser/parse-sql-query.d.ts.map +1 -0
- package/dist/parser/parse-sql-query.js +81 -0
- package/dist/parser/parse-sql-query.js.map +1 -0
- package/dist/parser/parse-sql-select.d.ts +57 -0
- package/dist/parser/parse-sql-select.d.ts.map +1 -0
- package/dist/parser/parse-sql-select.js +247 -0
- package/dist/parser/parse-sql-select.js.map +1 -0
- package/dist/parser/parse-sql-statement.d.ts +54 -0
- package/dist/parser/parse-sql-statement.d.ts.map +1 -0
- package/dist/parser/parse-sql-statement.js +207 -0
- package/dist/parser/parse-sql-statement.js.map +1 -0
- package/dist/parser/query-types.d.ts +56 -0
- package/dist/parser/query-types.d.ts.map +1 -0
- package/dist/parser/query-types.js +5 -0
- package/dist/parser/query-types.js.map +1 -0
- package/dist/parser/sql-parameters.d.ts +29 -0
- package/dist/parser/sql-parameters.d.ts.map +1 -0
- package/dist/parser/sql-parameters.js +110 -0
- package/dist/parser/sql-parameters.js.map +1 -0
- package/dist/parser/sql-parse-error.d.ts +13 -0
- package/dist/parser/sql-parse-error.d.ts.map +1 -0
- package/dist/parser/sql-parse-error.js +20 -0
- package/dist/parser/sql-parse-error.js.map +1 -0
- package/dist/parser/sql-predicate-schema.d.ts +164 -0
- package/dist/parser/sql-predicate-schema.d.ts.map +1 -0
- package/dist/parser/sql-predicate-schema.js +249 -0
- package/dist/parser/sql-predicate-schema.js.map +1 -0
- package/dist/parser/sql-predicate-types.d.ts +48 -0
- package/dist/parser/sql-predicate-types.d.ts.map +1 -0
- package/dist/parser/sql-predicate-types.js +9 -0
- package/dist/parser/sql-predicate-types.js.map +1 -0
- package/dist/parser/sql-predicate.d.ts +6 -0
- package/dist/parser/sql-predicate.d.ts.map +1 -0
- package/dist/parser/sql-predicate.js +8 -0
- package/dist/parser/sql-predicate.js.map +1 -0
- package/dist/scales/arrow-domains.d.ts +22 -0
- package/dist/scales/arrow-domains.d.ts.map +1 -0
- package/dist/scales/arrow-domains.js +166 -0
- package/dist/scales/arrow-domains.js.map +1 -0
- package/dist/scales/arrow-scales.d.ts +44 -0
- package/dist/scales/arrow-scales.d.ts.map +1 -0
- package/dist/scales/arrow-scales.js +255 -0
- package/dist/scales/arrow-scales.js.map +1 -0
- package/dist/scales/compile-scale.d.ts +2 -0
- package/dist/scales/compile-scale.d.ts.map +1 -0
- package/dist/scales/compile-scale.js +5 -0
- package/dist/scales/compile-scale.js.map +1 -0
- package/dist/scales/conformance.cjs +180 -0
- package/dist/scales/conformance.cjs.map +6 -0
- package/dist/scales/conformance.d.ts +26 -0
- package/dist/scales/conformance.d.ts.map +1 -0
- package/dist/scales/conformance.js +164 -0
- package/dist/scales/conformance.js.map +1 -0
- package/dist/scales/hdr-colors.d.ts +2 -0
- package/dist/scales/hdr-colors.d.ts.map +1 -0
- package/dist/scales/hdr-colors.js +5 -0
- package/dist/scales/hdr-colors.js.map +1 -0
- package/dist/scales/index.cjs +4334 -0
- package/dist/scales/index.cjs.map +6 -0
- package/dist/scales/index.d.ts +10 -0
- package/dist/scales/index.d.ts.map +1 -0
- package/dist/scales/index.js +11 -0
- package/dist/scales/index.js.map +1 -0
- package/dist/scales/palettes.d.ts +2 -0
- package/dist/scales/palettes.d.ts.map +1 -0
- package/dist/scales/palettes.js +5 -0
- package/dist/scales/palettes.js.map +1 -0
- package/dist/scales/parse-sql-scale.d.ts +27 -0
- package/dist/scales/parse-sql-scale.d.ts.map +1 -0
- package/dist/scales/parse-sql-scale.js +257 -0
- package/dist/scales/parse-sql-scale.js.map +1 -0
- package/dist/scales/quantile-sketch.d.ts +2 -0
- package/dist/scales/quantile-sketch.d.ts.map +1 -0
- package/dist/scales/quantile-sketch.js +5 -0
- package/dist/scales/quantile-sketch.js.map +1 -0
- package/dist/scales/scale-types.d.ts +2 -0
- package/dist/scales/scale-types.d.ts.map +1 -0
- package/dist/scales/scale-types.js +5 -0
- package/dist/scales/scale-types.js.map +1 -0
- package/dist/scales/streaming-domains.d.ts +18 -0
- package/dist/scales/streaming-domains.d.ts.map +1 -0
- package/dist/scales/streaming-domains.js +115 -0
- package/dist/scales/streaming-domains.js.map +1 -0
- package/package.json +92 -4
- package/src/arrow/aggregates.ts +386 -0
- package/src/arrow/buffers.ts +177 -0
- package/src/arrow/cast.ts +160 -0
- package/src/arrow/compiler.ts +1778 -0
- package/src/arrow/float16.ts +25 -0
- package/src/arrow/index.ts +22 -0
- package/src/arrow/like.ts +72 -0
- package/src/arrow/list-values.ts +221 -0
- package/src/arrow/predicate.ts +19 -0
- package/src/arrow/query-arrow-aggregate-snapshots.ts +107 -0
- package/src/arrow/query-arrow-sql.ts +255 -0
- package/src/arrow/query-arrow-table.ts +298 -0
- package/src/arrow/query-worker-entry.ts +8 -0
- package/src/arrow/query-worker.ts +194 -0
- package/src/arrow/registry.ts +87 -0
- package/src/arrow/relational-query.ts +247 -0
- package/src/arrow/replace-arrow-columns.ts +71 -0
- package/src/arrow/signature.ts +50 -0
- package/src/arrow/temporal-cast.ts +66 -0
- package/src/arrow/types.ts +76 -0
- package/src/arrow/unnest-arrow-column.ts +145 -0
- package/src/arrow/worker-entry.ts +10 -0
- package/src/arrow/worker.ts +143 -0
- package/src/color-functions/index.ts +405 -0
- package/src/diagnostic-log.ts +20 -0
- package/src/expressions/ast.ts +143 -0
- package/src/expressions/columnar-predicate.ts +293 -0
- package/src/expressions/evaluation-error.ts +29 -0
- package/src/expressions/expression-eval.ts +489 -0
- package/src/expressions/function-libraries.ts +126 -0
- package/src/expressions/function-registry.ts +171 -0
- package/src/expressions/get.ts +37 -0
- package/src/expressions/index.ts +40 -0
- package/src/expressions/numeric-functions.ts +99 -0
- package/src/expressions/parse-expression-string.ts +80 -0
- package/src/expressions/predicate-expression.ts +128 -0
- package/src/expressions/scalar-type.ts +39 -0
- package/src/expressions/vendor/jsep/LICENSE.txt +23 -0
- package/src/expressions/vendor/jsep/runtime.ts +949 -0
- package/src/expressions/vendor/jsep/types.ts +191 -0
- package/src/index.ts +54 -0
- package/src/math-functions/index.ts +235 -0
- package/src/parser/bind-sql-predicate.ts +21 -0
- package/src/parser/index.ts +43 -0
- package/src/parser/parameter-snapshots.ts +103 -0
- package/src/parser/parse-sql-expression.ts +500 -0
- package/src/parser/parse-sql-predicate.ts +454 -0
- package/src/parser/parse-sql-query.ts +90 -0
- package/src/parser/parse-sql-select.ts +272 -0
- package/src/parser/parse-sql-statement.ts +260 -0
- package/src/parser/query-types.ts +71 -0
- package/src/parser/sql-parameters.ts +174 -0
- package/src/parser/sql-parse-error.ts +33 -0
- package/src/parser/sql-predicate-schema.ts +279 -0
- package/src/parser/sql-predicate-types.ts +91 -0
- package/src/parser/sql-predicate.ts +28 -0
- package/src/scales/arrow-domains.ts +190 -0
- package/src/scales/arrow-scales.ts +363 -0
- package/src/scales/compile-scale.ts +5 -0
- package/src/scales/conformance.ts +186 -0
- package/src/scales/hdr-colors.ts +5 -0
- package/src/scales/index.ts +37 -0
- package/src/scales/palettes.ts +5 -0
- package/src/scales/parse-sql-scale.ts +300 -0
- package/src/scales/quantile-sketch.ts +5 -0
- package/src/scales/scale-types.ts +5 -0
- package/src/scales/streaming-domains.ts +139 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quantile-sketch.js","sourceRoot":"","sources":["../../src/scales/quantile-sketch.ts"],"names":[],"mappings":"AAAA,UAAU;AACV,+BAA+B;AAC/B,4DAA4D;AAE5D,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scale-types.d.ts","sourceRoot":"","sources":["../../src/scales/scale-types.ts"],"names":[],"mappings":"AAIA,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scale-types.js","sourceRoot":"","sources":["../../src/scales/scale-types.ts"],"names":[],"mappings":"AAAA,UAAU;AACV,+BAA+B;AAC/B,4DAA4D;AAE5D,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { RecordBatch } from 'apache-arrow';
|
|
2
|
+
import type { ScaleDomain, ScaleValue } from "./scale-types.js";
|
|
3
|
+
export type StreamingDomainOptions = {
|
|
4
|
+
/** Maximum retained weighted samples per referenced percentile column. */
|
|
5
|
+
quantileCapacity?: number;
|
|
6
|
+
/** Also keep quantile sketches for these columns, e.g. SCALE_QUANTILE inputs. */
|
|
7
|
+
quantileColumns?: readonly string[];
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Incremental domain accumulator. Each Arrow batch is visited once per
|
|
11
|
+
* referenced column, even when multiple visual scales depend on that column.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createStreamingDomainAccumulator(domains: readonly ScaleDomain[], options?: StreamingDomainOptions): {
|
|
14
|
+
update(batch: RecordBatch): void;
|
|
15
|
+
result(): readonly (readonly ScaleValue[])[];
|
|
16
|
+
quantileBreaks(column: string, classes: number): readonly number[];
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=streaming-domains.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streaming-domains.d.ts","sourceRoot":"","sources":["../../src/scales/streaming-domains.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAC,WAAW,EAAE,UAAU,EAAC,yBAAsB;AAa3D,MAAM,MAAM,sBAAsB,GAAG;IACnC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,GAAE,sBAA2B,GACnC;IACD,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC,MAAM,IAAI,SAAS,CAAC,SAAS,UAAU,EAAE,CAAC,EAAE,CAAC;IAC7C,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;CACpE,CAsGA"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
import { arrowScaleValue } from "./arrow-domains.js";
|
|
5
|
+
import { ScaleQuantileSketch } from "./quantile-sketch.js";
|
|
6
|
+
/**
|
|
7
|
+
* Incremental domain accumulator. Each Arrow batch is visited once per
|
|
8
|
+
* referenced column, even when multiple visual scales depend on that column.
|
|
9
|
+
*/
|
|
10
|
+
export function createStreamingDomainAccumulator(domains, options = {}) {
|
|
11
|
+
const columns = new Map();
|
|
12
|
+
const quantileColumns = new Set(options.quantileColumns ?? []);
|
|
13
|
+
const streamingDomains = domains.map(domain => {
|
|
14
|
+
if (domain.type === 'expression-extent') {
|
|
15
|
+
throw new Error('Streaming scale domains require computed values in a projected column');
|
|
16
|
+
}
|
|
17
|
+
return domain;
|
|
18
|
+
});
|
|
19
|
+
for (const domain of streamingDomains) {
|
|
20
|
+
if (domain.type === 'fixed')
|
|
21
|
+
continue;
|
|
22
|
+
const previous = columns.get(domain.column);
|
|
23
|
+
if (previous) {
|
|
24
|
+
if ((domain.type === 'percentile' || quantileColumns.has(domain.column)) &&
|
|
25
|
+
!previous.sketch) {
|
|
26
|
+
previous.sketch = new ScaleQuantileSketch(options.quantileCapacity);
|
|
27
|
+
}
|
|
28
|
+
if (domain.type === 'categories')
|
|
29
|
+
previous.categories ?? (previous.categories = new Set());
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
columns.set(domain.column, {
|
|
33
|
+
min: Infinity,
|
|
34
|
+
max: -Infinity,
|
|
35
|
+
seen: 0,
|
|
36
|
+
...(domain.type === 'categories' ? { categories: new Set() } : {}),
|
|
37
|
+
...(domain.type === 'percentile' || quantileColumns.has(domain.column)
|
|
38
|
+
? { sketch: new ScaleQuantileSketch(options.quantileCapacity) }
|
|
39
|
+
: {})
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
for (const name of quantileColumns) {
|
|
43
|
+
if (!columns.has(name))
|
|
44
|
+
columns.set(name, {
|
|
45
|
+
min: Infinity,
|
|
46
|
+
max: -Infinity,
|
|
47
|
+
seen: 0,
|
|
48
|
+
sketch: new ScaleQuantileSketch(options.quantileCapacity)
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
update(batch) {
|
|
53
|
+
for (const [name, state] of columns) {
|
|
54
|
+
const vector = batch.getChild(name);
|
|
55
|
+
if (!vector)
|
|
56
|
+
throw new Error(`Missing scale column: ${name}`);
|
|
57
|
+
for (let row = 0; row < vector.length; row++) {
|
|
58
|
+
const value = arrowScaleValue(vector.get(row));
|
|
59
|
+
if (value === null || value === undefined)
|
|
60
|
+
continue;
|
|
61
|
+
if (typeof value === 'string') {
|
|
62
|
+
state.categories?.add(value);
|
|
63
|
+
}
|
|
64
|
+
else if (typeof value === 'number' && Number.isFinite(value)) {
|
|
65
|
+
state.categories?.add(value);
|
|
66
|
+
state.seen++;
|
|
67
|
+
state.min = Math.min(state.min, value);
|
|
68
|
+
state.max = Math.max(state.max, value);
|
|
69
|
+
state.sketch?.add(value);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
throw new Error(`Scale column ${name} has unsupported value at row ${row}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
result() {
|
|
78
|
+
return streamingDomains.map(domain => {
|
|
79
|
+
if (domain.type === 'fixed')
|
|
80
|
+
return [...domain.values];
|
|
81
|
+
const state = columns.get(domain.column);
|
|
82
|
+
if (domain.type === 'categories') {
|
|
83
|
+
const labels = [...state.categories];
|
|
84
|
+
return domain.order === 'first-seen'
|
|
85
|
+
? labels
|
|
86
|
+
: labels.sort((a, b) => typeof a === 'number' && typeof b === 'number' ? a - b : a < b ? -1 : a > b ? 1 : 0);
|
|
87
|
+
}
|
|
88
|
+
if (!state.seen)
|
|
89
|
+
return [];
|
|
90
|
+
if (domain.type === 'percentile') {
|
|
91
|
+
return [state.sketch.quantile(domain.lower), state.sketch.quantile(domain.upper)];
|
|
92
|
+
}
|
|
93
|
+
if (domain.type === 'symmetric') {
|
|
94
|
+
const radius = Math.max(Math.abs(state.min - domain.pivot), Math.abs(state.max - domain.pivot));
|
|
95
|
+
return [domain.pivot - radius, domain.pivot + radius];
|
|
96
|
+
}
|
|
97
|
+
if (domain.includeZero)
|
|
98
|
+
return [Math.min(state.min, 0), Math.max(state.max, 0)];
|
|
99
|
+
return [state.min, state.max];
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
quantileBreaks(column, classes) {
|
|
103
|
+
if (!Number.isSafeInteger(classes) || classes < 2) {
|
|
104
|
+
throw new Error('Scale class count must be at least two');
|
|
105
|
+
}
|
|
106
|
+
const sketch = columns.get(column)?.sketch;
|
|
107
|
+
if (!sketch)
|
|
108
|
+
throw new Error(`Column ${column} has no quantile sketch`);
|
|
109
|
+
if (!sketch.length)
|
|
110
|
+
return [];
|
|
111
|
+
return Array.from({ length: classes - 1 }, (_, i) => sketch.quantile((i + 1) / classes));
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=streaming-domains.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streaming-domains.js","sourceRoot":"","sources":["../../src/scales/streaming-domains.ts"],"names":[],"mappings":"AAAA,UAAU;AACV,+BAA+B;AAC/B,4DAA4D;AAI5D,OAAO,EAAC,eAAe,EAAC,2BAAwB;AAEhD,OAAO,EAAC,mBAAmB,EAAC,6BAA0B;AAiBtD;;;GAGG;AACH,MAAM,UAAU,gCAAgC,CAC9C,OAA+B,EAC/B,UAAkC,EAAE;IAMpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QAC5C,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,QAAQ,EAAE,CAAC;YACb,IACE,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACpE,CAAC,QAAQ,CAAC,MAAM,EAChB,CAAC;gBACD,QAAQ,CAAC,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACtE,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY;gBAAE,QAAQ,CAAC,UAAU,KAAnB,QAAQ,CAAC,UAAU,GAAK,IAAI,GAAG,EAAE,EAAC;YACpE,SAAS;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;YACzB,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,CAAC,QAAQ;YACd,IAAI,EAAE,CAAC;YACP,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,EAAC,UAAU,EAAE,IAAI,GAAG,EAAc,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;gBACpE,CAAC,CAAC,EAAC,MAAM,EAAE,IAAI,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAC;gBAC7D,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;gBAChB,GAAG,EAAE,QAAQ;gBACb,GAAG,EAAE,CAAC,QAAQ;gBACd,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,IAAI,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,CAAC;aAC1D,CAAC,CAAC;IACP,CAAC;IAED,OAAO;QACL,MAAM,CAAC,KAAK;YACV,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;gBACpC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;gBAC9D,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;oBAC7C,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC/C,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;wBAAE,SAAS;oBACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;wBAC9B,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC/B,CAAC;yBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC/D,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC7B,KAAK,CAAC,IAAI,EAAE,CAAC;wBACb,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACvC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACvC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC;yBAAM,CAAC;wBACN,MAAM,IAAI,KAAK,CAAC,gBAAgB,IAAI,iCAAiC,GAAG,EAAE,CAAC,CAAC;oBAC9E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM;YACJ,OAAO,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACnC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;oBAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC;gBAC1C,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACjC,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,UAAW,CAAC,CAAC;oBACtC,OAAO,MAAM,CAAC,KAAK,KAAK,YAAY;wBAClC,CAAC,CAAC,MAAM;wBACR,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACnB,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACpF,CAAC;gBACR,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,IAAI;oBAAE,OAAO,EAAE,CAAC;gBAC3B,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACjC,OAAO,CAAC,KAAK,CAAC,MAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,KAAK,CAAC,MAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CACnC,CAAC;oBACF,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;gBACxD,CAAC;gBACD,IAAI,MAAM,CAAC,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;gBAChF,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC;QACD,cAAc,CAAC,MAAM,EAAE,OAAO;YAC5B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;YAC3C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,UAAU,MAAM,yBAAyB,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,OAAO,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,OAAO,GAAG,CAAC,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAE,CAAC,CAAC;QAC1F,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,94 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@math.gl/sql",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "5.0.0-beta.12",
|
|
4
|
+
"description": "Lightweight SQL query engine for Apache Arrow tables",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"publishConfig": {
|
|
8
|
+
"access": "public"
|
|
9
|
+
},
|
|
10
|
+
"repository": {
|
|
11
|
+
"type": "git",
|
|
12
|
+
"url": "https://github.com/visgl/math.gl.git"
|
|
13
|
+
},
|
|
14
|
+
"types": "dist/index.d.ts",
|
|
15
|
+
"main": "dist/index.cjs",
|
|
16
|
+
"module": "dist/index.js",
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"src",
|
|
20
|
+
"!src/**/README.md",
|
|
21
|
+
"README.md",
|
|
22
|
+
"LICENSE",
|
|
23
|
+
"THIRD-PARTY-NOTICES.md"
|
|
24
|
+
],
|
|
25
|
+
"exports": {
|
|
26
|
+
".": {
|
|
27
|
+
"types": "./dist/index.d.ts",
|
|
28
|
+
"import": "./dist/index.js",
|
|
29
|
+
"require": "./dist/index.cjs"
|
|
30
|
+
},
|
|
31
|
+
"./scales": {
|
|
32
|
+
"types": "./dist/scales/index.d.ts",
|
|
33
|
+
"import": "./dist/scales/index.js",
|
|
34
|
+
"require": "./dist/scales/index.cjs"
|
|
35
|
+
},
|
|
36
|
+
"./scales/conformance": {
|
|
37
|
+
"types": "./dist/scales/conformance.d.ts",
|
|
38
|
+
"import": "./dist/scales/conformance.js",
|
|
39
|
+
"require": "./dist/scales/conformance.cjs"
|
|
40
|
+
},
|
|
41
|
+
"./parser": {
|
|
42
|
+
"types": "./dist/parser/index.d.ts",
|
|
43
|
+
"import": "./dist/parser/index.js",
|
|
44
|
+
"require": "./dist/parser/index.cjs"
|
|
45
|
+
},
|
|
46
|
+
"./expressions": {
|
|
47
|
+
"types": "./dist/expressions/index.d.ts",
|
|
48
|
+
"import": "./dist/expressions/index.js",
|
|
49
|
+
"require": "./dist/expressions/index.cjs"
|
|
50
|
+
},
|
|
51
|
+
"./arrow": {
|
|
52
|
+
"types": "./dist/arrow/index.d.ts",
|
|
53
|
+
"import": "./dist/arrow/index.js",
|
|
54
|
+
"require": "./dist/arrow/index.cjs"
|
|
55
|
+
},
|
|
56
|
+
"./worker": {
|
|
57
|
+
"types": "./dist/arrow/worker.d.ts",
|
|
58
|
+
"import": "./dist/arrow/worker.js",
|
|
59
|
+
"require": "./dist/arrow/worker.cjs"
|
|
60
|
+
},
|
|
61
|
+
"./worker-entry.js": "./dist/arrow/worker-entry.js",
|
|
62
|
+
"./query-worker-entry.js": "./dist/arrow/query-worker-entry.js",
|
|
63
|
+
"./math-functions": {
|
|
64
|
+
"types": "./dist/math-functions/index.d.ts",
|
|
65
|
+
"import": "./dist/math-functions/index.js",
|
|
66
|
+
"require": "./dist/math-functions/index.cjs"
|
|
67
|
+
},
|
|
68
|
+
"./color-functions": {
|
|
69
|
+
"types": "./dist/color-functions/index.d.ts",
|
|
70
|
+
"import": "./dist/color-functions/index.js",
|
|
71
|
+
"require": "./dist/color-functions/index.cjs"
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
"sideEffects": [
|
|
75
|
+
"./dist/arrow/worker-entry.js",
|
|
76
|
+
"./src/arrow/worker-entry.ts",
|
|
77
|
+
"./dist/arrow/query-worker-entry.js",
|
|
78
|
+
"./src/arrow/query-worker-entry.ts"
|
|
79
|
+
],
|
|
80
|
+
"dependencies": {
|
|
81
|
+
"@math.gl/core": "5.0.0-beta.12",
|
|
82
|
+
"@math.gl/geometry": "5.0.0-beta.12",
|
|
83
|
+
"@math.gl/tasks": "5.0.0-beta.12"
|
|
84
|
+
},
|
|
85
|
+
"peerDependencies": {
|
|
86
|
+
"apache-arrow": ">=17.0.0"
|
|
87
|
+
},
|
|
88
|
+
"peerDependenciesMeta": {
|
|
89
|
+
"apache-arrow": {
|
|
90
|
+
"optional": true
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
"gitHead": "f48cd096d5f3b61dced52d5fcd5906a7e9541b12"
|
|
94
|
+
}
|
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// math.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DataType, type Table, type RecordBatch} from 'apache-arrow';
|
|
6
|
+
import {compareUTF8} from '@math.gl/core/utf8';
|
|
7
|
+
import {ColumnReader, valueType} from './buffers';
|
|
8
|
+
import {compileArrowExpression} from './compiler';
|
|
9
|
+
import type {
|
|
10
|
+
ArrowExpression,
|
|
11
|
+
ArrowExpressionOptions,
|
|
12
|
+
ArrowExpressionValue,
|
|
13
|
+
CompiledArrowExpression
|
|
14
|
+
} from './types';
|
|
15
|
+
|
|
16
|
+
type Operation =
|
|
17
|
+
| 'COUNT'
|
|
18
|
+
| 'SUM'
|
|
19
|
+
| 'MIN'
|
|
20
|
+
| 'MAX'
|
|
21
|
+
| 'AVG'
|
|
22
|
+
| 'EVERY'
|
|
23
|
+
| 'BOOL_AND'
|
|
24
|
+
| 'BOOL_OR'
|
|
25
|
+
| 'BIT_AND'
|
|
26
|
+
| 'BIT_OR'
|
|
27
|
+
| 'BIT_XOR';
|
|
28
|
+
|
|
29
|
+
/** One scalar reduction. Omit the COUNT expression for COUNT(*). */
|
|
30
|
+
export type ArrowAggregate =
|
|
31
|
+
| {readonly operation: 'COUNT'; readonly expression?: ArrowExpression}
|
|
32
|
+
| {readonly operation: Exclude<Operation, 'COUNT'>; readonly expression: ArrowExpression};
|
|
33
|
+
|
|
34
|
+
export interface ArrowAggregateAccumulator<K extends string> {
|
|
35
|
+
/** Adds a batch. A failed expression leaves the previous state intact. */
|
|
36
|
+
update(batch: RecordBatch, options?: {signal?: AbortSignal}): void;
|
|
37
|
+
/** Adds the partial state of another accumulator from this compiled plan. */
|
|
38
|
+
merge(other: ArrowAggregateAccumulator<K>): void;
|
|
39
|
+
/** Returns a snapshot. Further batches may change the next result. */
|
|
40
|
+
result(): Record<K, ArrowExpressionValue>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Incrementally reduces rows into groups without storing the input rows. */
|
|
44
|
+
export interface ArrowGroupedAggregateAccumulator<K extends string> {
|
|
45
|
+
/** Adds one batch atomically. An expression error or cancellation leaves the previous state intact. */
|
|
46
|
+
update(batch: RecordBatch, options?: {signal?: AbortSignal}): void;
|
|
47
|
+
/** Returns a detached snapshot in first-seen key order. Null keys form one group. */
|
|
48
|
+
result(): ReadonlyMap<ArrowExpressionValue, Record<K, ArrowExpressionValue>>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A schema-bound plan for scalar or grouped reductions. */
|
|
52
|
+
export interface CompiledArrowAggregates<K extends string> {
|
|
53
|
+
/** Starts a grouped reduction using one scalar key expression. */
|
|
54
|
+
createGroupedAccumulator(groupBy: ArrowExpression): ArrowGroupedAggregateAccumulator<K>;
|
|
55
|
+
createAccumulator(): ArrowAggregateAccumulator<K>;
|
|
56
|
+
evaluate(
|
|
57
|
+
input: Table | Iterable<RecordBatch>,
|
|
58
|
+
options?: {signal?: AbortSignal}
|
|
59
|
+
): Record<K, ArrowExpressionValue>;
|
|
60
|
+
evaluateInBatches(
|
|
61
|
+
input: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
|
|
62
|
+
options?: {signal?: AbortSignal}
|
|
63
|
+
): Promise<Record<K, ArrowExpressionValue>>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
type Value = Exclude<ArrowExpressionValue, null | string> | Uint8Array;
|
|
67
|
+
type State = {
|
|
68
|
+
count: bigint;
|
|
69
|
+
sum: number | bigint;
|
|
70
|
+
values: Map<Operation, Value>;
|
|
71
|
+
};
|
|
72
|
+
type Group = {
|
|
73
|
+
expression?: CompiledArrowExpression;
|
|
74
|
+
operations: Set<Operation>;
|
|
75
|
+
integer: boolean;
|
|
76
|
+
wide: boolean;
|
|
77
|
+
utf8: boolean;
|
|
78
|
+
sums: boolean;
|
|
79
|
+
reductions: Operation[];
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const reductions = new Set<Operation>([
|
|
83
|
+
'MIN',
|
|
84
|
+
'MAX',
|
|
85
|
+
'EVERY',
|
|
86
|
+
'BOOL_AND',
|
|
87
|
+
'BOOL_OR',
|
|
88
|
+
'BIT_AND',
|
|
89
|
+
'BIT_OR',
|
|
90
|
+
'BIT_XOR'
|
|
91
|
+
]);
|
|
92
|
+
|
|
93
|
+
/** Compiles named scalar aggregates. Identical expression strings or shared ASTs run once per batch. */
|
|
94
|
+
export function compileArrowAggregates<K extends string>(
|
|
95
|
+
aggregates: Readonly<Record<K, ArrowAggregate>>,
|
|
96
|
+
options: ArrowExpressionOptions
|
|
97
|
+
): CompiledArrowAggregates<K> {
|
|
98
|
+
const groups: Group[] = [];
|
|
99
|
+
const byExpression = new Map<ArrowExpression | undefined, Group>();
|
|
100
|
+
const outputs = (Object.entries(aggregates) as [K, ArrowAggregate][]).map(([name, aggregate]) => {
|
|
101
|
+
const {operation, expression} = aggregate;
|
|
102
|
+
if (!reductions.has(operation) && !['COUNT', 'SUM', 'AVG'].includes(operation))
|
|
103
|
+
throw new Error(`Unsupported Arrow aggregate: ${operation}`);
|
|
104
|
+
if (expression === undefined && operation !== 'COUNT')
|
|
105
|
+
throw new Error(`${operation} requires an expression`);
|
|
106
|
+
let group = byExpression.get(expression);
|
|
107
|
+
if (!group) {
|
|
108
|
+
const compiled =
|
|
109
|
+
expression === undefined ? undefined : compileArrowExpression(expression, options);
|
|
110
|
+
const type = compiled && valueType(compiled.outputField.type);
|
|
111
|
+
group = {
|
|
112
|
+
expression: compiled,
|
|
113
|
+
operations: new Set(),
|
|
114
|
+
integer: Boolean(type && DataType.isInt(type)),
|
|
115
|
+
wide: Boolean(type && DataType.isInt(type) && type.bitWidth === 64),
|
|
116
|
+
utf8: Boolean(type && DataType.isUtf8(type)),
|
|
117
|
+
sums: false,
|
|
118
|
+
reductions: []
|
|
119
|
+
};
|
|
120
|
+
byExpression.set(expression, group);
|
|
121
|
+
groups.push(group);
|
|
122
|
+
}
|
|
123
|
+
const type = group.expression && valueType(group.expression.outputField.type);
|
|
124
|
+
if (type && !DataType.isNull(type) && operation !== 'COUNT') {
|
|
125
|
+
const numeric = group.integer || DataType.isFloat(type);
|
|
126
|
+
const supported = operation.startsWith('BIT_')
|
|
127
|
+
? group.integer
|
|
128
|
+
: operation === 'EVERY' || operation.startsWith('BOOL_')
|
|
129
|
+
? DataType.isBool(type)
|
|
130
|
+
: numeric || (group.utf8 && (operation === 'MIN' || operation === 'MAX'));
|
|
131
|
+
if (!supported) throw new Error(`${operation} does not support Arrow type ${type}`);
|
|
132
|
+
}
|
|
133
|
+
group.operations.add(operation);
|
|
134
|
+
return {name, operation, group};
|
|
135
|
+
});
|
|
136
|
+
for (const group of groups) {
|
|
137
|
+
group.sums = group.operations.has('SUM') || group.operations.has('AVG');
|
|
138
|
+
group.reductions = [...group.operations].filter(operation => reductions.has(operation));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Materializes one aggregate result without exposing accumulator state. */
|
|
142
|
+
const resultFor = (states: Map<Group, State>): Record<K, ArrowExpressionValue> =>
|
|
143
|
+
Object.fromEntries(
|
|
144
|
+
outputs.map(({name, operation, group}) => [
|
|
145
|
+
name,
|
|
146
|
+
finish(operation, group, states.get(group)!)
|
|
147
|
+
])
|
|
148
|
+
) as Record<K, ArrowExpressionValue>;
|
|
149
|
+
|
|
150
|
+
const accumulators = new WeakMap<ArrowAggregateAccumulator<K>, Map<Group, State>>();
|
|
151
|
+
const createAccumulator = (): ArrowAggregateAccumulator<K> => {
|
|
152
|
+
const states = new Map(groups.map(group => [group, emptyState(group)]));
|
|
153
|
+
const accumulator: ArrowAggregateAccumulator<K> = {
|
|
154
|
+
update(batch, {signal} = {}) {
|
|
155
|
+
signal?.throwIfAborted();
|
|
156
|
+
const partials = groups.map(group => summarize(group, batch, signal)[0]!);
|
|
157
|
+
signal?.throwIfAborted();
|
|
158
|
+
groups.forEach((group, index) => mergeState(group, states.get(group)!, partials[index]!));
|
|
159
|
+
},
|
|
160
|
+
merge(other) {
|
|
161
|
+
const source = accumulators.get(other);
|
|
162
|
+
if (!source || other === accumulator)
|
|
163
|
+
throw new Error('Merge requires another accumulator from the same Arrow aggregate plan');
|
|
164
|
+
for (const group of groups) mergeState(group, states.get(group)!, source.get(group)!);
|
|
165
|
+
},
|
|
166
|
+
result() {
|
|
167
|
+
return resultFor(states);
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
accumulators.set(accumulator, states);
|
|
171
|
+
return accumulator;
|
|
172
|
+
};
|
|
173
|
+
return {
|
|
174
|
+
createAccumulator,
|
|
175
|
+
createGroupedAccumulator(groupBy) {
|
|
176
|
+
const keyExpression = compileArrowExpression(groupBy, options);
|
|
177
|
+
const type = valueType(keyExpression.outputField.type);
|
|
178
|
+
if (
|
|
179
|
+
!(
|
|
180
|
+
DataType.isInt(type) ||
|
|
181
|
+
DataType.isFloat(type) ||
|
|
182
|
+
DataType.isBool(type) ||
|
|
183
|
+
DataType.isUtf8(type) ||
|
|
184
|
+
DataType.isNull(type)
|
|
185
|
+
)
|
|
186
|
+
) {
|
|
187
|
+
throw new Error(`Unsupported Arrow aggregate group key type: ${type}`);
|
|
188
|
+
}
|
|
189
|
+
const statesByKey = new Map<ArrowExpressionValue, Map<Group, State>>();
|
|
190
|
+
return {
|
|
191
|
+
update(batch, {signal} = {}) {
|
|
192
|
+
const partition = partitionRows(keyExpression, batch, signal);
|
|
193
|
+
const partials = groups.map(group => summarize(group, batch, signal, partition));
|
|
194
|
+
signal?.throwIfAborted();
|
|
195
|
+
for (const [key, index] of partition.indexByKey) {
|
|
196
|
+
let states = statesByKey.get(key);
|
|
197
|
+
if (!states) {
|
|
198
|
+
states = new Map(groups.map(group => [group, emptyState(group)]));
|
|
199
|
+
statesByKey.set(key, states);
|
|
200
|
+
}
|
|
201
|
+
groups.forEach((group, groupIndex) =>
|
|
202
|
+
mergeState(group, states.get(group)!, partials[groupIndex]![index]!)
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
result() {
|
|
207
|
+
return new Map(Array.from(statesByKey, ([key, states]) => [key, resultFor(states)]));
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
},
|
|
211
|
+
evaluate(input, evaluation) {
|
|
212
|
+
evaluation?.signal?.throwIfAborted();
|
|
213
|
+
const accumulator = createAccumulator();
|
|
214
|
+
for (const batch of 'batches' in input ? input.batches : input)
|
|
215
|
+
accumulator.update(batch, evaluation);
|
|
216
|
+
return accumulator.result();
|
|
217
|
+
},
|
|
218
|
+
async evaluateInBatches(input, evaluation) {
|
|
219
|
+
evaluation?.signal?.throwIfAborted();
|
|
220
|
+
const accumulator = createAccumulator();
|
|
221
|
+
for await (const batch of input) accumulator.update(batch, evaluation);
|
|
222
|
+
evaluation?.signal?.throwIfAborted();
|
|
223
|
+
return accumulator.result();
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function emptyState(group: Group): State {
|
|
229
|
+
return {count: 0n, sum: group.integer ? 0n : 0, values: new Map()};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Dense batch-local group indices keep expression evaluation independent of group count. */
|
|
233
|
+
type RowPartition = {
|
|
234
|
+
indexByKey: ReadonlyMap<ArrowExpressionValue, number>;
|
|
235
|
+
indexByRow: Uint32Array;
|
|
236
|
+
rowCounts: readonly number[];
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
/** Resolves one group key per row and records first-seen order. */
|
|
240
|
+
function partitionRows(
|
|
241
|
+
expression: CompiledArrowExpression,
|
|
242
|
+
batch: RecordBatch,
|
|
243
|
+
signal?: AbortSignal
|
|
244
|
+
): RowPartition {
|
|
245
|
+
const reader = new ColumnReader(expression.evaluate(batch, {signal}));
|
|
246
|
+
const indexByKey = new Map<ArrowExpressionValue, number>();
|
|
247
|
+
const indexByRow = new Uint32Array(batch.numRows);
|
|
248
|
+
const rowCounts: number[] = [];
|
|
249
|
+
for (let row = 0; row < batch.numRows; row++) {
|
|
250
|
+
if (row % 1024 === 0) signal?.throwIfAborted();
|
|
251
|
+
const key = reader.get(row);
|
|
252
|
+
if (typeof key === 'number' && !Number.isFinite(key))
|
|
253
|
+
throw new Error('Arrow aggregate group keys must be finite');
|
|
254
|
+
let index = indexByKey.get(key);
|
|
255
|
+
if (index === undefined) {
|
|
256
|
+
index = indexByKey.size;
|
|
257
|
+
indexByKey.set(key, index);
|
|
258
|
+
rowCounts.push(0);
|
|
259
|
+
}
|
|
260
|
+
indexByRow[row] = index;
|
|
261
|
+
rowCounts[index]!++;
|
|
262
|
+
}
|
|
263
|
+
return {indexByKey, indexByRow, rowCounts};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Reduces one expression across the batch, optionally routing rows to groups. */
|
|
267
|
+
function summarize(
|
|
268
|
+
group: Group,
|
|
269
|
+
batch: RecordBatch,
|
|
270
|
+
signal?: AbortSignal,
|
|
271
|
+
partition?: RowPartition
|
|
272
|
+
): State[] {
|
|
273
|
+
const states = Array.from({length: partition?.indexByKey.size ?? 1}, () => emptyState(group));
|
|
274
|
+
if (!group.expression) {
|
|
275
|
+
for (let index = 0; index < states.length; index++)
|
|
276
|
+
states[index]!.count = BigInt(partition?.rowCounts[index] ?? batch.numRows);
|
|
277
|
+
return states;
|
|
278
|
+
}
|
|
279
|
+
const vector = group.expression.evaluate(batch, {signal});
|
|
280
|
+
const countOnly = !group.sums && !group.reductions.length;
|
|
281
|
+
if (countOnly && !partition) {
|
|
282
|
+
states[0]!.count = BigInt(vector.length - vector.nullCount);
|
|
283
|
+
return states;
|
|
284
|
+
}
|
|
285
|
+
const reader = new ColumnReader(vector);
|
|
286
|
+
const counts = Array.from({length: states.length}, () => 0);
|
|
287
|
+
for (let row = 0; row < vector.length; row++) {
|
|
288
|
+
if (row % 1024 === 0) signal?.throwIfAborted();
|
|
289
|
+
const index = partition?.indexByRow[row] ?? 0;
|
|
290
|
+
const state = states[index]!;
|
|
291
|
+
if (countOnly || group.utf8) {
|
|
292
|
+
if (!reader.isValid(row)) continue;
|
|
293
|
+
counts[index]!++;
|
|
294
|
+
if (countOnly) continue;
|
|
295
|
+
const bytes = reader.utf8(row);
|
|
296
|
+
for (const operation of group.reductions) {
|
|
297
|
+
const previous = state.values.get(operation) as Uint8Array | undefined;
|
|
298
|
+
if (winsUTF8(operation, bytes, previous))
|
|
299
|
+
state.values.set(operation, bytes[0].slice(bytes[1], bytes[2]));
|
|
300
|
+
}
|
|
301
|
+
} else {
|
|
302
|
+
const value = reader.get(row) as number | bigint | boolean | null;
|
|
303
|
+
if (value === null) continue;
|
|
304
|
+
counts[index]!++;
|
|
305
|
+
if (group.sums) state.sum = add(group, state.sum, value as number | bigint);
|
|
306
|
+
for (const operation of group.reductions) {
|
|
307
|
+
const operand = operation.startsWith('BIT_') ? BigInt(value) : value;
|
|
308
|
+
state.values.set(operation, reduce(operation, state.values.get(operation), operand));
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
states.forEach((state, index) => {
|
|
313
|
+
state.count = BigInt(counts[index]!);
|
|
314
|
+
});
|
|
315
|
+
return states;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function add(group: Group, left: number | bigint, right: number | bigint): number | bigint {
|
|
319
|
+
return group.integer ? BigInt(left) + BigInt(right) : Number(left) + Number(right);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function winsUTF8(
|
|
323
|
+
operation: Operation,
|
|
324
|
+
[bytes, start, end]: [Uint8Array, number, number],
|
|
325
|
+
previous: Uint8Array | undefined
|
|
326
|
+
): boolean {
|
|
327
|
+
if (!previous) return true;
|
|
328
|
+
const order = compareUTF8(bytes, start, end, previous, 0, previous.length);
|
|
329
|
+
return operation === 'MIN' ? order < 0 : order > 0;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function reduce(operation: Operation, left: Value | undefined, right: Value): Value {
|
|
333
|
+
if (left === undefined) return right;
|
|
334
|
+
switch (operation) {
|
|
335
|
+
case 'MIN':
|
|
336
|
+
return typeof left === 'number'
|
|
337
|
+
? Math.min(left, right as number)
|
|
338
|
+
: left < right
|
|
339
|
+
? left
|
|
340
|
+
: right;
|
|
341
|
+
case 'MAX':
|
|
342
|
+
return typeof left === 'number'
|
|
343
|
+
? Math.max(left, right as number)
|
|
344
|
+
: left > right
|
|
345
|
+
? left
|
|
346
|
+
: right;
|
|
347
|
+
case 'EVERY':
|
|
348
|
+
case 'BOOL_AND':
|
|
349
|
+
return Boolean(left && right);
|
|
350
|
+
case 'BOOL_OR':
|
|
351
|
+
return Boolean(left || right);
|
|
352
|
+
case 'BIT_AND':
|
|
353
|
+
return (left as bigint) & (right as bigint);
|
|
354
|
+
case 'BIT_OR':
|
|
355
|
+
return (left as bigint) | (right as bigint);
|
|
356
|
+
case 'BIT_XOR':
|
|
357
|
+
return (left as bigint) ^ (right as bigint);
|
|
358
|
+
default:
|
|
359
|
+
throw new Error(`Invalid Arrow reduction: ${operation}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function mergeState(group: Group, target: State, source: State): void {
|
|
364
|
+
if (!source.count) return;
|
|
365
|
+
target.count += source.count;
|
|
366
|
+
if (group.sums) target.sum = add(group, target.sum, source.sum);
|
|
367
|
+
for (const [operation, value] of source.values) {
|
|
368
|
+
const previous = target.values.get(operation);
|
|
369
|
+
if (value instanceof Uint8Array) {
|
|
370
|
+
if (winsUTF8(operation, [value, 0, value.length], previous as Uint8Array | undefined))
|
|
371
|
+
target.values.set(operation, value);
|
|
372
|
+
} else {
|
|
373
|
+
target.values.set(operation, reduce(operation, previous, value));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function finish(operation: Operation, group: Group, state: State): ArrowExpressionValue {
|
|
379
|
+
if (operation === 'COUNT') return state.count;
|
|
380
|
+
if (!state.count) return null;
|
|
381
|
+
if (operation === 'SUM') return state.sum;
|
|
382
|
+
if (operation === 'AVG') return Number(state.sum) / Number(state.count);
|
|
383
|
+
const value = state.values.get(operation)!;
|
|
384
|
+
if (value instanceof Uint8Array) return new TextDecoder().decode(value);
|
|
385
|
+
return operation.startsWith('BIT_') && !group.wide ? Number(value) : value;
|
|
386
|
+
}
|