@honua/react 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/columnar/apache-arrow.d.ts +17 -0
- package/columnar/apache-arrow.d.ts.map +1 -0
- package/columnar/apache-arrow.js +1036 -0
- package/columnar/batch-cache-conformance.d.ts +66 -0
- package/columnar/batch-cache-conformance.d.ts.map +1 -0
- package/columnar/batch-cache-conformance.js +317 -0
- package/columnar/batch-cache-indexeddb.d.ts +31 -0
- package/columnar/batch-cache-indexeddb.d.ts.map +1 -0
- package/columnar/batch-cache-indexeddb.js +148 -0
- package/columnar/batch-cache.d.ts +298 -0
- package/columnar/batch-cache.d.ts.map +1 -0
- package/columnar/batch-cache.js +757 -0
- package/columnar/geoarrow-aggregate.d.ts +178 -0
- package/columnar/geoarrow-aggregate.d.ts.map +1 -0
- package/columnar/geoarrow-aggregate.js +976 -0
- package/columnar/geoarrow-patch.d.ts +382 -0
- package/columnar/geoarrow-patch.d.ts.map +1 -0
- package/columnar/geoarrow-patch.js +1656 -0
- package/columnar/geoarrow-reproject.d.ts +22 -0
- package/columnar/geoarrow-reproject.d.ts.map +1 -0
- package/columnar/geoarrow-reproject.js +119 -0
- package/columnar/geoarrow-serialization.d.ts +68 -0
- package/columnar/geoarrow-serialization.d.ts.map +1 -0
- package/columnar/geoarrow-serialization.js +331 -0
- package/columnar/geoarrow-types.d.ts +271 -0
- package/columnar/geoarrow-types.d.ts.map +1 -0
- package/columnar/geoarrow-types.js +15 -0
- package/columnar/geoarrow.d.ts +52 -0
- package/columnar/geoarrow.d.ts.map +1 -0
- package/columnar/geoarrow.js +1469 -0
- package/columnar/index.d.ts +43 -0
- package/columnar/index.d.ts.map +1 -0
- package/columnar/index.js +29 -0
- package/columnar/operators.d.ts +70 -0
- package/columnar/operators.d.ts.map +1 -0
- package/columnar/operators.js +307 -0
- package/columnar/result.d.ts +369 -0
- package/columnar/result.d.ts.map +1 -0
- package/columnar/result.js +815 -0
- package/columnar/telemetry.d.ts +152 -0
- package/columnar/telemetry.d.ts.map +1 -0
- package/columnar/telemetry.js +265 -0
- package/columnar/transfer.d.ts +28 -0
- package/columnar/transfer.d.ts.map +1 -0
- package/columnar/transfer.js +669 -0
- package/columnar/types.d.ts +175 -0
- package/columnar/types.d.ts.map +1 -0
- package/columnar/types.js +22 -0
- package/columnar/worker.d.ts +157 -0
- package/columnar/worker.d.ts.map +1 -0
- package/columnar/worker.js +1059 -0
- package/connect-url-safety.d.ts +53 -0
- package/connect-url-safety.js +189 -0
- package/contract/pmtiles.d.ts +6 -5
- package/contract/pmtiles.d.ts.map +1 -1
- package/contract/pmtiles.js +30 -5
- package/core/client.d.ts.map +1 -1
- package/core/client.js +87 -19
- package/core/deprecation-notice.d.ts +28 -0
- package/core/deprecation-notice.d.ts.map +1 -0
- package/core/deprecation-notice.js +22 -0
- package/core/esri-geojson.d.ts.map +1 -1
- package/core/esri-geojson.js +10 -6
- package/core/optional-grpc-peer.d.ts +15 -0
- package/core/optional-grpc-peer.d.ts.map +1 -0
- package/core/optional-grpc-peer.js +60 -0
- package/esri-compat/basemap-gallery.d.ts +2 -0
- package/esri-compat/basemap-gallery.d.ts.map +1 -1
- package/esri-compat/basemap-gallery.js +12 -0
- package/esri-compat/basemap.d.ts +4 -0
- package/esri-compat/basemap.d.ts.map +1 -1
- package/esri-compat/basemap.js +70 -0
- package/esri-compat/bookmarks.d.ts +2 -0
- package/esri-compat/bookmarks.d.ts.map +1 -1
- package/esri-compat/bookmarks.js +12 -0
- package/esri-compat/controls.d.ts +18 -0
- package/esri-compat/controls.d.ts.map +1 -1
- package/esri-compat/controls.js +120 -57
- package/esri-compat/directions.d.ts +4 -0
- package/esri-compat/directions.d.ts.map +1 -1
- package/esri-compat/directions.js +30 -20
- package/esri-compat/editor.d.ts +4 -0
- package/esri-compat/editor.d.ts.map +1 -1
- package/esri-compat/editor.js +16 -0
- package/esri-compat/esri-config.d.ts +2 -1
- package/esri-compat/esri-config.d.ts.map +1 -1
- package/esri-compat/feature-effect.d.ts +23 -0
- package/esri-compat/feature-effect.d.ts.map +1 -0
- package/esri-compat/feature-effect.js +12 -0
- package/esri-compat/feature-form.d.ts +6 -0
- package/esri-compat/feature-form.d.ts.map +1 -1
- package/esri-compat/feature-form.js +63 -0
- package/esri-compat/feature-layer.d.ts +21 -2
- package/esri-compat/feature-layer.d.ts.map +1 -1
- package/esri-compat/feature-layer.js +172 -1
- package/esri-compat/feature-table.d.ts +5 -0
- package/esri-compat/feature-table.d.ts.map +1 -1
- package/esri-compat/feature-table.js +23 -0
- package/esri-compat/identity-manager.d.ts +1 -1
- package/esri-compat/identity-manager.js +1 -1
- package/esri-compat/legend.d.ts +2 -0
- package/esri-compat/legend.d.ts.map +1 -1
- package/esri-compat/legend.js +98 -0
- package/esri-compat/locator.d.ts +1 -1
- package/esri-compat/locator.js +1 -1
- package/esri-compat/map-view-mount.d.ts +1 -0
- package/esri-compat/map-view-mount.d.ts.map +1 -1
- package/esri-compat/map-view-mount.js +46 -25
- package/esri-compat/map-view.d.ts +13 -0
- package/esri-compat/map-view.d.ts.map +1 -1
- package/esri-compat/map-view.js +94 -8
- package/esri-compat/map.d.ts +5 -1
- package/esri-compat/map.d.ts.map +1 -1
- package/esri-compat/map.js +10 -0
- package/esri-compat/measurement.d.ts +2 -0
- package/esri-compat/measurement.d.ts.map +1 -1
- package/esri-compat/measurement.js +14 -0
- package/esri-compat/point.d.ts +7 -0
- package/esri-compat/point.d.ts.map +1 -1
- package/esri-compat/point.js +33 -3
- package/esri-compat/popup.d.ts +2 -0
- package/esri-compat/popup.d.ts.map +1 -1
- package/esri-compat/popup.js +12 -0
- package/esri-compat/portal.d.ts +1 -1
- package/esri-compat/portal.js +1 -1
- package/esri-compat/print.d.ts +3 -0
- package/esri-compat/print.d.ts.map +1 -1
- package/esri-compat/print.js +15 -0
- package/esri-compat/search.d.ts +4 -0
- package/esri-compat/search.d.ts.map +1 -1
- package/esri-compat/search.js +25 -28
- package/esri-compat/sketch.d.ts +2 -0
- package/esri-compat/sketch.d.ts.map +1 -1
- package/esri-compat/sketch.js +12 -0
- package/esri-compat/widget-host.d.ts +46 -17
- package/esri-compat/widget-host.d.ts.map +1 -1
- package/esri-compat/widget-host.js +84 -20
- package/esri-compat/widget-shell-hosts.d.ts +61 -0
- package/esri-compat/widget-shell-hosts.d.ts.map +1 -0
- package/esri-compat/widget-shell-hosts.js +76 -0
- package/geocoding/index.d.ts +114 -0
- package/geocoding/index.d.ts.map +1 -0
- package/geocoding/index.js +293 -0
- package/geocoding/provider-http.d.ts +37 -0
- package/geocoding/provider-http.d.ts.map +1 -0
- package/geocoding/provider-http.js +93 -0
- package/geocoding/provider.d.ts +145 -0
- package/geocoding/provider.d.ts.map +1 -0
- package/geocoding/provider.js +41 -0
- package/geocoding/providers/honua.d.ts +28 -0
- package/geocoding/providers/honua.d.ts.map +1 -0
- package/geocoding/providers/honua.js +68 -0
- package/geocoding/providers/nominatim.d.ts +47 -0
- package/geocoding/providers/nominatim.d.ts.map +1 -0
- package/geocoding/providers/nominatim.js +105 -0
- package/geocoding/providers/pelias.d.ts +40 -0
- package/geocoding/providers/pelias.d.ts.map +1 -0
- package/geocoding/providers/pelias.js +115 -0
- package/geocoding/providers/photon.d.ts +39 -0
- package/geocoding/providers/photon.d.ts.map +1 -0
- package/geocoding/providers/photon.js +107 -0
- package/geometry/convert.d.ts +13 -0
- package/geometry/convert.d.ts.map +1 -0
- package/geometry/convert.js +12 -0
- package/geometry/index.d.ts +17 -0
- package/geometry/index.d.ts.map +1 -0
- package/geometry/index.js +15 -0
- package/geometry/internal.d.ts +25 -0
- package/geometry/internal.d.ts.map +1 -0
- package/geometry/internal.js +40 -0
- package/geometry/ops.d.ts +71 -0
- package/geometry/ops.d.ts.map +1 -0
- package/geometry/ops.js +147 -0
- package/geometry/project.d.ts +40 -0
- package/geometry/project.d.ts.map +1 -0
- package/geometry/project.js +88 -0
- package/geometry/types.d.ts +36 -0
- package/geometry/types.d.ts.map +1 -0
- package/geometry/types.js +12 -0
- package/kernel/renderer.d.ts +82 -0
- package/kernel/renderer.js +1 -0
- package/offline/digest.d.ts +15 -0
- package/offline/digest.js +58 -0
- package/offline/quota.d.ts +161 -0
- package/offline/quota.js +370 -0
- package/offline/types.d.ts +349 -0
- package/offline/types.js +75 -0
- package/package.json +14 -2
- package/widget-capabilities.d.ts +67 -0
- package/widget-capabilities.js +152 -0
|
@@ -0,0 +1,976 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded, deterministic reduction of a normative Honua GeoArrow batch.
|
|
3
|
+
*
|
|
4
|
+
* Every other columnar operation is row-preserving or row-dropping. This one
|
|
5
|
+
* reduces: it scans only the packed buffer views returned by
|
|
6
|
+
* {@link inspectGeoArrowBatch}, accumulates into group-indexed scalars, and
|
|
7
|
+
* emits a result batch whose row count is the group count. No input row is ever
|
|
8
|
+
* materialized as an object, `decodeGeoArrowBatch` is never called, and peak
|
|
9
|
+
* retained memory is bounded by the group ceiling rather than the row count.
|
|
10
|
+
*
|
|
11
|
+
* The result is a dependency-free Honua columnar batch carrying the
|
|
12
|
+
* `honua.aggregate` layout (see {@link readGeoArrowAggregateBatch}) rather than
|
|
13
|
+
* a GeoArrow batch, because the normative GeoArrow layout admits exactly one
|
|
14
|
+
* geometry column plus optional temporal, dictionary, and feature-id columns
|
|
15
|
+
* and therefore has nowhere to carry metric columns.
|
|
16
|
+
*
|
|
17
|
+
* ## Determinism
|
|
18
|
+
*
|
|
19
|
+
* A reduction that is not deterministic is not cacheable, and this one is the
|
|
20
|
+
* first columnar operation whose output is small enough to be worth caching.
|
|
21
|
+
* Three things could otherwise make the same input produce different output.
|
|
22
|
+
*
|
|
23
|
+
* 1. **Worker scheduling.** The scan yields to the host task queue so a cancel
|
|
24
|
+
* can land, but the arithmetic never observes the scheduler: accumulation is
|
|
25
|
+
* one sequential pass over the packed buffers, and a yield only suspends it.
|
|
26
|
+
* The yield cadence is therefore a pure performance knob — varying
|
|
27
|
+
* `yieldIntervalRows` cannot change a single output byte.
|
|
28
|
+
* 2. **Group ordering.** Groups are emitted ascending by key with the declared
|
|
29
|
+
* null group last, never in first-seen or hash-iteration order, so two
|
|
30
|
+
* batches holding the same rows in a different layout produce the same
|
|
31
|
+
* output row order. Duplicate dictionary entries encoding the same string
|
|
32
|
+
* are canonicalized to one group rather than to whichever index appeared
|
|
33
|
+
* first.
|
|
34
|
+
* 3. **Floating-point associativity.** `count`, `min`, and `max` are
|
|
35
|
+
* order-independent by construction. `sum` and `mean` are not, and a naive
|
|
36
|
+
* running total makes the reported value depend on the order the rows were
|
|
37
|
+
* visited in: `[1e16, 1, 1, -1e16]` accumulates left to right to `0` when
|
|
38
|
+
* the exact answer is `2`. Both use Kahan–Babuška–Neumaier compensation
|
|
39
|
+
* instead, carrying every addition's discarded low-order bits in a second
|
|
40
|
+
* group-indexed accumulator and folding them in once at the end. That is a
|
|
41
|
+
* bound on the error rather than a proof of bit-identity under every
|
|
42
|
+
* conceivable permutation, but it removes the cancellation family that makes
|
|
43
|
+
* naive accumulation order-dependent in practice.
|
|
44
|
+
*
|
|
45
|
+
* @experimental
|
|
46
|
+
*/
|
|
47
|
+
import { HonuaGeoArrowError, } from "./geoarrow-types.js";
|
|
48
|
+
import { inspectGeoArrowBatch } from "./geoarrow.js";
|
|
49
|
+
import { createColumnarBatch } from "./transfer.js";
|
|
50
|
+
/** Honua's dependency-free aggregate-result layout, recorded in schema metadata. */
|
|
51
|
+
export const HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION = "1.0";
|
|
52
|
+
/** Default ceiling on distinct groups, including the declared null group. */
|
|
53
|
+
export const DEFAULT_GEOARROW_AGGREGATE_MAX_GROUPS = 65_536;
|
|
54
|
+
/**
|
|
55
|
+
* Rows scanned between cooperative abort checks. Well inside the 65,536-row
|
|
56
|
+
* ceiling the columnar data-plane contract requires.
|
|
57
|
+
*/
|
|
58
|
+
export const GEOARROW_AGGREGATE_ABORT_CHECK_ROWS = 8_192;
|
|
59
|
+
/** Rows scanned between cooperative event-loop yields, so a cancel can land. */
|
|
60
|
+
export const DEFAULT_GEOARROW_AGGREGATE_YIELD_ROWS = 16_384;
|
|
61
|
+
/**
|
|
62
|
+
* Inclusive magnitude ceiling for a spatial-grid cell index on either axis,
|
|
63
|
+
* chosen so a packed `(x, y)` cell key stays an exact safe integer.
|
|
64
|
+
*/
|
|
65
|
+
export const GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX = 33_554_432;
|
|
66
|
+
const GRID_KEY_STRIDE = GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX * 2;
|
|
67
|
+
const META = {
|
|
68
|
+
version: "honua.aggregate.version",
|
|
69
|
+
spec: "honua.aggregate.spec",
|
|
70
|
+
source: "honua.aggregate.source",
|
|
71
|
+
};
|
|
72
|
+
const KIND_COUNT = 0;
|
|
73
|
+
const KIND_SUM = 1;
|
|
74
|
+
const KIND_MIN = 2;
|
|
75
|
+
const KIND_MAX = 3;
|
|
76
|
+
const KIND_MEAN = 4;
|
|
77
|
+
const GROUP_DICTIONARY = 0;
|
|
78
|
+
const GROUP_TEMPORAL = 1;
|
|
79
|
+
const GROUP_GRID = 2;
|
|
80
|
+
const COLUMN_F64 = 0;
|
|
81
|
+
const COLUMN_U32 = 1;
|
|
82
|
+
const COLUMN_I64 = 2;
|
|
83
|
+
const LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
84
|
+
const TRUNCATION_SECONDS = Object.freeze({
|
|
85
|
+
second: 1,
|
|
86
|
+
minute: 60,
|
|
87
|
+
hour: 3_600,
|
|
88
|
+
day: 86_400,
|
|
89
|
+
});
|
|
90
|
+
const TICKS_PER_SECOND = Object.freeze({
|
|
91
|
+
second: 1,
|
|
92
|
+
millisecond: 1_000,
|
|
93
|
+
microsecond: 1_000_000,
|
|
94
|
+
nanosecond: 1_000_000_000,
|
|
95
|
+
});
|
|
96
|
+
const DIMENSION_ORDER = Object.freeze({
|
|
97
|
+
xy: Object.freeze(["x", "y"]),
|
|
98
|
+
xyz: Object.freeze(["x", "y", "z"]),
|
|
99
|
+
xym: Object.freeze(["x", "y", "m"]),
|
|
100
|
+
xyzm: Object.freeze(["x", "y", "z", "m"]),
|
|
101
|
+
});
|
|
102
|
+
const METRIC_KINDS = new Set(["count", "sum", "min", "max", "mean"]);
|
|
103
|
+
const METRIC_COLUMNS = new Set(["featureId", "temporal", "x", "y", "z", "m"]);
|
|
104
|
+
function fail(code, message, detail) {
|
|
105
|
+
throw new HonuaGeoArrowError(code, message, detail);
|
|
106
|
+
}
|
|
107
|
+
function requireTrimmed(value, label) {
|
|
108
|
+
if (typeof value !== "string" || value.length === 0 || value.trim() !== value) {
|
|
109
|
+
throw new TypeError(`GeoArrow aggregate ${label} must be a non-empty trimmed string`);
|
|
110
|
+
}
|
|
111
|
+
return value;
|
|
112
|
+
}
|
|
113
|
+
function requirePositiveInteger(value, label, fallback) {
|
|
114
|
+
if (value === undefined)
|
|
115
|
+
return fallback;
|
|
116
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
|
|
117
|
+
throw new TypeError(`GeoArrow aggregate ${label} must be a positive safe integer`);
|
|
118
|
+
}
|
|
119
|
+
return value;
|
|
120
|
+
}
|
|
121
|
+
function normalizeGroupSpec(group) {
|
|
122
|
+
if (typeof group !== "object" || group === null)
|
|
123
|
+
throw new TypeError("GeoArrow aggregate group must be an object");
|
|
124
|
+
const kind = group.kind;
|
|
125
|
+
if (kind === "dictionary")
|
|
126
|
+
return Object.freeze({ kind: "dictionary" });
|
|
127
|
+
if (kind === "temporal") {
|
|
128
|
+
const unit = group.unit;
|
|
129
|
+
if (typeof unit !== "string" || !(unit in TRUNCATION_SECONDS)) {
|
|
130
|
+
throw new TypeError("GeoArrow aggregate temporal group unit must be second, minute, hour, or day");
|
|
131
|
+
}
|
|
132
|
+
return Object.freeze({ kind: "temporal", unit });
|
|
133
|
+
}
|
|
134
|
+
if (kind === "grid") {
|
|
135
|
+
const raw = group.cellSize;
|
|
136
|
+
const size = typeof raw === "number" ? [raw, raw] : [raw?.[0], raw?.[1]];
|
|
137
|
+
if (!Number.isFinite(size[0]) || !Number.isFinite(size[1]) || size[0] <= 0 || size[1] <= 0) {
|
|
138
|
+
throw new TypeError("GeoArrow aggregate grid cellSize must be one or two finite positive numbers");
|
|
139
|
+
}
|
|
140
|
+
const rawOrigin = group.origin;
|
|
141
|
+
const origin = rawOrigin === undefined ? [0, 0] : [rawOrigin[0], rawOrigin[1]];
|
|
142
|
+
if (!Number.isFinite(origin[0]) || !Number.isFinite(origin[1])) {
|
|
143
|
+
throw new TypeError("GeoArrow aggregate grid origin must contain exactly two finite numbers");
|
|
144
|
+
}
|
|
145
|
+
return Object.freeze({ kind: "grid", cellSize: Object.freeze(size), origin: Object.freeze(origin) });
|
|
146
|
+
}
|
|
147
|
+
throw new TypeError("GeoArrow aggregate group kind must be dictionary, temporal, or grid");
|
|
148
|
+
}
|
|
149
|
+
function normalizeMetrics(metrics, reserved) {
|
|
150
|
+
if (!Array.isArray(metrics) || metrics.length === 0) {
|
|
151
|
+
throw new TypeError("GeoArrow aggregate metrics must be a non-empty array");
|
|
152
|
+
}
|
|
153
|
+
const seen = new Set();
|
|
154
|
+
const normalized = [];
|
|
155
|
+
for (const metric of metrics) {
|
|
156
|
+
if (typeof metric !== "object" || metric === null) {
|
|
157
|
+
throw new TypeError("GeoArrow aggregate metric must be an object");
|
|
158
|
+
}
|
|
159
|
+
const name = requireTrimmed(metric.name, "metric name");
|
|
160
|
+
if (seen.has(name))
|
|
161
|
+
throw new TypeError(`GeoArrow aggregate metric name is duplicated: ${name}`);
|
|
162
|
+
if (reserved.has(name))
|
|
163
|
+
throw new TypeError(`GeoArrow aggregate metric name collides with a group field: ${name}`);
|
|
164
|
+
seen.add(name);
|
|
165
|
+
if (typeof metric.kind !== "string" || !METRIC_KINDS.has(metric.kind)) {
|
|
166
|
+
throw new TypeError("GeoArrow aggregate metric kind must be count, sum, min, max, or mean");
|
|
167
|
+
}
|
|
168
|
+
if (metric.column !== undefined && !METRIC_COLUMNS.has(metric.column)) {
|
|
169
|
+
throw new TypeError("GeoArrow aggregate metric column must be featureId, temporal, x, y, z, or m");
|
|
170
|
+
}
|
|
171
|
+
if (metric.kind !== "count" && metric.column === undefined) {
|
|
172
|
+
throw new TypeError(`GeoArrow aggregate ${metric.kind} metric requires a column`);
|
|
173
|
+
}
|
|
174
|
+
normalized.push(Object.freeze({ name, kind: metric.kind, ...(metric.column === undefined ? {} : { column: metric.column }) }));
|
|
175
|
+
}
|
|
176
|
+
return Object.freeze(normalized);
|
|
177
|
+
}
|
|
178
|
+
function canonicalSpec(group, metrics, groupField, nullKeys, maxGroups) {
|
|
179
|
+
const canonicalGroup = group.kind === "dictionary"
|
|
180
|
+
? { kind: "dictionary" }
|
|
181
|
+
: group.kind === "temporal"
|
|
182
|
+
? { kind: "temporal", unit: group.unit }
|
|
183
|
+
: { kind: "grid", cellSize: group.cellSize, origin: group.origin ?? [0, 0] };
|
|
184
|
+
return JSON.stringify({
|
|
185
|
+
version: HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION,
|
|
186
|
+
groupField,
|
|
187
|
+
group: canonicalGroup,
|
|
188
|
+
metrics: metrics.map((metric) => ({ name: metric.name, kind: metric.kind, column: metric.column ?? null })),
|
|
189
|
+
nullKeys,
|
|
190
|
+
maxGroups,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
function isValidBit(validity, index) {
|
|
194
|
+
return validity === undefined || (validity[index >> 3] & (1 << (index & 7))) !== 0;
|
|
195
|
+
}
|
|
196
|
+
function buildColumnReader(inspection, column) {
|
|
197
|
+
if (column === "featureId") {
|
|
198
|
+
const featureIds = inspection.featureIds;
|
|
199
|
+
if (featureIds === undefined)
|
|
200
|
+
fail("invalid-input", "The batch has no feature-id column to aggregate.");
|
|
201
|
+
return {
|
|
202
|
+
type: COLUMN_U32,
|
|
203
|
+
f64: undefined,
|
|
204
|
+
u32: featureIds.values,
|
|
205
|
+
i32: undefined,
|
|
206
|
+
stride: 1,
|
|
207
|
+
offset: 0,
|
|
208
|
+
validity: undefined,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
if (column === "temporal") {
|
|
212
|
+
const temporal = inspection.temporal;
|
|
213
|
+
if (temporal === undefined)
|
|
214
|
+
fail("invalid-input", "The batch has no temporal column to aggregate.");
|
|
215
|
+
const halves = new Int32Array(temporal.values.buffer, temporal.values.byteOffset, temporal.values.length * 2);
|
|
216
|
+
return {
|
|
217
|
+
type: COLUMN_I64,
|
|
218
|
+
f64: undefined,
|
|
219
|
+
u32: undefined,
|
|
220
|
+
i32: halves,
|
|
221
|
+
stride: 2,
|
|
222
|
+
offset: 0,
|
|
223
|
+
validity: temporal.validity,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
const geometry = inspection.geometry;
|
|
227
|
+
if (geometry.kind !== "point") {
|
|
228
|
+
fail("unsupported-layout", `Coordinate metrics require a point geometry column; this batch is ${geometry.kind}.`, {
|
|
229
|
+
column,
|
|
230
|
+
kind: geometry.kind,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
const index = DIMENSION_ORDER[geometry.dimensions].indexOf(column);
|
|
234
|
+
if (index < 0) {
|
|
235
|
+
fail("invalid-input", `The geometry column has dimensions ${geometry.dimensions} and no ${column} ordinate.`);
|
|
236
|
+
}
|
|
237
|
+
if (geometry.coordinateLayout === "interleaved") {
|
|
238
|
+
const values = geometry.coordinates.interleaved;
|
|
239
|
+
if (values === undefined)
|
|
240
|
+
fail("invalid-batch", "Interleaved geometry is missing its coordinate buffer.");
|
|
241
|
+
return {
|
|
242
|
+
type: COLUMN_F64,
|
|
243
|
+
f64: values,
|
|
244
|
+
u32: undefined,
|
|
245
|
+
i32: undefined,
|
|
246
|
+
stride: DIMENSION_ORDER[geometry.dimensions].length,
|
|
247
|
+
offset: index,
|
|
248
|
+
validity: geometry.validity,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
const values = geometry.coordinates[column];
|
|
252
|
+
if (values === undefined)
|
|
253
|
+
fail("invalid-batch", `Separated geometry is missing its ${column} coordinate buffer.`);
|
|
254
|
+
return {
|
|
255
|
+
type: COLUMN_F64,
|
|
256
|
+
f64: values,
|
|
257
|
+
u32: undefined,
|
|
258
|
+
i32: undefined,
|
|
259
|
+
stride: 1,
|
|
260
|
+
offset: 0,
|
|
261
|
+
validity: geometry.validity,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function readColumn(reader, row) {
|
|
265
|
+
if (reader.type === COLUMN_F64)
|
|
266
|
+
return reader.f64[row * reader.stride + reader.offset];
|
|
267
|
+
if (reader.type === COLUMN_U32)
|
|
268
|
+
return reader.u32[row];
|
|
269
|
+
const base = row * 2;
|
|
270
|
+
const low = reader.i32[LITTLE_ENDIAN ? base : base + 1];
|
|
271
|
+
const high = reader.i32[LITTLE_ENDIAN ? base + 1 : base];
|
|
272
|
+
const value = high * 4_294_967_296 + (low >>> 0);
|
|
273
|
+
if (!Number.isSafeInteger(value)) {
|
|
274
|
+
fail("invalid-input", "A temporal value exceeds safe integer precision and cannot be aggregated.", { row });
|
|
275
|
+
}
|
|
276
|
+
return value;
|
|
277
|
+
}
|
|
278
|
+
/** Half-open vertex range backing one row's geometry. */
|
|
279
|
+
function vertexRange(geometry, row) {
|
|
280
|
+
if (geometry.kind === "point")
|
|
281
|
+
return [row, row + 1];
|
|
282
|
+
const start = geometry.offsets[row];
|
|
283
|
+
const end = geometry.offsets[row + 1];
|
|
284
|
+
if (geometry.kind === "linestring")
|
|
285
|
+
return [start, end];
|
|
286
|
+
if (end <= start)
|
|
287
|
+
return [0, 0];
|
|
288
|
+
return [geometry.ringOffsets[start], geometry.ringOffsets[end]];
|
|
289
|
+
}
|
|
290
|
+
function isEmptyPointRow(geometry, row) {
|
|
291
|
+
if (geometry.kind !== "point" || !isValidBit(geometry.validity, row))
|
|
292
|
+
return false;
|
|
293
|
+
const dimensions = DIMENSION_ORDER[geometry.dimensions];
|
|
294
|
+
if (geometry.coordinateLayout === "interleaved") {
|
|
295
|
+
const values = geometry.coordinates.interleaved;
|
|
296
|
+
const base = row * dimensions.length;
|
|
297
|
+
return dimensions.every((_, index) => Number.isNaN(values[base + index]));
|
|
298
|
+
}
|
|
299
|
+
return dimensions.every((dimension) => Number.isNaN(geometry.coordinates[dimension][row]));
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Yield to the host task queue so a queued worker cancel message is delivered.
|
|
303
|
+
* A microtask checkpoint is not enough: `postMessage` delivery is a task.
|
|
304
|
+
*/
|
|
305
|
+
function yieldToHost() {
|
|
306
|
+
return new Promise((resolve) => {
|
|
307
|
+
const immediate = globalThis.setImmediate;
|
|
308
|
+
if (typeof immediate === "function") {
|
|
309
|
+
immediate(resolve);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
if (typeof MessageChannel === "function") {
|
|
313
|
+
const channel = new MessageChannel();
|
|
314
|
+
channel.port1.onmessage = () => {
|
|
315
|
+
channel.port1.close();
|
|
316
|
+
channel.port2.close();
|
|
317
|
+
resolve();
|
|
318
|
+
};
|
|
319
|
+
channel.port1.start();
|
|
320
|
+
channel.port2.postMessage(undefined);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
setTimeout(resolve, 0);
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* One compensated addition. Returns the discarded low-order bits, which the
|
|
328
|
+
* caller accumulates alongside the running total.
|
|
329
|
+
*
|
|
330
|
+
* A naive `total += value` makes a reduction order-dependent: summing
|
|
331
|
+
* `[1e16, 1, 1, -1e16]` left to right yields `0` because each `1` falls off the
|
|
332
|
+
* bottom of `1e16`, and permuting the same four values yields yet other
|
|
333
|
+
* answers. Carrying the lost bits recovers the exact `2` from either order.
|
|
334
|
+
*/
|
|
335
|
+
function neumaierResidual(running, value, total) {
|
|
336
|
+
return Math.abs(running) >= Math.abs(value) ? running - total + value : value - total + running;
|
|
337
|
+
}
|
|
338
|
+
function bufferOf(id, role, field, view) {
|
|
339
|
+
return Object.freeze({
|
|
340
|
+
id,
|
|
341
|
+
role,
|
|
342
|
+
field,
|
|
343
|
+
data: view.buffer,
|
|
344
|
+
byteOffset: view.byteOffset,
|
|
345
|
+
byteLength: view.byteLength,
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
function buildUtf8Group(field, values, withNull) {
|
|
349
|
+
const count = values.length + (withNull ? 1 : 0);
|
|
350
|
+
const encoder = new TextEncoder();
|
|
351
|
+
const encoded = values.map((value) => encoder.encode(value));
|
|
352
|
+
let total = 0;
|
|
353
|
+
for (const bytes of encoded)
|
|
354
|
+
total += bytes.length;
|
|
355
|
+
const offsets = new Int32Array(count + 1);
|
|
356
|
+
const payload = new Uint8Array(total);
|
|
357
|
+
const validity = new Uint8Array((count + 7) >> 3);
|
|
358
|
+
let cursor = 0;
|
|
359
|
+
for (let index = 0; index < encoded.length; index += 1) {
|
|
360
|
+
payload.set(encoded[index], cursor);
|
|
361
|
+
cursor += encoded[index].length;
|
|
362
|
+
offsets[index + 1] = cursor;
|
|
363
|
+
validity[index >> 3] |= 1 << (index & 7);
|
|
364
|
+
}
|
|
365
|
+
for (let index = encoded.length; index < count; index += 1)
|
|
366
|
+
offsets[index + 1] = cursor;
|
|
367
|
+
return {
|
|
368
|
+
fields: [Object.freeze({ name: field, type: Object.freeze({ name: "utf8" }), nullable: true })],
|
|
369
|
+
buffers: [
|
|
370
|
+
bufferOf(`${field}.validity`, "validity", field, validity),
|
|
371
|
+
bufferOf(`${field}.offsets`, "offsets", field, offsets),
|
|
372
|
+
bufferOf(`${field}.values`, "values", field, payload),
|
|
373
|
+
],
|
|
374
|
+
orderingFields: [field],
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
function buildTemporalGroup(field, keys, withNull, unit, timezone) {
|
|
378
|
+
const count = keys.length + (withNull ? 1 : 0);
|
|
379
|
+
const values = new BigInt64Array(count);
|
|
380
|
+
const validity = new Uint8Array((count + 7) >> 3);
|
|
381
|
+
for (let index = 0; index < keys.length; index += 1) {
|
|
382
|
+
values[index] = BigInt(keys[index]);
|
|
383
|
+
validity[index >> 3] |= 1 << (index & 7);
|
|
384
|
+
}
|
|
385
|
+
return {
|
|
386
|
+
fields: [
|
|
387
|
+
Object.freeze({
|
|
388
|
+
name: field,
|
|
389
|
+
type: Object.freeze({
|
|
390
|
+
name: "timestamp",
|
|
391
|
+
parameters: Object.freeze({ unit, ...(timezone === undefined ? {} : { timezone }) }),
|
|
392
|
+
}),
|
|
393
|
+
nullable: true,
|
|
394
|
+
}),
|
|
395
|
+
],
|
|
396
|
+
buffers: [
|
|
397
|
+
bufferOf(`${field}.validity`, "validity", field, validity),
|
|
398
|
+
bufferOf(`${field}.values`, "values", field, values),
|
|
399
|
+
],
|
|
400
|
+
orderingFields: [field],
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
function buildGridGroup(field, keys, withNull) {
|
|
404
|
+
const count = keys.length + (withNull ? 1 : 0);
|
|
405
|
+
const xField = `${field}_x`;
|
|
406
|
+
const yField = `${field}_y`;
|
|
407
|
+
const xValues = new Int32Array(count);
|
|
408
|
+
const yValues = new Int32Array(count);
|
|
409
|
+
const xValidity = new Uint8Array((count + 7) >> 3);
|
|
410
|
+
const yValidity = new Uint8Array((count + 7) >> 3);
|
|
411
|
+
for (let index = 0; index < keys.length; index += 1) {
|
|
412
|
+
const key = keys[index];
|
|
413
|
+
xValues[index] = Math.floor(key / GRID_KEY_STRIDE) - GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX;
|
|
414
|
+
yValues[index] = (key % GRID_KEY_STRIDE) - GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX;
|
|
415
|
+
xValidity[index >> 3] |= 1 << (index & 7);
|
|
416
|
+
yValidity[index >> 3] |= 1 << (index & 7);
|
|
417
|
+
}
|
|
418
|
+
return {
|
|
419
|
+
fields: [
|
|
420
|
+
Object.freeze({ name: xField, type: Object.freeze({ name: "int32" }), nullable: true }),
|
|
421
|
+
Object.freeze({ name: yField, type: Object.freeze({ name: "int32" }), nullable: true }),
|
|
422
|
+
],
|
|
423
|
+
buffers: [
|
|
424
|
+
bufferOf(`${xField}.validity`, "validity", xField, xValidity),
|
|
425
|
+
bufferOf(`${xField}.values`, "values", xField, xValues),
|
|
426
|
+
bufferOf(`${yField}.validity`, "validity", yField, yValidity),
|
|
427
|
+
bufferOf(`${yField}.values`, "values", yField, yValues),
|
|
428
|
+
],
|
|
429
|
+
orderingFields: [xField, yField],
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Create a bounded GeoArrow aggregation for registration with
|
|
434
|
+
* `startColumnarWorkerHost`.
|
|
435
|
+
*
|
|
436
|
+
* The returned operation scans the input batch's packed buffers exactly once,
|
|
437
|
+
* accumulates into group-indexed scalars, and emits a `honua.aggregate` result
|
|
438
|
+
* batch. It fails closed — always before the output batch is allocated — when
|
|
439
|
+
* the group ceiling is exceeded, when a metric value is non-finite, and when a
|
|
440
|
+
* declared column does not exist in the batch. The input batch is never
|
|
441
|
+
* mutated.
|
|
442
|
+
*/
|
|
443
|
+
export function createGeoArrowAggregateOperation(options) {
|
|
444
|
+
if (typeof options !== "object" || options === null) {
|
|
445
|
+
throw new TypeError("GeoArrow aggregate options must be an object");
|
|
446
|
+
}
|
|
447
|
+
const id = requireTrimmed(options.id, "id");
|
|
448
|
+
const schemaId = requireTrimmed(options.schemaId, "schemaId");
|
|
449
|
+
const groupField = options.groupField === undefined ? "group" : requireTrimmed(options.groupField, "groupField");
|
|
450
|
+
const group = normalizeGroupSpec(options.group);
|
|
451
|
+
const groupFieldNames = group.kind === "grid" ? [`${groupField}_x`, `${groupField}_y`] : [groupField];
|
|
452
|
+
const metrics = normalizeMetrics(options.metrics, new Set(groupFieldNames));
|
|
453
|
+
const maxGroups = requirePositiveInteger(options.maxGroups, "maxGroups", DEFAULT_GEOARROW_AGGREGATE_MAX_GROUPS);
|
|
454
|
+
const nullKeys = options.nullKeys ?? "null-group";
|
|
455
|
+
if (nullKeys !== "null-group" && nullKeys !== "skip") {
|
|
456
|
+
throw new TypeError('GeoArrow aggregate nullKeys must be "null-group" or "skip"');
|
|
457
|
+
}
|
|
458
|
+
const yieldIntervalRows = requirePositiveInteger(options.yieldIntervalRows, "yieldIntervalRows", DEFAULT_GEOARROW_AGGREGATE_YIELD_ROWS);
|
|
459
|
+
const limits = options.limits ?? {};
|
|
460
|
+
const specJson = canonicalSpec(group, metrics, groupField, nullKeys, maxGroups);
|
|
461
|
+
return async (batch, context) => {
|
|
462
|
+
context.signal.throwIfAborted();
|
|
463
|
+
context.reportProgress(0, "inspect");
|
|
464
|
+
const inspection = inspectGeoArrowBatch(batch, limits);
|
|
465
|
+
const source = inspection.batch;
|
|
466
|
+
const identity = source.identity;
|
|
467
|
+
if (!identity)
|
|
468
|
+
fail("invalid-batch", "A GeoArrow aggregation requires the source batch identity.");
|
|
469
|
+
const rows = source.rowCount;
|
|
470
|
+
const geometry = inspection.geometry;
|
|
471
|
+
// Distinct scratch columns, read at most once per row and shared by metrics.
|
|
472
|
+
const columnNames = [];
|
|
473
|
+
for (const metric of metrics) {
|
|
474
|
+
if (metric.column !== undefined && !columnNames.includes(metric.column))
|
|
475
|
+
columnNames.push(metric.column);
|
|
476
|
+
}
|
|
477
|
+
const readers = columnNames.map((column) => buildColumnReader(inspection, column));
|
|
478
|
+
const columnCount = readers.length;
|
|
479
|
+
const columnValue = new Float64Array(columnCount);
|
|
480
|
+
const columnValid = new Uint8Array(columnCount);
|
|
481
|
+
const accumulators = metrics.map((metric) => ({
|
|
482
|
+
spec: metric,
|
|
483
|
+
kind: metric.kind === "count"
|
|
484
|
+
? KIND_COUNT
|
|
485
|
+
: metric.kind === "sum"
|
|
486
|
+
? KIND_SUM
|
|
487
|
+
: metric.kind === "min"
|
|
488
|
+
? KIND_MIN
|
|
489
|
+
: metric.kind === "max"
|
|
490
|
+
? KIND_MAX
|
|
491
|
+
: KIND_MEAN,
|
|
492
|
+
column: metric.column === undefined ? -1 : columnNames.indexOf(metric.column),
|
|
493
|
+
values: [],
|
|
494
|
+
compensation: [],
|
|
495
|
+
nonNull: [],
|
|
496
|
+
}));
|
|
497
|
+
const accumulatorCount = accumulators.length;
|
|
498
|
+
const groupKind = group.kind === "dictionary" ? GROUP_DICTIONARY : group.kind === "temporal" ? GROUP_TEMPORAL : GROUP_GRID;
|
|
499
|
+
// Group-mode preparation. Every allocation here is bounded by the
|
|
500
|
+
// dictionary cardinality or by a constant, never by the row count.
|
|
501
|
+
let dictionaryValues = [];
|
|
502
|
+
let canonicalOfIndex = new Int32Array(0);
|
|
503
|
+
let groupOfCanonical = new Int32Array(0);
|
|
504
|
+
let dictionaryIndices = new Int32Array(0);
|
|
505
|
+
let dictionaryValidity;
|
|
506
|
+
let temporalReader;
|
|
507
|
+
let temporalStep = 1;
|
|
508
|
+
let gridSizeX = 1;
|
|
509
|
+
let gridSizeY = 1;
|
|
510
|
+
let gridOriginX = 0;
|
|
511
|
+
let gridOriginY = 0;
|
|
512
|
+
let interleaved = false;
|
|
513
|
+
let coordinateStride = 2;
|
|
514
|
+
let coordinateX = new Float64Array(0);
|
|
515
|
+
let coordinateY = new Float64Array(0);
|
|
516
|
+
const numericGroups = new Map();
|
|
517
|
+
const groupKeys = [];
|
|
518
|
+
if (groupKind === GROUP_DICTIONARY) {
|
|
519
|
+
const dictionary = inspection.dictionary;
|
|
520
|
+
if (dictionary === undefined)
|
|
521
|
+
fail("invalid-input", "Dictionary grouping requires a dictionary column.");
|
|
522
|
+
const size = dictionary.offsets.length - 1;
|
|
523
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
524
|
+
const decoded = new Array(size);
|
|
525
|
+
canonicalOfIndex = new Int32Array(size);
|
|
526
|
+
const firstByValue = new Map();
|
|
527
|
+
for (let index = 0; index < size; index += 1) {
|
|
528
|
+
const value = decoder.decode(dictionary.values.subarray(dictionary.offsets[index], dictionary.offsets[index + 1]));
|
|
529
|
+
decoded[index] = value;
|
|
530
|
+
const existing = firstByValue.get(value);
|
|
531
|
+
if (existing === undefined) {
|
|
532
|
+
firstByValue.set(value, index);
|
|
533
|
+
canonicalOfIndex[index] = index;
|
|
534
|
+
}
|
|
535
|
+
else
|
|
536
|
+
canonicalOfIndex[index] = existing;
|
|
537
|
+
}
|
|
538
|
+
dictionaryValues = decoded;
|
|
539
|
+
groupOfCanonical = new Int32Array(size).fill(-1);
|
|
540
|
+
dictionaryIndices = dictionary.indices;
|
|
541
|
+
dictionaryValidity = dictionary.validity;
|
|
542
|
+
}
|
|
543
|
+
else if (groupKind === GROUP_TEMPORAL) {
|
|
544
|
+
const temporal = inspection.temporal;
|
|
545
|
+
if (temporal === undefined)
|
|
546
|
+
fail("invalid-input", "Temporal grouping requires a temporal column.");
|
|
547
|
+
const ticks = TICKS_PER_SECOND[temporal.unit];
|
|
548
|
+
if (ticks === undefined)
|
|
549
|
+
fail("unsupported-layout", `Unsupported temporal unit: ${temporal.unit}`);
|
|
550
|
+
temporalStep = ticks * TRUNCATION_SECONDS[group.unit];
|
|
551
|
+
temporalReader = buildColumnReader(inspection, "temporal");
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
const grid = group;
|
|
555
|
+
const cell = grid.cellSize;
|
|
556
|
+
gridSizeX = cell[0];
|
|
557
|
+
gridSizeY = cell[1];
|
|
558
|
+
gridOriginX = grid.origin?.[0] ?? 0;
|
|
559
|
+
gridOriginY = grid.origin?.[1] ?? 0;
|
|
560
|
+
interleaved = geometry.coordinateLayout === "interleaved";
|
|
561
|
+
coordinateStride = DIMENSION_ORDER[geometry.dimensions].length;
|
|
562
|
+
if (interleaved) {
|
|
563
|
+
const values = geometry.coordinates.interleaved;
|
|
564
|
+
if (values === undefined)
|
|
565
|
+
fail("invalid-batch", "Interleaved geometry is missing its coordinate buffer.");
|
|
566
|
+
coordinateX = values;
|
|
567
|
+
coordinateY = values;
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
const x = geometry.coordinates.x;
|
|
571
|
+
const y = geometry.coordinates.y;
|
|
572
|
+
if (x === undefined || y === undefined)
|
|
573
|
+
fail("invalid-batch", "Separated geometry is missing x or y.");
|
|
574
|
+
coordinateX = x;
|
|
575
|
+
coordinateY = y;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
// Group-indexed accumulators, bounded by `maxGroups` and not by row count.
|
|
579
|
+
let groupCount = 0;
|
|
580
|
+
let nullGroup = -1;
|
|
581
|
+
let skippedRows = 0;
|
|
582
|
+
const openGroup = () => {
|
|
583
|
+
if (groupCount >= maxGroups) {
|
|
584
|
+
fail("group-limit-exceeded", `The aggregation produced more than ${maxGroups} groups; raise maxGroups or coarsen the group key.`, { maxGroups });
|
|
585
|
+
}
|
|
586
|
+
const ordinal = groupCount;
|
|
587
|
+
groupCount += 1;
|
|
588
|
+
for (let index = 0; index < accumulatorCount; index += 1) {
|
|
589
|
+
const accumulator = accumulators[index];
|
|
590
|
+
accumulator.values.push(accumulator.kind === KIND_MIN
|
|
591
|
+
? Number.POSITIVE_INFINITY
|
|
592
|
+
: accumulator.kind === KIND_MAX
|
|
593
|
+
? Number.NEGATIVE_INFINITY
|
|
594
|
+
: 0);
|
|
595
|
+
accumulator.compensation.push(0);
|
|
596
|
+
accumulator.nonNull.push(0);
|
|
597
|
+
}
|
|
598
|
+
return ordinal;
|
|
599
|
+
};
|
|
600
|
+
const openNullGroup = () => {
|
|
601
|
+
if (nullGroup < 0)
|
|
602
|
+
nullGroup = openGroup();
|
|
603
|
+
return nullGroup;
|
|
604
|
+
};
|
|
605
|
+
const gridCell = (x, y, row) => {
|
|
606
|
+
const cellX = Math.floor((x - gridOriginX) / gridSizeX);
|
|
607
|
+
const cellY = Math.floor((y - gridOriginY) / gridSizeY);
|
|
608
|
+
if (!Number.isFinite(cellX) ||
|
|
609
|
+
!Number.isFinite(cellY) ||
|
|
610
|
+
Math.abs(cellX) > GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX ||
|
|
611
|
+
Math.abs(cellY) > GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX) {
|
|
612
|
+
fail("invalid-input", "A spatial-grid cell index exceeded the addressable grid; increase cellSize.", {
|
|
613
|
+
row,
|
|
614
|
+
cellX,
|
|
615
|
+
cellY,
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
return ((cellX + GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX) * GRID_KEY_STRIDE +
|
|
619
|
+
(cellY + GEOARROW_AGGREGATE_MAX_GRID_CELL_INDEX));
|
|
620
|
+
};
|
|
621
|
+
let reportedProgress = 0.01;
|
|
622
|
+
let nextYield = yieldIntervalRows;
|
|
623
|
+
context.reportProgress(reportedProgress, "scan");
|
|
624
|
+
for (let row = 0; row < rows; row += 1) {
|
|
625
|
+
if ((row & (GEOARROW_AGGREGATE_ABORT_CHECK_ROWS - 1)) === 0)
|
|
626
|
+
context.signal.throwIfAborted();
|
|
627
|
+
if (row === nextYield) {
|
|
628
|
+
nextYield += yieldIntervalRows;
|
|
629
|
+
const fraction = 0.01 + (row / rows) * 0.98;
|
|
630
|
+
if (fraction - reportedProgress >= 0.01) {
|
|
631
|
+
reportedProgress = fraction;
|
|
632
|
+
context.reportProgress(fraction, "scan");
|
|
633
|
+
}
|
|
634
|
+
await yieldToHost();
|
|
635
|
+
context.signal.throwIfAborted();
|
|
636
|
+
}
|
|
637
|
+
const emptyPoint = isEmptyPointRow(geometry, row);
|
|
638
|
+
// Resolve the group ordinal from packed buffers only.
|
|
639
|
+
let ordinal;
|
|
640
|
+
if (groupKind === GROUP_DICTIONARY) {
|
|
641
|
+
if (!isValidBit(dictionaryValidity, row)) {
|
|
642
|
+
if (nullKeys === "skip") {
|
|
643
|
+
skippedRows += 1;
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
ordinal = openNullGroup();
|
|
647
|
+
}
|
|
648
|
+
else {
|
|
649
|
+
const raw = dictionaryIndices[row];
|
|
650
|
+
if (raw < 0 || raw >= canonicalOfIndex.length) {
|
|
651
|
+
fail("invalid-batch", "A dictionary index is outside the dictionary.", { row, index: raw });
|
|
652
|
+
}
|
|
653
|
+
const canonical = canonicalOfIndex[raw];
|
|
654
|
+
let assigned = groupOfCanonical[canonical];
|
|
655
|
+
if (assigned < 0) {
|
|
656
|
+
assigned = openGroup();
|
|
657
|
+
groupOfCanonical[canonical] = assigned;
|
|
658
|
+
groupKeys.push(canonical);
|
|
659
|
+
}
|
|
660
|
+
ordinal = assigned;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
else if (groupKind === GROUP_TEMPORAL) {
|
|
664
|
+
const reader = temporalReader;
|
|
665
|
+
if (!isValidBit(reader.validity, row)) {
|
|
666
|
+
if (nullKeys === "skip") {
|
|
667
|
+
skippedRows += 1;
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
ordinal = openNullGroup();
|
|
671
|
+
}
|
|
672
|
+
else {
|
|
673
|
+
const value = readColumn(reader, row);
|
|
674
|
+
const remainder = value % temporalStep;
|
|
675
|
+
let key = value - remainder;
|
|
676
|
+
if (remainder !== 0 && value < 0)
|
|
677
|
+
key -= temporalStep;
|
|
678
|
+
let assigned = numericGroups.get(key);
|
|
679
|
+
if (assigned === undefined) {
|
|
680
|
+
assigned = openGroup();
|
|
681
|
+
numericGroups.set(key, assigned);
|
|
682
|
+
groupKeys.push(key);
|
|
683
|
+
}
|
|
684
|
+
ordinal = assigned;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
else {
|
|
688
|
+
const range = vertexRange(geometry, row);
|
|
689
|
+
if (!isValidBit(geometry.validity, row) || range[1] <= range[0] || emptyPoint) {
|
|
690
|
+
if (nullKeys === "skip") {
|
|
691
|
+
skippedRows += 1;
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
ordinal = openNullGroup();
|
|
695
|
+
}
|
|
696
|
+
else {
|
|
697
|
+
let minX = Number.POSITIVE_INFINITY;
|
|
698
|
+
let minY = Number.POSITIVE_INFINITY;
|
|
699
|
+
let maxX = Number.NEGATIVE_INFINITY;
|
|
700
|
+
let maxY = Number.NEGATIVE_INFINITY;
|
|
701
|
+
for (let vertex = range[0]; vertex < range[1]; vertex += 1) {
|
|
702
|
+
const x = interleaved ? coordinateX[vertex * coordinateStride] : coordinateX[vertex];
|
|
703
|
+
const y = interleaved ? coordinateY[vertex * coordinateStride + 1] : coordinateY[vertex];
|
|
704
|
+
if (x < minX)
|
|
705
|
+
minX = x;
|
|
706
|
+
if (x > maxX)
|
|
707
|
+
maxX = x;
|
|
708
|
+
if (y < minY)
|
|
709
|
+
minY = y;
|
|
710
|
+
if (y > maxY)
|
|
711
|
+
maxY = y;
|
|
712
|
+
}
|
|
713
|
+
const key = gridCell((minX + maxX) / 2, (minY + maxY) / 2, row);
|
|
714
|
+
let assigned = numericGroups.get(key);
|
|
715
|
+
if (assigned === undefined) {
|
|
716
|
+
assigned = openGroup();
|
|
717
|
+
numericGroups.set(key, assigned);
|
|
718
|
+
groupKeys.push(key);
|
|
719
|
+
}
|
|
720
|
+
ordinal = assigned;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
for (let index = 0; index < columnCount; index += 1) {
|
|
724
|
+
const reader = readers[index];
|
|
725
|
+
if (!isValidBit(reader.validity, row) || (reader.type === COLUMN_F64 && emptyPoint)) {
|
|
726
|
+
columnValid[index] = 0;
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
const value = readColumn(reader, row);
|
|
730
|
+
// Backstop. `inspectGeoArrowBatch` already refuses a batch whose
|
|
731
|
+
// coordinate buffers hold a non-finite value, and the feature-id and
|
|
732
|
+
// temporal columns are integral, so a metric column that reaches here
|
|
733
|
+
// is finite under today's contract. Keep the guard anyway: it is the
|
|
734
|
+
// one thing standing between a widened column set and a poisoned sum.
|
|
735
|
+
if (!Number.isFinite(value)) {
|
|
736
|
+
fail("invalid-input", "A non-finite metric value cannot be aggregated deterministically.", {
|
|
737
|
+
row,
|
|
738
|
+
column: columnNames[index],
|
|
739
|
+
value,
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
columnValid[index] = 1;
|
|
743
|
+
columnValue[index] = value;
|
|
744
|
+
}
|
|
745
|
+
for (let index = 0; index < accumulatorCount; index += 1) {
|
|
746
|
+
const accumulator = accumulators[index];
|
|
747
|
+
if (accumulator.column < 0) {
|
|
748
|
+
accumulator.values[ordinal] += 1;
|
|
749
|
+
continue;
|
|
750
|
+
}
|
|
751
|
+
if (columnValid[accumulator.column] === 0)
|
|
752
|
+
continue;
|
|
753
|
+
const value = columnValue[accumulator.column];
|
|
754
|
+
accumulator.nonNull[ordinal] += 1;
|
|
755
|
+
if (accumulator.kind === KIND_COUNT)
|
|
756
|
+
accumulator.values[ordinal] += 1;
|
|
757
|
+
else if (accumulator.kind === KIND_SUM || accumulator.kind === KIND_MEAN) {
|
|
758
|
+
const running = accumulator.values[ordinal];
|
|
759
|
+
const total = running + value;
|
|
760
|
+
accumulator.compensation[ordinal] += neumaierResidual(running, value, total);
|
|
761
|
+
accumulator.values[ordinal] = total;
|
|
762
|
+
}
|
|
763
|
+
else if (accumulator.kind === KIND_MIN) {
|
|
764
|
+
if (value < accumulator.values[ordinal])
|
|
765
|
+
accumulator.values[ordinal] = value;
|
|
766
|
+
}
|
|
767
|
+
else if (value > accumulator.values[ordinal])
|
|
768
|
+
accumulator.values[ordinal] = value;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
context.signal.throwIfAborted();
|
|
772
|
+
context.reportProgress(0.99, "encode");
|
|
773
|
+
// Deterministic output ordering: ascending by group key, null group last.
|
|
774
|
+
const order = [];
|
|
775
|
+
for (let ordinal = 0; ordinal < groupCount; ordinal += 1)
|
|
776
|
+
if (ordinal !== nullGroup)
|
|
777
|
+
order.push(ordinal);
|
|
778
|
+
const keyOfOrdinal = new Map();
|
|
779
|
+
if (groupKind === GROUP_DICTIONARY) {
|
|
780
|
+
for (const canonical of groupKeys)
|
|
781
|
+
keyOfOrdinal.set(groupOfCanonical[canonical], canonical);
|
|
782
|
+
order.sort((left, right) => {
|
|
783
|
+
const a = dictionaryValues[keyOfOrdinal.get(left)];
|
|
784
|
+
const b = dictionaryValues[keyOfOrdinal.get(right)];
|
|
785
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
else {
|
|
789
|
+
for (const key of groupKeys)
|
|
790
|
+
keyOfOrdinal.set(numericGroups.get(key), key);
|
|
791
|
+
order.sort((left, right) => keyOfOrdinal.get(left) - keyOfOrdinal.get(right));
|
|
792
|
+
}
|
|
793
|
+
const sortedKeys = order.map((ordinal) => keyOfOrdinal.get(ordinal));
|
|
794
|
+
if (nullGroup >= 0)
|
|
795
|
+
order.push(nullGroup);
|
|
796
|
+
const groupColumns = groupKind === GROUP_DICTIONARY
|
|
797
|
+
? buildUtf8Group(groupField, sortedKeys.map((canonical) => dictionaryValues[canonical]), nullGroup >= 0)
|
|
798
|
+
: groupKind === GROUP_TEMPORAL
|
|
799
|
+
? buildTemporalGroup(groupField, sortedKeys, nullGroup >= 0, inspection.temporal.unit, inspection.temporal.timezone)
|
|
800
|
+
: buildGridGroup(groupField, sortedKeys, nullGroup >= 0);
|
|
801
|
+
const count = order.length;
|
|
802
|
+
const fields = [...groupColumns.fields];
|
|
803
|
+
const buffers = [...groupColumns.buffers];
|
|
804
|
+
for (let index = 0; index < accumulatorCount; index += 1) {
|
|
805
|
+
const accumulator = accumulators[index];
|
|
806
|
+
const name = accumulator.spec.name;
|
|
807
|
+
const nullable = accumulator.kind !== KIND_COUNT;
|
|
808
|
+
const values = new Float64Array(count);
|
|
809
|
+
const validity = nullable ? new Uint8Array((count + 7) >> 3) : undefined;
|
|
810
|
+
for (let slot = 0; slot < count; slot += 1) {
|
|
811
|
+
const ordinal = order[slot];
|
|
812
|
+
if (!nullable) {
|
|
813
|
+
values[slot] = accumulator.values[ordinal];
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
816
|
+
const nonNull = accumulator.nonNull[ordinal];
|
|
817
|
+
if (nonNull === 0)
|
|
818
|
+
continue;
|
|
819
|
+
// Fold the carried low-order bits back in exactly once, so `sum` and
|
|
820
|
+
// `mean` report the compensated total rather than the drifted one.
|
|
821
|
+
const total = accumulator.values[ordinal] + accumulator.compensation[ordinal];
|
|
822
|
+
values[slot] = accumulator.kind === KIND_MEAN ? total / nonNull : total;
|
|
823
|
+
validity[slot >> 3] |= 1 << (slot & 7);
|
|
824
|
+
}
|
|
825
|
+
fields.push(Object.freeze({ name, type: Object.freeze({ name: "float64" }), nullable }));
|
|
826
|
+
if (validity)
|
|
827
|
+
buffers.push(bufferOf(`${name}.validity`, "validity", name, validity));
|
|
828
|
+
buffers.push(bufferOf(`${name}.values`, "values", name, values));
|
|
829
|
+
}
|
|
830
|
+
const sourceJson = JSON.stringify({
|
|
831
|
+
batchId: source.id,
|
|
832
|
+
schemaId: source.schema.id,
|
|
833
|
+
rowCount: rows,
|
|
834
|
+
planId: identity.planId,
|
|
835
|
+
sourceId: identity.sourceId,
|
|
836
|
+
sourceVersion: identity.sourceVersion,
|
|
837
|
+
skippedRows,
|
|
838
|
+
});
|
|
839
|
+
const aggregatedIdentity = {
|
|
840
|
+
sourceId: identity.sourceId,
|
|
841
|
+
sourceVersion: identity.sourceVersion,
|
|
842
|
+
schemaVersion: schemaId,
|
|
843
|
+
planId: `honua.aggregate@${HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION}|${identity.planId}|${specJson}`,
|
|
844
|
+
authorizationScope: identity.authorizationScope,
|
|
845
|
+
ordering: Object.freeze({
|
|
846
|
+
stable: true,
|
|
847
|
+
keys: Object.freeze(groupColumns.orderingFields.map((field) => Object.freeze({ field, direction: "ascending", nulls: "last" }))),
|
|
848
|
+
}),
|
|
849
|
+
freshness: identity.freshness,
|
|
850
|
+
};
|
|
851
|
+
const output = createColumnarBatch({
|
|
852
|
+
id,
|
|
853
|
+
sequence: source.sequence,
|
|
854
|
+
rowCount: count,
|
|
855
|
+
schema: {
|
|
856
|
+
id: schemaId,
|
|
857
|
+
fields,
|
|
858
|
+
metadata: {
|
|
859
|
+
[META.version]: HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION,
|
|
860
|
+
[META.spec]: specJson,
|
|
861
|
+
[META.source]: sourceJson,
|
|
862
|
+
},
|
|
863
|
+
},
|
|
864
|
+
identity: aggregatedIdentity,
|
|
865
|
+
buffers,
|
|
866
|
+
}, limits);
|
|
867
|
+
context.reportProgress(1, "complete");
|
|
868
|
+
return output;
|
|
869
|
+
};
|
|
870
|
+
}
|
|
871
|
+
function typedView(buffer, ctor) {
|
|
872
|
+
if (buffer.byteOffset % ctor.BYTES_PER_ELEMENT !== 0 || buffer.byteLength % ctor.BYTES_PER_ELEMENT !== 0) {
|
|
873
|
+
fail("invalid-batch", `Buffer "${buffer.id}" is not component-aligned.`, { bufferId: buffer.id });
|
|
874
|
+
}
|
|
875
|
+
try {
|
|
876
|
+
return new ctor(buffer.data, buffer.byteOffset, buffer.byteLength / ctor.BYTES_PER_ELEMENT);
|
|
877
|
+
}
|
|
878
|
+
catch (cause) {
|
|
879
|
+
throw new HonuaGeoArrowError("invalid-batch", `Buffer "${buffer.id}" cannot be viewed with its normative component type.`, { bufferId: buffer.id }, { cause });
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* Decode a `honua.aggregate` result batch produced by
|
|
884
|
+
* {@link createGeoArrowAggregateOperation}.
|
|
885
|
+
*
|
|
886
|
+
* Materialization is bounded by the group count, which the producing operation
|
|
887
|
+
* already capped, so this is always safe to call on a result the SDK produced.
|
|
888
|
+
*/
|
|
889
|
+
export function readGeoArrowAggregateBatch(batch, limits = {}) {
|
|
890
|
+
let normalized;
|
|
891
|
+
try {
|
|
892
|
+
normalized = createColumnarBatch(batch, limits);
|
|
893
|
+
}
|
|
894
|
+
catch (cause) {
|
|
895
|
+
throw new HonuaGeoArrowError("invalid-batch", "The aggregate batch does not satisfy the columnar transport contract.", undefined, { cause });
|
|
896
|
+
}
|
|
897
|
+
const metadata = normalized.schema.metadata ?? {};
|
|
898
|
+
if (metadata[META.version] !== HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION) {
|
|
899
|
+
fail("unsupported-layout", `Batch must use Honua aggregate ${HONUA_GEOARROW_AGGREGATE_LAYOUT_VERSION}.`);
|
|
900
|
+
}
|
|
901
|
+
let spec;
|
|
902
|
+
let source;
|
|
903
|
+
try {
|
|
904
|
+
spec = JSON.parse(metadata[META.spec]);
|
|
905
|
+
source = JSON.parse(metadata[META.source]);
|
|
906
|
+
}
|
|
907
|
+
catch (cause) {
|
|
908
|
+
throw new HonuaGeoArrowError("invalid-batch", "The aggregate batch specification metadata is not readable.", undefined, { cause });
|
|
909
|
+
}
|
|
910
|
+
const buffers = new Map();
|
|
911
|
+
for (const buffer of normalized.buffers)
|
|
912
|
+
buffers.set(buffer.id, buffer);
|
|
913
|
+
const required = (bufferId) => {
|
|
914
|
+
const buffer = buffers.get(bufferId);
|
|
915
|
+
if (!buffer)
|
|
916
|
+
fail("invalid-batch", `The aggregate batch is missing buffer ${bufferId}.`);
|
|
917
|
+
return buffer;
|
|
918
|
+
};
|
|
919
|
+
const count = normalized.rowCount;
|
|
920
|
+
const field = spec.groupField;
|
|
921
|
+
const keys = new Array(count);
|
|
922
|
+
if (spec.group.kind === "dictionary") {
|
|
923
|
+
const validity = typedView(required(`${field}.validity`), Uint8Array);
|
|
924
|
+
const offsets = typedView(required(`${field}.offsets`), Int32Array);
|
|
925
|
+
const values = typedView(required(`${field}.values`), Uint8Array);
|
|
926
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
927
|
+
for (let index = 0; index < count; index += 1) {
|
|
928
|
+
keys[index] = isValidBit(validity, index)
|
|
929
|
+
? decoder.decode(values.subarray(offsets[index], offsets[index + 1]))
|
|
930
|
+
: null;
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
else if (spec.group.kind === "temporal") {
|
|
934
|
+
const validity = typedView(required(`${field}.validity`), Uint8Array);
|
|
935
|
+
const values = typedView(required(`${field}.values`), BigInt64Array);
|
|
936
|
+
for (let index = 0; index < count; index += 1) {
|
|
937
|
+
keys[index] = isValidBit(validity, index) ? Number(values[index]) : null;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
else {
|
|
941
|
+
const validity = typedView(required(`${field}_x.validity`), Uint8Array);
|
|
942
|
+
const xValues = typedView(required(`${field}_x.values`), Int32Array);
|
|
943
|
+
const yValues = typedView(required(`${field}_y.values`), Int32Array);
|
|
944
|
+
for (let index = 0; index < count; index += 1) {
|
|
945
|
+
keys[index] = isValidBit(validity, index) ? Object.freeze([xValues[index], yValues[index]]) : null;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
const metricViews = spec.metrics.map((metric) => ({
|
|
949
|
+
name: metric.name,
|
|
950
|
+
values: typedView(required(`${metric.name}.values`), Float64Array),
|
|
951
|
+
validity: buffers.has(`${metric.name}.validity`)
|
|
952
|
+
? typedView(required(`${metric.name}.validity`), Uint8Array)
|
|
953
|
+
: undefined,
|
|
954
|
+
}));
|
|
955
|
+
const rows = new Array(count);
|
|
956
|
+
for (let index = 0; index < count; index += 1) {
|
|
957
|
+
const values = {};
|
|
958
|
+
for (const view of metricViews)
|
|
959
|
+
values[view.name] = isValidBit(view.validity, index) ? view.values[index] : null;
|
|
960
|
+
rows[index] = Object.freeze({ key: keys[index] ?? null, metrics: Object.freeze(values) });
|
|
961
|
+
}
|
|
962
|
+
return Object.freeze({
|
|
963
|
+
batch: normalized,
|
|
964
|
+
group: Object.freeze(spec.group),
|
|
965
|
+
groupField: field,
|
|
966
|
+
metrics: Object.freeze(spec.metrics.map((metric) => Object.freeze({
|
|
967
|
+
name: metric.name,
|
|
968
|
+
kind: metric.kind,
|
|
969
|
+
...(metric.column === null ? {} : { column: metric.column }),
|
|
970
|
+
}))),
|
|
971
|
+
nullKeys: spec.nullKeys,
|
|
972
|
+
maxGroups: spec.maxGroups,
|
|
973
|
+
source: Object.freeze(source),
|
|
974
|
+
rows: Object.freeze(rows),
|
|
975
|
+
});
|
|
976
|
+
}
|