@loaders.gl/arrow 4.0.0 → 4.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +34 -26
- package/dist/arrow-writer.d.ts.map +1 -1
- package/dist/arrow-writer.js +6 -3
- package/dist/arrow-writer.js.map +1 -1
- package/dist/dist.dev.js +658 -6
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +40 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +189 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +19 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.js +138 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +1 -0
- package/dist/geoarrow/get-arrow-bounds.d.ts +11 -0
- package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -0
- package/dist/geoarrow/get-arrow-bounds.js +24 -0
- package/dist/geoarrow/get-arrow-bounds.js.map +1 -0
- package/dist/index.cjs +627 -32
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow-table-batch.d.ts +2 -2
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js +7 -7
- package/dist/lib/arrow-table-batch.js.map +1 -1
- package/dist/lib/arrow-table.d.ts +3 -3
- package/dist/lib/arrow-table.d.ts.map +1 -1
- package/dist/lib/arrow-table.js.map +1 -1
- package/dist/lib/encode-arrow.js +5 -5
- package/dist/lib/encode-arrow.js.map +1 -1
- package/dist/lib/parse-arrow-in-batches.js +3 -3
- package/dist/lib/parse-arrow-in-batches.js.map +1 -1
- package/dist/lib/parse-arrow-sync.d.ts.map +1 -1
- package/dist/lib/parse-arrow-sync.js +3 -3
- package/dist/lib/parse-arrow-sync.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts +3 -2
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js +9 -9
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/convert-arrow-schema.d.ts +19 -0
- package/dist/schema/convert-arrow-schema.d.ts.map +1 -0
- package/dist/schema/convert-arrow-schema.js +232 -0
- package/dist/schema/convert-arrow-schema.js.map +1 -0
- package/dist/{lib/convert-table.d.ts → tables/convert-arrow-to-table.d.ts} +4 -4
- package/dist/tables/convert-arrow-to-table.d.ts.map +1 -0
- package/dist/{lib/convert-table.js → tables/convert-arrow-to-table.js} +1 -1
- package/dist/tables/convert-arrow-to-table.js.map +1 -0
- package/dist/{schema → tables}/convert-table-to-arrow.d.ts.map +1 -1
- package/dist/{schema → tables}/convert-table-to-arrow.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/workers/arrow-worker.js.map +1 -1
- package/package.json +5 -4
- package/src/arrow-loader.ts +1 -0
- package/src/arrow-writer.ts +7 -3
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +260 -0
- package/src/geoarrow/convert-geoarrow-to-geojson.ts +192 -0
- package/src/geoarrow/get-arrow-bounds.ts +40 -0
- package/src/index.ts +33 -4
- package/src/lib/arrow-table-batch.ts +13 -23
- package/src/lib/arrow-table.ts +3 -3
- package/src/lib/encode-arrow.ts +8 -8
- package/src/lib/parse-arrow-in-batches.ts +4 -4
- package/src/lib/parse-arrow-sync.ts +6 -3
- package/src/schema/arrow-type-utils.ts +11 -29
- package/src/schema/convert-arrow-schema.ts +265 -0
- package/src/{lib/convert-table.ts → tables/convert-arrow-to-table.ts} +4 -3
- package/src/types.ts +3 -0
- package/src/workers/arrow-worker.ts +3 -0
- package/dist/lib/convert-table.d.ts.map +0 -1
- package/dist/lib/convert-table.js.map +0 -1
- package/dist/schema/convert-schema-arrow.d.ts +0 -13
- package/dist/schema/convert-schema-arrow.d.ts.map +0 -1
- package/dist/schema/convert-schema-arrow.js +0 -158
- package/dist/schema/convert-schema-arrow.js.map +0 -1
- package/src/schema/convert-schema-arrow.ts +0 -233
- /package/dist/{schema → tables}/convert-table-to-arrow.d.ts +0 -0
- /package/dist/{schema → tables}/convert-table-to-arrow.js +0 -0
- /package/src/{schema → tables}/convert-table-to-arrow.ts +0 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import * as arrow from 'apache-arrow';
|
|
5
|
+
import {BinaryFeatureCollection as BinaryFeatures} from '@loaders.gl/schema';
|
|
6
|
+
import {GeoArrowEncoding} from '@loaders.gl/gis';
|
|
7
|
+
import {updateBoundsFromGeoArrowSamples} from './get-arrow-bounds';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Binary data from geoarrow column and can be used by e.g. deck.gl GeojsonLayer
|
|
11
|
+
*/
|
|
12
|
+
export type BinaryDataFromGeoArrow = {
|
|
13
|
+
binaryGeometries: BinaryFeatures[];
|
|
14
|
+
bounds: [number, number, number, number];
|
|
15
|
+
featureTypes: {polygon: boolean; point: boolean; line: boolean};
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
type BinaryGeometryContent = {
|
|
19
|
+
featureIds: Uint32Array;
|
|
20
|
+
flatCoordinateArray: Float64Array;
|
|
21
|
+
nDim: number;
|
|
22
|
+
geomOffset: Int32Array;
|
|
23
|
+
geometryIndicies: Uint16Array;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
// binary geometry template, see deck.gl BinaryGeometry
|
|
27
|
+
export const BINARY_GEOMETRY_TEMPLATE = {
|
|
28
|
+
globalFeatureIds: {value: new Uint32Array(0), size: 1},
|
|
29
|
+
positions: {value: new Float32Array(0), size: 2},
|
|
30
|
+
properties: [],
|
|
31
|
+
numericProps: {},
|
|
32
|
+
featureIds: {value: new Uint32Array(0), size: 1}
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* get binary geometries from geoarrow column
|
|
37
|
+
*
|
|
38
|
+
* @param geoColumn the geoarrow column, e.g. arrowTable.getChildAt(geoColumnIndex)
|
|
39
|
+
* @param geoEncoding the geo encoding of the geoarrow column, e.g. getGeoArrowEncoding(arrowTable.schema, geoColumnName)
|
|
40
|
+
* @returns BinaryDataFromGeoArrow
|
|
41
|
+
*/
|
|
42
|
+
export function getBinaryGeometriesFromArrow(
|
|
43
|
+
geoColumn: arrow.Vector,
|
|
44
|
+
geoEncoding: GeoArrowEncoding
|
|
45
|
+
): BinaryDataFromGeoArrow {
|
|
46
|
+
const featureTypes = {
|
|
47
|
+
polygon: geoEncoding === 'geoarrow.multipolygon' || geoEncoding === 'geoarrow.polygon',
|
|
48
|
+
point: geoEncoding === 'geoarrow.multipoint' || geoEncoding === 'geoarrow.point',
|
|
49
|
+
line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const chunks = geoColumn.data;
|
|
53
|
+
let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
|
|
54
|
+
let globalFeatureIdOffset = 0;
|
|
55
|
+
const binaryGeometries: BinaryFeatures[] = [];
|
|
56
|
+
|
|
57
|
+
chunks.forEach((chunk) => {
|
|
58
|
+
const {featureIds, flatCoordinateArray, nDim, geomOffset} = getBinaryGeometriesFromChunk(
|
|
59
|
+
chunk,
|
|
60
|
+
geoEncoding
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
const numOfVertices = flatCoordinateArray.length / nDim;
|
|
64
|
+
const globalFeatureIds = new Uint32Array(numOfVertices);
|
|
65
|
+
for (let i = 0; i < numOfVertices; i++) {
|
|
66
|
+
globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const binaryContent = {
|
|
70
|
+
globalFeatureIds: {value: globalFeatureIds, size: 1},
|
|
71
|
+
positions: {
|
|
72
|
+
value: flatCoordinateArray,
|
|
73
|
+
size: nDim
|
|
74
|
+
},
|
|
75
|
+
featureIds: {value: featureIds, size: 1},
|
|
76
|
+
properties: [...Array(chunk.length).keys()].map((i) => ({
|
|
77
|
+
index: i + globalFeatureIdOffset
|
|
78
|
+
}))
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
// TODO: check if chunks are sequentially accessed
|
|
82
|
+
globalFeatureIdOffset += chunk.length;
|
|
83
|
+
|
|
84
|
+
// NOTE: deck.gl defines the BinaryFeatures structure must have points, lines, polygons even if they are empty
|
|
85
|
+
binaryGeometries.push({
|
|
86
|
+
shape: 'binary-feature-collection',
|
|
87
|
+
points: {
|
|
88
|
+
type: 'Point',
|
|
89
|
+
...BINARY_GEOMETRY_TEMPLATE,
|
|
90
|
+
...(featureTypes.point ? binaryContent : {})
|
|
91
|
+
},
|
|
92
|
+
lines: {
|
|
93
|
+
type: 'LineString',
|
|
94
|
+
...BINARY_GEOMETRY_TEMPLATE,
|
|
95
|
+
...(featureTypes.line ? binaryContent : {}),
|
|
96
|
+
pathIndices: {value: featureTypes.line ? geomOffset : new Uint16Array(0), size: 1}
|
|
97
|
+
},
|
|
98
|
+
polygons: {
|
|
99
|
+
type: 'Polygon',
|
|
100
|
+
...BINARY_GEOMETRY_TEMPLATE,
|
|
101
|
+
...(featureTypes.polygon ? binaryContent : {}),
|
|
102
|
+
polygonIndices: {
|
|
103
|
+
// TODO why deck.gl's tessellatePolygon performance is not good when using geometryIndicies
|
|
104
|
+
// even when there is no hole in any polygon
|
|
105
|
+
value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
|
|
106
|
+
size: 1
|
|
107
|
+
},
|
|
108
|
+
primitivePolygonIndices: {
|
|
109
|
+
value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
|
|
110
|
+
size: 1
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
bounds = updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
return {binaryGeometries, bounds, featureTypes};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* get binary geometries from geoarrow column
|
|
123
|
+
* @param chunk one chunk/batch of geoarrow column
|
|
124
|
+
* @param geoEncoding geo encoding of the geoarrow column
|
|
125
|
+
* @returns BinaryGeometryContent
|
|
126
|
+
*/
|
|
127
|
+
function getBinaryGeometriesFromChunk(
|
|
128
|
+
chunk: arrow.Data,
|
|
129
|
+
geoEncoding: GeoArrowEncoding
|
|
130
|
+
): BinaryGeometryContent {
|
|
131
|
+
switch (geoEncoding) {
|
|
132
|
+
case 'geoarrow.point':
|
|
133
|
+
case 'geoarrow.multipoint':
|
|
134
|
+
return getBinaryPointsFromChunk(chunk, geoEncoding);
|
|
135
|
+
case 'geoarrow.linestring':
|
|
136
|
+
case 'geoarrow.multilinestring':
|
|
137
|
+
return getBinaryLinesFromChunk(chunk, geoEncoding);
|
|
138
|
+
case 'geoarrow.polygon':
|
|
139
|
+
case 'geoarrow.multipolygon':
|
|
140
|
+
return getBinaryPolygonsFromChunk(chunk, geoEncoding);
|
|
141
|
+
default:
|
|
142
|
+
throw Error('invalid geoarrow encoding');
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* get binary polygons from geoarrow polygon column
|
|
148
|
+
* @param chunk one chunk of geoarrow polygon column
|
|
149
|
+
* @param geoEncoding the geo encoding of the geoarrow polygon column
|
|
150
|
+
* @returns BinaryGeometryContent
|
|
151
|
+
*/
|
|
152
|
+
function getBinaryPolygonsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
|
|
153
|
+
const isMultiPolygon = geoEncoding === 'geoarrow.multipolygon';
|
|
154
|
+
|
|
155
|
+
const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
|
|
156
|
+
const ringData = polygonData.children[0];
|
|
157
|
+
const pointData = ringData.children[0];
|
|
158
|
+
const coordData = pointData.children[0];
|
|
159
|
+
const nDim = pointData.stride;
|
|
160
|
+
const geomOffset = ringData.valueOffsets;
|
|
161
|
+
const flatCoordinateArray = coordData.values;
|
|
162
|
+
|
|
163
|
+
const geometryIndicies = new Uint16Array(chunk.length + 1);
|
|
164
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
165
|
+
geometryIndicies[i] = geomOffset[chunk.valueOffsets[i]];
|
|
166
|
+
}
|
|
167
|
+
geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
|
|
168
|
+
|
|
169
|
+
const numOfVertices = flatCoordinateArray.length / nDim;
|
|
170
|
+
const featureIds = new Uint32Array(numOfVertices);
|
|
171
|
+
for (let i = 0; i < chunk.length - 1; i++) {
|
|
172
|
+
const startIdx = geomOffset[chunk.valueOffsets[i]];
|
|
173
|
+
const endIdx = geomOffset[chunk.valueOffsets[i + 1]];
|
|
174
|
+
for (let j = startIdx; j < endIdx; j++) {
|
|
175
|
+
featureIds[j] = i;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
featureIds,
|
|
181
|
+
flatCoordinateArray,
|
|
182
|
+
nDim,
|
|
183
|
+
geomOffset,
|
|
184
|
+
geometryIndicies
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* get binary lines from geoarrow line column
|
|
190
|
+
* @param chunk one chunk/batch of geoarrow column
|
|
191
|
+
* @param geoEncoding the geo encoding of the geoarrow column
|
|
192
|
+
* @returns BinaryGeometryContent
|
|
193
|
+
*/
|
|
194
|
+
function getBinaryLinesFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
|
|
195
|
+
const isMultiLineString = geoEncoding === 'geoarrow.multilinestring';
|
|
196
|
+
|
|
197
|
+
const lineData = isMultiLineString ? chunk.children[0] : chunk;
|
|
198
|
+
const pointData = lineData.children[0];
|
|
199
|
+
const coordData = pointData.children[0];
|
|
200
|
+
|
|
201
|
+
const nDim = pointData.stride;
|
|
202
|
+
const geomOffset = lineData.valueOffsets;
|
|
203
|
+
const flatCoordinateArray = coordData.values;
|
|
204
|
+
|
|
205
|
+
// geometryIndicies is not needed for line string
|
|
206
|
+
const geometryIndicies = new Uint16Array(0);
|
|
207
|
+
|
|
208
|
+
const numOfVertices = flatCoordinateArray.length / nDim;
|
|
209
|
+
const featureIds = new Uint32Array(numOfVertices);
|
|
210
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
211
|
+
const startIdx = geomOffset[i];
|
|
212
|
+
const endIdx = geomOffset[i + 1];
|
|
213
|
+
for (let j = startIdx; j < endIdx; j++) {
|
|
214
|
+
featureIds[j] = i;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return {
|
|
219
|
+
featureIds,
|
|
220
|
+
flatCoordinateArray,
|
|
221
|
+
nDim,
|
|
222
|
+
geomOffset,
|
|
223
|
+
geometryIndicies
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* get binary points from geoarrow point column
|
|
229
|
+
* @param chunk one chunk/batch of geoarrow column
|
|
230
|
+
* @param geoEncoding geo encoding of the geoarrow column
|
|
231
|
+
* @returns BinaryGeometryContent
|
|
232
|
+
*/
|
|
233
|
+
function getBinaryPointsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
|
|
234
|
+
const isMultiPoint = geoEncoding === 'geoarrow.multipoint';
|
|
235
|
+
|
|
236
|
+
const pointData = isMultiPoint ? chunk.children[0] : chunk;
|
|
237
|
+
const coordData = pointData.children[0];
|
|
238
|
+
|
|
239
|
+
const nDim = pointData.stride;
|
|
240
|
+
const flatCoordinateArray = coordData.values;
|
|
241
|
+
|
|
242
|
+
// geometryIndices is not needed for point
|
|
243
|
+
const geometryIndicies = new Uint16Array(0);
|
|
244
|
+
// geomOffset is not needed for point
|
|
245
|
+
const geomOffset = new Int32Array(0);
|
|
246
|
+
|
|
247
|
+
const numOfVertices = flatCoordinateArray.length / nDim;
|
|
248
|
+
const featureIds = new Uint32Array(numOfVertices);
|
|
249
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
250
|
+
featureIds[i] = i;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
featureIds,
|
|
255
|
+
flatCoordinateArray,
|
|
256
|
+
nDim,
|
|
257
|
+
geomOffset,
|
|
258
|
+
geometryIndicies
|
|
259
|
+
};
|
|
260
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import * as arrow from 'apache-arrow';
|
|
5
|
+
import {
|
|
6
|
+
Feature,
|
|
7
|
+
MultiPolygon,
|
|
8
|
+
Position,
|
|
9
|
+
Polygon,
|
|
10
|
+
MultiPoint,
|
|
11
|
+
Point,
|
|
12
|
+
MultiLineString,
|
|
13
|
+
LineString
|
|
14
|
+
} from '@loaders.gl/schema';
|
|
15
|
+
import type {GeoArrowEncoding} from '@loaders.gl/gis';
|
|
16
|
+
|
|
17
|
+
type RawArrowFeature = {
|
|
18
|
+
encoding?: GeoArrowEncoding;
|
|
19
|
+
data: any;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* parse geometry from arrow data that is returned from processArrowData()
|
|
24
|
+
* NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
|
|
25
|
+
* it is currently only used for picking because currently deck.gl returns only the index of the feature
|
|
26
|
+
* So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
|
|
27
|
+
*
|
|
28
|
+
* @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
|
|
29
|
+
* @see processArrowData
|
|
30
|
+
* @returns Feature or null
|
|
31
|
+
*/
|
|
32
|
+
export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
|
|
33
|
+
const encoding = rawData.encoding?.toLowerCase();
|
|
34
|
+
const data = rawData.data;
|
|
35
|
+
if (!encoding || !data) {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
let geometry;
|
|
40
|
+
|
|
41
|
+
switch (encoding) {
|
|
42
|
+
case 'geoarrow.multipolygon':
|
|
43
|
+
geometry = arrowMultiPolygonToFeature(data);
|
|
44
|
+
break;
|
|
45
|
+
case 'geoarrow.polygon':
|
|
46
|
+
geometry = arrowPolygonToFeature(data);
|
|
47
|
+
break;
|
|
48
|
+
case 'geoarrow.multipoint':
|
|
49
|
+
geometry = arrowMultiPointToFeature(data);
|
|
50
|
+
break;
|
|
51
|
+
case 'geoarrow.point':
|
|
52
|
+
geometry = arrowPointToFeature(data);
|
|
53
|
+
break;
|
|
54
|
+
case 'geoarrow.multilinestring':
|
|
55
|
+
geometry = arrowMultiLineStringToFeature(data);
|
|
56
|
+
break;
|
|
57
|
+
case 'geoarrow.linestring':
|
|
58
|
+
geometry = arrowLineStringToFeature(data);
|
|
59
|
+
break;
|
|
60
|
+
default: {
|
|
61
|
+
throw Error(`GeoArrow encoding not supported ${encoding}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
type: 'Feature',
|
|
66
|
+
geometry,
|
|
67
|
+
properties: {}
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* convert Arrow MultiPolygon to geojson Feature
|
|
73
|
+
*/
|
|
74
|
+
function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {
|
|
75
|
+
const multiPolygon: Position[][][] = [];
|
|
76
|
+
for (let m = 0; m < arrowMultiPolygon.length; m++) {
|
|
77
|
+
const arrowPolygon = arrowMultiPolygon.get(m);
|
|
78
|
+
const polygon: Position[][] = [];
|
|
79
|
+
for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) {
|
|
80
|
+
const arrowRing = arrowPolygon?.get(i);
|
|
81
|
+
const ring: Position[] = [];
|
|
82
|
+
for (let j = 0; arrowRing && j < arrowRing.length; j++) {
|
|
83
|
+
const arrowCoord = arrowRing.get(j);
|
|
84
|
+
const coord: Position = Array.from(arrowCoord);
|
|
85
|
+
ring.push(coord);
|
|
86
|
+
}
|
|
87
|
+
polygon.push(ring);
|
|
88
|
+
}
|
|
89
|
+
multiPolygon.push(polygon);
|
|
90
|
+
}
|
|
91
|
+
const geometry: MultiPolygon = {
|
|
92
|
+
type: 'MultiPolygon',
|
|
93
|
+
coordinates: multiPolygon
|
|
94
|
+
};
|
|
95
|
+
return geometry;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* convert Arrow Polygon to geojson Feature
|
|
100
|
+
*/
|
|
101
|
+
function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
|
|
102
|
+
const polygon: Position[][] = [];
|
|
103
|
+
for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
|
|
104
|
+
const arrowRing = arrowPolygon.get(i);
|
|
105
|
+
const ring: Position[] = [];
|
|
106
|
+
for (let j = 0; arrowRing && j < arrowRing.length; j++) {
|
|
107
|
+
const arrowCoord = arrowRing.get(j);
|
|
108
|
+
const coords: Position = Array.from(arrowCoord);
|
|
109
|
+
ring.push(coords);
|
|
110
|
+
}
|
|
111
|
+
polygon.push(ring);
|
|
112
|
+
}
|
|
113
|
+
const geometry: Polygon = {
|
|
114
|
+
type: 'Polygon',
|
|
115
|
+
coordinates: polygon
|
|
116
|
+
};
|
|
117
|
+
return geometry;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* convert Arrow MultiPoint to geojson MultiPoint
|
|
122
|
+
*/
|
|
123
|
+
function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
|
|
124
|
+
const multiPoint: Position[] = [];
|
|
125
|
+
for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
|
|
126
|
+
const arrowPoint = arrowMultiPoint.get(i);
|
|
127
|
+
if (arrowPoint) {
|
|
128
|
+
const coord: Position = Array.from(arrowPoint);
|
|
129
|
+
multiPoint.push(coord);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const geometry: MultiPoint = {
|
|
133
|
+
type: 'MultiPoint',
|
|
134
|
+
coordinates: multiPoint
|
|
135
|
+
};
|
|
136
|
+
return geometry;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* convert Arrow Point to geojson Point
|
|
141
|
+
*/
|
|
142
|
+
function arrowPointToFeature(arrowPoint: arrow.Vector): Point {
|
|
143
|
+
const point: Position = Array.from(arrowPoint);
|
|
144
|
+
const geometry: Point = {
|
|
145
|
+
type: 'Point',
|
|
146
|
+
coordinates: point
|
|
147
|
+
};
|
|
148
|
+
return geometry;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* convert Arrow MultiLineString to geojson MultiLineString
|
|
153
|
+
*/
|
|
154
|
+
function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {
|
|
155
|
+
const multiLineString: Position[][] = [];
|
|
156
|
+
for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
|
|
157
|
+
const arrowLineString = arrowMultiLineString.get(i);
|
|
158
|
+
const lineString: Position[] = [];
|
|
159
|
+
for (let j = 0; arrowLineString && j < arrowLineString.length; j++) {
|
|
160
|
+
const arrowCoord = arrowLineString.get(j);
|
|
161
|
+
if (arrowCoord) {
|
|
162
|
+
const coords: Position = Array.from(arrowCoord);
|
|
163
|
+
lineString.push(coords);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
multiLineString.push(lineString);
|
|
167
|
+
}
|
|
168
|
+
const geometry: MultiLineString = {
|
|
169
|
+
type: 'MultiLineString',
|
|
170
|
+
coordinates: multiLineString
|
|
171
|
+
};
|
|
172
|
+
return geometry;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* convert Arrow LineString to geojson LineString
|
|
177
|
+
*/
|
|
178
|
+
function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
|
|
179
|
+
const lineString: Position[] = [];
|
|
180
|
+
for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
|
|
181
|
+
const arrowCoord = arrowLineString.get(i);
|
|
182
|
+
if (arrowCoord) {
|
|
183
|
+
const coords: Position = Array.from(arrowCoord);
|
|
184
|
+
lineString.push(coords);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const geometry: LineString = {
|
|
188
|
+
type: 'LineString',
|
|
189
|
+
coordinates: lineString
|
|
190
|
+
};
|
|
191
|
+
return geometry;
|
|
192
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Update bounds from geoarrow sample data
|
|
6
|
+
*
|
|
7
|
+
* @param flatCoords the flattend coordinates array from one chunk of geoarrow column
|
|
8
|
+
* @param nDim the number of dimensions of the coordinates
|
|
9
|
+
* @param bounds the bounds to be updated
|
|
10
|
+
* @param sampleSize how many samples to be used to update the bounds, default is 1000 per chunk
|
|
11
|
+
* @returns the updated bounds
|
|
12
|
+
*/
|
|
13
|
+
export function updateBoundsFromGeoArrowSamples(
|
|
14
|
+
flatCoords: Float64Array,
|
|
15
|
+
nDim: number,
|
|
16
|
+
bounds: [number, number, number, number],
|
|
17
|
+
sampleSize: number = 100
|
|
18
|
+
): [number, number, number, number] {
|
|
19
|
+
const numberOfFeatures = flatCoords.length / nDim;
|
|
20
|
+
const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
|
|
21
|
+
|
|
22
|
+
const newBounds: [number, number, number, number] = [...bounds];
|
|
23
|
+
for (let i = 0; i < numberOfFeatures; i += sampleStep) {
|
|
24
|
+
const lng = flatCoords[i * nDim];
|
|
25
|
+
const lat = flatCoords[i * nDim + 1];
|
|
26
|
+
if (lng < newBounds[0]) {
|
|
27
|
+
newBounds[0] = lng;
|
|
28
|
+
}
|
|
29
|
+
if (lat < newBounds[1]) {
|
|
30
|
+
newBounds[1] = lat;
|
|
31
|
+
}
|
|
32
|
+
if (lng > newBounds[2]) {
|
|
33
|
+
newBounds[2] = lng;
|
|
34
|
+
}
|
|
35
|
+
if (lat > newBounds[3]) {
|
|
36
|
+
newBounds[3] = lat;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return newBounds;
|
|
40
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
// loaders.gl, MIT
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
2
3
|
|
|
3
4
|
import type {LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
4
5
|
import type {ArrowLoaderOptions} from './arrow-loader';
|
|
@@ -12,12 +13,25 @@ import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
|
|
|
12
13
|
|
|
13
14
|
import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
|
|
14
15
|
|
|
15
|
-
//
|
|
16
|
+
// Make the ArrowBatch type available
|
|
17
|
+
TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
|
|
18
|
+
|
|
19
|
+
// TYPES
|
|
16
20
|
|
|
17
21
|
export {getArrowType} from './schema/arrow-type-utils';
|
|
18
22
|
|
|
19
|
-
//
|
|
20
|
-
|
|
23
|
+
// SCHEMA
|
|
24
|
+
|
|
25
|
+
export {
|
|
26
|
+
serializeArrowSchema,
|
|
27
|
+
deserializeArrowSchema,
|
|
28
|
+
serializeArrowMetadata,
|
|
29
|
+
deserializeArrowMetadata,
|
|
30
|
+
serializeArrowField,
|
|
31
|
+
deserializeArrowField,
|
|
32
|
+
serializeArrowType,
|
|
33
|
+
deserializeArrowType
|
|
34
|
+
} from './schema/convert-arrow-schema';
|
|
21
35
|
|
|
22
36
|
// Types
|
|
23
37
|
export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
|
|
@@ -44,3 +58,18 @@ export const ArrowLoader: LoaderWithParser<
|
|
|
44
58
|
parseSync,
|
|
45
59
|
parseInBatches: parseArrowInBatches
|
|
46
60
|
};
|
|
61
|
+
|
|
62
|
+
// Arrow Utils
|
|
63
|
+
export type {GeoArrowEncoding} from '@loaders.gl/gis';
|
|
64
|
+
// getGeometryColumnsFromArrowTable,
|
|
65
|
+
// getGeoArrowEncoding
|
|
66
|
+
|
|
67
|
+
export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
68
|
+
export {
|
|
69
|
+
BINARY_GEOMETRY_TEMPLATE,
|
|
70
|
+
getBinaryGeometriesFromArrow
|
|
71
|
+
} from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
72
|
+
|
|
73
|
+
export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
|
|
74
|
+
|
|
75
|
+
export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
|
|
@@ -1,19 +1,9 @@
|
|
|
1
1
|
import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
|
|
2
2
|
import type {ArrowTableBatch} from './arrow-table';
|
|
3
|
-
import
|
|
4
|
-
Table as ApacheArrowTable,
|
|
5
|
-
Schema,
|
|
6
|
-
Field,
|
|
7
|
-
RecordBatch,
|
|
8
|
-
Struct,
|
|
9
|
-
makeVector,
|
|
10
|
-
makeData,
|
|
11
|
-
Vector,
|
|
12
|
-
Float32
|
|
13
|
-
} from 'apache-arrow';
|
|
3
|
+
import * as arrow from 'apache-arrow';
|
|
14
4
|
|
|
15
5
|
export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
|
|
16
|
-
arrowSchema: Schema | null;
|
|
6
|
+
arrowSchema: arrow.Schema | null;
|
|
17
7
|
|
|
18
8
|
constructor(schema, options) {
|
|
19
9
|
super(schema, options);
|
|
@@ -30,10 +20,10 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
|
|
|
30
20
|
const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
|
|
31
21
|
|
|
32
22
|
// Create the record batch
|
|
33
|
-
const recordBatch = new RecordBatch(
|
|
23
|
+
const recordBatch = new arrow.RecordBatch(
|
|
34
24
|
this.arrowSchema,
|
|
35
|
-
makeData({
|
|
36
|
-
type: new Struct(this.arrowSchema.fields),
|
|
25
|
+
arrow.makeData({
|
|
26
|
+
type: new arrow.Struct(this.arrowSchema.fields),
|
|
37
27
|
children: arrowVectors.map(({data}) => data[0])
|
|
38
28
|
})
|
|
39
29
|
);
|
|
@@ -41,7 +31,7 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
|
|
|
41
31
|
return {
|
|
42
32
|
shape: 'arrow-table',
|
|
43
33
|
batchType: 'data',
|
|
44
|
-
data: new
|
|
34
|
+
data: new arrow.Table([recordBatch]),
|
|
45
35
|
length: batch.length
|
|
46
36
|
};
|
|
47
37
|
}
|
|
@@ -51,15 +41,15 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
|
|
|
51
41
|
}
|
|
52
42
|
|
|
53
43
|
// Convert from a simple loaders.gl schema to an Arrow schema
|
|
54
|
-
function getArrowSchema(schema): Schema {
|
|
55
|
-
const arrowFields: Field[] = [];
|
|
44
|
+
function getArrowSchema(schema): arrow.Schema {
|
|
45
|
+
const arrowFields: arrow.Field[] = [];
|
|
56
46
|
for (const key in schema) {
|
|
57
47
|
const field = schema[key];
|
|
58
48
|
if (field.type === Float32Array) {
|
|
59
49
|
// TODO - just store the original field as metadata?
|
|
60
50
|
const metadata = new Map(); // field;
|
|
61
|
-
// arrow: new Field(name, nullable, metadata)
|
|
62
|
-
const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
|
|
51
|
+
// arrow: new arrow.Field(name, nullable, metadata)
|
|
52
|
+
const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
|
|
63
53
|
arrowFields.push(arrowField);
|
|
64
54
|
}
|
|
65
55
|
}
|
|
@@ -67,16 +57,16 @@ function getArrowSchema(schema): Schema {
|
|
|
67
57
|
throw new Error('No arrow convertible fields');
|
|
68
58
|
}
|
|
69
59
|
|
|
70
|
-
return new Schema(arrowFields);
|
|
60
|
+
return new arrow.Schema(arrowFields);
|
|
71
61
|
}
|
|
72
62
|
|
|
73
63
|
// Convert from simple loaders.gl arrays to arrow vectors
|
|
74
|
-
function getArrowVectors(arrowSchema, data): Vector[] {
|
|
64
|
+
function getArrowVectors(arrowSchema, data): arrow.Vector[] {
|
|
75
65
|
const arrowVectors: any[] = [];
|
|
76
66
|
for (const field of arrowSchema.fields) {
|
|
77
67
|
const vector = data[field.name];
|
|
78
68
|
if (vector instanceof Float32Array) {
|
|
79
|
-
const arrowVector = makeVector(vector);
|
|
69
|
+
const arrowVector = arrow.makeVector(vector);
|
|
80
70
|
arrowVectors.push(arrowVector);
|
|
81
71
|
}
|
|
82
72
|
}
|
package/src/lib/arrow-table.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
4
|
import type {Batch, Schema} from '@loaders.gl/schema';
|
|
5
|
-
import type
|
|
5
|
+
import type * as arrow from 'apache-arrow';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* A table organized as an Apache Arrow table
|
|
@@ -11,7 +11,7 @@ import type {Table as ApacheArrowTable} from 'apache-arrow';
|
|
|
11
11
|
export type ArrowTable = {
|
|
12
12
|
shape: 'arrow-table';
|
|
13
13
|
schema?: Schema;
|
|
14
|
-
data:
|
|
14
|
+
data: arrow.Table;
|
|
15
15
|
};
|
|
16
16
|
|
|
17
17
|
/**
|
|
@@ -22,6 +22,6 @@ export type ArrowTableBatch = Batch & {
|
|
|
22
22
|
shape: 'arrow-table';
|
|
23
23
|
schemaType?: 'explicit' | 'deduced';
|
|
24
24
|
schema?: Schema;
|
|
25
|
-
data:
|
|
25
|
+
data: arrow.Table; // ApacheRecordBatch;
|
|
26
26
|
length: number;
|
|
27
27
|
};
|
package/src/lib/encode-arrow.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as arrow from 'apache-arrow';
|
|
2
2
|
import {AnyArrayType, VECTOR_TYPES} from '../types';
|
|
3
3
|
|
|
4
4
|
export type ColumnarTable = {
|
|
@@ -15,28 +15,28 @@ export type ColumnarTable = {
|
|
|
15
15
|
* @returns - encoded ArrayBuffer
|
|
16
16
|
*/
|
|
17
17
|
export function encodeArrowSync(data: ColumnarTable): ArrayBuffer {
|
|
18
|
-
const vectors: Record<string, Vector> = {};
|
|
18
|
+
const vectors: Record<string, arrow.Vector> = {};
|
|
19
19
|
for (const arrayData of data) {
|
|
20
20
|
const arrayVector = createVector(arrayData.array, arrayData.type);
|
|
21
21
|
vectors[arrayData.name] = arrayVector;
|
|
22
22
|
}
|
|
23
|
-
const table = new Table(vectors);
|
|
24
|
-
const arrowBuffer = tableToIPC(table);
|
|
23
|
+
const table = new arrow.Table(vectors);
|
|
24
|
+
const arrowBuffer = arrow.tableToIPC(table);
|
|
25
25
|
return arrowBuffer;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
|
-
* Create Arrow Vector from given data and vector type
|
|
29
|
+
* Create Arrow arrow.Vector from given data and vector type
|
|
30
30
|
* @param array {import('../types').AnyArrayType} - columns data
|
|
31
31
|
* @param type {number} - the writer options
|
|
32
32
|
* @return a vector of one of vector's types defined in the Apache Arrow library
|
|
33
33
|
*/
|
|
34
|
-
function createVector(array, type): Vector {
|
|
34
|
+
function createVector(array, type): arrow.Vector {
|
|
35
35
|
switch (type) {
|
|
36
36
|
case VECTOR_TYPES.DATE:
|
|
37
|
-
return vectorFromArray(array);
|
|
37
|
+
return arrow.vectorFromArray(array);
|
|
38
38
|
case VECTOR_TYPES.FLOAT:
|
|
39
39
|
default:
|
|
40
|
-
return vectorFromArray(array);
|
|
40
|
+
return arrow.vectorFromArray(array);
|
|
41
41
|
}
|
|
42
42
|
}
|