@loaders.gl/arrow 4.0.3 → 4.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts +6 -3
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js +9 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +272 -537
- package/dist/dist.dev.js +2399 -269
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +33 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +146 -17
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts +13 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts.map +1 -0
- package/dist/geoarrow/{convert-geoarrow-to-geojson.js → convert-geoarrow-to-geojson-geometry.js} +36 -25
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +1 -0
- package/dist/geoarrow-loader.d.ts +19 -0
- package/dist/geoarrow-loader.d.ts.map +1 -0
- package/dist/geoarrow-loader.js +23 -0
- package/dist/geoarrow-loader.js.map +1 -0
- package/dist/geoarrow-writer.d.ts +9 -0
- package/dist/geoarrow-writer.d.ts.map +1 -0
- package/dist/geoarrow-writer.js +19 -0
- package/dist/geoarrow-writer.js.map +1 -0
- package/dist/index.cjs +481 -255
- package/dist/index.d.ts +9 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -13
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js.map +1 -1
- package/dist/lib/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encode-arrow.js.map +1 -1
- package/dist/lib/encode-geoarrow.d.ts +15 -0
- package/dist/lib/encode-geoarrow.d.ts.map +1 -0
- package/dist/lib/encode-geoarrow.js +22 -0
- package/dist/lib/encode-geoarrow.js.map +1 -0
- package/dist/{lib → parsers}/parse-arrow-in-batches.d.ts +1 -1
- package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -0
- package/dist/parsers/parse-arrow-in-batches.js.map +1 -0
- package/dist/parsers/parse-arrow-sync.d.ts +6 -0
- package/dist/parsers/parse-arrow-sync.d.ts.map +1 -0
- package/dist/parsers/parse-arrow-sync.js +28 -0
- package/dist/parsers/parse-arrow-sync.js.map +1 -0
- package/dist/parsers/parse-geoarrow-in-batches.d.ts +6 -0
- package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -0
- package/dist/parsers/parse-geoarrow-in-batches.js +5 -0
- package/dist/parsers/parse-geoarrow-in-batches.js.map +1 -0
- package/dist/parsers/parse-geoarrow-sync.d.ts +6 -0
- package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -0
- package/dist/parsers/parse-geoarrow-sync.js +14 -0
- package/dist/parsers/parse-geoarrow-sync.js.map +1 -0
- package/dist/tables/convert-arrow-to-columnar-table.d.ts +8 -0
- package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -0
- package/dist/tables/convert-arrow-to-columnar-table.js +15 -0
- package/dist/tables/convert-arrow-to-columnar-table.js.map +1 -0
- package/dist/tables/convert-arrow-to-geojson-table.d.ts +16 -0
- package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -0
- package/dist/tables/convert-arrow-to-geojson-table.js +39 -0
- package/dist/tables/convert-arrow-to-geojson-table.js.map +1 -0
- package/dist/tables/convert-columnar-to-row-table.d.ts +7 -0
- package/dist/tables/convert-columnar-to-row-table.d.ts.map +1 -0
- package/dist/tables/convert-columnar-to-row-table.js +19 -0
- package/dist/tables/convert-columnar-to-row-table.js.map +1 -0
- package/dist/triangulate-on-worker.d.ts +36 -0
- package/dist/triangulate-on-worker.d.ts.map +1 -0
- package/dist/triangulate-on-worker.js +14 -0
- package/dist/triangulate-on-worker.js.map +1 -0
- package/dist/triangulation-worker.js +887 -0
- package/dist/workers/arrow-worker.js +1 -1
- package/dist/workers/arrow-worker.js.map +1 -1
- package/dist/workers/triangulation-worker-node.d.ts +2 -0
- package/dist/workers/triangulation-worker-node.d.ts.map +1 -0
- package/dist/workers/triangulation-worker-node.js +2 -0
- package/dist/workers/triangulation-worker-node.js.map +1 -0
- package/dist/workers/triangulation-worker.d.ts +2 -0
- package/dist/workers/triangulation-worker.d.ts.map +1 -0
- package/dist/workers/triangulation-worker.js +26 -0
- package/dist/workers/triangulation-worker.js.map +1 -0
- package/package.json +20 -12
- package/src/arrow-loader.ts +25 -3
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +247 -23
- package/src/geoarrow/{convert-geoarrow-to-geojson.ts → convert-geoarrow-to-geojson-geometry.ts} +58 -44
- package/src/geoarrow-loader.ts +51 -0
- package/src/geoarrow-writer.ts +41 -0
- package/src/index.ts +31 -37
- package/src/lib/arrow-table-batch.ts +3 -0
- package/src/lib/encode-arrow.ts +3 -0
- package/src/lib/encode-geoarrow.ts +45 -0
- package/src/{lib → parsers}/parse-arrow-in-batches.ts +4 -2
- package/src/parsers/parse-arrow-sync.ts +43 -0
- package/src/parsers/parse-geoarrow-in-batches.ts +15 -0
- package/src/parsers/parse-geoarrow-sync.ts +22 -0
- package/src/tables/convert-arrow-to-columnar-table.ts +30 -0
- package/src/tables/convert-arrow-to-geojson-table.ts +65 -0
- package/src/tables/convert-columnar-to-row-table.ts +30 -0
- package/src/triangulate-on-worker.ts +47 -0
- package/src/workers/arrow-worker.ts +1 -1
- package/src/workers/triangulation-worker-node.ts +4 -0
- package/src/workers/triangulation-worker.ts +39 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +0 -19
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +0 -1
- package/dist/lib/parse-arrow-in-batches.d.ts.map +0 -1
- package/dist/lib/parse-arrow-in-batches.js.map +0 -1
- package/dist/lib/parse-arrow-sync.d.ts +0 -5
- package/dist/lib/parse-arrow-sync.d.ts.map +0 -1
- package/dist/lib/parse-arrow-sync.js +0 -21
- package/dist/lib/parse-arrow-sync.js.map +0 -1
- package/dist/tables/convert-arrow-to-table.d.ts +0 -21
- package/dist/tables/convert-arrow-to-table.d.ts.map +0 -1
- package/dist/tables/convert-arrow-to-table.js +0 -37
- package/dist/tables/convert-arrow-to-table.js.map +0 -1
- package/src/lib/parse-arrow-sync.ts +0 -35
- package/src/tables/convert-arrow-to-table.ts +0 -68
- /package/dist/{lib → parsers}/parse-arrow-in-batches.js +0 -0
package/src/geoarrow/{convert-geoarrow-to-geojson.ts → convert-geoarrow-to-geojson-geometry.ts}
RENAMED
|
@@ -1,77 +1,95 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
|
-
import * as arrow from 'apache-arrow';
|
|
4
|
+
// import * as arrow from 'apache-arrow';
|
|
5
5
|
import {
|
|
6
|
-
Feature,
|
|
7
6
|
MultiPolygon,
|
|
8
7
|
Position,
|
|
9
8
|
Polygon,
|
|
10
9
|
MultiPoint,
|
|
11
10
|
Point,
|
|
12
11
|
MultiLineString,
|
|
13
|
-
LineString
|
|
12
|
+
LineString,
|
|
13
|
+
Geometry,
|
|
14
|
+
BinaryGeometry
|
|
14
15
|
} from '@loaders.gl/schema';
|
|
15
|
-
import type
|
|
16
|
-
|
|
17
|
-
type RawArrowFeature = {
|
|
18
|
-
encoding?: GeoArrowEncoding;
|
|
19
|
-
data: any;
|
|
20
|
-
};
|
|
16
|
+
import {binaryToGeometry, type GeoArrowEncoding} from '@loaders.gl/gis';
|
|
17
|
+
import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
|
|
21
18
|
|
|
22
19
|
/**
|
|
23
20
|
* parse geometry from arrow data that is returned from processArrowData()
|
|
24
|
-
* NOTE: this function could be
|
|
25
|
-
* it is currently
|
|
26
|
-
* So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
|
|
21
|
+
* NOTE: this function could be deduplicated with the binaryToFeature() in deck.gl,
|
|
22
|
+
* it is currently used for deck.gl picking because currently deck.gl returns only the index of the feature
|
|
27
23
|
*
|
|
28
|
-
* @param
|
|
29
|
-
* @
|
|
24
|
+
* @param data data extraced from arrow vector representing a geometry
|
|
25
|
+
* @param encoding the geoarrow encoding of the geometry column
|
|
30
26
|
* @returns Feature or null
|
|
31
27
|
*/
|
|
32
|
-
export function parseGeometryFromArrow(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
export function parseGeometryFromArrow(
|
|
29
|
+
arrowCellValue: any,
|
|
30
|
+
encoding?: GeoArrowEncoding
|
|
31
|
+
): Geometry | null {
|
|
32
|
+
// sanity
|
|
33
|
+
encoding = encoding?.toLowerCase() as GeoArrowEncoding;
|
|
34
|
+
if (!encoding || !arrowCellValue) {
|
|
36
35
|
return null;
|
|
37
36
|
}
|
|
38
37
|
|
|
39
|
-
let geometry;
|
|
38
|
+
let geometry: Geometry;
|
|
40
39
|
|
|
41
40
|
switch (encoding) {
|
|
42
41
|
case 'geoarrow.multipolygon':
|
|
43
|
-
geometry = arrowMultiPolygonToFeature(
|
|
42
|
+
geometry = arrowMultiPolygonToFeature(arrowCellValue);
|
|
44
43
|
break;
|
|
45
44
|
case 'geoarrow.polygon':
|
|
46
|
-
geometry = arrowPolygonToFeature(
|
|
45
|
+
geometry = arrowPolygonToFeature(arrowCellValue);
|
|
47
46
|
break;
|
|
48
47
|
case 'geoarrow.multipoint':
|
|
49
|
-
geometry = arrowMultiPointToFeature(
|
|
48
|
+
geometry = arrowMultiPointToFeature(arrowCellValue);
|
|
50
49
|
break;
|
|
51
50
|
case 'geoarrow.point':
|
|
52
|
-
geometry = arrowPointToFeature(
|
|
51
|
+
geometry = arrowPointToFeature(arrowCellValue);
|
|
53
52
|
break;
|
|
54
53
|
case 'geoarrow.multilinestring':
|
|
55
|
-
geometry = arrowMultiLineStringToFeature(
|
|
54
|
+
geometry = arrowMultiLineStringToFeature(arrowCellValue);
|
|
56
55
|
break;
|
|
57
56
|
case 'geoarrow.linestring':
|
|
58
|
-
geometry = arrowLineStringToFeature(
|
|
57
|
+
geometry = arrowLineStringToFeature(arrowCellValue);
|
|
58
|
+
break;
|
|
59
|
+
case 'geoarrow.wkb':
|
|
60
|
+
geometry = arrowWKBToFeature(arrowCellValue);
|
|
61
|
+
break;
|
|
62
|
+
case 'geoarrow.wkt':
|
|
63
|
+
geometry = arrowWKTToFeature(arrowCellValue);
|
|
59
64
|
break;
|
|
60
65
|
default: {
|
|
61
66
|
throw Error(`GeoArrow encoding not supported ${encoding}`);
|
|
62
67
|
}
|
|
63
68
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
|
|
70
|
+
return geometry;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function arrowWKBToFeature(arrowCellValue: any) {
|
|
74
|
+
// The actual WKB array buffer starts from byteOffset and ends at byteOffset + byteLength
|
|
75
|
+
const arrayBuffer: ArrayBuffer = arrowCellValue.buffer.slice(
|
|
76
|
+
arrowCellValue.byteOffset,
|
|
77
|
+
arrowCellValue.byteOffset + arrowCellValue.byteLength
|
|
78
|
+
);
|
|
79
|
+
const binaryGeometry = WKBLoader.parseSync?.(arrayBuffer)! as BinaryGeometry;
|
|
80
|
+
const geometry = binaryToGeometry(binaryGeometry);
|
|
81
|
+
return geometry;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function arrowWKTToFeature(arrowCellValue: any) {
|
|
85
|
+
const string: string = arrowCellValue;
|
|
86
|
+
return WKTLoader.parseTextSync?.(string)!;
|
|
69
87
|
}
|
|
70
88
|
|
|
71
89
|
/**
|
|
72
90
|
* convert Arrow MultiPolygon to geojson Feature
|
|
73
91
|
*/
|
|
74
|
-
function arrowMultiPolygonToFeature(arrowMultiPolygon:
|
|
92
|
+
function arrowMultiPolygonToFeature(arrowMultiPolygon: any): MultiPolygon {
|
|
75
93
|
const multiPolygon: Position[][][] = [];
|
|
76
94
|
for (let m = 0; m < arrowMultiPolygon.length; m++) {
|
|
77
95
|
const arrowPolygon = arrowMultiPolygon.get(m);
|
|
@@ -98,7 +116,7 @@ function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolyg
|
|
|
98
116
|
/**
|
|
99
117
|
* convert Arrow Polygon to geojson Feature
|
|
100
118
|
*/
|
|
101
|
-
function arrowPolygonToFeature(arrowPolygon:
|
|
119
|
+
function arrowPolygonToFeature(arrowPolygon: any): Polygon {
|
|
102
120
|
const polygon: Position[][] = [];
|
|
103
121
|
for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
|
|
104
122
|
const arrowRing = arrowPolygon.get(i);
|
|
@@ -120,7 +138,7 @@ function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
|
|
|
120
138
|
/**
|
|
121
139
|
* convert Arrow MultiPoint to geojson MultiPoint
|
|
122
140
|
*/
|
|
123
|
-
function arrowMultiPointToFeature(arrowMultiPoint:
|
|
141
|
+
function arrowMultiPointToFeature(arrowMultiPoint: any): MultiPoint {
|
|
124
142
|
const multiPoint: Position[] = [];
|
|
125
143
|
for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
|
|
126
144
|
const arrowPoint = arrowMultiPoint.get(i);
|
|
@@ -129,29 +147,27 @@ function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
|
|
|
129
147
|
multiPoint.push(coord);
|
|
130
148
|
}
|
|
131
149
|
}
|
|
132
|
-
|
|
150
|
+
return {
|
|
133
151
|
type: 'MultiPoint',
|
|
134
152
|
coordinates: multiPoint
|
|
135
153
|
};
|
|
136
|
-
return geometry;
|
|
137
154
|
}
|
|
138
155
|
|
|
139
156
|
/**
|
|
140
157
|
* convert Arrow Point to geojson Point
|
|
141
158
|
*/
|
|
142
|
-
function arrowPointToFeature(arrowPoint:
|
|
159
|
+
function arrowPointToFeature(arrowPoint: any): Point {
|
|
143
160
|
const point: Position = Array.from(arrowPoint);
|
|
144
|
-
|
|
161
|
+
return {
|
|
145
162
|
type: 'Point',
|
|
146
163
|
coordinates: point
|
|
147
164
|
};
|
|
148
|
-
return geometry;
|
|
149
165
|
}
|
|
150
166
|
|
|
151
167
|
/**
|
|
152
168
|
* convert Arrow MultiLineString to geojson MultiLineString
|
|
153
169
|
*/
|
|
154
|
-
function arrowMultiLineStringToFeature(arrowMultiLineString:
|
|
170
|
+
function arrowMultiLineStringToFeature(arrowMultiLineString: any): MultiLineString {
|
|
155
171
|
const multiLineString: Position[][] = [];
|
|
156
172
|
for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
|
|
157
173
|
const arrowLineString = arrowMultiLineString.get(i);
|
|
@@ -165,17 +181,16 @@ function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): Mult
|
|
|
165
181
|
}
|
|
166
182
|
multiLineString.push(lineString);
|
|
167
183
|
}
|
|
168
|
-
|
|
184
|
+
return {
|
|
169
185
|
type: 'MultiLineString',
|
|
170
186
|
coordinates: multiLineString
|
|
171
187
|
};
|
|
172
|
-
return geometry;
|
|
173
188
|
}
|
|
174
189
|
|
|
175
190
|
/**
|
|
176
191
|
* convert Arrow LineString to geojson LineString
|
|
177
192
|
*/
|
|
178
|
-
function arrowLineStringToFeature(arrowLineString:
|
|
193
|
+
function arrowLineStringToFeature(arrowLineString: any): LineString {
|
|
179
194
|
const lineString: Position[] = [];
|
|
180
195
|
for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
|
|
181
196
|
const arrowCoord = arrowLineString.get(i);
|
|
@@ -184,9 +199,8 @@ function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
|
|
|
184
199
|
lineString.push(coords);
|
|
185
200
|
}
|
|
186
201
|
}
|
|
187
|
-
|
|
202
|
+
return {
|
|
188
203
|
type: 'LineString',
|
|
189
204
|
coordinates: lineString
|
|
190
205
|
};
|
|
191
|
-
return geometry;
|
|
192
206
|
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {ArrowWorkerLoader} from './arrow-loader';
|
|
6
|
+
import type {GeoJSONTable, GeoJSONTableBatch, BinaryGeometry} from '@loaders.gl/schema';
|
|
7
|
+
import type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
|
|
8
|
+
import {parseGeoArrowSync} from './parsers/parse-geoarrow-sync';
|
|
9
|
+
import {parseGeoArrowInBatches} from './parsers/parse-geoarrow-in-batches';
|
|
10
|
+
|
|
11
|
+
export type GeoArrowLoaderOptions = LoaderOptions & {
|
|
12
|
+
arrow?: {
|
|
13
|
+
shape: 'arrow-table' | 'binary-geometry';
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/** ArrowJS table loader */
|
|
18
|
+
export const GeoArrowWorkerLoader: Loader<
|
|
19
|
+
ArrowTable | BinaryGeometry,
|
|
20
|
+
never,
|
|
21
|
+
GeoArrowLoaderOptions
|
|
22
|
+
> = {
|
|
23
|
+
...ArrowWorkerLoader,
|
|
24
|
+
options: {
|
|
25
|
+
arrow: {
|
|
26
|
+
shape: 'arrow-table'
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* GeoArrowLoader loads an Apache Arrow table, parses GeoArrow type extension data
|
|
33
|
+
* to convert it to a GeoJSON table or a BinaryGeometry
|
|
34
|
+
*/
|
|
35
|
+
export const GeoArrowLoader: LoaderWithParser<
|
|
36
|
+
ArrowTable | GeoJSONTable, // | BinaryGeometry,
|
|
37
|
+
ArrowTableBatch | GeoJSONTableBatch, // | BinaryGeometry,
|
|
38
|
+
GeoArrowLoaderOptions
|
|
39
|
+
> = {
|
|
40
|
+
...ArrowWorkerLoader,
|
|
41
|
+
options: {
|
|
42
|
+
arrow: {
|
|
43
|
+
shape: 'arrow-table'
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
parse: async (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>
|
|
47
|
+
parseGeoArrowSync(arraybuffer, options?.arrow),
|
|
48
|
+
parseSync: (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>
|
|
49
|
+
parseGeoArrowSync(arraybuffer, options?.arrow),
|
|
50
|
+
parseInBatches: parseGeoArrowInBatches
|
|
51
|
+
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// import type {} from '@loaders.gl/loader-utils';
|
|
2
|
+
|
|
3
|
+
import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
|
|
4
|
+
import {GeoJSONTable, BinaryGeometry} from '@loaders.gl/schema';
|
|
5
|
+
import {encodeGeoArrowSync} from './lib/encode-geoarrow';
|
|
6
|
+
|
|
7
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
8
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
9
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
10
|
+
|
|
11
|
+
type ArrowWriterOptions = WriterOptions & {
|
|
12
|
+
arrow?: {};
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/** Apache Arrow writer */
|
|
16
|
+
export const GeoArrowWriter: WriterWithEncoder<
|
|
17
|
+
GeoJSONTable | BinaryGeometry,
|
|
18
|
+
never,
|
|
19
|
+
ArrowWriterOptions
|
|
20
|
+
> = {
|
|
21
|
+
name: 'Apache Arrow',
|
|
22
|
+
id: 'arrow',
|
|
23
|
+
module: 'arrow',
|
|
24
|
+
version: VERSION,
|
|
25
|
+
extensions: ['arrow', 'feather'],
|
|
26
|
+
mimeTypes: [
|
|
27
|
+
'application/vnd.apache.arrow.file',
|
|
28
|
+
'application/vnd.apache.arrow.stream',
|
|
29
|
+
'application/octet-stream'
|
|
30
|
+
],
|
|
31
|
+
binary: true,
|
|
32
|
+
options: {},
|
|
33
|
+
encode: async function encodeArrow(data, options?): Promise<ArrayBuffer> {
|
|
34
|
+
// @ts-expect-error
|
|
35
|
+
return encodeGeoArrowSync(data);
|
|
36
|
+
},
|
|
37
|
+
encodeSync(data, options?) {
|
|
38
|
+
// @ts-expect-error
|
|
39
|
+
return encodeGeoArrowSync(data);
|
|
40
|
+
}
|
|
41
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -1,16 +1,8 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
|
-
import type {LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
5
4
|
import type {ArrowLoaderOptions} from './arrow-loader';
|
|
6
|
-
import type {ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
|
|
7
|
-
import type {ArrowTable} from './lib/arrow-table';
|
|
8
|
-
|
|
9
5
|
import {TableBatchBuilder} from '@loaders.gl/schema';
|
|
10
|
-
import {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';
|
|
11
|
-
import parseSync from './lib/parse-arrow-sync';
|
|
12
|
-
import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
|
|
13
|
-
|
|
14
6
|
import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
|
|
15
7
|
|
|
16
8
|
// Make the ArrowBatch type available
|
|
@@ -22,6 +14,21 @@ export {getArrowType} from './schema/arrow-type-utils';
|
|
|
22
14
|
|
|
23
15
|
// SCHEMA
|
|
24
16
|
|
|
17
|
+
// Types
|
|
18
|
+
export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
|
|
19
|
+
export {VECTOR_TYPES} from './types';
|
|
20
|
+
|
|
21
|
+
// Arrow loader / Writer
|
|
22
|
+
|
|
23
|
+
export type {ArrowLoaderOptions};
|
|
24
|
+
export {ArrowLoader, ArrowWorkerLoader} from './arrow-loader';
|
|
25
|
+
|
|
26
|
+
export {ArrowWriter} from './arrow-writer';
|
|
27
|
+
|
|
28
|
+
// Geoarrow loader
|
|
29
|
+
export {GeoArrowLoader, GeoArrowWorkerLoader} from './geoarrow-loader';
|
|
30
|
+
|
|
31
|
+
// Schema utils
|
|
25
32
|
export {
|
|
26
33
|
serializeArrowSchema,
|
|
27
34
|
deserializeArrowSchema,
|
|
@@ -33,43 +40,30 @@ export {
|
|
|
33
40
|
deserializeArrowType
|
|
34
41
|
} from './schema/convert-arrow-schema';
|
|
35
42
|
|
|
36
|
-
//
|
|
37
|
-
export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
|
|
38
|
-
export {VECTOR_TYPES} from './types';
|
|
39
|
-
|
|
40
|
-
// Arrow writer
|
|
41
|
-
|
|
42
|
-
export {ArrowWriter} from './arrow-writer';
|
|
43
|
-
|
|
44
|
-
// Arrow loader
|
|
45
|
-
|
|
46
|
-
export type {ArrowLoaderOptions};
|
|
47
|
-
export {ArrowWorkerLoader};
|
|
48
|
-
|
|
49
|
-
/** ArrowJS table loader */
|
|
50
|
-
export const ArrowLoader: LoaderWithParser<
|
|
51
|
-
ArrowTable | ColumnarTable | ObjectRowTable,
|
|
52
|
-
ArrowTableBatch,
|
|
53
|
-
ArrowLoaderOptions
|
|
54
|
-
> = {
|
|
55
|
-
...ArrowWorkerLoader,
|
|
56
|
-
parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>
|
|
57
|
-
parseSync(arraybuffer, options),
|
|
58
|
-
parseSync,
|
|
59
|
-
parseInBatches: parseArrowInBatches
|
|
60
|
-
};
|
|
43
|
+
// EXPERIMENTAL
|
|
61
44
|
|
|
62
45
|
// Arrow Utils
|
|
63
46
|
export type {GeoArrowEncoding} from '@loaders.gl/gis';
|
|
64
47
|
// getGeometryColumnsFromArrowTable,
|
|
65
48
|
// getGeoArrowEncoding
|
|
66
49
|
|
|
67
|
-
export type {
|
|
50
|
+
export type {
|
|
51
|
+
BinaryDataFromGeoArrow,
|
|
52
|
+
BinaryGeometriesFromArrowOptions
|
|
53
|
+
} from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
68
54
|
export {
|
|
69
55
|
BINARY_GEOMETRY_TEMPLATE,
|
|
70
|
-
getBinaryGeometriesFromArrow
|
|
56
|
+
getBinaryGeometriesFromArrow,
|
|
57
|
+
getTriangleIndices,
|
|
58
|
+
getMeanCentersFromBinaryGeometries
|
|
71
59
|
} from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
72
60
|
|
|
73
|
-
export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
|
|
74
|
-
|
|
75
61
|
export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
|
|
62
|
+
|
|
63
|
+
export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson-geometry';
|
|
64
|
+
|
|
65
|
+
export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
|
|
66
|
+
|
|
67
|
+
// EXPERIMENTAL WORKER
|
|
68
|
+
|
|
69
|
+
export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';
|
package/src/lib/encode-arrow.ts
CHANGED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import * as arrow from 'apache-arrow';
|
|
5
|
+
import {AnyArrayType, VECTOR_TYPES} from '../types';
|
|
6
|
+
|
|
7
|
+
export type ColumnarTable = {
|
|
8
|
+
name: string;
|
|
9
|
+
array: AnyArrayType;
|
|
10
|
+
type: number;
|
|
11
|
+
}[];
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Encodes set of arrays into the Apache Arrow columnar format
|
|
15
|
+
* https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
|
|
16
|
+
* @param data - columns data
|
|
17
|
+
* @param options - the writer options
|
|
18
|
+
* @returns - encoded ArrayBuffer
|
|
19
|
+
*/
|
|
20
|
+
export function encodeGeoArrowSync(data: ColumnarTable): ArrayBuffer {
|
|
21
|
+
const vectors: Record<string, arrow.Vector> = {};
|
|
22
|
+
for (const arrayData of data) {
|
|
23
|
+
const arrayVector = createVector(arrayData.array, arrayData.type);
|
|
24
|
+
vectors[arrayData.name] = arrayVector;
|
|
25
|
+
}
|
|
26
|
+
const table = new arrow.Table(vectors);
|
|
27
|
+
const arrowBuffer = arrow.tableToIPC(table);
|
|
28
|
+
return arrowBuffer;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Create Arrow arrow.Vector from given data and vector type
|
|
33
|
+
* @param array {import('../types').AnyArrayType} - columns data
|
|
34
|
+
* @param type {number} - the writer options
|
|
35
|
+
* @return a vector of one of vector's types defined in the Apache Arrow library
|
|
36
|
+
*/
|
|
37
|
+
function createVector(array, type): arrow.Vector {
|
|
38
|
+
switch (type) {
|
|
39
|
+
case VECTOR_TYPES.DATE:
|
|
40
|
+
return arrow.vectorFromArray(array);
|
|
41
|
+
case VECTOR_TYPES.FLOAT:
|
|
42
|
+
default:
|
|
43
|
+
return arrow.vectorFromArray(array);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {ArrowTableBatch} from '../lib/arrow-table';
|
|
3
5
|
import * as arrow from 'apache-arrow';
|
|
4
6
|
// import {isIterable} from '@loaders.gl/core';
|
|
5
7
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
|
|
5
|
+
import type {ArrowTable} from '../lib/arrow-table';
|
|
6
|
+
import {convertTable} from '@loaders.gl/schema';
|
|
7
|
+
import * as arrow from 'apache-arrow';
|
|
8
|
+
import {convertArrowToColumnarTable} from '../tables/convert-arrow-to-columnar-table';
|
|
9
|
+
import {serializeArrowSchema} from '../schema/convert-arrow-schema';
|
|
10
|
+
|
|
11
|
+
// Parses arrow to a columnar table
|
|
12
|
+
export function parseArrowSync(
|
|
13
|
+
arrayBuffer,
|
|
14
|
+
options?: {shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table'}
|
|
15
|
+
): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable {
|
|
16
|
+
const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
|
|
17
|
+
const arrowTable: ArrowTable = {
|
|
18
|
+
shape: 'arrow-table',
|
|
19
|
+
schema: serializeArrowSchema(apacheArrowTable.schema),
|
|
20
|
+
data: apacheArrowTable
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const shape = options?.shape || 'arrow-table';
|
|
24
|
+
switch (shape) {
|
|
25
|
+
case 'arrow-table':
|
|
26
|
+
return arrowTable;
|
|
27
|
+
|
|
28
|
+
case 'columnar-table':
|
|
29
|
+
return convertArrowToColumnarTable(arrowTable);
|
|
30
|
+
|
|
31
|
+
case 'object-row-table':
|
|
32
|
+
let columnarTable = convertArrowToColumnarTable(arrowTable);
|
|
33
|
+
return convertTable(columnarTable, 'object-row-table');
|
|
34
|
+
|
|
35
|
+
case 'array-row-table':
|
|
36
|
+
columnarTable = convertArrowToColumnarTable(arrowTable);
|
|
37
|
+
return convertTable(columnarTable, 'array-row-table');
|
|
38
|
+
|
|
39
|
+
default:
|
|
40
|
+
// TODO
|
|
41
|
+
throw new Error(shape);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {GeoJSONTableBatch} from '@loaders.gl/schema';
|
|
5
|
+
import type {ArrowTableBatch} from '../lib/arrow-table';
|
|
6
|
+
import {parseArrowInBatches} from './parse-arrow-in-batches';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
*/
|
|
10
|
+
export function parseGeoArrowInBatches(
|
|
11
|
+
asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
|
|
12
|
+
): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch> {
|
|
13
|
+
// | BinaryGeometry
|
|
14
|
+
return parseArrowInBatches(asyncIterator);
|
|
15
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {GeoJSONTable} from '@loaders.gl/schema';
|
|
5
|
+
import type {ArrowTable} from '../lib/arrow-table';
|
|
6
|
+
import {parseArrowSync} from './parse-arrow-sync';
|
|
7
|
+
import {convertArrowToGeoJSONTable} from '../tables/convert-arrow-to-geojson-table';
|
|
8
|
+
|
|
9
|
+
// Parses arrow to a columnar table
|
|
10
|
+
export function parseGeoArrowSync(
|
|
11
|
+
arrayBuffer,
|
|
12
|
+
options?: {shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry'}
|
|
13
|
+
): ArrowTable | GeoJSONTable {
|
|
14
|
+
// | BinaryGeometry
|
|
15
|
+
const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;
|
|
16
|
+
switch (options?.shape) {
|
|
17
|
+
case 'geojson-table':
|
|
18
|
+
return convertArrowToGeoJSONTable(table);
|
|
19
|
+
default:
|
|
20
|
+
return table;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {ColumnarTable} from '@loaders.gl/schema';
|
|
5
|
+
import type {ArrowTable} from '../lib/arrow-table';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Convert an Apache Arrow table to a ColumnarTable
|
|
9
|
+
* @note Currently does not convert schema
|
|
10
|
+
*/
|
|
11
|
+
export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
|
|
12
|
+
// TODO - avoid calling `getColumn` on columns we are not interested in?
|
|
13
|
+
// Add options object?
|
|
14
|
+
|
|
15
|
+
const arrowTable = table.data;
|
|
16
|
+
const columnarTable = {};
|
|
17
|
+
|
|
18
|
+
for (const field of arrowTable.schema.fields) {
|
|
19
|
+
// This (is intended to) coalesce all record batches into a single typed array
|
|
20
|
+
const arrowColumn = arrowTable.getChild(field.name);
|
|
21
|
+
const values = arrowColumn?.toArray();
|
|
22
|
+
columnarTable[field.name] = values;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return {
|
|
26
|
+
shape: 'columnar-table',
|
|
27
|
+
schema: table.schema,
|
|
28
|
+
data: columnarTable
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
4
|
+
import type {Feature, GeoJSONTable} from '@loaders.gl/schema';
|
|
5
|
+
import * as arrow from 'apache-arrow';
|
|
6
|
+
import type {ArrowTable} from '../lib/arrow-table';
|
|
7
|
+
import {serializeArrowSchema, parseGeometryFromArrow} from '@loaders.gl/arrow';
|
|
8
|
+
import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Wrap an apache arrow table in a loaders.gl table wrapper.
|
|
12
|
+
* From this additional conversions are available.
|
|
13
|
+
* @param arrowTable
|
|
14
|
+
* @returns
|
|
15
|
+
*/
|
|
16
|
+
export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
|
|
17
|
+
return {
|
|
18
|
+
shape: 'arrow-table',
|
|
19
|
+
schema: serializeArrowSchema(arrowTable.schema),
|
|
20
|
+
data: arrowTable
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Convert an Apache Arrow table to a GeoJSONTable
|
|
26
|
+
* @note Currently does not convert schema
|
|
27
|
+
*/
|
|
28
|
+
export function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable {
|
|
29
|
+
const arrowTable = table.data;
|
|
30
|
+
const schema = serializeArrowSchema(arrowTable.schema);
|
|
31
|
+
const geometryColumns = getGeometryColumnsFromSchema(schema);
|
|
32
|
+
|
|
33
|
+
// get encoding from geometryColumns['geometry']
|
|
34
|
+
const encoding = geometryColumns.geometry.encoding;
|
|
35
|
+
|
|
36
|
+
const features: Feature[] = [];
|
|
37
|
+
|
|
38
|
+
// Remove geometry columns
|
|
39
|
+
const propertyColumnNames = arrowTable.schema.fields
|
|
40
|
+
.map((field) => field.name)
|
|
41
|
+
// TODO - this deletes all geometry columns
|
|
42
|
+
.filter((name) => !(name in geometryColumns));
|
|
43
|
+
const propertiesTable = arrowTable.select(propertyColumnNames);
|
|
44
|
+
|
|
45
|
+
const arrowGeometryColumn = arrowTable.getChild('geometry');
|
|
46
|
+
|
|
47
|
+
for (let row = 0; row < arrowTable.numRows; row++) {
|
|
48
|
+
// get the geometry value from arrow geometry column
|
|
49
|
+
// Note that type can vary
|
|
50
|
+
const arrowGeometry = arrowGeometryColumn?.get(row);
|
|
51
|
+
// parse arrow geometry to geojson feature
|
|
52
|
+
const feature = parseGeometryFromArrow(arrowGeometry, encoding);
|
|
53
|
+
if (feature) {
|
|
54
|
+
const properties = propertiesTable.get(row)?.toJSON() || {};
|
|
55
|
+
features.push({type: 'Feature', geometry: feature, properties});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
shape: 'geojson-table',
|
|
61
|
+
type: 'FeatureCollection',
|
|
62
|
+
schema: table.schema,
|
|
63
|
+
features
|
|
64
|
+
};
|
|
65
|
+
}
|