@loaders.gl/arrow 4.0.0 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/arrow-loader.d.ts.map +1 -1
  2. package/dist/arrow-loader.js.map +1 -1
  3. package/dist/arrow-worker.js +34 -26
  4. package/dist/arrow-writer.d.ts.map +1 -1
  5. package/dist/arrow-writer.js +6 -3
  6. package/dist/arrow-writer.js.map +1 -1
  7. package/dist/dist.dev.js +658 -6
  8. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +40 -0
  9. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -0
  10. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +189 -0
  11. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -0
  12. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +19 -0
  13. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +1 -0
  14. package/dist/geoarrow/convert-geoarrow-to-geojson.js +138 -0
  15. package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +1 -0
  16. package/dist/geoarrow/get-arrow-bounds.d.ts +11 -0
  17. package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -0
  18. package/dist/geoarrow/get-arrow-bounds.js +24 -0
  19. package/dist/geoarrow/get-arrow-bounds.js.map +1 -0
  20. package/dist/index.cjs +627 -32
  21. package/dist/index.d.ts +6 -0
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +5 -1
  24. package/dist/index.js.map +1 -1
  25. package/dist/lib/arrow-table-batch.d.ts +2 -2
  26. package/dist/lib/arrow-table-batch.d.ts.map +1 -1
  27. package/dist/lib/arrow-table-batch.js +7 -7
  28. package/dist/lib/arrow-table-batch.js.map +1 -1
  29. package/dist/lib/arrow-table.d.ts +3 -3
  30. package/dist/lib/arrow-table.d.ts.map +1 -1
  31. package/dist/lib/arrow-table.js.map +1 -1
  32. package/dist/lib/encode-arrow.js +5 -5
  33. package/dist/lib/encode-arrow.js.map +1 -1
  34. package/dist/lib/parse-arrow-in-batches.js +3 -3
  35. package/dist/lib/parse-arrow-in-batches.js.map +1 -1
  36. package/dist/lib/parse-arrow-sync.d.ts.map +1 -1
  37. package/dist/lib/parse-arrow-sync.js +3 -3
  38. package/dist/lib/parse-arrow-sync.js.map +1 -1
  39. package/dist/schema/arrow-type-utils.d.ts +3 -2
  40. package/dist/schema/arrow-type-utils.d.ts.map +1 -1
  41. package/dist/schema/arrow-type-utils.js +9 -9
  42. package/dist/schema/arrow-type-utils.js.map +1 -1
  43. package/dist/schema/convert-arrow-schema.d.ts +19 -0
  44. package/dist/schema/convert-arrow-schema.d.ts.map +1 -0
  45. package/dist/schema/convert-arrow-schema.js +232 -0
  46. package/dist/schema/convert-arrow-schema.js.map +1 -0
  47. package/dist/{lib/convert-table.d.ts → tables/convert-arrow-to-table.d.ts} +4 -4
  48. package/dist/tables/convert-arrow-to-table.d.ts.map +1 -0
  49. package/dist/{lib/convert-table.js → tables/convert-arrow-to-table.js} +1 -1
  50. package/dist/tables/convert-arrow-to-table.js.map +1 -0
  51. package/dist/{schema → tables}/convert-table-to-arrow.d.ts.map +1 -1
  52. package/dist/{schema → tables}/convert-table-to-arrow.js.map +1 -1
  53. package/dist/types.d.ts.map +1 -1
  54. package/dist/types.js.map +1 -1
  55. package/dist/workers/arrow-worker.js.map +1 -1
  56. package/package.json +5 -4
  57. package/src/arrow-loader.ts +1 -0
  58. package/src/arrow-writer.ts +7 -3
  59. package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +260 -0
  60. package/src/geoarrow/convert-geoarrow-to-geojson.ts +192 -0
  61. package/src/geoarrow/get-arrow-bounds.ts +40 -0
  62. package/src/index.ts +33 -4
  63. package/src/lib/arrow-table-batch.ts +13 -23
  64. package/src/lib/arrow-table.ts +3 -3
  65. package/src/lib/encode-arrow.ts +8 -8
  66. package/src/lib/parse-arrow-in-batches.ts +4 -4
  67. package/src/lib/parse-arrow-sync.ts +6 -3
  68. package/src/schema/arrow-type-utils.ts +11 -29
  69. package/src/schema/convert-arrow-schema.ts +265 -0
  70. package/src/{lib/convert-table.ts → tables/convert-arrow-to-table.ts} +4 -3
  71. package/src/types.ts +3 -0
  72. package/src/workers/arrow-worker.ts +3 -0
  73. package/dist/lib/convert-table.d.ts.map +0 -1
  74. package/dist/lib/convert-table.js.map +0 -1
  75. package/dist/schema/convert-schema-arrow.d.ts +0 -13
  76. package/dist/schema/convert-schema-arrow.d.ts.map +0 -1
  77. package/dist/schema/convert-schema-arrow.js +0 -158
  78. package/dist/schema/convert-schema-arrow.js.map +0 -1
  79. package/src/schema/convert-schema-arrow.ts +0 -233
  80. /package/dist/{schema → tables}/convert-table-to-arrow.d.ts +0 -0
  81. /package/dist/{schema → tables}/convert-table-to-arrow.js +0 -0
  82. /package/src/{schema → tables}/convert-table-to-arrow.ts +0 -0
@@ -0,0 +1,260 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import * as arrow from 'apache-arrow';
5
+ import {BinaryFeatureCollection as BinaryFeatures} from '@loaders.gl/schema';
6
+ import {GeoArrowEncoding} from '@loaders.gl/gis';
7
+ import {updateBoundsFromGeoArrowSamples} from './get-arrow-bounds';
8
+
9
+ /**
10
+ * Binary data from geoarrow column and can be used by e.g. deck.gl GeojsonLayer
11
+ */
12
+ export type BinaryDataFromGeoArrow = {
13
+ binaryGeometries: BinaryFeatures[];
14
+ bounds: [number, number, number, number];
15
+ featureTypes: {polygon: boolean; point: boolean; line: boolean};
16
+ };
17
+
18
+ type BinaryGeometryContent = {
19
+ featureIds: Uint32Array;
20
+ flatCoordinateArray: Float64Array;
21
+ nDim: number;
22
+ geomOffset: Int32Array;
23
+ geometryIndicies: Uint16Array;
24
+ };
25
+
26
+ // binary geometry template, see deck.gl BinaryGeometry
27
+ export const BINARY_GEOMETRY_TEMPLATE = {
28
+ globalFeatureIds: {value: new Uint32Array(0), size: 1},
29
+ positions: {value: new Float32Array(0), size: 2},
30
+ properties: [],
31
+ numericProps: {},
32
+ featureIds: {value: new Uint32Array(0), size: 1}
33
+ };
34
+
35
+ /**
36
+ * get binary geometries from geoarrow column
37
+ *
38
+ * @param geoColumn the geoarrow column, e.g. arrowTable.getChildAt(geoColumnIndex)
39
+ * @param geoEncoding the geo encoding of the geoarrow column, e.g. getGeoArrowEncoding(arrowTable.schema, geoColumnName)
40
+ * @returns BinaryDataFromGeoArrow
41
+ */
42
+ export function getBinaryGeometriesFromArrow(
43
+ geoColumn: arrow.Vector,
44
+ geoEncoding: GeoArrowEncoding
45
+ ): BinaryDataFromGeoArrow {
46
+ const featureTypes = {
47
+ polygon: geoEncoding === 'geoarrow.multipolygon' || geoEncoding === 'geoarrow.polygon',
48
+ point: geoEncoding === 'geoarrow.multipoint' || geoEncoding === 'geoarrow.point',
49
+ line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
50
+ };
51
+
52
+ const chunks = geoColumn.data;
53
+ let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
54
+ let globalFeatureIdOffset = 0;
55
+ const binaryGeometries: BinaryFeatures[] = [];
56
+
57
+ chunks.forEach((chunk) => {
58
+ const {featureIds, flatCoordinateArray, nDim, geomOffset} = getBinaryGeometriesFromChunk(
59
+ chunk,
60
+ geoEncoding
61
+ );
62
+
63
+ const numOfVertices = flatCoordinateArray.length / nDim;
64
+ const globalFeatureIds = new Uint32Array(numOfVertices);
65
+ for (let i = 0; i < numOfVertices; i++) {
66
+ globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
67
+ }
68
+
69
+ const binaryContent = {
70
+ globalFeatureIds: {value: globalFeatureIds, size: 1},
71
+ positions: {
72
+ value: flatCoordinateArray,
73
+ size: nDim
74
+ },
75
+ featureIds: {value: featureIds, size: 1},
76
+ properties: [...Array(chunk.length).keys()].map((i) => ({
77
+ index: i + globalFeatureIdOffset
78
+ }))
79
+ };
80
+
81
+ // TODO: check if chunks are sequentially accessed
82
+ globalFeatureIdOffset += chunk.length;
83
+
84
+ // NOTE: deck.gl defines the BinaryFeatures structure must have points, lines, polygons even if they are empty
85
+ binaryGeometries.push({
86
+ shape: 'binary-feature-collection',
87
+ points: {
88
+ type: 'Point',
89
+ ...BINARY_GEOMETRY_TEMPLATE,
90
+ ...(featureTypes.point ? binaryContent : {})
91
+ },
92
+ lines: {
93
+ type: 'LineString',
94
+ ...BINARY_GEOMETRY_TEMPLATE,
95
+ ...(featureTypes.line ? binaryContent : {}),
96
+ pathIndices: {value: featureTypes.line ? geomOffset : new Uint16Array(0), size: 1}
97
+ },
98
+ polygons: {
99
+ type: 'Polygon',
100
+ ...BINARY_GEOMETRY_TEMPLATE,
101
+ ...(featureTypes.polygon ? binaryContent : {}),
102
+ polygonIndices: {
103
+ // TODO why deck.gl's tessellatePolygon performance is not good when using geometryIndicies
104
+ // even when there is no hole in any polygon
105
+ value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
106
+ size: 1
107
+ },
108
+ primitivePolygonIndices: {
109
+ value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
110
+ size: 1
111
+ }
112
+ }
113
+ });
114
+
115
+ bounds = updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
116
+ });
117
+
118
+ return {binaryGeometries, bounds, featureTypes};
119
+ }
120
+
121
+ /**
122
+ * get binary geometries from geoarrow column
123
+ * @param chunk one chunk/batch of geoarrow column
124
+ * @param geoEncoding geo encoding of the geoarrow column
125
+ * @returns BinaryGeometryContent
126
+ */
127
+ function getBinaryGeometriesFromChunk(
128
+ chunk: arrow.Data,
129
+ geoEncoding: GeoArrowEncoding
130
+ ): BinaryGeometryContent {
131
+ switch (geoEncoding) {
132
+ case 'geoarrow.point':
133
+ case 'geoarrow.multipoint':
134
+ return getBinaryPointsFromChunk(chunk, geoEncoding);
135
+ case 'geoarrow.linestring':
136
+ case 'geoarrow.multilinestring':
137
+ return getBinaryLinesFromChunk(chunk, geoEncoding);
138
+ case 'geoarrow.polygon':
139
+ case 'geoarrow.multipolygon':
140
+ return getBinaryPolygonsFromChunk(chunk, geoEncoding);
141
+ default:
142
+ throw Error('invalid geoarrow encoding');
143
+ }
144
+ }
145
+
146
+ /**
147
+ * get binary polygons from geoarrow polygon column
148
+ * @param chunk one chunk of geoarrow polygon column
149
+ * @param geoEncoding the geo encoding of the geoarrow polygon column
150
+ * @returns BinaryGeometryContent
151
+ */
152
+ function getBinaryPolygonsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
153
+ const isMultiPolygon = geoEncoding === 'geoarrow.multipolygon';
154
+
155
+ const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
156
+ const ringData = polygonData.children[0];
157
+ const pointData = ringData.children[0];
158
+ const coordData = pointData.children[0];
159
+ const nDim = pointData.stride;
160
+ const geomOffset = ringData.valueOffsets;
161
+ const flatCoordinateArray = coordData.values;
162
+
163
+ const geometryIndicies = new Uint16Array(chunk.length + 1);
164
+ for (let i = 0; i < chunk.length; i++) {
165
+ geometryIndicies[i] = geomOffset[chunk.valueOffsets[i]];
166
+ }
167
+ geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
168
+
169
+ const numOfVertices = flatCoordinateArray.length / nDim;
170
+ const featureIds = new Uint32Array(numOfVertices);
171
+ for (let i = 0; i < chunk.length - 1; i++) {
172
+ const startIdx = geomOffset[chunk.valueOffsets[i]];
173
+ const endIdx = geomOffset[chunk.valueOffsets[i + 1]];
174
+ for (let j = startIdx; j < endIdx; j++) {
175
+ featureIds[j] = i;
176
+ }
177
+ }
178
+
179
+ return {
180
+ featureIds,
181
+ flatCoordinateArray,
182
+ nDim,
183
+ geomOffset,
184
+ geometryIndicies
185
+ };
186
+ }
187
+
188
+ /**
189
+ * get binary lines from geoarrow line column
190
+ * @param chunk one chunk/batch of geoarrow column
191
+ * @param geoEncoding the geo encoding of the geoarrow column
192
+ * @returns BinaryGeometryContent
193
+ */
194
+ function getBinaryLinesFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
195
+ const isMultiLineString = geoEncoding === 'geoarrow.multilinestring';
196
+
197
+ const lineData = isMultiLineString ? chunk.children[0] : chunk;
198
+ const pointData = lineData.children[0];
199
+ const coordData = pointData.children[0];
200
+
201
+ const nDim = pointData.stride;
202
+ const geomOffset = lineData.valueOffsets;
203
+ const flatCoordinateArray = coordData.values;
204
+
205
+ // geometryIndicies is not needed for line string
206
+ const geometryIndicies = new Uint16Array(0);
207
+
208
+ const numOfVertices = flatCoordinateArray.length / nDim;
209
+ const featureIds = new Uint32Array(numOfVertices);
210
+ for (let i = 0; i < chunk.length; i++) {
211
+ const startIdx = geomOffset[i];
212
+ const endIdx = geomOffset[i + 1];
213
+ for (let j = startIdx; j < endIdx; j++) {
214
+ featureIds[j] = i;
215
+ }
216
+ }
217
+
218
+ return {
219
+ featureIds,
220
+ flatCoordinateArray,
221
+ nDim,
222
+ geomOffset,
223
+ geometryIndicies
224
+ };
225
+ }
226
+
227
+ /**
228
+ * get binary points from geoarrow point column
229
+ * @param chunk one chunk/batch of geoarrow column
230
+ * @param geoEncoding geo encoding of the geoarrow column
231
+ * @returns BinaryGeometryContent
232
+ */
233
+ function getBinaryPointsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
234
+ const isMultiPoint = geoEncoding === 'geoarrow.multipoint';
235
+
236
+ const pointData = isMultiPoint ? chunk.children[0] : chunk;
237
+ const coordData = pointData.children[0];
238
+
239
+ const nDim = pointData.stride;
240
+ const flatCoordinateArray = coordData.values;
241
+
242
+ // geometryIndices is not needed for point
243
+ const geometryIndicies = new Uint16Array(0);
244
+ // geomOffset is not needed for point
245
+ const geomOffset = new Int32Array(0);
246
+
247
+ const numOfVertices = flatCoordinateArray.length / nDim;
248
+ const featureIds = new Uint32Array(numOfVertices);
249
+ for (let i = 0; i < chunk.length; i++) {
250
+ featureIds[i] = i;
251
+ }
252
+
253
+ return {
254
+ featureIds,
255
+ flatCoordinateArray,
256
+ nDim,
257
+ geomOffset,
258
+ geometryIndicies
259
+ };
260
+ }
@@ -0,0 +1,192 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import * as arrow from 'apache-arrow';
5
+ import {
6
+ Feature,
7
+ MultiPolygon,
8
+ Position,
9
+ Polygon,
10
+ MultiPoint,
11
+ Point,
12
+ MultiLineString,
13
+ LineString
14
+ } from '@loaders.gl/schema';
15
+ import type {GeoArrowEncoding} from '@loaders.gl/gis';
16
+
17
+ type RawArrowFeature = {
18
+ encoding?: GeoArrowEncoding;
19
+ data: any;
20
+ };
21
+
22
+ /**
23
+ * parse geometry from arrow data that is returned from processArrowData()
24
+ * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
25
+ * it is currently only used for picking because currently deck.gl returns only the index of the feature
26
+ * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
27
+ *
28
+ * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
29
+ * @see processArrowData
30
+ * @returns Feature or null
31
+ */
32
+ export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
33
+ const encoding = rawData.encoding?.toLowerCase();
34
+ const data = rawData.data;
35
+ if (!encoding || !data) {
36
+ return null;
37
+ }
38
+
39
+ let geometry;
40
+
41
+ switch (encoding) {
42
+ case 'geoarrow.multipolygon':
43
+ geometry = arrowMultiPolygonToFeature(data);
44
+ break;
45
+ case 'geoarrow.polygon':
46
+ geometry = arrowPolygonToFeature(data);
47
+ break;
48
+ case 'geoarrow.multipoint':
49
+ geometry = arrowMultiPointToFeature(data);
50
+ break;
51
+ case 'geoarrow.point':
52
+ geometry = arrowPointToFeature(data);
53
+ break;
54
+ case 'geoarrow.multilinestring':
55
+ geometry = arrowMultiLineStringToFeature(data);
56
+ break;
57
+ case 'geoarrow.linestring':
58
+ geometry = arrowLineStringToFeature(data);
59
+ break;
60
+ default: {
61
+ throw Error(`GeoArrow encoding not supported ${encoding}`);
62
+ }
63
+ }
64
+ return {
65
+ type: 'Feature',
66
+ geometry,
67
+ properties: {}
68
+ };
69
+ }
70
+
71
+ /**
72
+ * convert Arrow MultiPolygon to geojson Feature
73
+ */
74
+ function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {
75
+ const multiPolygon: Position[][][] = [];
76
+ for (let m = 0; m < arrowMultiPolygon.length; m++) {
77
+ const arrowPolygon = arrowMultiPolygon.get(m);
78
+ const polygon: Position[][] = [];
79
+ for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) {
80
+ const arrowRing = arrowPolygon?.get(i);
81
+ const ring: Position[] = [];
82
+ for (let j = 0; arrowRing && j < arrowRing.length; j++) {
83
+ const arrowCoord = arrowRing.get(j);
84
+ const coord: Position = Array.from(arrowCoord);
85
+ ring.push(coord);
86
+ }
87
+ polygon.push(ring);
88
+ }
89
+ multiPolygon.push(polygon);
90
+ }
91
+ const geometry: MultiPolygon = {
92
+ type: 'MultiPolygon',
93
+ coordinates: multiPolygon
94
+ };
95
+ return geometry;
96
+ }
97
+
98
+ /**
99
+ * convert Arrow Polygon to geojson Feature
100
+ */
101
+ function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
102
+ const polygon: Position[][] = [];
103
+ for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
104
+ const arrowRing = arrowPolygon.get(i);
105
+ const ring: Position[] = [];
106
+ for (let j = 0; arrowRing && j < arrowRing.length; j++) {
107
+ const arrowCoord = arrowRing.get(j);
108
+ const coords: Position = Array.from(arrowCoord);
109
+ ring.push(coords);
110
+ }
111
+ polygon.push(ring);
112
+ }
113
+ const geometry: Polygon = {
114
+ type: 'Polygon',
115
+ coordinates: polygon
116
+ };
117
+ return geometry;
118
+ }
119
+
120
+ /**
121
+ * convert Arrow MultiPoint to geojson MultiPoint
122
+ */
123
+ function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
124
+ const multiPoint: Position[] = [];
125
+ for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
126
+ const arrowPoint = arrowMultiPoint.get(i);
127
+ if (arrowPoint) {
128
+ const coord: Position = Array.from(arrowPoint);
129
+ multiPoint.push(coord);
130
+ }
131
+ }
132
+ const geometry: MultiPoint = {
133
+ type: 'MultiPoint',
134
+ coordinates: multiPoint
135
+ };
136
+ return geometry;
137
+ }
138
+
139
+ /**
140
+ * convert Arrow Point to geojson Point
141
+ */
142
+ function arrowPointToFeature(arrowPoint: arrow.Vector): Point {
143
+ const point: Position = Array.from(arrowPoint);
144
+ const geometry: Point = {
145
+ type: 'Point',
146
+ coordinates: point
147
+ };
148
+ return geometry;
149
+ }
150
+
151
+ /**
152
+ * convert Arrow MultiLineString to geojson MultiLineString
153
+ */
154
+ function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {
155
+ const multiLineString: Position[][] = [];
156
+ for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
157
+ const arrowLineString = arrowMultiLineString.get(i);
158
+ const lineString: Position[] = [];
159
+ for (let j = 0; arrowLineString && j < arrowLineString.length; j++) {
160
+ const arrowCoord = arrowLineString.get(j);
161
+ if (arrowCoord) {
162
+ const coords: Position = Array.from(arrowCoord);
163
+ lineString.push(coords);
164
+ }
165
+ }
166
+ multiLineString.push(lineString);
167
+ }
168
+ const geometry: MultiLineString = {
169
+ type: 'MultiLineString',
170
+ coordinates: multiLineString
171
+ };
172
+ return geometry;
173
+ }
174
+
175
+ /**
176
+ * convert Arrow LineString to geojson LineString
177
+ */
178
+ function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
179
+ const lineString: Position[] = [];
180
+ for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
181
+ const arrowCoord = arrowLineString.get(i);
182
+ if (arrowCoord) {
183
+ const coords: Position = Array.from(arrowCoord);
184
+ lineString.push(coords);
185
+ }
186
+ }
187
+ const geometry: LineString = {
188
+ type: 'LineString',
189
+ coordinates: lineString
190
+ };
191
+ return geometry;
192
+ }
@@ -0,0 +1,40 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ /**
5
+ * Update bounds from geoarrow sample data
6
+ *
7
+ * @param flatCoords the flattend coordinates array from one chunk of geoarrow column
8
+ * @param nDim the number of dimensions of the coordinates
9
+ * @param bounds the bounds to be updated
10
+ * @param sampleSize how many samples to be used to update the bounds, default is 1000 per chunk
11
+ * @returns the updated bounds
12
+ */
13
+ export function updateBoundsFromGeoArrowSamples(
14
+ flatCoords: Float64Array,
15
+ nDim: number,
16
+ bounds: [number, number, number, number],
17
+ sampleSize: number = 100
18
+ ): [number, number, number, number] {
19
+ const numberOfFeatures = flatCoords.length / nDim;
20
+ const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
21
+
22
+ const newBounds: [number, number, number, number] = [...bounds];
23
+ for (let i = 0; i < numberOfFeatures; i += sampleStep) {
24
+ const lng = flatCoords[i * nDim];
25
+ const lat = flatCoords[i * nDim + 1];
26
+ if (lng < newBounds[0]) {
27
+ newBounds[0] = lng;
28
+ }
29
+ if (lat < newBounds[1]) {
30
+ newBounds[1] = lat;
31
+ }
32
+ if (lng > newBounds[2]) {
33
+ newBounds[2] = lng;
34
+ }
35
+ if (lat > newBounds[3]) {
36
+ newBounds[3] = lat;
37
+ }
38
+ }
39
+ return newBounds;
40
+ }
package/src/index.ts CHANGED
@@ -1,4 +1,5 @@
1
- // loaders.gl, MIT
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
2
3
 
3
4
  import type {LoaderWithParser} from '@loaders.gl/loader-utils';
4
5
  import type {ArrowLoaderOptions} from './arrow-loader';
@@ -12,12 +13,25 @@ import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
12
13
 
13
14
  import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
14
15
 
15
- // SCHEMA
16
+ // Make the ArrowBatch type available
17
+ TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
18
+
19
+ // TYPES
16
20
 
17
21
  export {getArrowType} from './schema/arrow-type-utils';
18
22
 
19
- // Make the ArrowBatch type available
20
- TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
23
+ // SCHEMA
24
+
25
+ export {
26
+ serializeArrowSchema,
27
+ deserializeArrowSchema,
28
+ serializeArrowMetadata,
29
+ deserializeArrowMetadata,
30
+ serializeArrowField,
31
+ deserializeArrowField,
32
+ serializeArrowType,
33
+ deserializeArrowType
34
+ } from './schema/convert-arrow-schema';
21
35
 
22
36
  // Types
23
37
  export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
@@ -44,3 +58,18 @@ export const ArrowLoader: LoaderWithParser<
44
58
  parseSync,
45
59
  parseInBatches: parseArrowInBatches
46
60
  };
61
+
62
+ // Arrow Utils
63
+ export type {GeoArrowEncoding} from '@loaders.gl/gis';
64
+ // getGeometryColumnsFromArrowTable,
65
+ // getGeoArrowEncoding
66
+
67
+ export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
68
+ export {
69
+ BINARY_GEOMETRY_TEMPLATE,
70
+ getBinaryGeometriesFromArrow
71
+ } from './geoarrow/convert-geoarrow-to-binary-geometry';
72
+
73
+ export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
74
+
75
+ export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
@@ -1,19 +1,9 @@
1
1
  import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
2
2
  import type {ArrowTableBatch} from './arrow-table';
3
- import {
4
- Table as ApacheArrowTable,
5
- Schema,
6
- Field,
7
- RecordBatch,
8
- Struct,
9
- makeVector,
10
- makeData,
11
- Vector,
12
- Float32
13
- } from 'apache-arrow';
3
+ import * as arrow from 'apache-arrow';
14
4
 
15
5
  export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
16
- arrowSchema: Schema | null;
6
+ arrowSchema: arrow.Schema | null;
17
7
 
18
8
  constructor(schema, options) {
19
9
  super(schema, options);
@@ -30,10 +20,10 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
30
20
  const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
31
21
 
32
22
  // Create the record batch
33
- const recordBatch = new RecordBatch(
23
+ const recordBatch = new arrow.RecordBatch(
34
24
  this.arrowSchema,
35
- makeData({
36
- type: new Struct(this.arrowSchema.fields),
25
+ arrow.makeData({
26
+ type: new arrow.Struct(this.arrowSchema.fields),
37
27
  children: arrowVectors.map(({data}) => data[0])
38
28
  })
39
29
  );
@@ -41,7 +31,7 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
41
31
  return {
42
32
  shape: 'arrow-table',
43
33
  batchType: 'data',
44
- data: new ApacheArrowTable([recordBatch]),
34
+ data: new arrow.Table([recordBatch]),
45
35
  length: batch.length
46
36
  };
47
37
  }
@@ -51,15 +41,15 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
51
41
  }
52
42
 
53
43
  // Convert from a simple loaders.gl schema to an Arrow schema
54
- function getArrowSchema(schema): Schema {
55
- const arrowFields: Field[] = [];
44
+ function getArrowSchema(schema): arrow.Schema {
45
+ const arrowFields: arrow.Field[] = [];
56
46
  for (const key in schema) {
57
47
  const field = schema[key];
58
48
  if (field.type === Float32Array) {
59
49
  // TODO - just store the original field as metadata?
60
50
  const metadata = new Map(); // field;
61
- // arrow: new Field(name, nullable, metadata)
62
- const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
51
+ // arrow: new arrow.Field(name, nullable, metadata)
52
+ const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
63
53
  arrowFields.push(arrowField);
64
54
  }
65
55
  }
@@ -67,16 +57,16 @@ function getArrowSchema(schema): Schema {
67
57
  throw new Error('No arrow convertible fields');
68
58
  }
69
59
 
70
- return new Schema(arrowFields);
60
+ return new arrow.Schema(arrowFields);
71
61
  }
72
62
 
73
63
  // Convert from simple loaders.gl arrays to arrow vectors
74
- function getArrowVectors(arrowSchema, data): Vector[] {
64
+ function getArrowVectors(arrowSchema, data): arrow.Vector[] {
75
65
  const arrowVectors: any[] = [];
76
66
  for (const field of arrowSchema.fields) {
77
67
  const vector = data[field.name];
78
68
  if (vector instanceof Float32Array) {
79
- const arrowVector = makeVector(vector);
69
+ const arrowVector = arrow.makeVector(vector);
80
70
  arrowVectors.push(arrowVector);
81
71
  }
82
72
  }
@@ -2,7 +2,7 @@
2
2
  // Copyright (c) vis.gl contributors
3
3
 
4
4
  import type {Batch, Schema} from '@loaders.gl/schema';
5
- import type {Table as ApacheArrowTable} from 'apache-arrow';
5
+ import type * as arrow from 'apache-arrow';
6
6
 
7
7
  /**
8
8
  * A table organized as an Apache Arrow table
@@ -11,7 +11,7 @@ import type {Table as ApacheArrowTable} from 'apache-arrow';
11
11
  export type ArrowTable = {
12
12
  shape: 'arrow-table';
13
13
  schema?: Schema;
14
- data: ApacheArrowTable;
14
+ data: arrow.Table;
15
15
  };
16
16
 
17
17
  /**
@@ -22,6 +22,6 @@ export type ArrowTableBatch = Batch & {
22
22
  shape: 'arrow-table';
23
23
  schemaType?: 'explicit' | 'deduced';
24
24
  schema?: Schema;
25
- data: ApacheArrowTable; // ApacheRecordBatch;
25
+ data: arrow.Table; // ApacheRecordBatch;
26
26
  length: number;
27
27
  };
@@ -1,4 +1,4 @@
1
- import {Table, Vector, tableToIPC, vectorFromArray} from 'apache-arrow';
1
+ import * as arrow from 'apache-arrow';
2
2
  import {AnyArrayType, VECTOR_TYPES} from '../types';
3
3
 
4
4
  export type ColumnarTable = {
@@ -15,28 +15,28 @@ export type ColumnarTable = {
15
15
  * @returns - encoded ArrayBuffer
16
16
  */
17
17
  export function encodeArrowSync(data: ColumnarTable): ArrayBuffer {
18
- const vectors: Record<string, Vector> = {};
18
+ const vectors: Record<string, arrow.Vector> = {};
19
19
  for (const arrayData of data) {
20
20
  const arrayVector = createVector(arrayData.array, arrayData.type);
21
21
  vectors[arrayData.name] = arrayVector;
22
22
  }
23
- const table = new Table(vectors);
24
- const arrowBuffer = tableToIPC(table);
23
+ const table = new arrow.Table(vectors);
24
+ const arrowBuffer = arrow.tableToIPC(table);
25
25
  return arrowBuffer;
26
26
  }
27
27
 
28
28
  /**
29
- * Create Arrow Vector from given data and vector type
29
+ * Create Arrow arrow.Vector from given data and vector type
30
30
  * @param array {import('../types').AnyArrayType} - columns data
31
31
  * @param type {number} - the writer options
32
32
  * @return a vector of one of vector's types defined in the Apache Arrow library
33
33
  */
34
- function createVector(array, type): Vector {
34
+ function createVector(array, type): arrow.Vector {
35
35
  switch (type) {
36
36
  case VECTOR_TYPES.DATE:
37
- return vectorFromArray(array);
37
+ return arrow.vectorFromArray(array);
38
38
  case VECTOR_TYPES.FLOAT:
39
39
  default:
40
- return vectorFromArray(array);
40
+ return arrow.vectorFromArray(array);
41
41
  }
42
42
  }