@loaders.gl/arrow 4.0.3 → 4.1.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/arrow-loader.d.ts +6 -3
  2. package/dist/arrow-loader.d.ts.map +1 -1
  3. package/dist/arrow-loader.js +9 -1
  4. package/dist/arrow-loader.js.map +1 -1
  5. package/dist/arrow-worker.js +272 -537
  6. package/dist/dist.dev.js +2399 -269
  7. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +33 -1
  8. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -1
  9. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +146 -17
  10. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -1
  11. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts +13 -0
  12. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts.map +1 -0
  13. package/dist/geoarrow/{convert-geoarrow-to-geojson.js → convert-geoarrow-to-geojson-geometry.js} +36 -25
  14. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +1 -0
  15. package/dist/geoarrow-loader.d.ts +19 -0
  16. package/dist/geoarrow-loader.d.ts.map +1 -0
  17. package/dist/geoarrow-loader.js +23 -0
  18. package/dist/geoarrow-loader.js.map +1 -0
  19. package/dist/geoarrow-writer.d.ts +9 -0
  20. package/dist/geoarrow-writer.d.ts.map +1 -0
  21. package/dist/geoarrow-writer.js +19 -0
  22. package/dist/geoarrow-writer.js.map +1 -0
  23. package/dist/index.cjs +481 -255
  24. package/dist/index.d.ts +9 -12
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +7 -13
  27. package/dist/index.js.map +1 -1
  28. package/dist/lib/arrow-table-batch.d.ts.map +1 -1
  29. package/dist/lib/arrow-table-batch.js.map +1 -1
  30. package/dist/lib/encode-arrow.d.ts.map +1 -1
  31. package/dist/lib/encode-arrow.js.map +1 -1
  32. package/dist/lib/encode-geoarrow.d.ts +15 -0
  33. package/dist/lib/encode-geoarrow.d.ts.map +1 -0
  34. package/dist/lib/encode-geoarrow.js +22 -0
  35. package/dist/lib/encode-geoarrow.js.map +1 -0
  36. package/dist/{lib → parsers}/parse-arrow-in-batches.d.ts +1 -1
  37. package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -0
  38. package/dist/parsers/parse-arrow-in-batches.js.map +1 -0
  39. package/dist/parsers/parse-arrow-sync.d.ts +6 -0
  40. package/dist/parsers/parse-arrow-sync.d.ts.map +1 -0
  41. package/dist/parsers/parse-arrow-sync.js +28 -0
  42. package/dist/parsers/parse-arrow-sync.js.map +1 -0
  43. package/dist/parsers/parse-geoarrow-in-batches.d.ts +6 -0
  44. package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -0
  45. package/dist/parsers/parse-geoarrow-in-batches.js +5 -0
  46. package/dist/parsers/parse-geoarrow-in-batches.js.map +1 -0
  47. package/dist/parsers/parse-geoarrow-sync.d.ts +6 -0
  48. package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -0
  49. package/dist/parsers/parse-geoarrow-sync.js +14 -0
  50. package/dist/parsers/parse-geoarrow-sync.js.map +1 -0
  51. package/dist/tables/convert-arrow-to-columnar-table.d.ts +8 -0
  52. package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -0
  53. package/dist/tables/convert-arrow-to-columnar-table.js +15 -0
  54. package/dist/tables/convert-arrow-to-columnar-table.js.map +1 -0
  55. package/dist/tables/convert-arrow-to-geojson-table.d.ts +16 -0
  56. package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -0
  57. package/dist/tables/convert-arrow-to-geojson-table.js +39 -0
  58. package/dist/tables/convert-arrow-to-geojson-table.js.map +1 -0
  59. package/dist/tables/convert-columnar-to-row-table.d.ts +7 -0
  60. package/dist/tables/convert-columnar-to-row-table.d.ts.map +1 -0
  61. package/dist/tables/convert-columnar-to-row-table.js +19 -0
  62. package/dist/tables/convert-columnar-to-row-table.js.map +1 -0
  63. package/dist/triangulate-on-worker.d.ts +36 -0
  64. package/dist/triangulate-on-worker.d.ts.map +1 -0
  65. package/dist/triangulate-on-worker.js +14 -0
  66. package/dist/triangulate-on-worker.js.map +1 -0
  67. package/dist/triangulation-worker.js +887 -0
  68. package/dist/workers/arrow-worker.js +1 -1
  69. package/dist/workers/arrow-worker.js.map +1 -1
  70. package/dist/workers/triangulation-worker-node.d.ts +2 -0
  71. package/dist/workers/triangulation-worker-node.d.ts.map +1 -0
  72. package/dist/workers/triangulation-worker-node.js +2 -0
  73. package/dist/workers/triangulation-worker-node.js.map +1 -0
  74. package/dist/workers/triangulation-worker.d.ts +2 -0
  75. package/dist/workers/triangulation-worker.d.ts.map +1 -0
  76. package/dist/workers/triangulation-worker.js +26 -0
  77. package/dist/workers/triangulation-worker.js.map +1 -0
  78. package/package.json +20 -12
  79. package/src/arrow-loader.ts +25 -3
  80. package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +247 -23
  81. package/src/geoarrow/{convert-geoarrow-to-geojson.ts → convert-geoarrow-to-geojson-geometry.ts} +58 -44
  82. package/src/geoarrow-loader.ts +51 -0
  83. package/src/geoarrow-writer.ts +41 -0
  84. package/src/index.ts +31 -37
  85. package/src/lib/arrow-table-batch.ts +3 -0
  86. package/src/lib/encode-arrow.ts +3 -0
  87. package/src/lib/encode-geoarrow.ts +45 -0
  88. package/src/{lib → parsers}/parse-arrow-in-batches.ts +4 -2
  89. package/src/parsers/parse-arrow-sync.ts +43 -0
  90. package/src/parsers/parse-geoarrow-in-batches.ts +15 -0
  91. package/src/parsers/parse-geoarrow-sync.ts +22 -0
  92. package/src/tables/convert-arrow-to-columnar-table.ts +30 -0
  93. package/src/tables/convert-arrow-to-geojson-table.ts +65 -0
  94. package/src/tables/convert-columnar-to-row-table.ts +30 -0
  95. package/src/triangulate-on-worker.ts +47 -0
  96. package/src/workers/arrow-worker.ts +1 -1
  97. package/src/workers/triangulation-worker-node.ts +4 -0
  98. package/src/workers/triangulation-worker.ts +39 -0
  99. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +0 -19
  100. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +0 -1
  101. package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +0 -1
  102. package/dist/lib/parse-arrow-in-batches.d.ts.map +0 -1
  103. package/dist/lib/parse-arrow-in-batches.js.map +0 -1
  104. package/dist/lib/parse-arrow-sync.d.ts +0 -5
  105. package/dist/lib/parse-arrow-sync.d.ts.map +0 -1
  106. package/dist/lib/parse-arrow-sync.js +0 -21
  107. package/dist/lib/parse-arrow-sync.js.map +0 -1
  108. package/dist/tables/convert-arrow-to-table.d.ts +0 -21
  109. package/dist/tables/convert-arrow-to-table.d.ts.map +0 -1
  110. package/dist/tables/convert-arrow-to-table.js +0 -37
  111. package/dist/tables/convert-arrow-to-table.js.map +0 -1
  112. package/src/lib/parse-arrow-sync.ts +0 -35
  113. package/src/tables/convert-arrow-to-table.ts +0 -68
  114. /package/dist/{lib → parsers}/parse-arrow-in-batches.js +0 -0
@@ -1,77 +1,95 @@
1
1
  // loaders.gl, MIT license
2
2
  // Copyright (c) vis.gl contributors
3
3
 
4
- import * as arrow from 'apache-arrow';
4
+ // import * as arrow from 'apache-arrow';
5
5
  import {
6
- Feature,
7
6
  MultiPolygon,
8
7
  Position,
9
8
  Polygon,
10
9
  MultiPoint,
11
10
  Point,
12
11
  MultiLineString,
13
- LineString
12
+ LineString,
13
+ Geometry,
14
+ BinaryGeometry
14
15
  } from '@loaders.gl/schema';
15
- import type {GeoArrowEncoding} from '@loaders.gl/gis';
16
-
17
- type RawArrowFeature = {
18
- encoding?: GeoArrowEncoding;
19
- data: any;
20
- };
16
+ import {binaryToGeometry, type GeoArrowEncoding} from '@loaders.gl/gis';
17
+ import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
21
18
 
22
19
  /**
23
20
  * parse geometry from arrow data that is returned from processArrowData()
24
- * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
25
- * it is currently only used for picking because currently deck.gl returns only the index of the feature
26
- * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
21
+ * NOTE: this function could be deduplicated with the binaryToFeature() in deck.gl,
22
+ * it is currently used for deck.gl picking because currently deck.gl returns only the index of the feature
27
23
  *
28
- * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
29
- * @see processArrowData
24
+ * @param data data extraced from arrow vector representing a geometry
25
+ * @param encoding the geoarrow encoding of the geometry column
30
26
  * @returns Feature or null
31
27
  */
32
- export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
33
- const encoding = rawData.encoding?.toLowerCase();
34
- const data = rawData.data;
35
- if (!encoding || !data) {
28
+ export function parseGeometryFromArrow(
29
+ arrowCellValue: any,
30
+ encoding?: GeoArrowEncoding
31
+ ): Geometry | null {
32
+ // sanity
33
+ encoding = encoding?.toLowerCase() as GeoArrowEncoding;
34
+ if (!encoding || !arrowCellValue) {
36
35
  return null;
37
36
  }
38
37
 
39
- let geometry;
38
+ let geometry: Geometry;
40
39
 
41
40
  switch (encoding) {
42
41
  case 'geoarrow.multipolygon':
43
- geometry = arrowMultiPolygonToFeature(data);
42
+ geometry = arrowMultiPolygonToFeature(arrowCellValue);
44
43
  break;
45
44
  case 'geoarrow.polygon':
46
- geometry = arrowPolygonToFeature(data);
45
+ geometry = arrowPolygonToFeature(arrowCellValue);
47
46
  break;
48
47
  case 'geoarrow.multipoint':
49
- geometry = arrowMultiPointToFeature(data);
48
+ geometry = arrowMultiPointToFeature(arrowCellValue);
50
49
  break;
51
50
  case 'geoarrow.point':
52
- geometry = arrowPointToFeature(data);
51
+ geometry = arrowPointToFeature(arrowCellValue);
53
52
  break;
54
53
  case 'geoarrow.multilinestring':
55
- geometry = arrowMultiLineStringToFeature(data);
54
+ geometry = arrowMultiLineStringToFeature(arrowCellValue);
56
55
  break;
57
56
  case 'geoarrow.linestring':
58
- geometry = arrowLineStringToFeature(data);
57
+ geometry = arrowLineStringToFeature(arrowCellValue);
58
+ break;
59
+ case 'geoarrow.wkb':
60
+ geometry = arrowWKBToFeature(arrowCellValue);
61
+ break;
62
+ case 'geoarrow.wkt':
63
+ geometry = arrowWKTToFeature(arrowCellValue);
59
64
  break;
60
65
  default: {
61
66
  throw Error(`GeoArrow encoding not supported ${encoding}`);
62
67
  }
63
68
  }
64
- return {
65
- type: 'Feature',
66
- geometry,
67
- properties: {}
68
- };
69
+
70
+ return geometry;
71
+ }
72
+
73
+ function arrowWKBToFeature(arrowCellValue: any) {
74
+ // The actual WKB array buffer starts from byteOffset and ends at byteOffset + byteLength
75
+ const arrayBuffer: ArrayBuffer = arrowCellValue.buffer.slice(
76
+ arrowCellValue.byteOffset,
77
+ arrowCellValue.byteOffset + arrowCellValue.byteLength
78
+ );
79
+ const binaryGeometry = WKBLoader.parseSync?.(arrayBuffer)! as BinaryGeometry;
80
+ const geometry = binaryToGeometry(binaryGeometry);
81
+ return geometry;
82
+ }
83
+
84
+ function arrowWKTToFeature(arrowCellValue: any) {
85
+ const string: string = arrowCellValue;
86
+ return WKTLoader.parseTextSync?.(string)!;
69
87
  }
70
88
 
71
89
  /**
72
90
  * convert Arrow MultiPolygon to geojson Feature
73
91
  */
74
- function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {
92
+ function arrowMultiPolygonToFeature(arrowMultiPolygon: any): MultiPolygon {
75
93
  const multiPolygon: Position[][][] = [];
76
94
  for (let m = 0; m < arrowMultiPolygon.length; m++) {
77
95
  const arrowPolygon = arrowMultiPolygon.get(m);
@@ -98,7 +116,7 @@ function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolyg
98
116
  /**
99
117
  * convert Arrow Polygon to geojson Feature
100
118
  */
101
- function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
119
+ function arrowPolygonToFeature(arrowPolygon: any): Polygon {
102
120
  const polygon: Position[][] = [];
103
121
  for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
104
122
  const arrowRing = arrowPolygon.get(i);
@@ -120,7 +138,7 @@ function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
120
138
  /**
121
139
  * convert Arrow MultiPoint to geojson MultiPoint
122
140
  */
123
- function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
141
+ function arrowMultiPointToFeature(arrowMultiPoint: any): MultiPoint {
124
142
  const multiPoint: Position[] = [];
125
143
  for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
126
144
  const arrowPoint = arrowMultiPoint.get(i);
@@ -129,29 +147,27 @@ function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
129
147
  multiPoint.push(coord);
130
148
  }
131
149
  }
132
- const geometry: MultiPoint = {
150
+ return {
133
151
  type: 'MultiPoint',
134
152
  coordinates: multiPoint
135
153
  };
136
- return geometry;
137
154
  }
138
155
 
139
156
  /**
140
157
  * convert Arrow Point to geojson Point
141
158
  */
142
- function arrowPointToFeature(arrowPoint: arrow.Vector): Point {
159
+ function arrowPointToFeature(arrowPoint: any): Point {
143
160
  const point: Position = Array.from(arrowPoint);
144
- const geometry: Point = {
161
+ return {
145
162
  type: 'Point',
146
163
  coordinates: point
147
164
  };
148
- return geometry;
149
165
  }
150
166
 
151
167
  /**
152
168
  * convert Arrow MultiLineString to geojson MultiLineString
153
169
  */
154
- function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {
170
+ function arrowMultiLineStringToFeature(arrowMultiLineString: any): MultiLineString {
155
171
  const multiLineString: Position[][] = [];
156
172
  for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
157
173
  const arrowLineString = arrowMultiLineString.get(i);
@@ -165,17 +181,16 @@ function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): Mult
165
181
  }
166
182
  multiLineString.push(lineString);
167
183
  }
168
- const geometry: MultiLineString = {
184
+ return {
169
185
  type: 'MultiLineString',
170
186
  coordinates: multiLineString
171
187
  };
172
- return geometry;
173
188
  }
174
189
 
175
190
  /**
176
191
  * convert Arrow LineString to geojson LineString
177
192
  */
178
- function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
193
+ function arrowLineStringToFeature(arrowLineString: any): LineString {
179
194
  const lineString: Position[] = [];
180
195
  for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
181
196
  const arrowCoord = arrowLineString.get(i);
@@ -184,9 +199,8 @@ function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
184
199
  lineString.push(coords);
185
200
  }
186
201
  }
187
- const geometry: LineString = {
202
+ return {
188
203
  type: 'LineString',
189
204
  coordinates: lineString
190
205
  };
191
- return geometry;
192
206
  }
@@ -0,0 +1,51 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
5
+ import {ArrowWorkerLoader} from './arrow-loader';
6
+ import type {GeoJSONTable, GeoJSONTableBatch, BinaryGeometry} from '@loaders.gl/schema';
7
+ import type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
8
+ import {parseGeoArrowSync} from './parsers/parse-geoarrow-sync';
9
+ import {parseGeoArrowInBatches} from './parsers/parse-geoarrow-in-batches';
10
+
11
+ export type GeoArrowLoaderOptions = LoaderOptions & {
12
+ arrow?: {
13
+ shape: 'arrow-table' | 'binary-geometry';
14
+ };
15
+ };
16
+
17
+ /** ArrowJS table loader */
18
+ export const GeoArrowWorkerLoader: Loader<
19
+ ArrowTable | BinaryGeometry,
20
+ never,
21
+ GeoArrowLoaderOptions
22
+ > = {
23
+ ...ArrowWorkerLoader,
24
+ options: {
25
+ arrow: {
26
+ shape: 'arrow-table'
27
+ }
28
+ }
29
+ };
30
+
31
+ /**
32
+ * GeoArrowLoader loads an Apache Arrow table, parses GeoArrow type extension data
33
+ * to convert it to a GeoJSON table or a BinaryGeometry
34
+ */
35
+ export const GeoArrowLoader: LoaderWithParser<
36
+ ArrowTable | GeoJSONTable, // | BinaryGeometry,
37
+ ArrowTableBatch | GeoJSONTableBatch, // | BinaryGeometry,
38
+ GeoArrowLoaderOptions
39
+ > = {
40
+ ...ArrowWorkerLoader,
41
+ options: {
42
+ arrow: {
43
+ shape: 'arrow-table'
44
+ }
45
+ },
46
+ parse: async (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>
47
+ parseGeoArrowSync(arraybuffer, options?.arrow),
48
+ parseSync: (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>
49
+ parseGeoArrowSync(arraybuffer, options?.arrow),
50
+ parseInBatches: parseGeoArrowInBatches
51
+ };
@@ -0,0 +1,41 @@
1
+ // import type {} from '@loaders.gl/loader-utils';
2
+
3
+ import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
4
+ import {GeoJSONTable, BinaryGeometry} from '@loaders.gl/schema';
5
+ import {encodeGeoArrowSync} from './lib/encode-geoarrow';
6
+
7
+ // __VERSION__ is injected by babel-plugin-version-inline
8
+ // @ts-ignore TS2304: Cannot find name '__VERSION__'.
9
+ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
10
+
11
+ type ArrowWriterOptions = WriterOptions & {
12
+ arrow?: {};
13
+ };
14
+
15
+ /** Apache Arrow writer */
16
+ export const GeoArrowWriter: WriterWithEncoder<
17
+ GeoJSONTable | BinaryGeometry,
18
+ never,
19
+ ArrowWriterOptions
20
+ > = {
21
+ name: 'Apache Arrow',
22
+ id: 'arrow',
23
+ module: 'arrow',
24
+ version: VERSION,
25
+ extensions: ['arrow', 'feather'],
26
+ mimeTypes: [
27
+ 'application/vnd.apache.arrow.file',
28
+ 'application/vnd.apache.arrow.stream',
29
+ 'application/octet-stream'
30
+ ],
31
+ binary: true,
32
+ options: {},
33
+ encode: async function encodeArrow(data, options?): Promise<ArrayBuffer> {
34
+ // @ts-expect-error
35
+ return encodeGeoArrowSync(data);
36
+ },
37
+ encodeSync(data, options?) {
38
+ // @ts-expect-error
39
+ return encodeGeoArrowSync(data);
40
+ }
41
+ };
package/src/index.ts CHANGED
@@ -1,16 +1,8 @@
1
1
  // loaders.gl, MIT license
2
2
  // Copyright (c) vis.gl contributors
3
3
 
4
- import type {LoaderWithParser} from '@loaders.gl/loader-utils';
5
4
  import type {ArrowLoaderOptions} from './arrow-loader';
6
- import type {ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
7
- import type {ArrowTable} from './lib/arrow-table';
8
-
9
5
  import {TableBatchBuilder} from '@loaders.gl/schema';
10
- import {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';
11
- import parseSync from './lib/parse-arrow-sync';
12
- import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
13
-
14
6
  import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
15
7
 
16
8
  // Make the ArrowBatch type available
@@ -22,6 +14,21 @@ export {getArrowType} from './schema/arrow-type-utils';
22
14
 
23
15
  // SCHEMA
24
16
 
17
+ // Types
18
+ export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
19
+ export {VECTOR_TYPES} from './types';
20
+
21
+ // Arrow loader / Writer
22
+
23
+ export type {ArrowLoaderOptions};
24
+ export {ArrowLoader, ArrowWorkerLoader} from './arrow-loader';
25
+
26
+ export {ArrowWriter} from './arrow-writer';
27
+
28
+ // Geoarrow loader
29
+ export {GeoArrowLoader, GeoArrowWorkerLoader} from './geoarrow-loader';
30
+
31
+ // Schema utils
25
32
  export {
26
33
  serializeArrowSchema,
27
34
  deserializeArrowSchema,
@@ -33,43 +40,30 @@ export {
33
40
  deserializeArrowType
34
41
  } from './schema/convert-arrow-schema';
35
42
 
36
- // Types
37
- export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
38
- export {VECTOR_TYPES} from './types';
39
-
40
- // Arrow writer
41
-
42
- export {ArrowWriter} from './arrow-writer';
43
-
44
- // Arrow loader
45
-
46
- export type {ArrowLoaderOptions};
47
- export {ArrowWorkerLoader};
48
-
49
- /** ArrowJS table loader */
50
- export const ArrowLoader: LoaderWithParser<
51
- ArrowTable | ColumnarTable | ObjectRowTable,
52
- ArrowTableBatch,
53
- ArrowLoaderOptions
54
- > = {
55
- ...ArrowWorkerLoader,
56
- parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>
57
- parseSync(arraybuffer, options),
58
- parseSync,
59
- parseInBatches: parseArrowInBatches
60
- };
43
+ // EXPERIMENTAL
61
44
 
62
45
  // Arrow Utils
63
46
  export type {GeoArrowEncoding} from '@loaders.gl/gis';
64
47
  // getGeometryColumnsFromArrowTable,
65
48
  // getGeoArrowEncoding
66
49
 
67
- export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
50
+ export type {
51
+ BinaryDataFromGeoArrow,
52
+ BinaryGeometriesFromArrowOptions
53
+ } from './geoarrow/convert-geoarrow-to-binary-geometry';
68
54
  export {
69
55
  BINARY_GEOMETRY_TEMPLATE,
70
- getBinaryGeometriesFromArrow
56
+ getBinaryGeometriesFromArrow,
57
+ getTriangleIndices,
58
+ getMeanCentersFromBinaryGeometries
71
59
  } from './geoarrow/convert-geoarrow-to-binary-geometry';
72
60
 
73
- export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
74
-
75
61
  export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
62
+
63
+ export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson-geometry';
64
+
65
+ export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
66
+
67
+ // EXPERIMENTAL WORKER
68
+
69
+ export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';
@@ -1,3 +1,6 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
1
4
  import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
2
5
  import type {ArrowTableBatch} from './arrow-table';
3
6
  import * as arrow from 'apache-arrow';
@@ -1,3 +1,6 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
1
4
  import * as arrow from 'apache-arrow';
2
5
  import {AnyArrayType, VECTOR_TYPES} from '../types';
3
6
 
@@ -0,0 +1,45 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import * as arrow from 'apache-arrow';
5
+ import {AnyArrayType, VECTOR_TYPES} from '../types';
6
+
7
+ export type ColumnarTable = {
8
+ name: string;
9
+ array: AnyArrayType;
10
+ type: number;
11
+ }[];
12
+
13
+ /**
14
+ * Encodes set of arrays into the Apache Arrow columnar format
15
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
16
+ * @param data - columns data
17
+ * @param options - the writer options
18
+ * @returns - encoded ArrayBuffer
19
+ */
20
+ export function encodeGeoArrowSync(data: ColumnarTable): ArrayBuffer {
21
+ const vectors: Record<string, arrow.Vector> = {};
22
+ for (const arrayData of data) {
23
+ const arrayVector = createVector(arrayData.array, arrayData.type);
24
+ vectors[arrayData.name] = arrayVector;
25
+ }
26
+ const table = new arrow.Table(vectors);
27
+ const arrowBuffer = arrow.tableToIPC(table);
28
+ return arrowBuffer;
29
+ }
30
+
31
+ /**
32
+ * Create Arrow arrow.Vector from given data and vector type
33
+ * @param array {import('../types').AnyArrayType} - columns data
34
+ * @param type {number} - the writer options
35
+ * @return a vector of one of vector's types defined in the Apache Arrow library
36
+ */
37
+ function createVector(array, type): arrow.Vector {
38
+ switch (type) {
39
+ case VECTOR_TYPES.DATE:
40
+ return arrow.vectorFromArray(array);
41
+ case VECTOR_TYPES.FLOAT:
42
+ default:
43
+ return arrow.vectorFromArray(array);
44
+ }
45
+ }
@@ -1,5 +1,7 @@
1
- // TODO - this import defeats the sophisticated typescript checking in ArrowJS
2
- import type {ArrowTableBatch} from './arrow-table';
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {ArrowTableBatch} from '../lib/arrow-table';
3
5
  import * as arrow from 'apache-arrow';
4
6
  // import {isIterable} from '@loaders.gl/core';
5
7
 
@@ -0,0 +1,43 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
5
+ import type {ArrowTable} from '../lib/arrow-table';
6
+ import {convertTable} from '@loaders.gl/schema';
7
+ import * as arrow from 'apache-arrow';
8
+ import {convertArrowToColumnarTable} from '../tables/convert-arrow-to-columnar-table';
9
+ import {serializeArrowSchema} from '../schema/convert-arrow-schema';
10
+
11
+ // Parses arrow to a columnar table
12
+ export function parseArrowSync(
13
+ arrayBuffer,
14
+ options?: {shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table'}
15
+ ): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable {
16
+ const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
17
+ const arrowTable: ArrowTable = {
18
+ shape: 'arrow-table',
19
+ schema: serializeArrowSchema(apacheArrowTable.schema),
20
+ data: apacheArrowTable
21
+ };
22
+
23
+ const shape = options?.shape || 'arrow-table';
24
+ switch (shape) {
25
+ case 'arrow-table':
26
+ return arrowTable;
27
+
28
+ case 'columnar-table':
29
+ return convertArrowToColumnarTable(arrowTable);
30
+
31
+ case 'object-row-table':
32
+ let columnarTable = convertArrowToColumnarTable(arrowTable);
33
+ return convertTable(columnarTable, 'object-row-table');
34
+
35
+ case 'array-row-table':
36
+ columnarTable = convertArrowToColumnarTable(arrowTable);
37
+ return convertTable(columnarTable, 'array-row-table');
38
+
39
+ default:
40
+ // TODO
41
+ throw new Error(shape);
42
+ }
43
+ }
@@ -0,0 +1,15 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {GeoJSONTableBatch} from '@loaders.gl/schema';
5
+ import type {ArrowTableBatch} from '../lib/arrow-table';
6
+ import {parseArrowInBatches} from './parse-arrow-in-batches';
7
+
8
+ /**
9
+ */
10
+ export function parseGeoArrowInBatches(
11
+ asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
12
+ ): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch> {
13
+ // | BinaryGeometry
14
+ return parseArrowInBatches(asyncIterator);
15
+ }
@@ -0,0 +1,22 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {GeoJSONTable} from '@loaders.gl/schema';
5
+ import type {ArrowTable} from '../lib/arrow-table';
6
+ import {parseArrowSync} from './parse-arrow-sync';
7
+ import {convertArrowToGeoJSONTable} from '../tables/convert-arrow-to-geojson-table';
8
+
9
+ // Parses arrow to a columnar table
10
+ export function parseGeoArrowSync(
11
+ arrayBuffer,
12
+ options?: {shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry'}
13
+ ): ArrowTable | GeoJSONTable {
14
+ // | BinaryGeometry
15
+ const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;
16
+ switch (options?.shape) {
17
+ case 'geojson-table':
18
+ return convertArrowToGeoJSONTable(table);
19
+ default:
20
+ return table;
21
+ }
22
+ }
@@ -0,0 +1,30 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {ColumnarTable} from '@loaders.gl/schema';
5
+ import type {ArrowTable} from '../lib/arrow-table';
6
+
7
+ /**
8
+ * Convert an Apache Arrow table to a ColumnarTable
9
+ * @note Currently does not convert schema
10
+ */
11
+ export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
12
+ // TODO - avoid calling `getColumn` on columns we are not interested in?
13
+ // Add options object?
14
+
15
+ const arrowTable = table.data;
16
+ const columnarTable = {};
17
+
18
+ for (const field of arrowTable.schema.fields) {
19
+ // This (is intended to) coalesce all record batches into a single typed array
20
+ const arrowColumn = arrowTable.getChild(field.name);
21
+ const values = arrowColumn?.toArray();
22
+ columnarTable[field.name] = values;
23
+ }
24
+
25
+ return {
26
+ shape: 'columnar-table',
27
+ schema: table.schema,
28
+ data: columnarTable
29
+ };
30
+ }
@@ -0,0 +1,65 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
4
+ import type {Feature, GeoJSONTable} from '@loaders.gl/schema';
5
+ import * as arrow from 'apache-arrow';
6
+ import type {ArrowTable} from '../lib/arrow-table';
7
+ import {serializeArrowSchema, parseGeometryFromArrow} from '@loaders.gl/arrow';
8
+ import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
9
+
10
+ /**
11
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
12
+ * From this additional conversions are available.
13
+ * @param arrowTable
14
+ * @returns
15
+ */
16
+ export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
17
+ return {
18
+ shape: 'arrow-table',
19
+ schema: serializeArrowSchema(arrowTable.schema),
20
+ data: arrowTable
21
+ };
22
+ }
23
+
24
+ /**
25
+ * Convert an Apache Arrow table to a GeoJSONTable
26
+ * @note Currently does not convert schema
27
+ */
28
+ export function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable {
29
+ const arrowTable = table.data;
30
+ const schema = serializeArrowSchema(arrowTable.schema);
31
+ const geometryColumns = getGeometryColumnsFromSchema(schema);
32
+
33
+ // get encoding from geometryColumns['geometry']
34
+ const encoding = geometryColumns.geometry.encoding;
35
+
36
+ const features: Feature[] = [];
37
+
38
+ // Remove geometry columns
39
+ const propertyColumnNames = arrowTable.schema.fields
40
+ .map((field) => field.name)
41
+ // TODO - this deletes all geometry columns
42
+ .filter((name) => !(name in geometryColumns));
43
+ const propertiesTable = arrowTable.select(propertyColumnNames);
44
+
45
+ const arrowGeometryColumn = arrowTable.getChild('geometry');
46
+
47
+ for (let row = 0; row < arrowTable.numRows; row++) {
48
+ // get the geometry value from arrow geometry column
49
+ // Note that type can vary
50
+ const arrowGeometry = arrowGeometryColumn?.get(row);
51
+ // parse arrow geometry to geojson feature
52
+ const feature = parseGeometryFromArrow(arrowGeometry, encoding);
53
+ if (feature) {
54
+ const properties = propertiesTable.get(row)?.toJSON() || {};
55
+ features.push({type: 'Feature', geometry: feature, properties});
56
+ }
57
+ }
58
+
59
+ return {
60
+ shape: 'geojson-table',
61
+ type: 'FeatureCollection',
62
+ schema: table.schema,
63
+ features
64
+ };
65
+ }