@loaders.gl/arrow 4.2.0-alpha.4 → 4.2.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/arrow-loader.d.ts +1 -1
  2. package/dist/arrow-loader.d.ts.map +1 -1
  3. package/dist/arrow-loader.js +30 -19
  4. package/dist/arrow-worker.js +4 -2
  5. package/dist/arrow-writer.d.ts +1 -1
  6. package/dist/arrow-writer.d.ts.map +1 -1
  7. package/dist/arrow-writer.js +23 -16
  8. package/dist/dist.dev.js +361 -333
  9. package/dist/dist.min.js +23 -0
  10. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +345 -288
  11. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js +151 -124
  12. package/dist/geoarrow/get-arrow-bounds.js +32 -22
  13. package/dist/geoarrow-loader.d.ts +1 -1
  14. package/dist/geoarrow-loader.d.ts.map +1 -1
  15. package/dist/geoarrow-loader.js +22 -15
  16. package/dist/geoarrow-writer.js +25 -16
  17. package/dist/index.cjs +55 -71
  18. package/dist/index.cjs.map +7 -0
  19. package/dist/index.d.ts +16 -16
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +8 -1
  22. package/dist/lib/arrow-table-batch.d.ts +1 -1
  23. package/dist/lib/arrow-table-batch.d.ts.map +1 -1
  24. package/dist/lib/arrow-table-batch.js +55 -51
  25. package/dist/lib/arrow-table.js +3 -1
  26. package/dist/lib/encode-arrow.d.ts +1 -1
  27. package/dist/lib/encode-arrow.d.ts.map +1 -1
  28. package/dist/lib/encode-arrow.js +31 -16
  29. package/dist/lib/encode-geoarrow.d.ts +1 -1
  30. package/dist/lib/encode-geoarrow.d.ts.map +1 -1
  31. package/dist/lib/encode-geoarrow.js +31 -16
  32. package/dist/parsers/parse-arrow-in-batches.d.ts +2 -2
  33. package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
  34. package/dist/parsers/parse-arrow-in-batches.js +49 -21
  35. package/dist/parsers/parse-arrow-sync.d.ts +1 -1
  36. package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
  37. package/dist/parsers/parse-arrow-sync.js +26 -22
  38. package/dist/parsers/parse-geoarrow-in-batches.d.ts +1 -1
  39. package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
  40. package/dist/parsers/parse-geoarrow-in-batches.js +7 -2
  41. package/dist/parsers/parse-geoarrow-sync.d.ts +1 -1
  42. package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
  43. package/dist/parsers/parse-geoarrow-sync.js +12 -10
  44. package/dist/schema/arrow-type-utils.js +36 -9
  45. package/dist/schema/convert-arrow-schema.js +253 -240
  46. package/dist/tables/convert-arrow-to-columnar-table.d.ts +1 -1
  47. package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
  48. package/dist/tables/convert-arrow-to-columnar-table.js +22 -13
  49. package/dist/tables/convert-arrow-to-geojson-table.d.ts +1 -1
  50. package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
  51. package/dist/tables/convert-arrow-to-geojson-table.js +47 -32
  52. package/dist/tables/convert-columnar-to-row-table.js +22 -16
  53. package/dist/tables/convert-table-to-arrow.js +58 -1
  54. package/dist/triangulate-on-worker.js +24 -19
  55. package/dist/triangulation-worker.js +6 -1
  56. package/dist/types.js +8 -6
  57. package/dist/workers/arrow-worker.js +3 -1
  58. package/dist/workers/hard-clone.js +93 -49
  59. package/dist/workers/triangulation-worker-node.d.ts +1 -1
  60. package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
  61. package/dist/workers/triangulation-worker-node.js +3 -1
  62. package/dist/workers/triangulation-worker.js +48 -48
  63. package/package.json +13 -8
  64. package/dist/arrow-loader.js.map +0 -1
  65. package/dist/arrow-writer.js.map +0 -1
  66. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +0 -1
  67. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +0 -1
  68. package/dist/geoarrow/get-arrow-bounds.js.map +0 -1
  69. package/dist/geoarrow-loader.js.map +0 -1
  70. package/dist/geoarrow-writer.js.map +0 -1
  71. package/dist/index.js.map +0 -1
  72. package/dist/lib/arrow-table-batch.js.map +0 -1
  73. package/dist/lib/arrow-table.js.map +0 -1
  74. package/dist/lib/encode-arrow.js.map +0 -1
  75. package/dist/lib/encode-geoarrow.js.map +0 -1
  76. package/dist/parsers/parse-arrow-in-batches.js.map +0 -1
  77. package/dist/parsers/parse-arrow-sync.js.map +0 -1
  78. package/dist/parsers/parse-geoarrow-in-batches.js.map +0 -1
  79. package/dist/parsers/parse-geoarrow-sync.js.map +0 -1
  80. package/dist/schema/arrow-type-utils.js.map +0 -1
  81. package/dist/schema/convert-arrow-schema.js.map +0 -1
  82. package/dist/tables/convert-arrow-to-columnar-table.js.map +0 -1
  83. package/dist/tables/convert-arrow-to-geojson-table.js.map +0 -1
  84. package/dist/tables/convert-columnar-to-row-table.js.map +0 -1
  85. package/dist/tables/convert-table-to-arrow.js.map +0 -1
  86. package/dist/triangulate-on-worker.js.map +0 -1
  87. package/dist/types.js.map +0 -1
  88. package/dist/workers/arrow-worker.js.map +0 -1
  89. package/dist/workers/hard-clone.js.map +0 -1
  90. package/dist/workers/triangulation-worker-node.js.map +0 -1
  91. package/dist/workers/triangulation-worker.js.map +0 -1
@@ -1,259 +1,272 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import * as arrow from 'apache-arrow';
5
+ /** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
2
6
  export function serializeArrowSchema(arrowSchema) {
3
- return {
4
- fields: arrowSchema.fields.map(arrowField => serializeArrowField(arrowField)),
5
- metadata: serializeArrowMetadata(arrowSchema.metadata)
6
- };
7
+ return {
8
+ fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
9
+ metadata: serializeArrowMetadata(arrowSchema.metadata)
10
+ };
7
11
  }
12
+ /** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
8
13
  export function deserializeArrowSchema(schema) {
9
- return new arrow.Schema(schema.fields.map(field => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
14
+ return new arrow.Schema(schema.fields.map((field) => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
10
15
  }
16
+ /** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
11
17
  export function serializeArrowMetadata(arrowMetadata) {
12
- return Object.fromEntries(arrowMetadata);
18
+ return Object.fromEntries(arrowMetadata);
13
19
  }
20
+ /** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
14
21
  export function deserializeArrowMetadata(metadata) {
15
- return metadata ? new Map(Object.entries(metadata)) : new Map();
22
+ return metadata ? new Map(Object.entries(metadata)) : new Map();
16
23
  }
24
+ /** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
17
25
  export function serializeArrowField(field) {
18
- return {
19
- name: field.name,
20
- type: serializeArrowType(field.type),
21
- nullable: field.nullable,
22
- metadata: serializeArrowMetadata(field.metadata)
23
- };
26
+ return {
27
+ name: field.name,
28
+ type: serializeArrowType(field.type),
29
+ nullable: field.nullable,
30
+ metadata: serializeArrowMetadata(field.metadata)
31
+ };
24
32
  }
33
+ /** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
25
34
  export function deserializeArrowField(field) {
26
- return new arrow.Field(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
35
+ return new arrow.Field(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
27
36
  }
37
+ /** Converts a serializable loaders.gl data type to hydrated arrow data type */
38
+ // eslint-disable-next-line complexity
28
39
  export function serializeArrowType(arrowType) {
29
- switch (arrowType.constructor) {
30
- case arrow.Null:
31
- return 'null';
32
- case arrow.Binary:
33
- return 'binary';
34
- case arrow.Bool:
35
- return 'bool';
36
- case arrow.Int:
37
- const intType = arrowType;
38
- return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
39
- case arrow.Int8:
40
- return 'int8';
41
- case arrow.Int16:
42
- return 'int16';
43
- case arrow.Int32:
44
- return 'int32';
45
- case arrow.Int64:
46
- return 'int64';
47
- case arrow.Uint8:
48
- return 'uint8';
49
- case arrow.Uint16:
50
- return 'uint16';
51
- case arrow.Uint32:
52
- return 'uint32';
53
- case arrow.Uint64:
54
- return 'uint64';
55
- case arrow.Float:
56
- const precision = arrowType.precision;
57
- switch (precision) {
58
- case arrow.Precision.HALF:
59
- return 'float16';
60
- case arrow.Precision.SINGLE:
61
- return 'float32';
62
- case arrow.Precision.DOUBLE:
63
- return 'float64';
40
+ switch (arrowType.constructor) {
41
+ case arrow.Null:
42
+ return 'null';
43
+ case arrow.Binary:
44
+ return 'binary';
45
+ case arrow.Bool:
46
+ return 'bool';
47
+ case arrow.Int:
48
+ const intType = arrowType;
49
+ return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
50
+ case arrow.Int8:
51
+ return 'int8';
52
+ case arrow.Int16:
53
+ return 'int16';
54
+ case arrow.Int32:
55
+ return 'int32';
56
+ case arrow.Int64:
57
+ return 'int64';
58
+ case arrow.Uint8:
59
+ return 'uint8';
60
+ case arrow.Uint16:
61
+ return 'uint16';
62
+ case arrow.Uint32:
63
+ return 'uint32';
64
+ case arrow.Uint64:
65
+ return 'uint64';
66
+ case arrow.Float:
67
+ const precision = arrowType.precision;
68
+ // return `float(precision + 1) * 16`;
69
+ switch (precision) {
70
+ case arrow.Precision.HALF:
71
+ return 'float16';
72
+ case arrow.Precision.SINGLE:
73
+ return 'float32';
74
+ case arrow.Precision.DOUBLE:
75
+ return 'float64';
76
+ default:
77
+ return 'float16';
78
+ }
79
+ case arrow.Float16:
80
+ return 'float16';
81
+ case arrow.Float32:
82
+ return 'float32';
83
+ case arrow.Float64:
84
+ return 'float64';
85
+ case arrow.Utf8:
86
+ return 'utf8';
87
+ case arrow.Decimal:
88
+ const decimal = arrowType;
89
+ return {
90
+ type: 'decimal',
91
+ bitWidth: decimal.bitWidth,
92
+ precision: decimal.precision,
93
+ scale: decimal.scale
94
+ };
95
+ case arrow.Date_:
96
+ const dateUnit = arrowType.unit;
97
+ return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
98
+ case arrow.DateDay:
99
+ return 'date-day';
100
+ case arrow.DateMillisecond:
101
+ return 'date-millisecond';
102
+ case arrow.Time:
103
+ const timeUnit = arrowType.unit;
104
+ switch (timeUnit) {
105
+ case arrow.TimeUnit.SECOND:
106
+ return 'time-second';
107
+ case arrow.TimeUnit.MILLISECOND:
108
+ return 'time-millisecond';
109
+ case arrow.TimeUnit.MICROSECOND:
110
+ return 'time-microsecond';
111
+ case arrow.TimeUnit.NANOSECOND:
112
+ return 'time-nanosecond';
113
+ default:
114
+ return 'time-second';
115
+ }
116
+ case arrow.TimeMillisecond:
117
+ return 'time-millisecond';
118
+ case arrow.TimeSecond:
119
+ return 'time-second';
120
+ case arrow.TimeMicrosecond:
121
+ return 'time-microsecond';
122
+ case arrow.TimeNanosecond:
123
+ return 'time-nanosecond';
124
+ case arrow.Timestamp:
125
+ const timeStampUnit = arrowType.unit;
126
+ switch (timeStampUnit) {
127
+ case arrow.TimeUnit.SECOND:
128
+ return 'timestamp-second';
129
+ case arrow.TimeUnit.MILLISECOND:
130
+ return 'timestamp-millisecond';
131
+ case arrow.TimeUnit.MICROSECOND:
132
+ return 'timestamp-microsecond';
133
+ case arrow.TimeUnit.NANOSECOND:
134
+ return 'timestamp-nanosecond';
135
+ default:
136
+ return 'timestamp-second';
137
+ }
138
+ case arrow.TimestampSecond:
139
+ return 'timestamp-second';
140
+ case arrow.TimestampMillisecond:
141
+ return 'timestamp-millisecond';
142
+ case arrow.TimestampMicrosecond:
143
+ return 'timestamp-microsecond';
144
+ case arrow.TimestampNanosecond:
145
+ return 'timestamp-nanosecond';
146
+ case arrow.Interval:
147
+ const intervalUnit = arrowType.unit;
148
+ switch (intervalUnit) {
149
+ case arrow.IntervalUnit.DAY_TIME:
150
+ return 'interval-daytime';
151
+ case arrow.IntervalUnit.YEAR_MONTH:
152
+ return 'interval-yearmonth';
153
+ default:
154
+ return 'interval-daytime';
155
+ }
156
+ case arrow.IntervalDayTime:
157
+ return 'interval-daytime';
158
+ case arrow.IntervalYearMonth:
159
+ return 'interval-yearmonth';
160
+ case arrow.Map_:
161
+ const mapType = arrowType;
162
+ return {
163
+ type: 'map',
164
+ keysSorted: mapType.keysSorted,
165
+ children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
166
+ };
167
+ case arrow.List:
168
+ const listType = arrowType;
169
+ const listField = listType.valueField;
170
+ return {
171
+ type: 'list',
172
+ children: [serializeArrowField(listField)]
173
+ };
174
+ case arrow.FixedSizeList:
175
+ const fixedSizeList = arrowType;
176
+ return {
177
+ type: 'fixed-size-list',
178
+ listSize: fixedSizeList.listSize,
179
+ children: [serializeArrowField(fixedSizeList.children[0])]
180
+ };
181
+ case arrow.Struct:
182
+ const structType = arrowType;
183
+ return {
184
+ type: 'struct',
185
+ children: structType.children.map((arrowField) => serializeArrowField(arrowField))
186
+ };
64
187
  default:
65
- return 'float16';
66
- }
67
- case arrow.Float16:
68
- return 'float16';
69
- case arrow.Float32:
70
- return 'float32';
71
- case arrow.Float64:
72
- return 'float64';
73
- case arrow.Utf8:
74
- return 'utf8';
75
- case arrow.Decimal:
76
- const decimal = arrowType;
77
- return {
78
- type: 'decimal',
79
- bitWidth: decimal.bitWidth,
80
- precision: decimal.precision,
81
- scale: decimal.scale
82
- };
83
- case arrow.Date_:
84
- const dateUnit = arrowType.unit;
85
- return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
86
- case arrow.DateDay:
87
- return 'date-day';
88
- case arrow.DateMillisecond:
89
- return 'date-millisecond';
90
- case arrow.Time:
91
- const timeUnit = arrowType.unit;
92
- switch (timeUnit) {
93
- case arrow.TimeUnit.SECOND:
94
- return 'time-second';
95
- case arrow.TimeUnit.MILLISECOND:
96
- return 'time-millisecond';
97
- case arrow.TimeUnit.MICROSECOND:
98
- return 'time-microsecond';
99
- case arrow.TimeUnit.NANOSECOND:
100
- return 'time-nanosecond';
101
- default:
102
- return 'time-second';
103
- }
104
- case arrow.TimeMillisecond:
105
- return 'time-millisecond';
106
- case arrow.TimeSecond:
107
- return 'time-second';
108
- case arrow.TimeMicrosecond:
109
- return 'time-microsecond';
110
- case arrow.TimeNanosecond:
111
- return 'time-nanosecond';
112
- case arrow.Timestamp:
113
- const timeStampUnit = arrowType.unit;
114
- switch (timeStampUnit) {
115
- case arrow.TimeUnit.SECOND:
116
- return 'timestamp-second';
117
- case arrow.TimeUnit.MILLISECOND:
118
- return 'timestamp-millisecond';
119
- case arrow.TimeUnit.MICROSECOND:
120
- return 'timestamp-microsecond';
121
- case arrow.TimeUnit.NANOSECOND:
122
- return 'timestamp-nanosecond';
123
- default:
124
- return 'timestamp-second';
125
- }
126
- case arrow.TimestampSecond:
127
- return 'timestamp-second';
128
- case arrow.TimestampMillisecond:
129
- return 'timestamp-millisecond';
130
- case arrow.TimestampMicrosecond:
131
- return 'timestamp-microsecond';
132
- case arrow.TimestampNanosecond:
133
- return 'timestamp-nanosecond';
134
- case arrow.Interval:
135
- const intervalUnit = arrowType.unit;
136
- switch (intervalUnit) {
137
- case arrow.IntervalUnit.DAY_TIME:
138
- return 'interval-daytime';
139
- case arrow.IntervalUnit.YEAR_MONTH:
140
- return 'interval-yearmonth';
141
- default:
142
- return 'interval-daytime';
143
- }
144
- case arrow.IntervalDayTime:
145
- return 'interval-daytime';
146
- case arrow.IntervalYearMonth:
147
- return 'interval-yearmonth';
148
- case arrow.Map_:
149
- const mapType = arrowType;
150
- return {
151
- type: 'map',
152
- keysSorted: mapType.keysSorted,
153
- children: mapType.children.map(arrowField => serializeArrowField(arrowField))
154
- };
155
- case arrow.List:
156
- const listType = arrowType;
157
- const listField = listType.valueField;
158
- return {
159
- type: 'list',
160
- children: [serializeArrowField(listField)]
161
- };
162
- case arrow.FixedSizeList:
163
- const fixedSizeList = arrowType;
164
- return {
165
- type: 'fixed-size-list',
166
- listSize: fixedSizeList.listSize,
167
- children: [serializeArrowField(fixedSizeList.children[0])]
168
- };
169
- case arrow.Struct:
170
- const structType = arrowType;
171
- return {
172
- type: 'struct',
173
- children: structType.children.map(arrowField => serializeArrowField(arrowField))
174
- };
175
- default:
176
- throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
177
- }
188
+ throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
189
+ }
178
190
  }
191
+ /** Converts a serializable loaders.gl data type to hydrated arrow data type */
192
+ // eslint-disable-next-line complexity
179
193
  export function deserializeArrowType(dataType) {
180
- if (typeof dataType === 'object') {
181
- switch (dataType.type) {
182
- case 'decimal':
183
- return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
184
- case 'map':
185
- let children = dataType.children.map(arrowField => deserializeArrowField(arrowField));
186
- return new arrow.Map_(children, dataType.keysSorted);
187
- case 'list':
188
- const field = deserializeArrowField(dataType.children[0]);
189
- return new arrow.List(field);
190
- case 'fixed-size-list':
191
- const child = deserializeArrowField(dataType.children[0]);
192
- return new arrow.FixedSizeList(dataType.listSize, child);
193
- case 'struct':
194
- children = dataType.children.map(arrowField => deserializeArrowField(arrowField));
195
- return new arrow.Struct(children);
196
- default:
197
- throw new Error('array type not supported');
194
+ if (typeof dataType === 'object') {
195
+ switch (dataType.type) {
196
+ case 'decimal':
197
+ return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
198
+ case 'map':
199
+ let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
200
+ return new arrow.Map_(children, dataType.keysSorted);
201
+ case 'list':
202
+ const field = deserializeArrowField(dataType.children[0]);
203
+ return new arrow.List(field);
204
+ case 'fixed-size-list':
205
+ const child = deserializeArrowField(dataType.children[0]);
206
+ return new arrow.FixedSizeList(dataType.listSize, child);
207
+ case 'struct':
208
+ children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
209
+ return new arrow.Struct(children);
210
+ default:
211
+ throw new Error('array type not supported');
212
+ }
213
+ }
214
+ switch (dataType) {
215
+ case 'null':
216
+ return new arrow.Null();
217
+ case 'binary':
218
+ return new arrow.Binary();
219
+ case 'bool':
220
+ return new arrow.Bool();
221
+ case 'int8':
222
+ return new arrow.Int8();
223
+ case 'int16':
224
+ return new arrow.Int16();
225
+ case 'int32':
226
+ return new arrow.Int32();
227
+ case 'int64':
228
+ return new arrow.Int64();
229
+ case 'uint8':
230
+ return new arrow.Uint8();
231
+ case 'uint16':
232
+ return new arrow.Uint16();
233
+ case 'uint32':
234
+ return new arrow.Uint32();
235
+ case 'uint64':
236
+ return new arrow.Uint64();
237
+ case 'float16':
238
+ return new arrow.Float16();
239
+ case 'float32':
240
+ return new arrow.Float32();
241
+ case 'float64':
242
+ return new arrow.Float64();
243
+ case 'utf8':
244
+ return new arrow.Utf8();
245
+ case 'date-day':
246
+ return new arrow.DateDay();
247
+ case 'date-millisecond':
248
+ return new arrow.DateMillisecond();
249
+ case 'time-second':
250
+ return new arrow.TimeSecond();
251
+ case 'time-millisecond':
252
+ return new arrow.TimeMillisecond();
253
+ case 'time-microsecond':
254
+ return new arrow.TimeMicrosecond();
255
+ case 'time-nanosecond':
256
+ return new arrow.TimeNanosecond();
257
+ case 'timestamp-second':
258
+ return new arrow.TimestampSecond();
259
+ case 'timestamp-millisecond':
260
+ return new arrow.TimestampMillisecond();
261
+ case 'timestamp-microsecond':
262
+ return new arrow.TimestampMicrosecond();
263
+ case 'timestamp-nanosecond':
264
+ return new arrow.TimestampNanosecond();
265
+ case 'interval-daytime':
266
+ return new arrow.IntervalDayTime();
267
+ case 'interval-yearmonth':
268
+ return new arrow.IntervalYearMonth();
269
+ default:
270
+ throw new Error('array type not supported');
198
271
  }
199
- }
200
- switch (dataType) {
201
- case 'null':
202
- return new arrow.Null();
203
- case 'binary':
204
- return new arrow.Binary();
205
- case 'bool':
206
- return new arrow.Bool();
207
- case 'int8':
208
- return new arrow.Int8();
209
- case 'int16':
210
- return new arrow.Int16();
211
- case 'int32':
212
- return new arrow.Int32();
213
- case 'int64':
214
- return new arrow.Int64();
215
- case 'uint8':
216
- return new arrow.Uint8();
217
- case 'uint16':
218
- return new arrow.Uint16();
219
- case 'uint32':
220
- return new arrow.Uint32();
221
- case 'uint64':
222
- return new arrow.Uint64();
223
- case 'float16':
224
- return new arrow.Float16();
225
- case 'float32':
226
- return new arrow.Float32();
227
- case 'float64':
228
- return new arrow.Float64();
229
- case 'utf8':
230
- return new arrow.Utf8();
231
- case 'date-day':
232
- return new arrow.DateDay();
233
- case 'date-millisecond':
234
- return new arrow.DateMillisecond();
235
- case 'time-second':
236
- return new arrow.TimeSecond();
237
- case 'time-millisecond':
238
- return new arrow.TimeMillisecond();
239
- case 'time-microsecond':
240
- return new arrow.TimeMicrosecond();
241
- case 'time-nanosecond':
242
- return new arrow.TimeNanosecond();
243
- case 'timestamp-second':
244
- return new arrow.TimestampSecond();
245
- case 'timestamp-millisecond':
246
- return new arrow.TimestampMillisecond();
247
- case 'timestamp-microsecond':
248
- return new arrow.TimestampMicrosecond();
249
- case 'timestamp-nanosecond':
250
- return new arrow.TimestampNanosecond();
251
- case 'interval-daytime':
252
- return new arrow.IntervalDayTime();
253
- case 'interval-yearmonth':
254
- return new arrow.IntervalYearMonth();
255
- default:
256
- throw new Error('array type not supported');
257
- }
258
272
  }
259
- //# sourceMappingURL=convert-arrow-schema.js.map
@@ -1,5 +1,5 @@
1
1
  import type { ColumnarTable } from '@loaders.gl/schema';
2
- import type { ArrowTable } from '../lib/arrow-table';
2
+ import type { ArrowTable } from "../lib/arrow-table.js";
3
3
  /**
4
4
  * Convert an Apache Arrow table to a ColumnarTable
5
5
  * @note Currently does not convert schema
@@ -1 +1 @@
1
- {"version":3,"file":"convert-arrow-to-columnar-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-columnar-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAEnD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAmB5E"}
1
+ {"version":3,"file":"convert-arrow-to-columnar-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-columnar-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAEnD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAmB5E"}
@@ -1,15 +1,24 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ /**
5
+ * Convert an Apache Arrow table to a ColumnarTable
6
+ * @note Currently does not convert schema
7
+ */
1
8
  export function convertArrowToColumnarTable(table) {
2
- const arrowTable = table.data;
3
- const columnarTable = {};
4
- for (const field of arrowTable.schema.fields) {
5
- const arrowColumn = arrowTable.getChild(field.name);
6
- const values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
7
- columnarTable[field.name] = values;
8
- }
9
- return {
10
- shape: 'columnar-table',
11
- schema: table.schema,
12
- data: columnarTable
13
- };
9
+ // TODO - avoid calling `getColumn` on columns we are not interested in?
10
+ // Add options object?
11
+ const arrowTable = table.data;
12
+ const columnarTable = {};
13
+ for (const field of arrowTable.schema.fields) {
14
+ // This (is intended to) coalesce all record batches into a single typed array
15
+ const arrowColumn = arrowTable.getChild(field.name);
16
+ const values = arrowColumn?.toArray();
17
+ columnarTable[field.name] = values;
18
+ }
19
+ return {
20
+ shape: 'columnar-table',
21
+ schema: table.schema,
22
+ data: columnarTable
23
+ };
14
24
  }
15
- //# sourceMappingURL=convert-arrow-to-columnar-table.js.map
@@ -1,6 +1,6 @@
1
1
  import type { GeoJSONTable } from '@loaders.gl/schema';
2
2
  import * as arrow from 'apache-arrow';
3
- import type { ArrowTable } from '../lib/arrow-table';
3
+ import type { ArrowTable } from "../lib/arrow-table.js";
4
4
  /**
5
5
  * Wrap an apache arrow table in a loaders.gl table wrapper.
6
6
  * From this additional conversions are available.
@@ -1 +1 @@
1
- {"version":3,"file":"convert-arrow-to-geojson-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-geojson-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAU,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAInD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAMlF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,UAAU,GAAG,YAAY,CAqC1E"}
1
+ {"version":3,"file":"convert-arrow-to-geojson-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-geojson-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAU,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAInD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAMlF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,UAAU,GAAG,YAAY,CAqC1E"}
@@ -1,39 +1,54 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { serializeArrowSchema, parseGeometryFromArrow } from '@loaders.gl/arrow';
2
5
  import { getGeometryColumnsFromSchema } from '@loaders.gl/gis';
6
+ /**
7
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
8
+ * From this additional conversions are available.
9
+ * @param arrowTable
10
+ * @returns
11
+ */
3
12
  export function convertApacheArrowToArrowTable(arrowTable) {
4
- return {
5
- shape: 'arrow-table',
6
- schema: serializeArrowSchema(arrowTable.schema),
7
- data: arrowTable
8
- };
13
+ return {
14
+ shape: 'arrow-table',
15
+ schema: serializeArrowSchema(arrowTable.schema),
16
+ data: arrowTable
17
+ };
9
18
  }
19
+ /**
20
+ * Convert an Apache Arrow table to a GeoJSONTable
21
+ * @note Currently does not convert schema
22
+ */
10
23
  export function convertArrowToGeoJSONTable(table) {
11
- const arrowTable = table.data;
12
- const schema = serializeArrowSchema(arrowTable.schema);
13
- const geometryColumns = getGeometryColumnsFromSchema(schema);
14
- const encoding = geometryColumns.geometry.encoding;
15
- const features = [];
16
- const propertyColumnNames = arrowTable.schema.fields.map(field => field.name).filter(name => !(name in geometryColumns));
17
- const propertiesTable = arrowTable.select(propertyColumnNames);
18
- const arrowGeometryColumn = arrowTable.getChild('geometry');
19
- for (let row = 0; row < arrowTable.numRows; row++) {
20
- const arrowGeometry = arrowGeometryColumn === null || arrowGeometryColumn === void 0 ? void 0 : arrowGeometryColumn.get(row);
21
- const feature = parseGeometryFromArrow(arrowGeometry, encoding);
22
- if (feature) {
23
- var _propertiesTable$get;
24
- const properties = ((_propertiesTable$get = propertiesTable.get(row)) === null || _propertiesTable$get === void 0 ? void 0 : _propertiesTable$get.toJSON()) || {};
25
- features.push({
26
- type: 'Feature',
27
- geometry: feature,
28
- properties
29
- });
24
+ const arrowTable = table.data;
25
+ const schema = serializeArrowSchema(arrowTable.schema);
26
+ const geometryColumns = getGeometryColumnsFromSchema(schema);
27
+ // get encoding from geometryColumns['geometry']
28
+ const encoding = geometryColumns.geometry.encoding;
29
+ const features = [];
30
+ // Remove geometry columns
31
+ const propertyColumnNames = arrowTable.schema.fields
32
+ .map((field) => field.name)
33
+ // TODO - this deletes all geometry columns
34
+ .filter((name) => !(name in geometryColumns));
35
+ const propertiesTable = arrowTable.select(propertyColumnNames);
36
+ const arrowGeometryColumn = arrowTable.getChild('geometry');
37
+ for (let row = 0; row < arrowTable.numRows; row++) {
38
+ // get the geometry value from arrow geometry column
39
+ // Note that type can vary
40
+ const arrowGeometry = arrowGeometryColumn?.get(row);
41
+ // parse arrow geometry to geojson feature
42
+ const feature = parseGeometryFromArrow(arrowGeometry, encoding);
43
+ if (feature) {
44
+ const properties = propertiesTable.get(row)?.toJSON() || {};
45
+ features.push({ type: 'Feature', geometry: feature, properties });
46
+ }
30
47
  }
31
- }
32
- return {
33
- shape: 'geojson-table',
34
- type: 'FeatureCollection',
35
- schema: table.schema,
36
- features
37
- };
48
+ return {
49
+ shape: 'geojson-table',
50
+ type: 'FeatureCollection',
51
+ schema: table.schema,
52
+ features
53
+ };
38
54
  }
39
- //# sourceMappingURL=convert-arrow-to-geojson-table.js.map