@loaders.gl/arrow 4.0.1 → 4.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-worker.js +33 -25
- package/dist/arrow-writer.d.ts +2 -2
- package/dist/arrow-writer.d.ts.map +1 -1
- package/dist/arrow-writer.js +6 -3
- package/dist/arrow-writer.js.map +1 -1
- package/dist/dist.dev.js +60 -29
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +18 -2
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +18 -19
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +1 -1
- package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -1
- package/dist/geoarrow/get-arrow-bounds.js +1 -1
- package/dist/geoarrow/get-arrow-bounds.js.map +1 -1
- package/dist/index.cjs +177 -137
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow-table-batch.d.ts +2 -2
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js +7 -7
- package/dist/lib/arrow-table-batch.js.map +1 -1
- package/dist/lib/arrow-table.d.ts +3 -3
- package/dist/lib/arrow-table.d.ts.map +1 -1
- package/dist/lib/arrow-table.js.map +1 -1
- package/dist/lib/encode-arrow.js +5 -5
- package/dist/lib/encode-arrow.js.map +1 -1
- package/dist/lib/parse-arrow-in-batches.js +3 -3
- package/dist/lib/parse-arrow-in-batches.js.map +1 -1
- package/dist/lib/parse-arrow-sync.js +2 -2
- package/dist/lib/parse-arrow-sync.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts +2 -2
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js +9 -9
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/convert-arrow-schema.d.ts +7 -7
- package/dist/schema/convert-arrow-schema.d.ts.map +1 -1
- package/dist/schema/convert-arrow-schema.js +113 -86
- package/dist/schema/convert-arrow-schema.js.map +1 -1
- package/dist/tables/convert-arrow-to-table.d.ts +2 -2
- package/dist/tables/convert-arrow-to-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-table.js.map +1 -1
- package/package.json +5 -5
- package/src/arrow-writer.ts +8 -5
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +26 -26
- package/src/geoarrow/convert-geoarrow-to-geojson.ts +7 -7
- package/src/geoarrow/get-arrow-bounds.ts +1 -2
- package/src/index.ts +6 -1
- package/src/lib/arrow-table-batch.ts +13 -23
- package/src/lib/arrow-table.ts +3 -3
- package/src/lib/encode-arrow.ts +8 -8
- package/src/lib/parse-arrow-in-batches.ts +4 -4
- package/src/lib/parse-arrow-sync.ts +2 -2
- package/src/schema/arrow-type-utils.ts +10 -29
- package/src/schema/convert-arrow-schema.ts +126 -145
- package/src/tables/convert-arrow-to-table.ts +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
@@ -23,6 +33,7 @@ __export(src_exports, {
|
|
|
23
33
|
ArrowLoader: () => ArrowLoader2,
|
|
24
34
|
ArrowWorkerLoader: () => ArrowLoader,
|
|
25
35
|
ArrowWriter: () => ArrowWriter,
|
|
36
|
+
BINARY_GEOMETRY_TEMPLATE: () => BINARY_GEOMETRY_TEMPLATE,
|
|
26
37
|
VECTOR_TYPES: () => VECTOR_TYPES,
|
|
27
38
|
deserializeArrowField: () => deserializeArrowField,
|
|
28
39
|
deserializeArrowMetadata: () => deserializeArrowMetadata,
|
|
@@ -34,7 +45,8 @@ __export(src_exports, {
|
|
|
34
45
|
serializeArrowField: () => serializeArrowField,
|
|
35
46
|
serializeArrowMetadata: () => serializeArrowMetadata,
|
|
36
47
|
serializeArrowSchema: () => serializeArrowSchema,
|
|
37
|
-
serializeArrowType: () => serializeArrowType
|
|
48
|
+
serializeArrowType: () => serializeArrowType,
|
|
49
|
+
updateBoundsFromGeoArrowSamples: () => updateBoundsFromGeoArrowSamples
|
|
38
50
|
});
|
|
39
51
|
module.exports = __toCommonJS(src_exports);
|
|
40
52
|
var import_schema3 = require("@loaders.gl/schema");
|
|
@@ -65,7 +77,7 @@ var ArrowLoader = {
|
|
|
65
77
|
|
|
66
78
|
// src/lib/parse-arrow-sync.ts
|
|
67
79
|
var import_schema = require("@loaders.gl/schema");
|
|
68
|
-
var
|
|
80
|
+
var arrow = __toESM(require("apache-arrow"), 1);
|
|
69
81
|
|
|
70
82
|
// src/tables/convert-arrow-to-table.ts
|
|
71
83
|
function convertApacheArrowToArrowTable(arrowTable) {
|
|
@@ -91,7 +103,7 @@ function convertArrowToColumnarTable(table) {
|
|
|
91
103
|
// src/lib/parse-arrow-sync.ts
|
|
92
104
|
function parseArrowSync(arrayBuffer, options) {
|
|
93
105
|
var _a;
|
|
94
|
-
const apacheArrowTable =
|
|
106
|
+
const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
|
|
95
107
|
const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
|
|
96
108
|
const shape = ((_a = options == null ? void 0 : options.arrow) == null ? void 0 : _a.shape) || "arrow-table";
|
|
97
109
|
switch (shape) {
|
|
@@ -108,16 +120,16 @@ function parseArrowSync(arrayBuffer, options) {
|
|
|
108
120
|
}
|
|
109
121
|
|
|
110
122
|
// src/lib/parse-arrow-in-batches.ts
|
|
111
|
-
var
|
|
123
|
+
var arrow2 = __toESM(require("apache-arrow"), 1);
|
|
112
124
|
function parseArrowInBatches(asyncIterator) {
|
|
113
125
|
async function* makeArrowAsyncIterator() {
|
|
114
|
-
const readers =
|
|
126
|
+
const readers = arrow2.RecordBatchReader.readAll(asyncIterator);
|
|
115
127
|
for await (const reader of readers) {
|
|
116
128
|
for await (const recordBatch of reader) {
|
|
117
129
|
const arrowTabledBatch = {
|
|
118
130
|
shape: "arrow-table",
|
|
119
131
|
batchType: "data",
|
|
120
|
-
data: new
|
|
132
|
+
data: new arrow2.Table([recordBatch]),
|
|
121
133
|
length: recordBatch.data.length
|
|
122
134
|
};
|
|
123
135
|
yield arrowTabledBatch;
|
|
@@ -130,7 +142,7 @@ function parseArrowInBatches(asyncIterator) {
|
|
|
130
142
|
|
|
131
143
|
// src/lib/arrow-table-batch.ts
|
|
132
144
|
var import_schema2 = require("@loaders.gl/schema");
|
|
133
|
-
var
|
|
145
|
+
var arrow3 = __toESM(require("apache-arrow"), 1);
|
|
134
146
|
var ArrowTableBatchAggregator = class extends import_schema2.ColumnarTableBatchAggregator {
|
|
135
147
|
constructor(schema, options) {
|
|
136
148
|
super(schema, options);
|
|
@@ -141,17 +153,17 @@ var ArrowTableBatchAggregator = class extends import_schema2.ColumnarTableBatchA
|
|
|
141
153
|
if (batch) {
|
|
142
154
|
this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
|
|
143
155
|
const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
|
|
144
|
-
const recordBatch = new
|
|
156
|
+
const recordBatch = new arrow3.RecordBatch(
|
|
145
157
|
this.arrowSchema,
|
|
146
|
-
|
|
147
|
-
type: new
|
|
158
|
+
arrow3.makeData({
|
|
159
|
+
type: new arrow3.Struct(this.arrowSchema.fields),
|
|
148
160
|
children: arrowVectors.map(({ data }) => data[0])
|
|
149
161
|
})
|
|
150
162
|
);
|
|
151
163
|
return {
|
|
152
164
|
shape: "arrow-table",
|
|
153
165
|
batchType: "data",
|
|
154
|
-
data: new
|
|
166
|
+
data: new arrow3.Table([recordBatch]),
|
|
155
167
|
length: batch.length
|
|
156
168
|
};
|
|
157
169
|
}
|
|
@@ -164,21 +176,21 @@ function getArrowSchema(schema) {
|
|
|
164
176
|
const field = schema[key];
|
|
165
177
|
if (field.type === Float32Array) {
|
|
166
178
|
const metadata = /* @__PURE__ */ new Map();
|
|
167
|
-
const arrowField = new
|
|
179
|
+
const arrowField = new arrow3.Field(field.name, new arrow3.Float32(), field.nullable, metadata);
|
|
168
180
|
arrowFields.push(arrowField);
|
|
169
181
|
}
|
|
170
182
|
}
|
|
171
183
|
if (arrowFields.length === 0) {
|
|
172
184
|
throw new Error("No arrow convertible fields");
|
|
173
185
|
}
|
|
174
|
-
return new
|
|
186
|
+
return new arrow3.Schema(arrowFields);
|
|
175
187
|
}
|
|
176
188
|
function getArrowVectors(arrowSchema, data) {
|
|
177
189
|
const arrowVectors = [];
|
|
178
190
|
for (const field of arrowSchema.fields) {
|
|
179
191
|
const vector = data[field.name];
|
|
180
192
|
if (vector instanceof Float32Array) {
|
|
181
|
-
const arrowVector =
|
|
193
|
+
const arrowVector = arrow3.makeVector(vector);
|
|
182
194
|
arrowVectors.push(arrowVector);
|
|
183
195
|
}
|
|
184
196
|
}
|
|
@@ -189,32 +201,32 @@ function getArrowVectors(arrowSchema, data) {
|
|
|
189
201
|
}
|
|
190
202
|
|
|
191
203
|
// src/schema/arrow-type-utils.ts
|
|
192
|
-
var
|
|
204
|
+
var arrow4 = __toESM(require("apache-arrow"), 1);
|
|
193
205
|
function getArrowType(array) {
|
|
194
206
|
switch (array.constructor) {
|
|
195
207
|
case Int8Array:
|
|
196
|
-
return new
|
|
208
|
+
return new arrow4.Int8();
|
|
197
209
|
case Uint8Array:
|
|
198
|
-
return new
|
|
210
|
+
return new arrow4.Uint8();
|
|
199
211
|
case Int16Array:
|
|
200
|
-
return new
|
|
212
|
+
return new arrow4.Int16();
|
|
201
213
|
case Uint16Array:
|
|
202
|
-
return new
|
|
214
|
+
return new arrow4.Uint16();
|
|
203
215
|
case Int32Array:
|
|
204
|
-
return new
|
|
216
|
+
return new arrow4.Int32();
|
|
205
217
|
case Uint32Array:
|
|
206
|
-
return new
|
|
218
|
+
return new arrow4.Uint32();
|
|
207
219
|
case Float32Array:
|
|
208
|
-
return new
|
|
220
|
+
return new arrow4.Float32();
|
|
209
221
|
case Float64Array:
|
|
210
|
-
return new
|
|
222
|
+
return new arrow4.Float64();
|
|
211
223
|
default:
|
|
212
224
|
throw new Error("array type not supported");
|
|
213
225
|
}
|
|
214
226
|
}
|
|
215
227
|
|
|
216
228
|
// src/schema/convert-arrow-schema.ts
|
|
217
|
-
var
|
|
229
|
+
var arrow5 = __toESM(require("apache-arrow"), 1);
|
|
218
230
|
function serializeArrowSchema(arrowSchema) {
|
|
219
231
|
return {
|
|
220
232
|
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
@@ -222,7 +234,7 @@ function serializeArrowSchema(arrowSchema) {
|
|
|
222
234
|
};
|
|
223
235
|
}
|
|
224
236
|
function deserializeArrowSchema(schema) {
|
|
225
|
-
return new
|
|
237
|
+
return new arrow5.Schema(
|
|
226
238
|
schema.fields.map((field) => deserializeArrowField(field)),
|
|
227
239
|
deserializeArrowMetadata(schema.metadata)
|
|
228
240
|
);
|
|
@@ -242,7 +254,7 @@ function serializeArrowField(field) {
|
|
|
242
254
|
};
|
|
243
255
|
}
|
|
244
256
|
function deserializeArrowField(field) {
|
|
245
|
-
return new
|
|
257
|
+
return new arrow5.Field(
|
|
246
258
|
field.name,
|
|
247
259
|
deserializeArrowType(field.type),
|
|
248
260
|
field.nullable,
|
|
@@ -251,204 +263,231 @@ function deserializeArrowField(field) {
|
|
|
251
263
|
}
|
|
252
264
|
function serializeArrowType(arrowType) {
|
|
253
265
|
switch (arrowType.constructor) {
|
|
254
|
-
case
|
|
266
|
+
case arrow5.Null:
|
|
255
267
|
return "null";
|
|
256
|
-
case
|
|
268
|
+
case arrow5.Binary:
|
|
257
269
|
return "binary";
|
|
258
|
-
case
|
|
270
|
+
case arrow5.Bool:
|
|
259
271
|
return "bool";
|
|
260
|
-
case
|
|
272
|
+
case arrow5.Int:
|
|
261
273
|
const intType = arrowType;
|
|
262
274
|
return `${intType.isSigned ? "u" : ""}int${intType.bitWidth}`;
|
|
263
|
-
case
|
|
275
|
+
case arrow5.Int8:
|
|
264
276
|
return "int8";
|
|
265
|
-
case
|
|
277
|
+
case arrow5.Int16:
|
|
266
278
|
return "int16";
|
|
267
|
-
case
|
|
279
|
+
case arrow5.Int32:
|
|
268
280
|
return "int32";
|
|
269
|
-
case
|
|
281
|
+
case arrow5.Int64:
|
|
270
282
|
return "int64";
|
|
271
|
-
case
|
|
283
|
+
case arrow5.Uint8:
|
|
272
284
|
return "uint8";
|
|
273
|
-
case
|
|
285
|
+
case arrow5.Uint16:
|
|
274
286
|
return "uint16";
|
|
275
|
-
case
|
|
287
|
+
case arrow5.Uint32:
|
|
276
288
|
return "uint32";
|
|
277
|
-
case
|
|
289
|
+
case arrow5.Uint64:
|
|
278
290
|
return "uint64";
|
|
279
|
-
case
|
|
291
|
+
case arrow5.Float:
|
|
280
292
|
const precision = arrowType.precision;
|
|
281
293
|
switch (precision) {
|
|
282
|
-
case
|
|
294
|
+
case arrow5.Precision.HALF:
|
|
283
295
|
return "float16";
|
|
284
|
-
case
|
|
296
|
+
case arrow5.Precision.SINGLE:
|
|
285
297
|
return "float32";
|
|
286
|
-
case
|
|
298
|
+
case arrow5.Precision.DOUBLE:
|
|
287
299
|
return "float64";
|
|
288
300
|
default:
|
|
289
301
|
return "float16";
|
|
290
302
|
}
|
|
291
|
-
case
|
|
303
|
+
case arrow5.Float16:
|
|
292
304
|
return "float16";
|
|
293
|
-
case
|
|
305
|
+
case arrow5.Float32:
|
|
294
306
|
return "float32";
|
|
295
|
-
case
|
|
307
|
+
case arrow5.Float64:
|
|
296
308
|
return "float64";
|
|
297
|
-
case
|
|
309
|
+
case arrow5.Utf8:
|
|
298
310
|
return "utf8";
|
|
299
|
-
case
|
|
311
|
+
case arrow5.Decimal:
|
|
312
|
+
const decimal = arrowType;
|
|
313
|
+
return {
|
|
314
|
+
type: "decimal",
|
|
315
|
+
bitWidth: decimal.bitWidth,
|
|
316
|
+
precision: decimal.precision,
|
|
317
|
+
scale: decimal.scale
|
|
318
|
+
};
|
|
319
|
+
case arrow5.Date_:
|
|
300
320
|
const dateUnit = arrowType.unit;
|
|
301
|
-
return dateUnit ===
|
|
302
|
-
case
|
|
321
|
+
return dateUnit === arrow5.DateUnit.DAY ? "date-day" : "date-millisecond";
|
|
322
|
+
case arrow5.DateDay:
|
|
303
323
|
return "date-day";
|
|
304
|
-
case
|
|
324
|
+
case arrow5.DateMillisecond:
|
|
305
325
|
return "date-millisecond";
|
|
306
|
-
case
|
|
326
|
+
case arrow5.Time:
|
|
307
327
|
const timeUnit = arrowType.unit;
|
|
308
328
|
switch (timeUnit) {
|
|
309
|
-
case
|
|
329
|
+
case arrow5.TimeUnit.SECOND:
|
|
310
330
|
return "time-second";
|
|
311
|
-
case
|
|
331
|
+
case arrow5.TimeUnit.MILLISECOND:
|
|
312
332
|
return "time-millisecond";
|
|
313
|
-
case
|
|
333
|
+
case arrow5.TimeUnit.MICROSECOND:
|
|
314
334
|
return "time-microsecond";
|
|
315
|
-
case
|
|
335
|
+
case arrow5.TimeUnit.NANOSECOND:
|
|
316
336
|
return "time-nanosecond";
|
|
317
337
|
default:
|
|
318
338
|
return "time-second";
|
|
319
339
|
}
|
|
320
|
-
case
|
|
340
|
+
case arrow5.TimeMillisecond:
|
|
321
341
|
return "time-millisecond";
|
|
322
|
-
case
|
|
342
|
+
case arrow5.TimeSecond:
|
|
323
343
|
return "time-second";
|
|
324
|
-
case
|
|
344
|
+
case arrow5.TimeMicrosecond:
|
|
325
345
|
return "time-microsecond";
|
|
326
|
-
case
|
|
346
|
+
case arrow5.TimeNanosecond:
|
|
327
347
|
return "time-nanosecond";
|
|
328
|
-
case
|
|
348
|
+
case arrow5.Timestamp:
|
|
329
349
|
const timeStampUnit = arrowType.unit;
|
|
330
350
|
switch (timeStampUnit) {
|
|
331
|
-
case
|
|
351
|
+
case arrow5.TimeUnit.SECOND:
|
|
332
352
|
return "timestamp-second";
|
|
333
|
-
case
|
|
353
|
+
case arrow5.TimeUnit.MILLISECOND:
|
|
334
354
|
return "timestamp-millisecond";
|
|
335
|
-
case
|
|
355
|
+
case arrow5.TimeUnit.MICROSECOND:
|
|
336
356
|
return "timestamp-microsecond";
|
|
337
|
-
case
|
|
357
|
+
case arrow5.TimeUnit.NANOSECOND:
|
|
338
358
|
return "timestamp-nanosecond";
|
|
339
359
|
default:
|
|
340
360
|
return "timestamp-second";
|
|
341
361
|
}
|
|
342
|
-
case
|
|
362
|
+
case arrow5.TimestampSecond:
|
|
343
363
|
return "timestamp-second";
|
|
344
|
-
case
|
|
364
|
+
case arrow5.TimestampMillisecond:
|
|
345
365
|
return "timestamp-millisecond";
|
|
346
|
-
case
|
|
366
|
+
case arrow5.TimestampMicrosecond:
|
|
347
367
|
return "timestamp-microsecond";
|
|
348
|
-
case
|
|
368
|
+
case arrow5.TimestampNanosecond:
|
|
349
369
|
return "timestamp-nanosecond";
|
|
350
|
-
case
|
|
370
|
+
case arrow5.Interval:
|
|
351
371
|
const intervalUnit = arrowType.unit;
|
|
352
372
|
switch (intervalUnit) {
|
|
353
|
-
case
|
|
373
|
+
case arrow5.IntervalUnit.DAY_TIME:
|
|
354
374
|
return "interval-daytime";
|
|
355
|
-
case
|
|
375
|
+
case arrow5.IntervalUnit.YEAR_MONTH:
|
|
356
376
|
return "interval-yearmonth";
|
|
357
377
|
default:
|
|
358
378
|
return "interval-daytime";
|
|
359
379
|
}
|
|
360
|
-
case
|
|
380
|
+
case arrow5.IntervalDayTime:
|
|
361
381
|
return "interval-daytime";
|
|
362
|
-
case
|
|
382
|
+
case arrow5.IntervalYearMonth:
|
|
363
383
|
return "interval-yearmonth";
|
|
364
|
-
case
|
|
384
|
+
case arrow5.Map_:
|
|
385
|
+
const mapType = arrowType;
|
|
386
|
+
return {
|
|
387
|
+
type: "map",
|
|
388
|
+
keysSorted: mapType.keysSorted,
|
|
389
|
+
children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
390
|
+
};
|
|
391
|
+
case arrow5.List:
|
|
365
392
|
const listType = arrowType;
|
|
366
393
|
const listField = listType.valueField;
|
|
367
394
|
return {
|
|
368
395
|
type: "list",
|
|
369
396
|
children: [serializeArrowField(listField)]
|
|
370
397
|
};
|
|
371
|
-
case
|
|
398
|
+
case arrow5.FixedSizeList:
|
|
399
|
+
const fixedSizeList = arrowType;
|
|
372
400
|
return {
|
|
373
401
|
type: "fixed-size-list",
|
|
374
|
-
listSize:
|
|
375
|
-
children: [serializeArrowField(
|
|
402
|
+
listSize: fixedSizeList.listSize,
|
|
403
|
+
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
404
|
+
};
|
|
405
|
+
case arrow5.Struct:
|
|
406
|
+
const structType = arrowType;
|
|
407
|
+
return {
|
|
408
|
+
type: "struct",
|
|
409
|
+
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
376
410
|
};
|
|
377
411
|
default:
|
|
378
|
-
throw new Error(
|
|
412
|
+
throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
|
|
379
413
|
}
|
|
380
414
|
}
|
|
381
415
|
function deserializeArrowType(dataType) {
|
|
382
416
|
if (typeof dataType === "object") {
|
|
383
417
|
switch (dataType.type) {
|
|
418
|
+
case "decimal":
|
|
419
|
+
return new arrow5.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
|
|
420
|
+
case "map":
|
|
421
|
+
let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
|
|
422
|
+
return new arrow5.Map_(children, dataType.keysSorted);
|
|
384
423
|
case "list":
|
|
385
424
|
const field = deserializeArrowField(dataType.children[0]);
|
|
386
|
-
return new
|
|
425
|
+
return new arrow5.List(field);
|
|
387
426
|
case "fixed-size-list":
|
|
388
427
|
const child = deserializeArrowField(dataType.children[0]);
|
|
389
|
-
return new
|
|
428
|
+
return new arrow5.FixedSizeList(dataType.listSize, child);
|
|
390
429
|
case "struct":
|
|
391
|
-
|
|
392
|
-
return new
|
|
430
|
+
children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
|
|
431
|
+
return new arrow5.Struct(children);
|
|
393
432
|
default:
|
|
394
433
|
throw new Error("array type not supported");
|
|
395
434
|
}
|
|
396
435
|
}
|
|
397
436
|
switch (dataType) {
|
|
398
437
|
case "null":
|
|
399
|
-
return new
|
|
438
|
+
return new arrow5.Null();
|
|
400
439
|
case "binary":
|
|
401
|
-
return new
|
|
440
|
+
return new arrow5.Binary();
|
|
402
441
|
case "bool":
|
|
403
|
-
return new
|
|
442
|
+
return new arrow5.Bool();
|
|
404
443
|
case "int8":
|
|
405
|
-
return new
|
|
444
|
+
return new arrow5.Int8();
|
|
406
445
|
case "int16":
|
|
407
|
-
return new
|
|
446
|
+
return new arrow5.Int16();
|
|
408
447
|
case "int32":
|
|
409
|
-
return new
|
|
448
|
+
return new arrow5.Int32();
|
|
410
449
|
case "int64":
|
|
411
|
-
return new
|
|
450
|
+
return new arrow5.Int64();
|
|
412
451
|
case "uint8":
|
|
413
|
-
return new
|
|
452
|
+
return new arrow5.Uint8();
|
|
414
453
|
case "uint16":
|
|
415
|
-
return new
|
|
454
|
+
return new arrow5.Uint16();
|
|
416
455
|
case "uint32":
|
|
417
|
-
return new
|
|
456
|
+
return new arrow5.Uint32();
|
|
418
457
|
case "uint64":
|
|
419
|
-
return new
|
|
458
|
+
return new arrow5.Uint64();
|
|
420
459
|
case "float16":
|
|
421
|
-
return new
|
|
460
|
+
return new arrow5.Float16();
|
|
422
461
|
case "float32":
|
|
423
|
-
return new
|
|
462
|
+
return new arrow5.Float32();
|
|
424
463
|
case "float64":
|
|
425
|
-
return new
|
|
464
|
+
return new arrow5.Float64();
|
|
426
465
|
case "utf8":
|
|
427
|
-
return new
|
|
466
|
+
return new arrow5.Utf8();
|
|
428
467
|
case "date-day":
|
|
429
|
-
return new
|
|
468
|
+
return new arrow5.DateDay();
|
|
430
469
|
case "date-millisecond":
|
|
431
|
-
return new
|
|
470
|
+
return new arrow5.DateMillisecond();
|
|
432
471
|
case "time-second":
|
|
433
|
-
return new
|
|
472
|
+
return new arrow5.TimeSecond();
|
|
434
473
|
case "time-millisecond":
|
|
435
|
-
return new
|
|
474
|
+
return new arrow5.TimeMillisecond();
|
|
436
475
|
case "time-microsecond":
|
|
437
|
-
return new
|
|
476
|
+
return new arrow5.TimeMicrosecond();
|
|
438
477
|
case "time-nanosecond":
|
|
439
|
-
return new
|
|
478
|
+
return new arrow5.TimeNanosecond();
|
|
440
479
|
case "timestamp-second":
|
|
441
|
-
return new
|
|
480
|
+
return new arrow5.TimestampSecond();
|
|
442
481
|
case "timestamp-millisecond":
|
|
443
|
-
return new
|
|
482
|
+
return new arrow5.TimestampMillisecond();
|
|
444
483
|
case "timestamp-microsecond":
|
|
445
|
-
return new
|
|
484
|
+
return new arrow5.TimestampMicrosecond();
|
|
446
485
|
case "timestamp-nanosecond":
|
|
447
|
-
return new
|
|
486
|
+
return new arrow5.TimestampNanosecond();
|
|
448
487
|
case "interval-daytime":
|
|
449
|
-
return new
|
|
488
|
+
return new arrow5.IntervalDayTime();
|
|
450
489
|
case "interval-yearmonth":
|
|
451
|
-
return new
|
|
490
|
+
return new arrow5.IntervalYearMonth();
|
|
452
491
|
default:
|
|
453
492
|
throw new Error("array type not supported");
|
|
454
493
|
}
|
|
@@ -462,24 +501,24 @@ var VECTOR_TYPES = /* @__PURE__ */ ((VECTOR_TYPES2) => {
|
|
|
462
501
|
})(VECTOR_TYPES || {});
|
|
463
502
|
|
|
464
503
|
// src/lib/encode-arrow.ts
|
|
465
|
-
var
|
|
504
|
+
var arrow6 = __toESM(require("apache-arrow"), 1);
|
|
466
505
|
function encodeArrowSync(data) {
|
|
467
506
|
const vectors = {};
|
|
468
507
|
for (const arrayData of data) {
|
|
469
508
|
const arrayVector = createVector(arrayData.array, arrayData.type);
|
|
470
509
|
vectors[arrayData.name] = arrayVector;
|
|
471
510
|
}
|
|
472
|
-
const table = new
|
|
473
|
-
const arrowBuffer =
|
|
511
|
+
const table = new arrow6.Table(vectors);
|
|
512
|
+
const arrowBuffer = arrow6.tableToIPC(table);
|
|
474
513
|
return arrowBuffer;
|
|
475
514
|
}
|
|
476
515
|
function createVector(array, type) {
|
|
477
516
|
switch (type) {
|
|
478
517
|
case 1 /* DATE */:
|
|
479
|
-
return
|
|
518
|
+
return arrow6.vectorFromArray(array);
|
|
480
519
|
case 0 /* FLOAT */:
|
|
481
520
|
default:
|
|
482
|
-
return
|
|
521
|
+
return arrow6.vectorFromArray(array);
|
|
483
522
|
}
|
|
484
523
|
}
|
|
485
524
|
|
|
@@ -496,11 +535,14 @@ var ArrowWriter = {
|
|
|
496
535
|
"application/vnd.apache.arrow.stream",
|
|
497
536
|
"application/octet-stream"
|
|
498
537
|
],
|
|
499
|
-
|
|
538
|
+
binary: true,
|
|
539
|
+
options: {},
|
|
540
|
+
encode: async function encodeArrow(data, options) {
|
|
500
541
|
return encodeArrowSync(data);
|
|
501
542
|
},
|
|
502
|
-
|
|
503
|
-
|
|
543
|
+
encodeSync(data, options) {
|
|
544
|
+
return encodeArrowSync(data);
|
|
545
|
+
}
|
|
504
546
|
};
|
|
505
547
|
|
|
506
548
|
// src/geoarrow/get-arrow-bounds.ts
|
|
@@ -511,7 +553,7 @@ function updateBoundsFromGeoArrowSamples(flatCoords, nDim, bounds, sampleSize =
|
|
|
511
553
|
for (let i = 0; i < numberOfFeatures; i += sampleStep) {
|
|
512
554
|
const lng = flatCoords[i * nDim];
|
|
513
555
|
const lat = flatCoords[i * nDim + 1];
|
|
514
|
-
if (lng <
|
|
556
|
+
if (lng < newBounds[0]) {
|
|
515
557
|
newBounds[0] = lng;
|
|
516
558
|
}
|
|
517
559
|
if (lat < newBounds[1]) {
|
|
@@ -542,18 +584,16 @@ function getBinaryGeometriesFromArrow(geoColumn, geoEncoding) {
|
|
|
542
584
|
line: geoEncoding === "geoarrow.multilinestring" || geoEncoding === "geoarrow.linestring"
|
|
543
585
|
};
|
|
544
586
|
const chunks = geoColumn.data;
|
|
545
|
-
|
|
587
|
+
let bounds = [Infinity, Infinity, -Infinity, -Infinity];
|
|
546
588
|
let globalFeatureIdOffset = 0;
|
|
547
589
|
const binaryGeometries = [];
|
|
548
|
-
|
|
549
|
-
const geometries = chunks[c];
|
|
590
|
+
chunks.forEach((chunk) => {
|
|
550
591
|
const { featureIds, flatCoordinateArray, nDim, geomOffset } = getBinaryGeometriesFromChunk(
|
|
551
|
-
|
|
592
|
+
chunk,
|
|
552
593
|
geoEncoding
|
|
553
594
|
);
|
|
554
|
-
const
|
|
555
|
-
|
|
556
|
-
for (let i = 0; i < numOfVertices; i++) {
|
|
595
|
+
const globalFeatureIds = new Uint32Array(featureIds.length);
|
|
596
|
+
for (let i = 0; i < featureIds.length; i++) {
|
|
557
597
|
globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
|
|
558
598
|
}
|
|
559
599
|
const binaryContent = {
|
|
@@ -563,12 +603,11 @@ function getBinaryGeometriesFromArrow(geoColumn, geoEncoding) {
|
|
|
563
603
|
size: nDim
|
|
564
604
|
},
|
|
565
605
|
featureIds: { value: featureIds, size: 1 },
|
|
566
|
-
|
|
567
|
-
properties: [...Array(geometries.length).keys()].map((i) => ({
|
|
606
|
+
properties: [...Array(chunk.length).keys()].map((i) => ({
|
|
568
607
|
index: i + globalFeatureIdOffset
|
|
569
608
|
}))
|
|
570
609
|
};
|
|
571
|
-
globalFeatureIdOffset +=
|
|
610
|
+
globalFeatureIdOffset += chunk.length;
|
|
572
611
|
binaryGeometries.push({
|
|
573
612
|
shape: "binary-feature-collection",
|
|
574
613
|
points: {
|
|
@@ -598,8 +637,8 @@ function getBinaryGeometriesFromArrow(geoColumn, geoEncoding) {
|
|
|
598
637
|
}
|
|
599
638
|
}
|
|
600
639
|
});
|
|
601
|
-
updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
|
|
602
|
-
}
|
|
640
|
+
bounds = updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
|
|
641
|
+
});
|
|
603
642
|
return { binaryGeometries, bounds, featureTypes };
|
|
604
643
|
}
|
|
605
644
|
function getBinaryGeometriesFromChunk(chunk, geoEncoding) {
|
|
@@ -620,22 +659,23 @@ function getBinaryGeometriesFromChunk(chunk, geoEncoding) {
|
|
|
620
659
|
function getBinaryPolygonsFromChunk(chunk, geoEncoding) {
|
|
621
660
|
const isMultiPolygon = geoEncoding === "geoarrow.multipolygon";
|
|
622
661
|
const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
|
|
662
|
+
const polygonOffset = polygonData.valueOffsets;
|
|
663
|
+
const partData = isMultiPolygon ? chunk.valueOffsets.map((i) => polygonOffset.at(i) || i) : chunk.valueOffsets;
|
|
623
664
|
const ringData = polygonData.children[0];
|
|
624
665
|
const pointData = ringData.children[0];
|
|
625
666
|
const coordData = pointData.children[0];
|
|
626
667
|
const nDim = pointData.stride;
|
|
627
668
|
const geomOffset = ringData.valueOffsets;
|
|
628
669
|
const flatCoordinateArray = coordData.values;
|
|
629
|
-
const geometryIndicies = new Uint16Array(
|
|
630
|
-
for (let i = 0; i <
|
|
631
|
-
geometryIndicies[i] = geomOffset[
|
|
670
|
+
const geometryIndicies = new Uint16Array(polygonOffset.length);
|
|
671
|
+
for (let i = 0; i < polygonOffset.length; i++) {
|
|
672
|
+
geometryIndicies[i] = geomOffset[polygonOffset[i]];
|
|
632
673
|
}
|
|
633
|
-
geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
|
|
634
674
|
const numOfVertices = flatCoordinateArray.length / nDim;
|
|
635
675
|
const featureIds = new Uint32Array(numOfVertices);
|
|
636
|
-
for (let i = 0; i <
|
|
637
|
-
const startIdx = geomOffset[
|
|
638
|
-
const endIdx = geomOffset[
|
|
676
|
+
for (let i = 0; i < partData.length - 1; i++) {
|
|
677
|
+
const startIdx = geomOffset[partData[i]];
|
|
678
|
+
const endIdx = geomOffset[partData[i + 1]];
|
|
639
679
|
for (let j = startIdx; j < endIdx; j++) {
|
|
640
680
|
featureIds[j] = i;
|
|
641
681
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { ArrowWorkerLoader };
|
|
|
14
14
|
export declare const ArrowLoader: LoaderWithParser<ArrowTable | ColumnarTable | ObjectRowTable, ArrowTableBatch, ArrowLoaderOptions>;
|
|
15
15
|
export type { GeoArrowEncoding } from '@loaders.gl/gis';
|
|
16
16
|
export type { BinaryDataFromGeoArrow } from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
17
|
-
export { getBinaryGeometriesFromArrow } from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
17
|
+
export { BINARY_GEOMETRY_TEMPLATE, getBinaryGeometriesFromArrow } from './geoarrow/convert-geoarrow-to-binary-geometry';
|
|
18
18
|
export { parseGeometryFromArrow } from './geoarrow/convert-geoarrow-to-geojson';
|
|
19
|
+
export { updateBoundsFromGeoArrowSamples } from './geoarrow/get-arrow-bounds';
|
|
19
20
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,eAAe,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAGlD,OAAO,EAAC,WAAW,IAAI,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAWhE,OAAO,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAIvD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAGvC,YAAY,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAIrC,OAAO,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAI3C,YAAY,EAAC,kBAAkB,EAAC,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,CAAC;AAE3B,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,EAC3C,eAAe,EACf,kBAAkB,CAOnB,CAAC;AAGF,YAAY,EAAC,gBAAgB,EAAC,MAAM,iBAAiB,CAAC;AAItD,YAAY,EAAC,sBAAsB,EAAC,MAAM,gDAAgD,CAAC;AAC3F,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,eAAe,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAGlD,OAAO,EAAC,WAAW,IAAI,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAWhE,OAAO,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAIvD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAGvC,YAAY,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAIrC,OAAO,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAI3C,YAAY,EAAC,kBAAkB,EAAC,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,CAAC;AAE3B,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,EAC3C,eAAe,EACf,kBAAkB,CAOnB,CAAC;AAGF,YAAY,EAAC,gBAAgB,EAAC,MAAM,iBAAiB,CAAC;AAItD,YAAY,EAAC,sBAAsB,EAAC,MAAM,gDAAgD,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,4BAA4B,EAC7B,MAAM,gDAAgD,CAAC;AAExD,OAAO,EAAC,sBAAsB,EAAC,MAAM,wCAAwC,CAAC;AAE9E,OAAO,EAAC,+BAA+B,EAAC,MAAM,6BAA6B,CAAC"}
|