@loaders.gl/arrow 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts +1 -1
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js +30 -19
- package/dist/arrow-worker.js +4 -2
- package/dist/arrow-writer.d.ts +1 -1
- package/dist/arrow-writer.d.ts.map +1 -1
- package/dist/arrow-writer.js +23 -16
- package/dist/dist.dev.js +361 -333
- package/dist/dist.min.js +23 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +345 -288
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js +151 -124
- package/dist/geoarrow/get-arrow-bounds.js +32 -22
- package/dist/geoarrow-loader.d.ts +1 -1
- package/dist/geoarrow-loader.d.ts.map +1 -1
- package/dist/geoarrow-loader.js +22 -15
- package/dist/geoarrow-writer.js +25 -16
- package/dist/index.cjs +55 -71
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +16 -16
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/lib/arrow-table-batch.d.ts +1 -1
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js +55 -51
- package/dist/lib/arrow-table.js +3 -1
- package/dist/lib/encode-arrow.d.ts +1 -1
- package/dist/lib/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encode-arrow.js +31 -16
- package/dist/lib/encode-geoarrow.d.ts +1 -1
- package/dist/lib/encode-geoarrow.d.ts.map +1 -1
- package/dist/lib/encode-geoarrow.js +31 -16
- package/dist/parsers/parse-arrow-in-batches.d.ts +2 -2
- package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.js +49 -21
- package/dist/parsers/parse-arrow-sync.d.ts +1 -1
- package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-sync.js +26 -22
- package/dist/parsers/parse-geoarrow-in-batches.d.ts +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.js +7 -2
- package/dist/parsers/parse-geoarrow-sync.d.ts +1 -1
- package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.js +12 -10
- package/dist/schema/arrow-type-utils.js +36 -9
- package/dist/schema/convert-arrow-schema.js +253 -240
- package/dist/tables/convert-arrow-to-columnar-table.d.ts +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.js +22 -13
- package/dist/tables/convert-arrow-to-geojson-table.d.ts +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.js +47 -32
- package/dist/tables/convert-columnar-to-row-table.js +22 -16
- package/dist/tables/convert-table-to-arrow.js +58 -1
- package/dist/triangulate-on-worker.js +24 -19
- package/dist/triangulation-worker.js +6 -1
- package/dist/types.js +8 -6
- package/dist/workers/arrow-worker.js +3 -1
- package/dist/workers/hard-clone.js +93 -49
- package/dist/workers/triangulation-worker-node.d.ts +1 -1
- package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
- package/dist/workers/triangulation-worker-node.js +3 -1
- package/dist/workers/triangulation-worker.js +48 -48
- package/package.json +13 -8
- package/dist/arrow-loader.js.map +0 -1
- package/dist/arrow-writer.js.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +0 -1
- package/dist/geoarrow/get-arrow-bounds.js.map +0 -1
- package/dist/geoarrow-loader.js.map +0 -1
- package/dist/geoarrow-writer.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/arrow-table-batch.js.map +0 -1
- package/dist/lib/arrow-table.js.map +0 -1
- package/dist/lib/encode-arrow.js.map +0 -1
- package/dist/lib/encode-geoarrow.js.map +0 -1
- package/dist/parsers/parse-arrow-in-batches.js.map +0 -1
- package/dist/parsers/parse-arrow-sync.js.map +0 -1
- package/dist/parsers/parse-geoarrow-in-batches.js.map +0 -1
- package/dist/parsers/parse-geoarrow-sync.js.map +0 -1
- package/dist/schema/arrow-type-utils.js.map +0 -1
- package/dist/schema/convert-arrow-schema.js.map +0 -1
- package/dist/tables/convert-arrow-to-columnar-table.js.map +0 -1
- package/dist/tables/convert-arrow-to-geojson-table.js.map +0 -1
- package/dist/tables/convert-columnar-to-row-table.js.map +0 -1
- package/dist/tables/convert-table-to-arrow.js.map +0 -1
- package/dist/triangulate-on-worker.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/workers/arrow-worker.js.map +0 -1
- package/dist/workers/hard-clone.js.map +0 -1
- package/dist/workers/triangulation-worker-node.js.map +0 -1
- package/dist/workers/triangulation-worker.js.map +0 -1
|
@@ -1,62 +1,66 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
|
|
2
5
|
import * as arrow from 'apache-arrow';
|
|
3
6
|
export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
7
|
+
arrowSchema;
|
|
8
|
+
constructor(schema, options) {
|
|
9
|
+
super(schema, options);
|
|
10
|
+
this.arrowSchema = null;
|
|
11
|
+
}
|
|
12
|
+
getBatch() {
|
|
13
|
+
const batch = super.getBatch();
|
|
14
|
+
if (batch) {
|
|
15
|
+
// Get the arrow schema
|
|
16
|
+
this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
|
|
17
|
+
// Get arrow format vectors
|
|
18
|
+
const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
|
|
19
|
+
// Create the record batch
|
|
20
|
+
const recordBatch = new arrow.RecordBatch(this.arrowSchema, arrow.makeData({
|
|
21
|
+
type: new arrow.Struct(this.arrowSchema.fields),
|
|
22
|
+
children: arrowVectors.map(({ data }) => data[0])
|
|
23
|
+
}));
|
|
24
|
+
return {
|
|
25
|
+
shape: 'arrow-table',
|
|
26
|
+
batchType: 'data',
|
|
27
|
+
data: new arrow.Table([recordBatch]),
|
|
28
|
+
length: batch.length
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
29
32
|
}
|
|
30
|
-
return null;
|
|
31
|
-
}
|
|
32
33
|
}
|
|
34
|
+
// Convert from a simple loaders.gl schema to an Arrow schema
|
|
33
35
|
function getArrowSchema(schema) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
const arrowFields = [];
|
|
37
|
+
for (const key in schema) {
|
|
38
|
+
const field = schema[key];
|
|
39
|
+
if (field.type === Float32Array) {
|
|
40
|
+
// TODO - just store the original field as metadata?
|
|
41
|
+
const metadata = new Map(); // field;
|
|
42
|
+
// arrow: new arrow.Field(name, nullable, metadata)
|
|
43
|
+
const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
|
|
44
|
+
arrowFields.push(arrowField);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (arrowFields.length === 0) {
|
|
48
|
+
throw new Error('No arrow convertible fields');
|
|
41
49
|
}
|
|
42
|
-
|
|
43
|
-
if (arrowFields.length === 0) {
|
|
44
|
-
throw new Error('No arrow convertible fields');
|
|
45
|
-
}
|
|
46
|
-
return new arrow.Schema(arrowFields);
|
|
50
|
+
return new arrow.Schema(arrowFields);
|
|
47
51
|
}
|
|
52
|
+
// Convert from simple loaders.gl arrays to arrow vectors
|
|
48
53
|
function getArrowVectors(arrowSchema, data) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
54
|
+
const arrowVectors = [];
|
|
55
|
+
for (const field of arrowSchema.fields) {
|
|
56
|
+
const vector = data[field.name];
|
|
57
|
+
if (vector instanceof Float32Array) {
|
|
58
|
+
const arrowVector = arrow.makeVector(vector);
|
|
59
|
+
arrowVectors.push(arrowVector);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (arrowSchema.fields.length !== arrowVectors.length) {
|
|
63
|
+
throw new Error('Some columns not arrow convertible');
|
|
55
64
|
}
|
|
56
|
-
|
|
57
|
-
if (arrowSchema.fields.length !== arrowVectors.length) {
|
|
58
|
-
throw new Error('Some columns not arrow convertible');
|
|
59
|
-
}
|
|
60
|
-
return arrowVectors;
|
|
65
|
+
return arrowVectors;
|
|
61
66
|
}
|
|
62
|
-
//# sourceMappingURL=arrow-table-batch.js.map
|
package/dist/lib/arrow-table.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,
|
|
1
|
+
{"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAShE"}
|
package/dist/lib/encode-arrow.js
CHANGED
|
@@ -1,22 +1,37 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import * as arrow from 'apache-arrow';
|
|
2
5
|
import { VECTOR_TYPES } from "../types.js";
|
|
6
|
+
/**
|
|
7
|
+
* Encodes set of arrays into the Apache Arrow columnar format
|
|
8
|
+
* https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
|
|
9
|
+
* @param data - columns data
|
|
10
|
+
* @param options - the writer options
|
|
11
|
+
* @returns - encoded ArrayBuffer
|
|
12
|
+
*/
|
|
3
13
|
export function encodeArrowSync(data) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
14
|
+
const vectors = {};
|
|
15
|
+
for (const arrayData of data) {
|
|
16
|
+
const arrayVector = createVector(arrayData.array, arrayData.type);
|
|
17
|
+
vectors[arrayData.name] = arrayVector;
|
|
18
|
+
}
|
|
19
|
+
const table = new arrow.Table(vectors);
|
|
20
|
+
const arrowBuffer = arrow.tableToIPC(table);
|
|
21
|
+
return arrowBuffer;
|
|
12
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Create Arrow arrow.Vector from given data and vector type
|
|
25
|
+
* @param array {import('../types').AnyArrayType} - columns data
|
|
26
|
+
* @param type {number} - the writer options
|
|
27
|
+
* @return a vector of one of vector's types defined in the Apache Arrow library
|
|
28
|
+
*/
|
|
13
29
|
function createVector(array, type) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
30
|
+
switch (type) {
|
|
31
|
+
case VECTOR_TYPES.DATE:
|
|
32
|
+
return arrow.vectorFromArray(array);
|
|
33
|
+
case VECTOR_TYPES.FLOAT:
|
|
34
|
+
default:
|
|
35
|
+
return arrow.vectorFromArray(array);
|
|
36
|
+
}
|
|
21
37
|
}
|
|
22
|
-
//# sourceMappingURL=encode-arrow.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,
|
|
1
|
+
{"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CASnE"}
|
|
@@ -1,22 +1,37 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import * as arrow from 'apache-arrow';
|
|
2
5
|
import { VECTOR_TYPES } from "../types.js";
|
|
6
|
+
/**
|
|
7
|
+
* Encodes set of arrays into the Apache Arrow columnar format
|
|
8
|
+
* https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
|
|
9
|
+
* @param data - columns data
|
|
10
|
+
* @param options - the writer options
|
|
11
|
+
* @returns - encoded ArrayBuffer
|
|
12
|
+
*/
|
|
3
13
|
export function encodeGeoArrowSync(data) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
14
|
+
const vectors = {};
|
|
15
|
+
for (const arrayData of data) {
|
|
16
|
+
const arrayVector = createVector(arrayData.array, arrayData.type);
|
|
17
|
+
vectors[arrayData.name] = arrayVector;
|
|
18
|
+
}
|
|
19
|
+
const table = new arrow.Table(vectors);
|
|
20
|
+
const arrowBuffer = arrow.tableToIPC(table);
|
|
21
|
+
return arrowBuffer;
|
|
12
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Create Arrow arrow.Vector from given data and vector type
|
|
25
|
+
* @param array {import('../types').AnyArrayType} - columns data
|
|
26
|
+
* @param type {number} - the writer options
|
|
27
|
+
* @return a vector of one of vector's types defined in the Apache Arrow library
|
|
28
|
+
*/
|
|
13
29
|
function createVector(array, type) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
30
|
+
switch (type) {
|
|
31
|
+
case VECTOR_TYPES.DATE:
|
|
32
|
+
return arrow.vectorFromArray(array);
|
|
33
|
+
case VECTOR_TYPES.FLOAT:
|
|
34
|
+
default:
|
|
35
|
+
return arrow.vectorFromArray(array);
|
|
36
|
+
}
|
|
21
37
|
}
|
|
22
|
-
//# sourceMappingURL=encode-geoarrow.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ArrowTableBatch } from
|
|
2
|
-
import { ArrowLoaderOptions } from
|
|
1
|
+
import type { ArrowTableBatch } from "../lib/arrow-table.js";
|
|
2
|
+
import { ArrowLoaderOptions } from "../arrow-loader.js";
|
|
3
3
|
/**
|
|
4
4
|
*/
|
|
5
5
|
export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: ArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,8BAA2B;AAExD,OAAO,EAAC,kBAAkB,EAAC,2BAAwB;AAGnD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC,eAAe,CAAC,CA0ChC"}
|
|
@@ -1,27 +1,55 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import * as arrow from 'apache-arrow';
|
|
5
|
+
// import {isIterable} from '@loaders.gl/core';
|
|
6
|
+
/**
|
|
7
|
+
*/
|
|
2
8
|
export function parseArrowInBatches(asyncIterator, options) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
9
|
+
// Creates the appropriate arrow.RecordBatchReader subclasses from the input
|
|
10
|
+
// This will also close the underlying source in case of early termination or errors
|
|
11
|
+
// As an optimization, return a non-async iterator
|
|
12
|
+
/*
|
|
13
|
+
if (isIterable(readers)) {
|
|
14
|
+
function* makeArrowIterator() {
|
|
15
|
+
for (const reader of readers) {
|
|
16
|
+
for (const batch of reader) {
|
|
17
|
+
yield processBatch(batch, reader);
|
|
18
|
+
}
|
|
19
|
+
break; // only processing one stream of batches
|
|
13
20
|
}
|
|
14
|
-
const arrowTabledBatch = {
|
|
15
|
-
shape: 'arrow-table',
|
|
16
|
-
batchType: 'data',
|
|
17
|
-
data: new arrow.Table([recordBatch]),
|
|
18
|
-
length: recordBatch.data.length
|
|
19
|
-
};
|
|
20
|
-
yield arrowTabledBatch;
|
|
21
21
|
}
|
|
22
|
-
|
|
22
|
+
const arrowIterator = makeArrowIterator();
|
|
23
23
|
}
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
*/
|
|
25
|
+
async function* makeArrowAsyncIterator() {
|
|
26
|
+
// @ts-ignore
|
|
27
|
+
const readers = arrow.RecordBatchReader.readAll(asyncIterator);
|
|
28
|
+
for await (const reader of readers) {
|
|
29
|
+
for await (const recordBatch of reader) {
|
|
30
|
+
// use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
|
|
31
|
+
if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
|
|
32
|
+
await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
|
|
33
|
+
}
|
|
34
|
+
const arrowTabledBatch = {
|
|
35
|
+
shape: 'arrow-table',
|
|
36
|
+
batchType: 'data',
|
|
37
|
+
data: new arrow.Table([recordBatch]),
|
|
38
|
+
length: recordBatch.data.length
|
|
39
|
+
};
|
|
40
|
+
// processBatch(recordBatch);
|
|
41
|
+
yield arrowTabledBatch;
|
|
42
|
+
}
|
|
43
|
+
break; // only processing one stream of batches
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return makeArrowAsyncIterator(); // as AsyncIterator<ArrowTableBatch>;
|
|
26
47
|
}
|
|
27
|
-
|
|
48
|
+
// function processBatch(batch: RecordBatch): ArrowTableBatch {
|
|
49
|
+
// const values = {};
|
|
50
|
+
// batch.schema.fields.forEach(({name}, index) => {
|
|
51
|
+
// values[name] = batch.getChildAt(index)?.toArray();
|
|
52
|
+
// });
|
|
53
|
+
// return {
|
|
54
|
+
// };
|
|
55
|
+
// }
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ArrayRowTable, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
|
|
2
|
-
import type { ArrowTable } from
|
|
2
|
+
import type { ArrowTable } from "../lib/arrow-table.js";
|
|
3
3
|
export declare function parseArrowSync(arrayBuffer: any, options?: {
|
|
4
4
|
shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table';
|
|
5
5
|
}): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAC,UAAU,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAOnD,wBAAgB,cAAc,CAC5B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,iBAAiB,CAAA;CAAC,GAC5F,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,CA4B7D"}
|
|
@@ -1,28 +1,32 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { convertTable } from '@loaders.gl/schema';
|
|
2
5
|
import * as arrow from 'apache-arrow';
|
|
3
6
|
import { convertArrowToColumnarTable } from "../tables/convert-arrow-to-columnar-table.js";
|
|
4
7
|
import { serializeArrowSchema } from "../schema/convert-arrow-schema.js";
|
|
8
|
+
// Parses arrow to a columnar table
|
|
5
9
|
export function parseArrowSync(arrayBuffer, options) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
10
|
+
const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
|
|
11
|
+
const arrowTable = {
|
|
12
|
+
shape: 'arrow-table',
|
|
13
|
+
schema: serializeArrowSchema(apacheArrowTable.schema),
|
|
14
|
+
data: apacheArrowTable
|
|
15
|
+
};
|
|
16
|
+
const shape = options?.shape || 'arrow-table';
|
|
17
|
+
switch (shape) {
|
|
18
|
+
case 'arrow-table':
|
|
19
|
+
return arrowTable;
|
|
20
|
+
case 'columnar-table':
|
|
21
|
+
return convertArrowToColumnarTable(arrowTable);
|
|
22
|
+
case 'object-row-table':
|
|
23
|
+
let columnarTable = convertArrowToColumnarTable(arrowTable);
|
|
24
|
+
return convertTable(columnarTable, 'object-row-table');
|
|
25
|
+
case 'array-row-table':
|
|
26
|
+
columnarTable = convertArrowToColumnarTable(arrowTable);
|
|
27
|
+
return convertTable(columnarTable, 'array-row-table');
|
|
28
|
+
default:
|
|
29
|
+
// TODO
|
|
30
|
+
throw new Error(shape);
|
|
31
|
+
}
|
|
27
32
|
}
|
|
28
|
-
//# sourceMappingURL=parse-arrow-sync.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { GeoJSONTableBatch } from '@loaders.gl/schema';
|
|
2
|
-
import type { ArrowTableBatch } from
|
|
2
|
+
import type { ArrowTableBatch } from "../lib/arrow-table.js";
|
|
3
3
|
/**
|
|
4
4
|
*/
|
|
5
5
|
export declare function parseGeoArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-geoarrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,eAAe,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-geoarrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,eAAe,EAAC,8BAA2B;AAGxD;GACG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAGpD"}
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { parseArrowInBatches } from "./parse-arrow-in-batches.js";
|
|
5
|
+
/**
|
|
6
|
+
*/
|
|
2
7
|
export function parseGeoArrowInBatches(asyncIterator) {
|
|
3
|
-
|
|
8
|
+
// | BinaryGeometry
|
|
9
|
+
return parseArrowInBatches(asyncIterator);
|
|
4
10
|
}
|
|
5
|
-
//# sourceMappingURL=parse-geoarrow-in-batches.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { GeoJSONTable } from '@loaders.gl/schema';
|
|
2
|
-
import type { ArrowTable } from
|
|
2
|
+
import type { ArrowTable } from "../lib/arrow-table.js";
|
|
3
3
|
export declare function parseGeoArrowSync(arrayBuffer: any, options?: {
|
|
4
4
|
shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry';
|
|
5
5
|
}): ArrowTable | GeoJSONTable;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-geoarrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-geoarrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAKnD,wBAAgB,iBAAiB,CAC/B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,eAAe,GAAG,iBAAiB,CAAA;CAAC,GACtE,UAAU,GAAG,YAAY,CAS3B"}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { parseArrowSync } from "./parse-arrow-sync.js";
|
|
2
5
|
import { convertArrowToGeoJSONTable } from "../tables/convert-arrow-to-geojson-table.js";
|
|
6
|
+
// Parses arrow to a columnar table
|
|
3
7
|
export function parseGeoArrowSync(arrayBuffer, options) {
|
|
4
|
-
|
|
5
|
-
shape: 'arrow-table'
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
8
|
+
// | BinaryGeometry
|
|
9
|
+
const table = parseArrowSync(arrayBuffer, { shape: 'arrow-table' });
|
|
10
|
+
switch (options?.shape) {
|
|
11
|
+
case 'geojson-table':
|
|
12
|
+
return convertArrowToGeoJSONTable(table);
|
|
13
|
+
default:
|
|
14
|
+
return table;
|
|
15
|
+
}
|
|
13
16
|
}
|
|
14
|
-
//# sourceMappingURL=parse-geoarrow-sync.js.map
|
|
@@ -1,24 +1,51 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import * as arrow from 'apache-arrow';
|
|
5
|
+
/** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
|
|
2
6
|
export function getArrowType(array) {
|
|
7
|
+
switch (array.constructor) {
|
|
8
|
+
case Int8Array:
|
|
9
|
+
return new arrow.Int8();
|
|
10
|
+
case Uint8Array:
|
|
11
|
+
return new arrow.Uint8();
|
|
12
|
+
case Int16Array:
|
|
13
|
+
return new arrow.Int16();
|
|
14
|
+
case Uint16Array:
|
|
15
|
+
return new arrow.Uint16();
|
|
16
|
+
case Int32Array:
|
|
17
|
+
return new arrow.Int32();
|
|
18
|
+
case Uint32Array:
|
|
19
|
+
return new arrow.Uint32();
|
|
20
|
+
case Float32Array:
|
|
21
|
+
return new arrow.Float32();
|
|
22
|
+
case Float64Array:
|
|
23
|
+
return new arrow.Float64();
|
|
24
|
+
default:
|
|
25
|
+
throw new Error('array type not supported');
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/*
|
|
29
|
+
export function getArrowVector(array: TypedArray): AbstractVector {
|
|
3
30
|
switch (array.constructor) {
|
|
4
31
|
case Int8Array:
|
|
5
|
-
return
|
|
32
|
+
return Int8Vector.from(array);
|
|
6
33
|
case Uint8Array:
|
|
7
|
-
return
|
|
34
|
+
return Uint8Vector.from(array);
|
|
8
35
|
case Int16Array:
|
|
9
|
-
return
|
|
36
|
+
return Int16Vector.from(array);
|
|
10
37
|
case Uint16Array:
|
|
11
|
-
return
|
|
38
|
+
return Uint16Vector.from(array);
|
|
12
39
|
case Int32Array:
|
|
13
|
-
return
|
|
40
|
+
return Int32Vector.from(array);
|
|
14
41
|
case Uint32Array:
|
|
15
|
-
return
|
|
42
|
+
return Uint32Vector.from(array);
|
|
16
43
|
case Float32Array:
|
|
17
|
-
return
|
|
44
|
+
return Float32Vector.from(array);
|
|
18
45
|
case Float64Array:
|
|
19
|
-
return
|
|
46
|
+
return Float64Vector.from(array);
|
|
20
47
|
default:
|
|
21
48
|
throw new Error('array type not supported');
|
|
22
49
|
}
|
|
23
50
|
}
|
|
24
|
-
|
|
51
|
+
*/
|