@loaders.gl/arrow 4.2.0-alpha.3 → 4.2.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/arrow-loader.d.ts +1 -1
  2. package/dist/arrow-loader.d.ts.map +1 -1
  3. package/dist/arrow-loader.js +30 -19
  4. package/dist/arrow-worker.js +1 -1
  5. package/dist/arrow-writer.d.ts +1 -1
  6. package/dist/arrow-writer.d.ts.map +1 -1
  7. package/dist/arrow-writer.js +23 -16
  8. package/dist/dist.dev.js +370 -375
  9. package/dist/dist.min.js +23 -0
  10. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +345 -288
  11. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js +151 -124
  12. package/dist/geoarrow/get-arrow-bounds.js +32 -22
  13. package/dist/geoarrow-loader.d.ts +1 -1
  14. package/dist/geoarrow-loader.d.ts.map +1 -1
  15. package/dist/geoarrow-loader.js +22 -15
  16. package/dist/geoarrow-writer.js +25 -16
  17. package/dist/index.cjs +54 -71
  18. package/dist/index.cjs.map +7 -0
  19. package/dist/index.d.ts +16 -16
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +8 -1
  22. package/dist/lib/arrow-table-batch.d.ts +1 -1
  23. package/dist/lib/arrow-table-batch.d.ts.map +1 -1
  24. package/dist/lib/arrow-table-batch.js +54 -51
  25. package/dist/lib/arrow-table.js +3 -1
  26. package/dist/lib/encode-arrow.d.ts +1 -1
  27. package/dist/lib/encode-arrow.d.ts.map +1 -1
  28. package/dist/lib/encode-arrow.js +31 -16
  29. package/dist/lib/encode-geoarrow.d.ts +1 -1
  30. package/dist/lib/encode-geoarrow.d.ts.map +1 -1
  31. package/dist/lib/encode-geoarrow.js +31 -16
  32. package/dist/parsers/parse-arrow-in-batches.d.ts +2 -2
  33. package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
  34. package/dist/parsers/parse-arrow-in-batches.js +49 -21
  35. package/dist/parsers/parse-arrow-sync.d.ts +1 -1
  36. package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
  37. package/dist/parsers/parse-arrow-sync.js +26 -22
  38. package/dist/parsers/parse-geoarrow-in-batches.d.ts +1 -1
  39. package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
  40. package/dist/parsers/parse-geoarrow-in-batches.js +7 -2
  41. package/dist/parsers/parse-geoarrow-sync.d.ts +1 -1
  42. package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
  43. package/dist/parsers/parse-geoarrow-sync.js +12 -10
  44. package/dist/schema/arrow-type-utils.js +36 -9
  45. package/dist/schema/convert-arrow-schema.js +253 -240
  46. package/dist/tables/convert-arrow-to-columnar-table.d.ts +1 -1
  47. package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
  48. package/dist/tables/convert-arrow-to-columnar-table.js +22 -13
  49. package/dist/tables/convert-arrow-to-geojson-table.d.ts +1 -1
  50. package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
  51. package/dist/tables/convert-arrow-to-geojson-table.js +47 -32
  52. package/dist/tables/convert-columnar-to-row-table.js +22 -16
  53. package/dist/tables/convert-table-to-arrow.js +58 -1
  54. package/dist/triangulate-on-worker.js +24 -19
  55. package/dist/types.js +8 -6
  56. package/dist/workers/arrow-worker.js +3 -1
  57. package/dist/workers/hard-clone.js +93 -49
  58. package/dist/workers/triangulation-worker-node.d.ts +1 -1
  59. package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
  60. package/dist/workers/triangulation-worker-node.js +3 -1
  61. package/dist/workers/triangulation-worker.js +48 -48
  62. package/package.json +12 -8
  63. package/dist/arrow-loader.js.map +0 -1
  64. package/dist/arrow-writer.js.map +0 -1
  65. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +0 -1
  66. package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +0 -1
  67. package/dist/geoarrow/get-arrow-bounds.js.map +0 -1
  68. package/dist/geoarrow-loader.js.map +0 -1
  69. package/dist/geoarrow-writer.js.map +0 -1
  70. package/dist/index.js.map +0 -1
  71. package/dist/lib/arrow-table-batch.js.map +0 -1
  72. package/dist/lib/arrow-table.js.map +0 -1
  73. package/dist/lib/encode-arrow.js.map +0 -1
  74. package/dist/lib/encode-geoarrow.js.map +0 -1
  75. package/dist/parsers/parse-arrow-in-batches.js.map +0 -1
  76. package/dist/parsers/parse-arrow-sync.js.map +0 -1
  77. package/dist/parsers/parse-geoarrow-in-batches.js.map +0 -1
  78. package/dist/parsers/parse-geoarrow-sync.js.map +0 -1
  79. package/dist/schema/arrow-type-utils.js.map +0 -1
  80. package/dist/schema/convert-arrow-schema.js.map +0 -1
  81. package/dist/tables/convert-arrow-to-columnar-table.js.map +0 -1
  82. package/dist/tables/convert-arrow-to-geojson-table.js.map +0 -1
  83. package/dist/tables/convert-columnar-to-row-table.js.map +0 -1
  84. package/dist/tables/convert-table-to-arrow.js.map +0 -1
  85. package/dist/triangulate-on-worker.js.map +0 -1
  86. package/dist/types.js.map +0 -1
  87. package/dist/workers/arrow-worker.js.map +0 -1
  88. package/dist/workers/hard-clone.js.map +0 -1
  89. package/dist/workers/triangulation-worker-node.js.map +0 -1
  90. package/dist/workers/triangulation-worker.js.map +0 -1
@@ -1,62 +1,65 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
2
5
  import * as arrow from 'apache-arrow';
3
6
  export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
4
- constructor(schema, options) {
5
- super(schema, options);
6
- this.arrowSchema = void 0;
7
- this.arrowSchema = null;
8
- }
9
- getBatch() {
10
- const batch = super.getBatch();
11
- if (batch) {
12
- this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
13
- const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
14
- const recordBatch = new arrow.RecordBatch(this.arrowSchema, arrow.makeData({
15
- type: new arrow.Struct(this.arrowSchema.fields),
16
- children: arrowVectors.map(_ref => {
17
- let {
18
- data
19
- } = _ref;
20
- return data[0];
21
- })
22
- }));
23
- return {
24
- shape: 'arrow-table',
25
- batchType: 'data',
26
- data: new arrow.Table([recordBatch]),
27
- length: batch.length
28
- };
7
+ constructor(schema, options) {
8
+ super(schema, options);
9
+ this.arrowSchema = null;
10
+ }
11
+ getBatch() {
12
+ const batch = super.getBatch();
13
+ if (batch) {
14
+ // Get the arrow schema
15
+ this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
16
+ // Get arrow format vectors
17
+ const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
18
+ // Create the record batch
19
+ const recordBatch = new arrow.RecordBatch(this.arrowSchema, arrow.makeData({
20
+ type: new arrow.Struct(this.arrowSchema.fields),
21
+ children: arrowVectors.map(({ data }) => data[0])
22
+ }));
23
+ return {
24
+ shape: 'arrow-table',
25
+ batchType: 'data',
26
+ data: new arrow.Table([recordBatch]),
27
+ length: batch.length
28
+ };
29
+ }
30
+ return null;
29
31
  }
30
- return null;
31
- }
32
32
  }
33
+ // Convert from a simple loaders.gl schema to an Arrow schema
33
34
  function getArrowSchema(schema) {
34
- const arrowFields = [];
35
- for (const key in schema) {
36
- const field = schema[key];
37
- if (field.type === Float32Array) {
38
- const metadata = new Map();
39
- const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
40
- arrowFields.push(arrowField);
35
+ const arrowFields = [];
36
+ for (const key in schema) {
37
+ const field = schema[key];
38
+ if (field.type === Float32Array) {
39
+ // TODO - just store the original field as metadata?
40
+ const metadata = new Map(); // field;
41
+ // arrow: new arrow.Field(name, nullable, metadata)
42
+ const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
43
+ arrowFields.push(arrowField);
44
+ }
45
+ }
46
+ if (arrowFields.length === 0) {
47
+ throw new Error('No arrow convertible fields');
41
48
  }
42
- }
43
- if (arrowFields.length === 0) {
44
- throw new Error('No arrow convertible fields');
45
- }
46
- return new arrow.Schema(arrowFields);
49
+ return new arrow.Schema(arrowFields);
47
50
  }
51
+ // Convert from simple loaders.gl arrays to arrow vectors
48
52
  function getArrowVectors(arrowSchema, data) {
49
- const arrowVectors = [];
50
- for (const field of arrowSchema.fields) {
51
- const vector = data[field.name];
52
- if (vector instanceof Float32Array) {
53
- const arrowVector = arrow.makeVector(vector);
54
- arrowVectors.push(arrowVector);
53
+ const arrowVectors = [];
54
+ for (const field of arrowSchema.fields) {
55
+ const vector = data[field.name];
56
+ if (vector instanceof Float32Array) {
57
+ const arrowVector = arrow.makeVector(vector);
58
+ arrowVectors.push(arrowVector);
59
+ }
60
+ }
61
+ if (arrowSchema.fields.length !== arrowVectors.length) {
62
+ throw new Error('Some columns not arrow convertible');
55
63
  }
56
- }
57
- if (arrowSchema.fields.length !== arrowVectors.length) {
58
- throw new Error('Some columns not arrow convertible');
59
- }
60
- return arrowVectors;
64
+ return arrowVectors;
61
65
  }
62
- //# sourceMappingURL=arrow-table-batch.js.map
@@ -1,2 +1,4 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  export {};
2
- //# sourceMappingURL=arrow-table.js.map
@@ -1,4 +1,4 @@
1
- import { AnyArrayType } from '../types';
1
+ import { AnyArrayType } from "../types.js";
2
2
  export type ColumnarTable = {
3
3
  name: string;
4
4
  array: AnyArrayType;
@@ -1 +1 @@
1
- {"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,MAAM,UAAU,CAAC;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAShE"}
1
+ {"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAShE"}
@@ -1,22 +1,37 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import * as arrow from 'apache-arrow';
2
5
  import { VECTOR_TYPES } from "../types.js";
6
+ /**
7
+ * Encodes set of arrays into the Apache Arrow columnar format
8
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
9
+ * @param data - columns data
10
+ * @param options - the writer options
11
+ * @returns - encoded ArrayBuffer
12
+ */
3
13
  export function encodeArrowSync(data) {
4
- const vectors = {};
5
- for (const arrayData of data) {
6
- const arrayVector = createVector(arrayData.array, arrayData.type);
7
- vectors[arrayData.name] = arrayVector;
8
- }
9
- const table = new arrow.Table(vectors);
10
- const arrowBuffer = arrow.tableToIPC(table);
11
- return arrowBuffer;
14
+ const vectors = {};
15
+ for (const arrayData of data) {
16
+ const arrayVector = createVector(arrayData.array, arrayData.type);
17
+ vectors[arrayData.name] = arrayVector;
18
+ }
19
+ const table = new arrow.Table(vectors);
20
+ const arrowBuffer = arrow.tableToIPC(table);
21
+ return arrowBuffer;
12
22
  }
23
+ /**
24
+ * Create Arrow arrow.Vector from given data and vector type
25
+ * @param array {import('../types').AnyArrayType} - columns data
26
+ * @param type {number} - the writer options
27
+ * @return a vector of one of vector's types defined in the Apache Arrow library
28
+ */
13
29
  function createVector(array, type) {
14
- switch (type) {
15
- case VECTOR_TYPES.DATE:
16
- return arrow.vectorFromArray(array);
17
- case VECTOR_TYPES.FLOAT:
18
- default:
19
- return arrow.vectorFromArray(array);
20
- }
30
+ switch (type) {
31
+ case VECTOR_TYPES.DATE:
32
+ return arrow.vectorFromArray(array);
33
+ case VECTOR_TYPES.FLOAT:
34
+ default:
35
+ return arrow.vectorFromArray(array);
36
+ }
21
37
  }
22
- //# sourceMappingURL=encode-arrow.js.map
@@ -1,4 +1,4 @@
1
- import { AnyArrayType } from '../types';
1
+ import { AnyArrayType } from "../types.js";
2
2
  export type ColumnarTable = {
3
3
  name: string;
4
4
  array: AnyArrayType;
@@ -1 +1 @@
1
- {"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,MAAM,UAAU,CAAC;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CASnE"}
1
+ {"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CASnE"}
@@ -1,22 +1,37 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import * as arrow from 'apache-arrow';
2
5
  import { VECTOR_TYPES } from "../types.js";
6
+ /**
7
+ * Encodes set of arrays into the Apache Arrow columnar format
8
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
9
+ * @param data - columns data
10
+ * @param options - the writer options
11
+ * @returns - encoded ArrayBuffer
12
+ */
3
13
  export function encodeGeoArrowSync(data) {
4
- const vectors = {};
5
- for (const arrayData of data) {
6
- const arrayVector = createVector(arrayData.array, arrayData.type);
7
- vectors[arrayData.name] = arrayVector;
8
- }
9
- const table = new arrow.Table(vectors);
10
- const arrowBuffer = arrow.tableToIPC(table);
11
- return arrowBuffer;
14
+ const vectors = {};
15
+ for (const arrayData of data) {
16
+ const arrayVector = createVector(arrayData.array, arrayData.type);
17
+ vectors[arrayData.name] = arrayVector;
18
+ }
19
+ const table = new arrow.Table(vectors);
20
+ const arrowBuffer = arrow.tableToIPC(table);
21
+ return arrowBuffer;
12
22
  }
23
+ /**
24
+ * Create Arrow arrow.Vector from given data and vector type
25
+ * @param array {import('../types').AnyArrayType} - columns data
26
+ * @param type {number} - the writer options
27
+ * @return a vector of one of vector's types defined in the Apache Arrow library
28
+ */
13
29
  function createVector(array, type) {
14
- switch (type) {
15
- case VECTOR_TYPES.DATE:
16
- return arrow.vectorFromArray(array);
17
- case VECTOR_TYPES.FLOAT:
18
- default:
19
- return arrow.vectorFromArray(array);
20
- }
30
+ switch (type) {
31
+ case VECTOR_TYPES.DATE:
32
+ return arrow.vectorFromArray(array);
33
+ case VECTOR_TYPES.FLOAT:
34
+ default:
35
+ return arrow.vectorFromArray(array);
36
+ }
21
37
  }
22
- //# sourceMappingURL=encode-geoarrow.js.map
@@ -1,5 +1,5 @@
1
- import type { ArrowTableBatch } from '../lib/arrow-table';
2
- import { ArrowLoaderOptions } from '../arrow-loader';
1
+ import type { ArrowTableBatch } from "../lib/arrow-table.js";
2
+ import { ArrowLoaderOptions } from "../arrow-loader.js";
3
3
  /**
4
4
  */
5
5
  export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: ArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
@@ -1 +1 @@
1
- {"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAGnD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC,eAAe,CAAC,CA0ChC"}
1
+ {"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,8BAA2B;AAExD,OAAO,EAAC,kBAAkB,EAAC,2BAAwB;AAGnD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC,eAAe,CAAC,CA0ChC"}
@@ -1,27 +1,55 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import * as arrow from 'apache-arrow';
5
+ // import {isIterable} from '@loaders.gl/core';
6
+ /**
7
+ */
2
8
  export function parseArrowInBatches(asyncIterator, options) {
3
- async function* makeArrowAsyncIterator() {
4
- const readers = arrow.RecordBatchReader.readAll(asyncIterator);
5
- for await (const reader of readers) {
6
- for await (const recordBatch of reader) {
7
- var _options$arrow, _options$arrow2;
8
- if ((options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.batchDebounceMs) !== undefined && (options === null || options === void 0 ? void 0 : (_options$arrow2 = options.arrow) === null || _options$arrow2 === void 0 ? void 0 : _options$arrow2.batchDebounceMs) > 0) {
9
- await new Promise(resolve => {
10
- var _options$arrow3;
11
- return setTimeout(resolve, ((_options$arrow3 = options.arrow) === null || _options$arrow3 === void 0 ? void 0 : _options$arrow3.batchDebounceMs) || 0);
12
- });
9
+ // Creates the appropriate arrow.RecordBatchReader subclasses from the input
10
+ // This will also close the underlying source in case of early termination or errors
11
+ // As an optimization, return a non-async iterator
12
+ /*
13
+ if (isIterable(readers)) {
14
+ function* makeArrowIterator() {
15
+ for (const reader of readers) {
16
+ for (const batch of reader) {
17
+ yield processBatch(batch, reader);
18
+ }
19
+ break; // only processing one stream of batches
13
20
  }
14
- const arrowTabledBatch = {
15
- shape: 'arrow-table',
16
- batchType: 'data',
17
- data: new arrow.Table([recordBatch]),
18
- length: recordBatch.data.length
19
- };
20
- yield arrowTabledBatch;
21
21
  }
22
- break;
22
+ const arrowIterator = makeArrowIterator();
23
23
  }
24
- }
25
- return makeArrowAsyncIterator();
24
+ */
25
+ async function* makeArrowAsyncIterator() {
26
+ // @ts-ignore
27
+ const readers = arrow.RecordBatchReader.readAll(asyncIterator);
28
+ for await (const reader of readers) {
29
+ for await (const recordBatch of reader) {
30
+ // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
31
+ if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
32
+ await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
33
+ }
34
+ const arrowTabledBatch = {
35
+ shape: 'arrow-table',
36
+ batchType: 'data',
37
+ data: new arrow.Table([recordBatch]),
38
+ length: recordBatch.data.length
39
+ };
40
+ // processBatch(recordBatch);
41
+ yield arrowTabledBatch;
42
+ }
43
+ break; // only processing one stream of batches
44
+ }
45
+ }
46
+ return makeArrowAsyncIterator(); // as AsyncIterator<ArrowTableBatch>;
26
47
  }
27
- //# sourceMappingURL=parse-arrow-in-batches.js.map
48
+ // function processBatch(batch: RecordBatch): ArrowTableBatch {
49
+ // const values = {};
50
+ // batch.schema.fields.forEach(({name}, index) => {
51
+ // values[name] = batch.getChildAt(index)?.toArray();
52
+ // });
53
+ // return {
54
+ // };
55
+ // }
@@ -1,5 +1,5 @@
1
1
  import type { ArrayRowTable, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
2
- import type { ArrowTable } from '../lib/arrow-table';
2
+ import type { ArrowTable } from "../lib/arrow-table.js";
3
3
  export declare function parseArrowSync(arrayBuffer: any, options?: {
4
4
  shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table';
5
5
  }): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable;
@@ -1 +1 @@
1
- {"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAOnD,wBAAgB,cAAc,CAC5B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,iBAAiB,CAAA;CAAC,GAC5F,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,CA4B7D"}
1
+ {"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAOnD,wBAAgB,cAAc,CAC5B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,iBAAiB,CAAA;CAAC,GAC5F,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,CA4B7D"}
@@ -1,28 +1,32 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { convertTable } from '@loaders.gl/schema';
2
5
  import * as arrow from 'apache-arrow';
3
6
  import { convertArrowToColumnarTable } from "../tables/convert-arrow-to-columnar-table.js";
4
7
  import { serializeArrowSchema } from "../schema/convert-arrow-schema.js";
8
+ // Parses arrow to a columnar table
5
9
  export function parseArrowSync(arrayBuffer, options) {
6
- const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
7
- const arrowTable = {
8
- shape: 'arrow-table',
9
- schema: serializeArrowSchema(apacheArrowTable.schema),
10
- data: apacheArrowTable
11
- };
12
- const shape = (options === null || options === void 0 ? void 0 : options.shape) || 'arrow-table';
13
- switch (shape) {
14
- case 'arrow-table':
15
- return arrowTable;
16
- case 'columnar-table':
17
- return convertArrowToColumnarTable(arrowTable);
18
- case 'object-row-table':
19
- let columnarTable = convertArrowToColumnarTable(arrowTable);
20
- return convertTable(columnarTable, 'object-row-table');
21
- case 'array-row-table':
22
- columnarTable = convertArrowToColumnarTable(arrowTable);
23
- return convertTable(columnarTable, 'array-row-table');
24
- default:
25
- throw new Error(shape);
26
- }
10
+ const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
11
+ const arrowTable = {
12
+ shape: 'arrow-table',
13
+ schema: serializeArrowSchema(apacheArrowTable.schema),
14
+ data: apacheArrowTable
15
+ };
16
+ const shape = options?.shape || 'arrow-table';
17
+ switch (shape) {
18
+ case 'arrow-table':
19
+ return arrowTable;
20
+ case 'columnar-table':
21
+ return convertArrowToColumnarTable(arrowTable);
22
+ case 'object-row-table':
23
+ let columnarTable = convertArrowToColumnarTable(arrowTable);
24
+ return convertTable(columnarTable, 'object-row-table');
25
+ case 'array-row-table':
26
+ columnarTable = convertArrowToColumnarTable(arrowTable);
27
+ return convertTable(columnarTable, 'array-row-table');
28
+ default:
29
+ // TODO
30
+ throw new Error(shape);
31
+ }
27
32
  }
28
- //# sourceMappingURL=parse-arrow-sync.js.map
@@ -1,5 +1,5 @@
1
1
  import type { GeoJSONTableBatch } from '@loaders.gl/schema';
2
- import type { ArrowTableBatch } from '../lib/arrow-table';
2
+ import type { ArrowTableBatch } from "../lib/arrow-table.js";
3
3
  /**
4
4
  */
5
5
  export declare function parseGeoArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch>;
@@ -1 +1 @@
1
- {"version":3,"file":"parse-geoarrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAGxD;GACG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAGpD"}
1
+ {"version":3,"file":"parse-geoarrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,eAAe,EAAC,8BAA2B;AAGxD;GACG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAGpD"}
@@ -1,5 +1,10 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { parseArrowInBatches } from "./parse-arrow-in-batches.js";
5
+ /**
6
+ */
2
7
  export function parseGeoArrowInBatches(asyncIterator) {
3
- return parseArrowInBatches(asyncIterator);
8
+ // | BinaryGeometry
9
+ return parseArrowInBatches(asyncIterator);
4
10
  }
5
- //# sourceMappingURL=parse-geoarrow-in-batches.js.map
@@ -1,5 +1,5 @@
1
1
  import type { GeoJSONTable } from '@loaders.gl/schema';
2
- import type { ArrowTable } from '../lib/arrow-table';
2
+ import type { ArrowTable } from "../lib/arrow-table.js";
3
3
  export declare function parseGeoArrowSync(arrayBuffer: any, options?: {
4
4
  shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry';
5
5
  }): ArrowTable | GeoJSONTable;
@@ -1 +1 @@
1
- {"version":3,"file":"parse-geoarrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAKnD,wBAAgB,iBAAiB,CAC/B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,eAAe,GAAG,iBAAiB,CAAA;CAAC,GACtE,UAAU,GAAG,YAAY,CAS3B"}
1
+ {"version":3,"file":"parse-geoarrow-sync.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-geoarrow-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAKnD,wBAAgB,iBAAiB,CAC/B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,eAAe,GAAG,iBAAiB,CAAA;CAAC,GACtE,UAAU,GAAG,YAAY,CAS3B"}
@@ -1,14 +1,16 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { parseArrowSync } from "./parse-arrow-sync.js";
2
5
  import { convertArrowToGeoJSONTable } from "../tables/convert-arrow-to-geojson-table.js";
6
+ // Parses arrow to a columnar table
3
7
  export function parseGeoArrowSync(arrayBuffer, options) {
4
- const table = parseArrowSync(arrayBuffer, {
5
- shape: 'arrow-table'
6
- });
7
- switch (options === null || options === void 0 ? void 0 : options.shape) {
8
- case 'geojson-table':
9
- return convertArrowToGeoJSONTable(table);
10
- default:
11
- return table;
12
- }
8
+ // | BinaryGeometry
9
+ const table = parseArrowSync(arrayBuffer, { shape: 'arrow-table' });
10
+ switch (options?.shape) {
11
+ case 'geojson-table':
12
+ return convertArrowToGeoJSONTable(table);
13
+ default:
14
+ return table;
15
+ }
13
16
  }
14
- //# sourceMappingURL=parse-geoarrow-sync.js.map
@@ -1,24 +1,51 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import * as arrow from 'apache-arrow';
5
+ /** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
2
6
  export function getArrowType(array) {
7
+ switch (array.constructor) {
8
+ case Int8Array:
9
+ return new arrow.Int8();
10
+ case Uint8Array:
11
+ return new arrow.Uint8();
12
+ case Int16Array:
13
+ return new arrow.Int16();
14
+ case Uint16Array:
15
+ return new arrow.Uint16();
16
+ case Int32Array:
17
+ return new arrow.Int32();
18
+ case Uint32Array:
19
+ return new arrow.Uint32();
20
+ case Float32Array:
21
+ return new arrow.Float32();
22
+ case Float64Array:
23
+ return new arrow.Float64();
24
+ default:
25
+ throw new Error('array type not supported');
26
+ }
27
+ }
28
+ /*
29
+ export function getArrowVector(array: TypedArray): AbstractVector {
3
30
  switch (array.constructor) {
4
31
  case Int8Array:
5
- return new arrow.Int8();
32
+ return Int8Vector.from(array);
6
33
  case Uint8Array:
7
- return new arrow.Uint8();
34
+ return Uint8Vector.from(array);
8
35
  case Int16Array:
9
- return new arrow.Int16();
36
+ return Int16Vector.from(array);
10
37
  case Uint16Array:
11
- return new arrow.Uint16();
38
+ return Uint16Vector.from(array);
12
39
  case Int32Array:
13
- return new arrow.Int32();
40
+ return Int32Vector.from(array);
14
41
  case Uint32Array:
15
- return new arrow.Uint32();
42
+ return Uint32Vector.from(array);
16
43
  case Float32Array:
17
- return new arrow.Float32();
44
+ return Float32Vector.from(array);
18
45
  case Float64Array:
19
- return new arrow.Float64();
46
+ return Float64Vector.from(array);
20
47
  default:
21
48
  throw new Error('array type not supported');
22
49
  }
23
50
  }
24
- //# sourceMappingURL=arrow-type-utils.js.map
51
+ */