@loaders.gl/arrow 4.4.0-alpha.2 → 4.4.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.js +1 -0
- package/dist/arrow-loader.js.map +1 -0
- package/dist/arrow-worker.js +29 -3
- package/dist/arrow-writer.js +2 -1
- package/dist/arrow-writer.js.map +1 -0
- package/dist/dist.dev.js +1509 -1458
- package/dist/dist.min.js +5 -5
- package/dist/exports/arrow-format.js +1 -0
- package/dist/exports/arrow-format.js.map +1 -0
- package/dist/exports/arrow-loader.js +2 -1
- package/dist/exports/arrow-loader.js.map +1 -0
- package/dist/exports/geoarrow-loader.js +1 -0
- package/dist/exports/geoarrow-loader.js.map +1 -0
- package/dist/geoarrow-loader.js +1 -0
- package/dist/geoarrow-loader.js.map +1 -0
- package/dist/geoarrow-writer.js +2 -1
- package/dist/geoarrow-writer.js.map +1 -0
- package/dist/index.cjs +7 -5
- package/dist/index.cjs.map +4 -4
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/encoders/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow.js +3 -1
- package/dist/lib/encoders/encode-arrow.js.map +1 -0
- package/dist/lib/encoders/encode-geoarrow.d.ts.map +1 -1
- package/dist/lib/encoders/encode-geoarrow.js +3 -1
- package/dist/lib/encoders/encode-geoarrow.js.map +1 -0
- package/dist/lib/parsers/parse-arrow.d.ts +1 -1
- package/dist/lib/parsers/parse-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-arrow.js +3 -1
- package/dist/lib/parsers/parse-arrow.js.map +1 -0
- package/dist/lib/parsers/parse-geoarrow.d.ts +1 -1
- package/dist/lib/parsers/parse-geoarrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoarrow.js +1 -0
- package/dist/lib/parsers/parse-geoarrow.js.map +1 -0
- package/dist/lib/types.js +1 -0
- package/dist/lib/types.js.map +1 -0
- package/dist/triangulate-on-worker.js +2 -1
- package/dist/triangulate-on-worker.js.map +1 -0
- package/dist/workers/arrow-worker.js +1 -0
- package/dist/workers/arrow-worker.js.map +1 -0
- package/dist/workers/hard-clone.js +1 -0
- package/dist/workers/hard-clone.js.map +1 -0
- package/dist/workers/triangulation-worker-node.js +1 -0
- package/dist/workers/triangulation-worker-node.js.map +1 -0
- package/dist/workers/triangulation-worker.js +1 -0
- package/dist/workers/triangulation-worker.js.map +1 -0
- package/package.json +11 -8
- package/src/lib/encoders/encode-arrow.ts +2 -1
- package/src/lib/encoders/encode-geoarrow.ts +2 -1
- package/src/lib/parsers/parse-arrow.ts +5 -2
- package/src/lib/parsers/parse-geoarrow.ts +3 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arrow-format.js","sourceRoot":"","sources":["../../src/exports/arrow-format.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,2BAA2B;AAC3B,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,IAAI,EAAE,cAAc;IACpB,EAAE,EAAE,OAAO;IACX,MAAM,EAAE,OAAO;IACf,QAAQ,EAAE,OAAO;IACjB,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;IAChC,SAAS,EAAE;QACT,mCAAmC;QACnC,qCAAqC;QACrC,0BAA0B;KAC3B;IACD,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,CAAC,OAAO,CAAC;CACS,CAAC"}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
5
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
-
const VERSION = typeof "4.4.0-alpha.
|
|
6
|
+
const VERSION = typeof "4.4.0-alpha.9" !== 'undefined' ? "4.4.0-alpha.9" : 'latest';
|
|
7
7
|
/** ArrowJS table loader */
|
|
8
8
|
export const ArrowWorkerLoader = {
|
|
9
9
|
dataType: null,
|
|
@@ -28,3 +28,4 @@ export const ArrowWorkerLoader = {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
};
|
|
31
|
+
//# sourceMappingURL=arrow-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arrow-loader.js","sourceRoot":"","sources":["../../src/exports/arrow-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAe5E,2BAA2B;AAC3B,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,QAAQ,EAAE,IAA6B;IACvC,SAAS,EAAE,IAAa;IAExB,IAAI,EAAE,cAAc;IACpB,EAAE,EAAE,OAAO;IACX,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,OAAO;IAChB,gBAAgB;IAChB,QAAQ,EAAE,OAAO;IACjB,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;IAChC,SAAS,EAAE;QACT,mCAAmC;QACnC,qCAAqC;QACrC,0BAA0B;KAC3B;IACD,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,OAAO,EAAE;QACP,KAAK,EAAE;YACL,KAAK,EAAE,gBAAgB;SACxB;KACF;CAC+D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoarrow-loader.js","sourceRoot":"","sources":["../../src/exports/geoarrow-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,iBAAiB,EAAC,0BAAuB;AAQjD,2BAA2B;AAC3B,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,GAAG,iBAAiB;IACpB,OAAO,EAAE;QACP,KAAK,EAAE;YACL,KAAK,EAAE,aAAa;SACrB;KACF;CAC6F,CAAC"}
|
package/dist/geoarrow-loader.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoarrow-loader.js","sourceRoot":"","sources":["../src/geoarrow-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AASpC,OAAO,EAAC,iBAAiB,EAAE,sBAAsB,EAAC,wCAAqC;AAEvF,OAAO,EAAC,oBAAoB,EAAC,qCAAkC;AAE/D;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,GAAG,oBAAoB;IAEvB,KAAK,EAAE,KAAK,EAAE,WAAwB,EAAE,OAA+B,EAAE,EAAE,CACzE,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC;IAChD,SAAS,EAAE,CAAC,WAAwB,EAAE,OAA+B,EAAE,EAAE,CACvE,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC;IAChD,cAAc,EAAE,sBAAsB;CAKvC,CAAC"}
|
package/dist/geoarrow-writer.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { encodeGeoArrowSync } from "./lib/encoders/encode-geoarrow.js";
|
|
3
3
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
4
4
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
5
|
-
const VERSION = typeof "4.4.0-alpha.
|
|
5
|
+
const VERSION = typeof "4.4.0-alpha.9" !== 'undefined' ? "4.4.0-alpha.9" : 'latest';
|
|
6
6
|
/** Apache Arrow writer */
|
|
7
7
|
export const GeoArrowWriter = {
|
|
8
8
|
name: 'Apache Arrow',
|
|
@@ -26,3 +26,4 @@ export const GeoArrowWriter = {
|
|
|
26
26
|
return encodeGeoArrowSync(data);
|
|
27
27
|
}
|
|
28
28
|
};
|
|
29
|
+
//# sourceMappingURL=geoarrow-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoarrow-writer.js","sourceRoot":"","sources":["../src/geoarrow-writer.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAIlD,OAAO,EAAC,kBAAkB,EAAC,0CAAuC;AAElE,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAM5E,0BAA0B;AAC1B,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,cAAc;IACpB,EAAE,EAAE,OAAO;IACX,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,OAAO;IAChB,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;IAChC,SAAS,EAAE;QACT,mCAAmC;QACnC,qCAAqC;QACrC,0BAA0B;KAC3B;IACD,MAAM,EAAE,IAAI;IACZ,OAAO,EAAE,EAAE;IACX,MAAM,EAAE,KAAK,UAAU,WAAW,CAAC,IAAI,EAAE,OAAQ;QAC/C,mBAAmB;QACnB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,UAAU,CAAC,IAAI,EAAE,OAAQ;QACvB,mBAAmB;QACnB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CAC6F,CAAC"}
|
package/dist/index.cjs
CHANGED
|
@@ -68,7 +68,7 @@ var ArrowFormat = {
|
|
|
68
68
|
};
|
|
69
69
|
|
|
70
70
|
// dist/exports/arrow-loader.js
|
|
71
|
-
var VERSION = true ? "4.4.0-alpha.
|
|
71
|
+
var VERSION = true ? "4.4.0-alpha.9" : "latest";
|
|
72
72
|
var ArrowWorkerLoader = {
|
|
73
73
|
dataType: null,
|
|
74
74
|
batchType: null,
|
|
@@ -96,6 +96,7 @@ var ArrowWorkerLoader = {
|
|
|
96
96
|
// dist/lib/parsers/parse-arrow.js
|
|
97
97
|
var arrow = __toESM(require("apache-arrow"), 1);
|
|
98
98
|
var import_schema_utils = require("@loaders.gl/schema-utils");
|
|
99
|
+
var import_loader_utils = require("@loaders.gl/loader-utils");
|
|
99
100
|
function parseArrowSync(arrayBuffer, options) {
|
|
100
101
|
const shape = (options == null ? void 0 : options.shape) || "arrow-table";
|
|
101
102
|
const arrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
|
|
@@ -104,7 +105,7 @@ function parseArrowSync(arrayBuffer, options) {
|
|
|
104
105
|
function parseArrowInBatches(asyncIterator, options) {
|
|
105
106
|
async function* makeArrowAsyncIterator() {
|
|
106
107
|
var _a, _b;
|
|
107
|
-
const readers = arrow.RecordBatchReader.readAll(asyncIterator);
|
|
108
|
+
const readers = arrow.RecordBatchReader.readAll((0, import_loader_utils.toArrayBufferIterator)(asyncIterator));
|
|
108
109
|
for await (const reader of readers) {
|
|
109
110
|
for await (const recordBatch of reader) {
|
|
110
111
|
if (((_a = options == null ? void 0 : options.arrow) == null ? void 0 : _a.batchDebounceMs) !== void 0 && ((_b = options == null ? void 0 : options.arrow) == null ? void 0 : _b.batchDebounceMs) > 0) {
|
|
@@ -137,6 +138,7 @@ var ArrowLoader = {
|
|
|
137
138
|
|
|
138
139
|
// dist/lib/encoders/encode-arrow.js
|
|
139
140
|
var arrow2 = __toESM(require("apache-arrow"), 1);
|
|
141
|
+
var import_loader_utils2 = require("@loaders.gl/loader-utils");
|
|
140
142
|
function encodeArrowSync(data) {
|
|
141
143
|
const vectors = {};
|
|
142
144
|
for (const arrayData of data) {
|
|
@@ -145,7 +147,7 @@ function encodeArrowSync(data) {
|
|
|
145
147
|
}
|
|
146
148
|
const table = new arrow2.Table(vectors);
|
|
147
149
|
const arrowBuffer = arrow2.tableToIPC(table);
|
|
148
|
-
return arrowBuffer;
|
|
150
|
+
return (0, import_loader_utils2.ensureArrayBuffer)(arrowBuffer);
|
|
149
151
|
}
|
|
150
152
|
function createVector(array, type) {
|
|
151
153
|
switch (type) {
|
|
@@ -158,7 +160,7 @@ function createVector(array, type) {
|
|
|
158
160
|
}
|
|
159
161
|
|
|
160
162
|
// dist/arrow-writer.js
|
|
161
|
-
var VERSION2 = true ? "4.4.0-alpha.
|
|
163
|
+
var VERSION2 = true ? "4.4.0-alpha.9" : "latest";
|
|
162
164
|
var ArrowWriter = {
|
|
163
165
|
name: "Apache Arrow",
|
|
164
166
|
id: "arrow",
|
|
@@ -260,7 +262,7 @@ function cloneBuffer(arr, force) {
|
|
|
260
262
|
|
|
261
263
|
// dist/triangulate-on-worker.js
|
|
262
264
|
var import_worker_utils = require("@loaders.gl/worker-utils");
|
|
263
|
-
var VERSION3 = true ? "4.4.0-alpha.
|
|
265
|
+
var VERSION3 = true ? "4.4.0-alpha.9" : "latest";
|
|
264
266
|
var TriangulationWorker = {
|
|
265
267
|
id: "triangulation",
|
|
266
268
|
name: "Triangulate",
|
package/dist/index.cjs.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
|
-
"sources": ["index.
|
|
4
|
-
"sourcesContent": ["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// Types\nexport { VECTOR_TYPES } from \"./lib/types.js\";\n// Arrow loader / Writer\nexport { ArrowFormat } from \"./exports/arrow-format.js\";\nexport { ArrowWorkerLoader } from \"./exports/arrow-loader.js\";\nexport { ArrowLoader } from \"./arrow-loader.js\";\nexport { ArrowWriter } from \"./arrow-writer.js\";\n// Geoarrow loader\nexport { GeoArrowWorkerLoader } from \"./exports/geoarrow-loader.js\";\nexport { GeoArrowLoader } from \"./geoarrow-loader.js\";\n// EXPERIMENTAL\n// Arrow Utils\n// getGeometryColumnsFromArrowTable,\n// getGeoArrowEncoding\n// EXPERIMENTAL WORKER\nexport { hardClone } from \"./workers/hard-clone.js\";\nexport { TriangulationWorker, triangulateOnWorker, parseGeoArrowOnWorker } from \"./triangulate-on-worker.js\";\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nexport var VECTOR_TYPES;\n(function (VECTOR_TYPES) {\n VECTOR_TYPES[VECTOR_TYPES[\"FLOAT\"] = 0] = \"FLOAT\";\n VECTOR_TYPES[VECTOR_TYPES[\"DATE\"] = 1] = \"DATE\";\n})(VECTOR_TYPES || (VECTOR_TYPES = {}));\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n/** ArrowJS table loader */\nexport const ArrowFormat = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW']\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof \"4.4.0-alpha.1\" !== 'undefined' ? \"4.4.0-alpha.1\" : 'latest';\n/** ArrowJS table loader */\nexport const ArrowWorkerLoader = {\n dataType: null,\n batchType: null,\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n // worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: {\n arrow: {\n shape: 'columnar-table'\n }\n }\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport * as arrow from 'apache-arrow';\nimport { convertArrowToTable } from '@loaders.gl/schema-utils';\n/** Parses arrow to a loaders.gl table. Defaults to `arrow-table` */\nexport function parseArrowSync(arrayBuffer, options) {\n const shape = options?.shape || 'arrow-table';\n const arrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);\n return convertArrowToTable(arrowTable, shape);\n}\nexport function parseArrowInBatches(asyncIterator, options) {\n // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n // This will also close the underlying source in case of early termination or errors\n // As an optimization, return a non-async iterator\n /*\n if (isIterable(readers)) {\n function* makeArrowIterator() {\n for (const reader of readers) {\n for (const batch of reader) {\n yield processBatch(batch, reader);\n }\n break; // only processing one stream of batches\n }\n }\n const arrowIterator = makeArrowIterator();\n }\n */\n async function* makeArrowAsyncIterator() {\n // @ts-ignore\n const readers = arrow.RecordBatchReader.readAll(asyncIterator);\n for await (const reader of readers) {\n for await (const recordBatch of reader) {\n // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)\n if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {\n await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));\n }\n const arrowTabledBatch = {\n shape: 'arrow-table',\n batchType: 'data',\n data: new arrow.Table([recordBatch]),\n length: recordBatch.data.length\n };\n // processBatch(recordBatch);\n yield arrowTabledBatch;\n }\n break; // only processing one stream of batches\n }\n }\n return makeArrowAsyncIterator(); // as AsyncIterator<ArrowTableBatch>;\n}\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n// const values = {};\n// batch.schema.fields.forEach(({name}, index) => {\n// values[name] = batch.getChildAt(index)?.toArray();\n// });\n// return {\n// };\n// }\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport { parseArrowSync, parseArrowInBatches } from \"./lib/parsers/parse-arrow.js\";\nimport { ArrowWorkerLoader } from \"./exports/arrow-loader.js\";\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n ...ArrowWorkerLoader,\n parse: async (arraybuffer, options) => parseArrowSync(arraybuffer, options?.arrow),\n parseSync: (arraybuffer, options) => parseArrowSync(arraybuffer, options?.arrow),\n parseInBatches: parseArrowInBatches\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport * as arrow from 'apache-arrow';\nimport { VECTOR_TYPES } from \"../types.js\";\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data) {\n const vectors = {};\n for (const arrayData of data) {\n const arrayVector = createVector(arrayData.array, arrayData.type);\n vectors[arrayData.name] = arrayVector;\n }\n const table = new arrow.Table(vectors);\n const arrowBuffer = arrow.tableToIPC(table);\n return arrowBuffer;\n}\n/**\n * Create Arrow arrow.Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type) {\n switch (type) {\n case VECTOR_TYPES.DATE:\n return arrow.vectorFromArray(array);\n case VECTOR_TYPES.FLOAT:\n default:\n return arrow.vectorFromArray(array);\n }\n}\n", "// import type {} from '@loaders.gl/loader-utils';\nimport { encodeArrowSync } from \"./lib/encoders/encode-arrow.js\";\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof \"4.4.0-alpha.1\" !== 'undefined' ? \"4.4.0-alpha.1\" : 'latest';\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n options: {},\n encode: async function encodeArrow(data, options) {\n return encodeArrowSync(data);\n },\n encodeSync(data, options) {\n return encodeArrowSync(data);\n }\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport { ArrowWorkerLoader } from \"./arrow-loader.js\";\n/** ArrowJS table loader */\nexport const GeoArrowWorkerLoader = {\n ...ArrowWorkerLoader,\n options: {\n arrow: {\n shape: 'arrow-table'\n }\n }\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport { parseArrowSync, parseArrowInBatches } from \"./parse-arrow.js\";\nimport { convertGeoArrowToTable } from '@loaders.gl/gis';\n// Parses arrow to a columnar table\nexport function parseGeoArrowSync(arrayBuffer, options) {\n // | BinaryGeometry\n const table = parseArrowSync(arrayBuffer, { shape: 'arrow-table' });\n switch (options?.shape) {\n case 'geojson-table':\n return convertGeoArrowToTable(table.data, 'geojson-table');\n default:\n return table;\n }\n}\n/**\n */\nexport function parseGeoArrowInBatches(asyncIterator) {\n // | BinaryGeometry\n return parseArrowInBatches(asyncIterator);\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport { parseGeoArrowSync, parseGeoArrowInBatches } from \"./lib/parsers/parse-geoarrow.js\";\nimport { GeoArrowWorkerLoader } from \"./exports/geoarrow-loader.js\";\n/**\n * GeoArrowLoader loads an Apache Arrow table, parses GeoArrow type extension data\n * to convert it to a GeoJSON table or a BinaryGeometry\n */\nexport const GeoArrowLoader = {\n ...GeoArrowWorkerLoader,\n parse: async (arraybuffer, options) => parseGeoArrowSync(arraybuffer, options?.arrow),\n parseSync: (arraybuffer, options) => parseGeoArrowSync(arraybuffer, options?.arrow),\n parseInBatches: parseGeoArrowInBatches\n};\n", "import * as arrow from 'apache-arrow';\nexport function hardClone(data, force = false) {\n // Check if `data` is an arrow.Vector\n if ('data' in data) {\n return new arrow.Vector(data.data.map((data) => hardClone(data, force)));\n }\n // Clone each of the children, recursively\n const clonedChildren = [];\n for (const childData of data.children) {\n clonedChildren.push(hardClone(childData, force));\n }\n // Clone the dictionary if there is one\n let clonedDictionary;\n if (data.dictionary !== undefined) {\n clonedDictionary = hardClone(data.dictionary, force);\n }\n // Buffers can have up to four entries. Each of these can be `undefined` for\n // one or more array types.\n //\n // - OFFSET: value offsets for variable size list types\n // - DATA: the underlying data\n // - VALIDITY: the null buffer. This may be empty or undefined if all elements\n // are non-null/valid.\n // - TYPE: type ids for a union type.\n const clonedBuffers = {\n [arrow.BufferType.OFFSET]: cloneBuffer(data.buffers[arrow.BufferType.OFFSET], force),\n [arrow.BufferType.DATA]: cloneBuffer(data.buffers[arrow.BufferType.DATA], force),\n [arrow.BufferType.VALIDITY]: cloneBuffer(data.buffers[arrow.BufferType.VALIDITY], force),\n [arrow.BufferType.TYPE]: cloneBuffer(data.buffers[arrow.BufferType.TYPE], force)\n };\n // Note: the data.offset is passed on so that a sliced Data instance will not\n // be \"un-sliced\". However keep in mind that this means we're cloning the\n // _original backing buffer_, not only the portion of the Data that was\n // sliced.\n return new arrow.Data(data.type, data.offset, data.length, \n // @ts-expect-error _nullCount is protected. We're using it here to mimic\n // `Data.clone`\n data._nullCount, clonedBuffers, clonedChildren, clonedDictionary);\n}\n/**\n * Test whether an arrow.Data instance is a slice of a larger `ArrayBuffer`.\n */\nexport function isShared(data) {\n // Loop over arrow.Vector\n if ('data' in data) {\n return data.data.some((data) => isShared(data));\n }\n // Check child data\n for (const childData of data.children) {\n if (isShared(childData)) {\n return true;\n }\n }\n // Check dictionary\n if (data.dictionary !== undefined) {\n if (isShared(data.dictionary)) {\n return true;\n }\n }\n const bufferTypes = [\n arrow.BufferType.OFFSET,\n arrow.BufferType.DATA,\n arrow.BufferType.VALIDITY,\n arrow.BufferType.TYPE\n ];\n for (const bufferType of bufferTypes) {\n if (data.buffers[bufferType] !== undefined && isTypedArraySliced(data.buffers[bufferType])) {\n return true;\n }\n }\n return false;\n}\n/**\n * Returns true if the current typed array is a partial slice on a larger\n * ArrayBuffer\n */\nfunction isTypedArraySliced(arr) {\n return !(arr.byteOffset === 0 && arr.byteLength === arr.buffer.byteLength);\n}\n/**\n * If a slice of a larger ArrayBuffer, clone to a fresh `ArrayBuffer`.\n *\n * If `force` is `true`, always clone the array, even if not shared.\n */\nfunction cloneBuffer(arr, force) {\n // Not all buffer types are defined for every type of Arrow array. E.g.\n // `arrow.BufferType.TYPE` is only defined for the Union type.\n if (arr === undefined) {\n return arr;\n }\n // The current array is not a part of a larger ArrayBuffer, don't clone it\n if (!force && !isTypedArraySliced(arr)) {\n return arr;\n }\n // Note: TypedArray.slice() **copies** into a new ArrayBuffer\n // @ts-expect-error 'Uint8Array' is assignable to the constraint of type 'A',\n // but 'A' could be instantiated with a different subtype of constraint\n // 'TypedArray'\n // We know from arr.slice that it will always return the same\n return arr.slice();\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\nimport { processOnWorker } from '@loaders.gl/worker-utils';\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof \"4.4.0-alpha.1\" !== 'undefined' ? \"4.4.0-alpha.1\" : 'latest';\n/**\n * Worker for tessellating geometries. Normally called through triangulateOnWorker\n */\nexport const TriangulationWorker = {\n id: 'triangulation',\n name: 'Triangulate',\n module: 'arrow',\n version: VERSION,\n options: {}\n};\n/**\n * Triangulate a set of polygons on worker, type safe API\n */\nexport function triangulateOnWorker(data, options = {}) {\n return processOnWorker(TriangulationWorker, { ...data, operation: 'triangulate' }, options);\n}\n/**\n * Parse GeoArrow geometry colum on worker, type safe API\n */\nexport function parseGeoArrowOnWorker(data, options = {}) {\n return processOnWorker(TriangulationWorker, { ...data, operation: 'parse-geoarrow' }, options);\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA
|
|
6
|
-
"names": ["VECTOR_TYPES", "_a", "arrow", "VERSION", "arrow", "data", "VERSION"]
|
|
3
|
+
"sources": ["../src/index.ts", "../src/lib/types.ts", "../src/exports/arrow-format.ts", "../src/exports/arrow-loader.ts", "../src/lib/parsers/parse-arrow.ts", "../src/arrow-loader.ts", "../src/lib/encoders/encode-arrow.ts", "../src/arrow-writer.ts", "../src/exports/geoarrow-loader.ts", "../src/lib/parsers/parse-geoarrow.ts", "../src/geoarrow-loader.ts", "../src/workers/hard-clone.ts", "../src/triangulate-on-worker.ts"],
|
|
4
|
+
"sourcesContent": ["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// Types\nexport {VECTOR_TYPES} from './lib/types';\n\n// Arrow loader / Writer\n\nexport {ArrowFormat} from './exports/arrow-format';\n\nexport type {ArrowLoaderOptions} from './exports/arrow-loader';\nexport {ArrowWorkerLoader} from './exports/arrow-loader';\nexport {ArrowLoader} from './arrow-loader';\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Geoarrow loader\nexport {GeoArrowWorkerLoader} from './exports/geoarrow-loader';\nexport {GeoArrowLoader} from './geoarrow-loader';\n\n// EXPERIMENTAL\n\n// Arrow Utils\n// getGeometryColumnsFromArrowTable,\n// getGeoArrowEncoding\n\n// EXPERIMENTAL WORKER\nexport {hardClone} from './workers/hard-clone';\n\nexport type {ParseGeoArrowInput, ParseGeoArrowResult} from './triangulate-on-worker';\nexport {\n TriangulationWorker,\n triangulateOnWorker,\n parseGeoArrowOnWorker\n} from './triangulate-on-worker';\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\ntype TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n FLOAT,\n DATE\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Format} from '@loaders.gl/loader-utils';\n\n/** ArrowJS table loader */\nexport const ArrowFormat = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW']\n} as const satisfies Format;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ArrowTable} from '@loaders.gl/schema';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\n/** ArrowLoader options */\nexport type ArrowLoaderOptions = LoaderOptions & {\n /** ArrowLoader options */\n arrow?: {\n /** Shape of returned data */\n shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';\n /** Debounce time between batches (prevent excessive numbers of small batches) */\n batchDebounceMs?: number;\n /** Override the URL to the worker bundle (by default loads from unpkg.com) */\n workerUrl?: string;\n };\n};\n\n/** ArrowJS table loader */\nexport const ArrowWorkerLoader = {\n dataType: null as unknown as ArrowTable,\n batchType: null as never,\n\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n // worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: {\n arrow: {\n shape: 'columnar-table'\n }\n }\n} as const satisfies Loader<ArrowTable, never, ArrowLoaderOptions>;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport type {Table, ArrowTableBatch} from '@loaders.gl/schema';\nimport {ArrowLoaderOptions} from '../../exports/arrow-loader';\nimport {convertArrowToTable} from '@loaders.gl/schema-utils';\nimport {toArrayBufferIterator} from '@loaders.gl/loader-utils';\n\n/** Parses arrow to a loaders.gl table. Defaults to `arrow-table` */\nexport function parseArrowSync(arrayBuffer, options?: {shape?: Table['shape']}): Table {\n const shape = options?.shape || 'arrow-table';\n const arrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);\n return convertArrowToTable(arrowTable, shape);\n}\n\nexport function parseArrowInBatches(\n asyncIterator:\n | AsyncIterable<ArrayBufferLike | ArrayBufferView>\n | Iterable<ArrayBufferLike | ArrayBufferView>,\n options?: ArrowLoaderOptions\n): AsyncIterable<ArrowTableBatch> {\n // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n // This will also close the underlying source in case of early termination or errors\n\n // As an optimization, return a non-async iterator\n /*\n if (isIterable(readers)) {\n function* makeArrowIterator() {\n for (const reader of readers) {\n for (const batch of reader) {\n yield processBatch(batch, reader);\n }\n break; // only processing one stream of batches\n }\n }\n const arrowIterator = makeArrowIterator();\n }\n */\n\n async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n // @ts-ignore\n const readers = arrow.RecordBatchReader.readAll(toArrayBufferIterator(asyncIterator));\n for await (const reader of readers) {\n for await (const recordBatch of reader) {\n // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)\n if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {\n await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));\n }\n const arrowTabledBatch: ArrowTableBatch = {\n shape: 'arrow-table',\n batchType: 'data',\n data: new arrow.Table([recordBatch]),\n length: recordBatch.data.length\n };\n // processBatch(recordBatch);\n yield arrowTabledBatch;\n }\n break; // only processing one stream of batches\n }\n }\n\n return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n// const values = {};\n// batch.schema.fields.forEach(({name}, index) => {\n// values[name] = batch.getChildAt(index)?.toArray();\n// });\n// return {\n// };\n// }\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {Table, ArrowTableBatch} from '@loaders.gl/schema';\nimport {parseArrowSync, parseArrowInBatches} from './lib/parsers/parse-arrow';\n\nimport type {ArrowLoaderOptions} from './exports/arrow-loader';\nimport {ArrowWorkerLoader} from './exports/arrow-loader';\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n ...ArrowWorkerLoader,\n parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseArrowSync(arraybuffer, options?.arrow),\n parseSync: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseArrowSync(arraybuffer, options?.arrow),\n parseInBatches: parseArrowInBatches\n} as const satisfies LoaderWithParser<Table, ArrowTableBatch, ArrowLoaderOptions>;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\nimport {ensureArrayBuffer} from '@loaders.gl/loader-utils';\n\nexport type ColumnarTable = {\n name: string;\n array: AnyArrayType;\n type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n const vectors: Record<string, arrow.Vector> = {};\n for (const arrayData of data) {\n const arrayVector = createVector(arrayData.array, arrayData.type);\n vectors[arrayData.name] = arrayVector;\n }\n const table = new arrow.Table(vectors);\n const arrowBuffer = arrow.tableToIPC(table);\n return ensureArrayBuffer(arrowBuffer);\n}\n\n/**\n * Create Arrow arrow.Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type): arrow.Vector {\n switch (type) {\n case VECTOR_TYPES.DATE:\n return arrow.vectorFromArray(array);\n case VECTOR_TYPES.FLOAT:\n default:\n return arrow.vectorFromArray(array);\n }\n}\n", "// import type {} from '@loaders.gl/loader-utils';\n\nimport type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTable, encodeArrowSync} from './lib/encoders/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions & {\n arrow?: {};\n};\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n options: {},\n encode: async function encodeArrow(data, options?): Promise<ArrayBuffer> {\n return encodeArrowSync(data);\n },\n encodeSync(data, options?) {\n return encodeArrowSync(data);\n }\n} as const satisfies WriterWithEncoder<ColumnarTable, never, ArrowWriterOptions>;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ArrowTable, ArrowTableBatch, BinaryGeometry} from '@loaders.gl/schema';\nimport {ArrowWorkerLoader} from './arrow-loader';\n\nexport type GeoArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape?: 'arrow-table' | 'binary-geometry';\n };\n};\n\n/** ArrowJS table loader */\nexport const GeoArrowWorkerLoader = {\n ...ArrowWorkerLoader,\n options: {\n arrow: {\n shape: 'arrow-table'\n }\n }\n} as const satisfies Loader<ArrowTable | BinaryGeometry, ArrowTableBatch, GeoArrowLoaderOptions>;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {\n GeoJSONTable,\n GeoJSONTableBatch,\n ArrowTable,\n ArrowTableBatch\n} from '@loaders.gl/schema';\nimport {parseArrowSync, parseArrowInBatches} from './parse-arrow';\nimport {convertGeoArrowToTable} from '@loaders.gl/gis';\n\n// Parses arrow to a columnar table\nexport function parseGeoArrowSync(\n arrayBuffer,\n options?: {shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry'}\n): ArrowTable | GeoJSONTable {\n // | BinaryGeometry\n const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;\n switch (options?.shape) {\n case 'geojson-table':\n return convertGeoArrowToTable(table.data, 'geojson-table');\n default:\n return table;\n }\n}\n\n/**\n */\nexport function parseGeoArrowInBatches(\n asyncIterator:\n | AsyncIterable<ArrayBufferLike | ArrayBufferView>\n | Iterable<ArrayBufferLike | ArrayBufferView>\n): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch> {\n // | BinaryGeometry\n return parseArrowInBatches(asyncIterator);\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {\n ArrowTable,\n ArrowTableBatch,\n GeoJSONTable,\n GeoJSONTableBatch\n} from '@loaders.gl/schema';\nimport {parseGeoArrowSync, parseGeoArrowInBatches} from './lib/parsers/parse-geoarrow';\nimport type {GeoArrowLoaderOptions} from './exports/geoarrow-loader';\nimport {GeoArrowWorkerLoader} from './exports/geoarrow-loader';\n\n/**\n * GeoArrowLoader loads an Apache Arrow table, parses GeoArrow type extension data\n * to convert it to a GeoJSON table or a BinaryGeometry\n */\nexport const GeoArrowLoader = {\n ...GeoArrowWorkerLoader,\n\n parse: async (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>\n parseGeoArrowSync(arraybuffer, options?.arrow),\n parseSync: (arraybuffer: ArrayBuffer, options?: GeoArrowLoaderOptions) =>\n parseGeoArrowSync(arraybuffer, options?.arrow),\n parseInBatches: parseGeoArrowInBatches\n} as const satisfies LoaderWithParser<\n ArrowTable | GeoJSONTable, // | BinaryGeometry,\n ArrowTableBatch | GeoJSONTableBatch, // | BinaryGeometry,\n GeoArrowLoaderOptions\n>;\n", "import * as arrow from 'apache-arrow';\nimport type {Buffers} from 'apache-arrow/data';\n\ntype TypedArray =\n | Uint8Array\n | Uint8ClampedArray\n | Uint16Array\n | Uint32Array\n | Int8Array\n | Int16Array\n | Int32Array\n | Float32Array\n | Float64Array;\n\n/**\n * Clone an Arrow JS Data or Vector, detaching from an existing ArrayBuffer if\n * it is shared with other.\n *\n * The purpose of this function is to enable transferring a `Data` instance,\n * e.g. to a web worker, without neutering any other data.\n *\n * Any internal buffers that are a slice of a larger `ArrayBuffer` (i.e. where\n * the typed array's `byteOffset` is not `0` and where its `byteLength` does not\n * match its `array.buffer.byteLength`) are copied into new `ArrayBuffers`.\n *\n * If `force` is `true`, always clone internal buffers, even if not shared. If\n * the default, `false`, any internal buffers that are **not** a slice of a\n * larger `ArrayBuffer` will not be copied.\n */\nexport function hardClone<T extends arrow.DataType>(\n input: arrow.Data<T>,\n force?: boolean\n): arrow.Data<T>;\nexport function hardClone<T extends arrow.DataType>(\n input: arrow.Vector<T>,\n force?: boolean\n): arrow.Vector<T>;\n\nexport function hardClone<T extends arrow.DataType>(\n data: arrow.Data<T> | arrow.Vector<T>,\n force: boolean = false\n): arrow.Data<T> | arrow.Vector<T> {\n // Check if `data` is an arrow.Vector\n if ('data' in data) {\n return new arrow.Vector(data.data.map((data) => hardClone(data, force)));\n }\n\n // Clone each of the children, recursively\n const clonedChildren: arrow.Data[] = [];\n for (const childData of data.children) {\n clonedChildren.push(hardClone(childData, force));\n }\n\n // Clone the dictionary if there is one\n let clonedDictionary: arrow.Vector | undefined;\n if (data.dictionary !== undefined) {\n clonedDictionary = hardClone(data.dictionary, force);\n }\n\n // Buffers can have up to four entries. Each of these can be `undefined` for\n // one or more array types.\n //\n // - OFFSET: value offsets for variable size list types\n // - DATA: the underlying data\n // - VALIDITY: the null buffer. This may be empty or undefined if all elements\n // are non-null/valid.\n // - TYPE: type ids for a union type.\n const clonedBuffers: Buffers<T> = {\n [arrow.BufferType.OFFSET]: cloneBuffer(data.buffers[arrow.BufferType.OFFSET], force),\n [arrow.BufferType.DATA]: cloneBuffer(data.buffers[arrow.BufferType.DATA], force),\n [arrow.BufferType.VALIDITY]: cloneBuffer(data.buffers[arrow.BufferType.VALIDITY], force),\n [arrow.BufferType.TYPE]: cloneBuffer(data.buffers[arrow.BufferType.TYPE], force)\n };\n\n // Note: the data.offset is passed on so that a sliced Data instance will not\n // be \"un-sliced\". However keep in mind that this means we're cloning the\n // _original backing buffer_, not only the portion of the Data that was\n // sliced.\n return new arrow.Data(\n data.type,\n data.offset,\n data.length,\n // @ts-expect-error _nullCount is protected. We're using it here to mimic\n // `Data.clone`\n data._nullCount,\n clonedBuffers,\n clonedChildren,\n clonedDictionary\n );\n}\n\n/**\n * Test whether an arrow.Data instance is a slice of a larger `ArrayBuffer`.\n */\nexport function isShared<T extends arrow.DataType>(data: arrow.Data<T> | arrow.Vector<T>): boolean {\n // Loop over arrow.Vector\n if ('data' in data) {\n return data.data.some((data) => isShared(data));\n }\n\n // Check child data\n for (const childData of data.children) {\n if (isShared(childData)) {\n return true;\n }\n }\n\n // Check dictionary\n if (data.dictionary !== undefined) {\n if (isShared(data.dictionary)) {\n return true;\n }\n }\n\n const bufferTypes = [\n arrow.BufferType.OFFSET,\n arrow.BufferType.DATA,\n arrow.BufferType.VALIDITY,\n arrow.BufferType.TYPE\n ];\n for (const bufferType of bufferTypes) {\n if (data.buffers[bufferType] !== undefined && isTypedArraySliced(data.buffers[bufferType])) {\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Returns true if the current typed array is a partial slice on a larger\n * ArrayBuffer\n */\nfunction isTypedArraySliced(arr: TypedArray): boolean {\n return !(arr.byteOffset === 0 && arr.byteLength === arr.buffer.byteLength);\n}\n\n/**\n * If a slice of a larger ArrayBuffer, clone to a fresh `ArrayBuffer`.\n *\n * If `force` is `true`, always clone the array, even if not shared.\n */\nfunction cloneBuffer<A extends TypedArray | undefined>(arr: A, force: boolean): A {\n // Not all buffer types are defined for every type of Arrow array. E.g.\n // `arrow.BufferType.TYPE` is only defined for the Union type.\n if (arr === undefined) {\n return arr;\n }\n\n // The current array is not a part of a larger ArrayBuffer, don't clone it\n if (!force && !isTypedArraySliced(arr)) {\n return arr;\n }\n\n // Note: TypedArray.slice() **copies** into a new ArrayBuffer\n\n // @ts-expect-error 'Uint8Array' is assignable to the constraint of type 'A',\n // but 'A' could be instantiated with a different subtype of constraint\n // 'TypedArray'\n // We know from arr.slice that it will always return the same\n return arr.slice();\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport type {WorkerOptions} from '@loaders.gl/worker-utils';\nimport {processOnWorker} from '@loaders.gl/worker-utils';\nimport type {GeoArrowEncoding} from '@loaders.gl/geoarrow';\nimport {BinaryDataFromGeoArrow} from '@loaders.gl/gis';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type TriangulationWorkerInput =\n | ({operation: 'triangulate'} & TriangulateInput)\n | ParseGeoArrowInput\n | {operation: 'test'; data: any};\n\nexport type TriangulationWorkerOutput =\n | ({operation: 'triangulate'} & TriangulateResult)\n | ({operation: 'parse-geoarrow'} & ParseGeoArrowResult)\n | {operation: 'test'; data: any};\n\ntype GeoArrowChunkData = {\n type: arrow.DataType;\n offset: number;\n length: number;\n nullCount: number;\n buffers: any;\n children: arrow.Data[];\n dictionary?: arrow.Vector;\n};\n\nexport type ParseGeoArrowInput = {\n operation: 'parse-geoarrow';\n chunkData: GeoArrowChunkData;\n chunkIndex: number;\n chunkOffset: number;\n geometryEncoding: GeoArrowEncoding;\n calculateMeanCenters: boolean;\n triangle: boolean;\n};\n\nexport type ParseGeoArrowResult = {\n chunkIndex: number;\n binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null;\n};\n\n/** Input data for operation: 'triangulate' */\nexport type TriangulateInput = {\n polygonIndices: Uint16Array;\n primitivePolygonIndices: Int32Array;\n flatCoordinateArray: Float64Array;\n nDim: number;\n};\n\n/** Result type for operation: 'triangulate' */\nexport type TriangulateResult = TriangulateInput & {\n triangleIndices?: Uint32Array;\n};\n\n/**\n * Worker for tessellating geometries. Normally called through triangulateOnWorker\n */\nexport const TriangulationWorker = {\n id: 'triangulation',\n name: 'Triangulate',\n module: 'arrow',\n version: VERSION,\n options: {}\n};\n\n/**\n * Triangulate a set of polygons on worker, type safe API\n */\nexport function triangulateOnWorker(\n data: TriangulateInput,\n options: WorkerOptions = {}\n): Promise<TriangulateResult> {\n return processOnWorker(TriangulationWorker, {...data, operation: 'triangulate'}, options);\n}\n\n/**\n * Parse GeoArrow geometry colum on worker, type safe API\n */\nexport function parseGeoArrowOnWorker(\n data: ParseGeoArrowInput,\n options: WorkerOptions = {}\n): Promise<ParseGeoArrowResult> {\n return processOnWorker(TriangulationWorker, {...data, operation: 'parse-geoarrow'}, options);\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;;;;;;;;;;ACYA,IAAY;CAAZ,SAAYA,eAAY;AACtB,EAAAA,cAAAA,cAAA,OAAA,IAAA,CAAA,IAAA;AACA,EAAAA,cAAAA,cAAA,MAAA,IAAA,CAAA,IAAA;AACF,GAHY,iBAAA,eAAY,CAAA,EAAA;;;ACLjB,IAAM,cAAc;EACzB,MAAM;EACN,IAAI;EACJ,QAAQ;EACR,UAAU;EACV,YAAY,CAAC,SAAS,SAAS;EAC/B,WAAW;IACT;IACA;IACA;;EAEF,QAAQ;EACR,OAAO,CAAC,OAAO;;;;ACVjB,IAAM,UAAU,OAAoC,kBAAe;AAgB5D,IAAM,oBAAoB;EAC/B,UAAU;EACV,WAAW;EAEX,MAAM;EACN,IAAI;EACJ,QAAQ;EACR,SAAS;;EAET,UAAU;EACV,YAAY,CAAC,SAAS,SAAS;EAC/B,WAAW;IACT;IACA;IACA;;EAEF,QAAQ;EACR,OAAO,CAAC,OAAO;EACf,SAAS;IACP,OAAO;MACL,OAAO;;;;;;ACzCb,YAAuB;AAGvB,0BAAkC;AAClC,0BAAoC;AAG9B,SAAU,eAAe,aAAa,SAAkC;AAC5E,QAAM,SAAQ,mCAAS,UAAS;AAChC,QAAM,aAAmB,mBAAa,CAAC,IAAI,WAAW,WAAW,CAAC,CAAC;AACnE,aAAO,yCAAoB,YAAY,KAAK;AAC9C;AAEM,SAAU,oBACd,eAGA,SAA4B;AAoB5B,kBAAgB,yBAAsB;AAzCxC;AA2CI,UAAM,UAAgB,wBAAkB,YAAQ,2CAAsB,aAAa,CAAC;AACpF,qBAAiB,UAAU,SAAS;AAClC,uBAAiB,eAAe,QAAQ;AAEtC,cAAI,wCAAS,UAAT,mBAAgB,qBAAoB,YAAa,wCAAS,UAAT,mBAAgB,mBAAkB,GAAG;AACxF,gBAAM,IAAI,QAAQ,CAAC,YAAS;AAhDtC,gBAAAC;AAgDyC,8BAAW,WAASA,MAAA,QAAQ,UAAR,gBAAAA,IAAe,oBAAmB,CAAC;WAAC;QACzF;AACA,cAAM,mBAAoC;UACxC,OAAO;UACP,WAAW;UACX,MAAM,IAAU,YAAM,CAAC,WAAW,CAAC;UACnC,QAAQ,YAAY,KAAK;;AAG3B,cAAM;MACR;AACA;IACF;EACF;AAEA,SAAO,uBAAsB;AAC/B;;;ACpDO,IAAM,cAAc;EACzB,GAAG;EACH,OAAO,OAAO,aAA0B,YACtC,eAAe,aAAa,mCAAS,KAAK;EAC5C,WAAW,CAAC,aAA0B,YACpC,eAAe,aAAa,mCAAS,KAAK;EAC5C,gBAAgB;;;;ACdlB,IAAAC,SAAuB;AAEvB,IAAAC,uBAAgC;AAe1B,SAAU,gBAAgB,MAAmB;AACjD,QAAM,UAAwC,CAAA;AAC9C,aAAW,aAAa,MAAM;AAC5B,UAAM,cAAc,aAAa,UAAU,OAAO,UAAU,IAAI;AAChE,YAAQ,UAAU,IAAI,IAAI;EAC5B;AACA,QAAM,QAAQ,IAAU,aAAM,OAAO;AACrC,QAAM,cAAoB,kBAAW,KAAK;AAC1C,aAAO,wCAAkB,WAAW;AACtC;AAQA,SAAS,aAAa,OAAO,MAAI;AAC/B,UAAQ,MAAM;IACZ,KAAK,aAAa;AAChB,aAAa,uBAAgB,KAAK;IACpC,KAAK,aAAa;IAClB;AACE,aAAa,uBAAgB,KAAK;EACtC;AACF;;;ACvCA,IAAMC,WAAU,OAAoC,kBAAe;AAO5D,IAAM,cAAc;EACzB,MAAM;EACN,IAAI;EACJ,QAAQ;EACR,SAASA;EACT,YAAY,CAAC,SAAS,SAAS;EAC/B,WAAW;IACT;IACA;IACA;;EAEF,QAAQ;EACR,SAAS,CAAA;EACT,QAAQ,eAAe,YAAY,MAAM,SAAQ;AAC/C,WAAO,gBAAgB,IAAI;EAC7B;EACA,WAAW,MAAM,SAAQ;AACvB,WAAO,gBAAgB,IAAI;EAC7B;;;;ACjBK,IAAM,uBAAuB;EAClC,GAAG;EACH,SAAS;IACP,OAAO;MACL,OAAO;;;;;;ACRb,iBAAqC;AAG/B,SAAU,kBACd,aACA,SAAuE;AAGvE,QAAM,QAAQ,eAAe,aAAa,EAAC,OAAO,cAAa,CAAC;AAChE,UAAQ,mCAAS,OAAO;IACtB,KAAK;AACH,iBAAO,mCAAuB,MAAM,MAAM,eAAe;IAC3D;AACE,aAAO;EACX;AACF;AAIM,SAAU,uBACd,eAE+C;AAG/C,SAAO,oBAAoB,aAAa;AAC1C;;;AClBO,IAAM,iBAAiB;EAC5B,GAAG;EAEH,OAAO,OAAO,aAA0B,YACtC,kBAAkB,aAAa,mCAAS,KAAK;EAC/C,WAAW,CAAC,aAA0B,YACpC,kBAAkB,aAAa,mCAAS,KAAK;EAC/C,gBAAgB;;;;AC1BlB,IAAAC,SAAuB;AAsCjB,SAAU,UACd,MACA,QAAiB,OAAK;AAGtB,MAAI,UAAU,MAAM;AAClB,WAAO,IAAU,cAAO,KAAK,KAAK,IAAI,CAACC,UAAS,UAAUA,OAAM,KAAK,CAAC,CAAC;EACzE;AAGA,QAAM,iBAA+B,CAAA;AACrC,aAAW,aAAa,KAAK,UAAU;AACrC,mBAAe,KAAK,UAAU,WAAW,KAAK,CAAC;EACjD;AAGA,MAAI;AACJ,MAAI,KAAK,eAAe,QAAW;AACjC,uBAAmB,UAAU,KAAK,YAAY,KAAK;EACrD;AAUA,QAAM,gBAA4B;IAChC,CAAO,kBAAW,MAAM,GAAG,YAAY,KAAK,QAAc,kBAAW,MAAM,GAAG,KAAK;IACnF,CAAO,kBAAW,IAAI,GAAG,YAAY,KAAK,QAAc,kBAAW,IAAI,GAAG,KAAK;IAC/E,CAAO,kBAAW,QAAQ,GAAG,YAAY,KAAK,QAAc,kBAAW,QAAQ,GAAG,KAAK;IACvF,CAAO,kBAAW,IAAI,GAAG,YAAY,KAAK,QAAc,kBAAW,IAAI,GAAG,KAAK;;AAOjF,SAAO,IAAU;IACf,KAAK;IACL,KAAK;IACL,KAAK;;;IAGL,KAAK;IACL;IACA;IACA;EAAgB;AAEpB;AA4CA,SAAS,mBAAmB,KAAe;AACzC,SAAO,EAAE,IAAI,eAAe,KAAK,IAAI,eAAe,IAAI,OAAO;AACjE;AAOA,SAAS,YAA8C,KAAQ,OAAc;AAG3E,MAAI,QAAQ,QAAW;AACrB,WAAO;EACT;AAGA,MAAI,CAAC,SAAS,CAAC,mBAAmB,GAAG,GAAG;AACtC,WAAO;EACT;AAQA,SAAO,IAAI,MAAK;AAClB;;;AC3JA,0BAA8B;AAM9B,IAAMC,WAAU,OAAoC,kBAAe;AAqD5D,IAAM,sBAAsB;EACjC,IAAI;EACJ,MAAM;EACN,QAAQ;EACR,SAASA;EACT,SAAS,CAAA;;AAML,SAAU,oBACd,MACA,UAAyB,CAAA,GAAE;AAE3B,aAAO,qCAAgB,qBAAqB,EAAC,GAAG,MAAM,WAAW,cAAa,GAAG,OAAO;AAC1F;AAKM,SAAU,sBACd,MACA,UAAyB,CAAA,GAAE;AAE3B,aAAO,qCAAgB,qBAAqB,EAAC,GAAG,MAAM,WAAW,iBAAgB,GAAG,OAAO;AAC7F;",
|
|
6
|
+
"names": ["VECTOR_TYPES", "_a", "arrow", "import_loader_utils", "VERSION", "arrow", "data", "VERSION"]
|
|
7
7
|
}
|
package/dist/index.js
CHANGED
|
@@ -18,3 +18,4 @@ export { GeoArrowLoader } from "./geoarrow-loader.js";
|
|
|
18
18
|
// EXPERIMENTAL WORKER
|
|
19
19
|
export { hardClone } from "./workers/hard-clone.js";
|
|
20
20
|
export { TriangulationWorker, triangulateOnWorker, parseGeoArrowOnWorker } from "./triangulate-on-worker.js";
|
|
21
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,QAAQ;AACR,OAAO,EAAC,YAAY,EAAC,uBAAoB;AAEzC,wBAAwB;AAExB,OAAO,EAAC,WAAW,EAAC,kCAA+B;AAGnD,OAAO,EAAC,iBAAiB,EAAC,kCAA+B;AACzD,OAAO,EAAC,WAAW,EAAC,0BAAuB;AAE3C,OAAO,EAAC,WAAW,EAAC,0BAAuB;AAE3C,kBAAkB;AAClB,OAAO,EAAC,oBAAoB,EAAC,qCAAkC;AAC/D,OAAO,EAAC,cAAc,EAAC,6BAA0B;AAEjD,eAAe;AAEf,cAAc;AACd,oCAAoC;AACpC,sBAAsB;AAEtB,sBAAsB;AACtB,OAAO,EAAC,SAAS,EAAC,gCAA6B;AAG/C,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,mCAAgC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;
|
|
1
|
+
{"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAGpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAShE"}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
import * as arrow from 'apache-arrow';
|
|
5
5
|
import { VECTOR_TYPES } from "../types.js";
|
|
6
|
+
import { ensureArrayBuffer } from '@loaders.gl/loader-utils';
|
|
6
7
|
/**
|
|
7
8
|
* Encodes set of arrays into the Apache Arrow columnar format
|
|
8
9
|
* https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
|
|
@@ -18,7 +19,7 @@ export function encodeArrowSync(data) {
|
|
|
18
19
|
}
|
|
19
20
|
const table = new arrow.Table(vectors);
|
|
20
21
|
const arrowBuffer = arrow.tableToIPC(table);
|
|
21
|
-
return arrowBuffer;
|
|
22
|
+
return ensureArrayBuffer(arrowBuffer);
|
|
22
23
|
}
|
|
23
24
|
/**
|
|
24
25
|
* Create Arrow arrow.Vector from given data and vector type
|
|
@@ -35,3 +36,4 @@ function createVector(array, type) {
|
|
|
35
36
|
return arrow.vectorFromArray(array);
|
|
36
37
|
}
|
|
37
38
|
}
|
|
39
|
+
//# sourceMappingURL=encode-arrow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encode-arrow.js","sourceRoot":"","sources":["../../../src/lib/encoders/encode-arrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,EAAe,YAAY,EAAC,oBAAiB;AACpD,OAAO,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAQ3D;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,IAAmB;IACjD,MAAM,OAAO,GAAiC,EAAE,CAAC;IACjD,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAClE,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC;IACxC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,iBAAiB,CAAC,WAAW,CAAC,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAK,EAAE,IAAI;IAC/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC,IAAI;YACpB,OAAO,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,YAAY,CAAC,KAAK,CAAC;QACxB;YACE,OAAO,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;
|
|
1
|
+
{"version":3,"file":"encode-geoarrow.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-geoarrow.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,YAAY,EAAe,oBAAiB;AAGpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CASnE"}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
import * as arrow from 'apache-arrow';
|
|
5
5
|
import { VECTOR_TYPES } from "../types.js";
|
|
6
|
+
import { ensureArrayBuffer } from '@loaders.gl/loader-utils';
|
|
6
7
|
/**
|
|
7
8
|
* Encodes set of arrays into the Apache Arrow columnar format
|
|
8
9
|
* https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
|
|
@@ -18,7 +19,7 @@ export function encodeGeoArrowSync(data) {
|
|
|
18
19
|
}
|
|
19
20
|
const table = new arrow.Table(vectors);
|
|
20
21
|
const arrowBuffer = arrow.tableToIPC(table);
|
|
21
|
-
return arrowBuffer;
|
|
22
|
+
return ensureArrayBuffer(arrowBuffer);
|
|
22
23
|
}
|
|
23
24
|
/**
|
|
24
25
|
* Create Arrow arrow.Vector from given data and vector type
|
|
@@ -35,3 +36,4 @@ function createVector(array, type) {
|
|
|
35
36
|
return arrow.vectorFromArray(array);
|
|
36
37
|
}
|
|
37
38
|
}
|
|
39
|
+
//# sourceMappingURL=encode-geoarrow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encode-geoarrow.js","sourceRoot":"","sources":["../../../src/lib/encoders/encode-geoarrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,EAAe,YAAY,EAAC,oBAAiB;AACpD,OAAO,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAQ3D;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAmB;IACpD,MAAM,OAAO,GAAiC,EAAE,CAAC;IACjD,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAClE,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC;IACxC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,iBAAiB,CAAC,WAAW,CAAC,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAK,EAAE,IAAI;IAC/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC,IAAI;YACpB,OAAO,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,YAAY,CAAC,KAAK,CAAC;QACxB;YACE,OAAO,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;AACH,CAAC"}
|
|
@@ -4,5 +4,5 @@ import { ArrowLoaderOptions } from "../../exports/arrow-loader.js";
|
|
|
4
4
|
export declare function parseArrowSync(arrayBuffer: any, options?: {
|
|
5
5
|
shape?: Table['shape'];
|
|
6
6
|
}): Table;
|
|
7
|
-
export declare function parseArrowInBatches(asyncIterator: AsyncIterable<
|
|
7
|
+
export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>, options?: ArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
|
|
8
8
|
//# sourceMappingURL=parse-arrow.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,KAAK,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAC,kBAAkB,EAAC,sCAAmC;
|
|
1
|
+
{"version":3,"file":"parse-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,KAAK,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAC,kBAAkB,EAAC,sCAAmC;AAI9D,oEAAoE;AACpE,wBAAgB,cAAc,CAAC,WAAW,KAAA,EAAE,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;CAAC,GAAG,KAAK,CAIrF;AAED,wBAAgB,mBAAmB,CACjC,aAAa,EACT,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC,eAAe,CAAC,CA0ChC"}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
import * as arrow from 'apache-arrow';
|
|
5
5
|
import { convertArrowToTable } from '@loaders.gl/schema-utils';
|
|
6
|
+
import { toArrayBufferIterator } from '@loaders.gl/loader-utils';
|
|
6
7
|
/** Parses arrow to a loaders.gl table. Defaults to `arrow-table` */
|
|
7
8
|
export function parseArrowSync(arrayBuffer, options) {
|
|
8
9
|
const shape = options?.shape || 'arrow-table';
|
|
@@ -28,7 +29,7 @@ export function parseArrowInBatches(asyncIterator, options) {
|
|
|
28
29
|
*/
|
|
29
30
|
async function* makeArrowAsyncIterator() {
|
|
30
31
|
// @ts-ignore
|
|
31
|
-
const readers = arrow.RecordBatchReader.readAll(asyncIterator);
|
|
32
|
+
const readers = arrow.RecordBatchReader.readAll(toArrayBufferIterator(asyncIterator));
|
|
32
33
|
for await (const reader of readers) {
|
|
33
34
|
for await (const recordBatch of reader) {
|
|
34
35
|
// use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
|
|
@@ -57,3 +58,4 @@ export function parseArrowInBatches(asyncIterator, options) {
|
|
|
57
58
|
// return {
|
|
58
59
|
// };
|
|
59
60
|
// }
|
|
61
|
+
//# sourceMappingURL=parse-arrow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-arrow.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-arrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAGtC,OAAO,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAC,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AAE/D,oEAAoE;AACpE,MAAM,UAAU,cAAc,CAAC,WAAW,EAAE,OAAkC;IAC5E,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO,mBAAmB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,aAE+C,EAC/C,OAA4B;IAE5B,4EAA4E;IAC5E,oFAAoF;IAEpF,kDAAkD;IAClD;;;;;;;;;;;;MAYE;IAEF,KAAK,SAAS,CAAC,CAAC,sBAAsB;QACpC,aAAa;QACb,MAAM,OAAO,GAAG,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC,CAAC;QACtF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YACnC,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,MAAM,EAAE,CAAC;gBACvC,uGAAuG;gBACvG,IAAI,OAAO,EAAE,KAAK,EAAE,eAAe,KAAK,SAAS,IAAI,OAAO,EAAE,KAAK,EAAE,eAAe,GAAG,CAAC,EAAE,CAAC;oBACzF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3F,CAAC;gBACD,MAAM,gBAAgB,GAAoB;oBACxC,KAAK,EAAE,aAAa;oBACpB,SAAS,EAAE,MAAM;oBACjB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;oBACpC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM;iBAChC,CAAC;gBACF,6BAA6B;gBAC7B,MAAM,gBAAgB,CAAC;YACzB,CAAC;YACD,MAAM,CAAC,wCAAwC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,sBAAsB,EAAS,CAAC,CAAC,qCAAqC;AAC/E,CAAC;AAED,+DAA+D;AAC/D,uBAAuB;AACvB,qDAAqD;AACrD,yDAAyD;AACzD,QAAQ;AACR,aAAa;AACb,OAAO;AACP,IAAI"}
|
|
@@ -4,5 +4,5 @@ export declare function parseGeoArrowSync(arrayBuffer: any, options?: {
|
|
|
4
4
|
}): ArrowTable | GeoJSONTable;
|
|
5
5
|
/**
|
|
6
6
|
*/
|
|
7
|
-
export declare function parseGeoArrowInBatches(asyncIterator: AsyncIterable<
|
|
7
|
+
export declare function parseGeoArrowInBatches(asyncIterator: AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch>;
|
|
8
8
|
//# sourceMappingURL=parse-geoarrow.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-geoarrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoarrow.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAK5B,wBAAgB,iBAAiB,CAC/B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,eAAe,GAAG,iBAAiB,CAAA;CAAC,GACtE,UAAU,GAAG,YAAY,CAS3B;AAED;GACG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,
|
|
1
|
+
{"version":3,"file":"parse-geoarrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoarrow.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAK5B,wBAAgB,iBAAiB,CAC/B,WAAW,KAAA,EACX,OAAO,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,aAAa,GAAG,eAAe,GAAG,iBAAiB,CAAA;CAAC,GACtE,UAAU,GAAG,YAAY,CAS3B;AAED;GACG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,EACT,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,GAC9C,aAAa,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAGpD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-geoarrow.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoarrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAQpC,OAAO,EAAC,cAAc,EAAE,mBAAmB,EAAC,yBAAsB;AAClE,OAAO,EAAC,sBAAsB,EAAC,MAAM,iBAAiB,CAAC;AAEvD,mCAAmC;AACnC,MAAM,UAAU,iBAAiB,CAC/B,WAAW,EACX,OAAuE;IAEvE,mBAAmB;IACnB,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,EAAE,EAAC,KAAK,EAAE,aAAa,EAAC,CAAe,CAAC;IAChF,QAAQ,OAAO,EAAE,KAAK,EAAE,CAAC;QACvB,KAAK,eAAe;YAClB,OAAO,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QAC7D;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED;GACG;AACH,MAAM,UAAU,sBAAsB,CACpC,aAE+C;IAE/C,mBAAmB;IACnB,OAAO,mBAAmB,CAAC,aAAa,CAAC,CAAC;AAC5C,CAAC"}
|
package/dist/lib/types.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAUpC,MAAM,CAAN,IAAY,YAGX;AAHD,WAAY,YAAY;IACtB,iDAAK,CAAA;IACL,+CAAI,CAAA;AACN,CAAC,EAHW,YAAY,KAAZ,YAAY,QAGvB"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { processOnWorker } from '@loaders.gl/worker-utils';
|
|
5
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof "4.4.0-alpha.
|
|
7
|
+
const VERSION = typeof "4.4.0-alpha.9" !== 'undefined' ? "4.4.0-alpha.9" : 'latest';
|
|
8
8
|
/**
|
|
9
9
|
* Worker for tessellating geometries. Normally called through triangulateOnWorker
|
|
10
10
|
*/
|
|
@@ -27,3 +27,4 @@ export function triangulateOnWorker(data, options = {}) {
|
|
|
27
27
|
export function parseGeoArrowOnWorker(data, options = {}) {
|
|
28
28
|
return processOnWorker(TriangulationWorker, { ...data, operation: 'parse-geoarrow' }, options);
|
|
29
29
|
}
|
|
30
|
+
//# sourceMappingURL=triangulate-on-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"triangulate-on-worker.js","sourceRoot":"","sources":["../src/triangulate-on-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,eAAe,EAAC,MAAM,0BAA0B,CAAC;AAIzD,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAkD5E;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,EAAE,EAAE,eAAe;IACnB,IAAI,EAAE,aAAa;IACnB,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE,EAAE;CACZ,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAsB,EACtB,UAAyB,EAAE;IAE3B,OAAO,eAAe,CAAC,mBAAmB,EAAE,EAAC,GAAG,IAAI,EAAE,SAAS,EAAE,aAAa,EAAC,EAAE,OAAO,CAAC,CAAC;AAC5F,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAwB,EACxB,UAAyB,EAAE;IAE3B,OAAO,eAAe,CAAC,mBAAmB,EAAE,EAAC,GAAG,IAAI,EAAE,SAAS,EAAE,gBAAgB,EAAC,EAAE,OAAO,CAAC,CAAC;AAC/F,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arrow-worker.js","sourceRoot":"","sources":["../../src/workers/arrow-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAC,WAAW,EAAC,2BAAwB;AAE5C,kBAAkB,CAAC,WAAW,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hard-clone.js","sourceRoot":"","sources":["../../src/workers/hard-clone.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAsCtC,MAAM,UAAU,SAAS,CACvB,IAAqC,EACrC,QAAiB,KAAK;IAEtB,qCAAqC;IACrC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,0CAA0C;IAC1C,MAAM,cAAc,GAAiB,EAAE,CAAC;IACxC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACtC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,uCAAuC;IACvC,IAAI,gBAA0C,CAAC;IAC/C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;IAED,4EAA4E;IAC5E,2BAA2B;IAC3B,EAAE;IACF,uDAAuD;IACvD,8BAA8B;IAC9B,8EAA8E;IAC9E,wBAAwB;IACxB,qCAAqC;IACrC,MAAM,aAAa,GAAe;QAChC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC;QACpF,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC;QAChF,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACxF,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC;KACjF,CAAC;IAEF,6EAA6E;IAC7E,yEAAyE;IACzE,uEAAuE;IACvE,UAAU;IACV,OAAO,IAAI,KAAK,CAAC,IAAI,CACnB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM;IACX,yEAAyE;IACzE,eAAe;IACf,IAAI,CAAC,UAAU,EACf,aAAa,EACb,cAAc,EACd,gBAAgB,CACjB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAA2B,IAAqC;IACtF,yBAAyB;IACzB,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,mBAAmB;IACnB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG;QAClB,KAAK,CAAC,UAAU,CAAC,MAAM;QACvB,KAAK,CAAC,UAAU,CAAC,IAAI;QACrB,KAAK,CAAC,UAAU,CAAC,QAAQ;QACzB,KAAK,CAAC,UAAU,CAAC,IAAI;KACtB,CAAC;IACF,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,SAAS,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAC3F,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,GAAe;IACzC,OAAO,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAmC,GAAM,EAAE,KAAc;IAC3E,uEAAuE;IACvE,8DAA8D;IAC9D,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,0EAA0E;IAC1E,IAAI,CAAC,KAAK,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,6DAA6D;IAE7D,6EAA6E;IAC7E,uEAAuE;IACvE,eAAe;IACf,6DAA6D;IAC7D,OAAO,GAAG,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"triangulation-worker-node.js","sourceRoot":"","sources":["../../src/workers/triangulation-worker-node.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,mCAAgC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"triangulation-worker.js","sourceRoot":"","sources":["../../src/workers/triangulation-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AACtD,OAAO,EACL,kBAAkB,EAClB,wCAAwC,EAEzC,MAAM,iBAAiB,CAAC;AASzB,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACxC,MAAM,KAAK,GAAG,IAAgC,CAAC;IAC/C,MAAM,SAAS,GAAG,KAAK,EAAE,SAAS,CAAC;IACnC,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,aAAa;YAChB,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChC,KAAK,gBAAgB;YACnB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClC;YACE,MAAM,IAAI,KAAK,CACb,8CAA8C,SAAS,0BAA0B,CAClF,CAAC;IACN,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,SAAS,gBAAgB,CAAC,IAAsB;IAC9C,+BAA+B;IAC/B,0BAA0B;IAC1B,8BAA8B;IAC9B,MAAM,eAAe,GAAG,kBAAkB,CACxC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,mBAAmB,EACxB,IAAI,CAAC,IAAI,CACV,CAAC;IACF,OAAO,EAAC,GAAG,IAAI,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAC,eAAe,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,kBAAkB,CAAC,IAAwB;IAClD,IAAI,sBAAsB,GAAkC,IAAI,CAAC;IACjE,MAAM,EAAC,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,QAAQ,EAAC,GAC1F,IAAI,CAAC;IACP,qDAAqD;IACrD,MAAM,SAAS,GAAG,IAAI,KAAK,CAAC,IAAI,CAC9B,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,SAAS,EACnB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,QAAQ,EAClB,SAAS,CAAC,UAAU,CACrB,CAAC;IACF,yCAAyC;IACzC,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACnD,IAAI,cAAc,EAAE,CAAC;QACnB,uFAAuF;QACvF,MAAM,OAAO,GAAG,EAAC,oBAAoB,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAC,CAAC;QAC7E,sBAAsB,GAAG,wCAAwC,CAC/D,cAAc,EACd,gBAAgB,EAChB,OAAO,CACR,CAAC;QACF,0DAA0D;QAC1D,OAAO;YACL,sBAAsB;YACtB,UAAU;SACX,CAAC;IACJ,CAAC;IACD,OAAO;QACL,sBAAsB;QACtB,UAAU;KACX,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/arrow",
|
|
3
|
-
"version": "4.4.0-alpha.
|
|
3
|
+
"version": "4.4.0-alpha.9",
|
|
4
4
|
"description": "Simple columnar table loader for the Apache Arrow format",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -36,6 +36,9 @@
|
|
|
36
36
|
"./exports/*": {
|
|
37
37
|
"types": "./dist/exports/*.d.ts",
|
|
38
38
|
"import": "./dist/exports/*.js"
|
|
39
|
+
},
|
|
40
|
+
"./arrow-worker.js": {
|
|
41
|
+
"import": "./dist/arrow-worker.js"
|
|
39
42
|
}
|
|
40
43
|
},
|
|
41
44
|
"sideEffects": false,
|
|
@@ -57,17 +60,17 @@
|
|
|
57
60
|
"build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
|
|
58
61
|
},
|
|
59
62
|
"dependencies": {
|
|
60
|
-
"@loaders.gl/gis": "4.4.0-alpha.
|
|
61
|
-
"@loaders.gl/loader-utils": "4.4.0-alpha.
|
|
62
|
-
"@loaders.gl/schema": "4.4.0-alpha.
|
|
63
|
-
"@loaders.gl/schema-utils": "4.4.0-alpha.
|
|
64
|
-
"@loaders.gl/wkt": "4.4.0-alpha.
|
|
65
|
-
"@loaders.gl/worker-utils": "4.4.0-alpha.
|
|
63
|
+
"@loaders.gl/gis": "4.4.0-alpha.9",
|
|
64
|
+
"@loaders.gl/loader-utils": "4.4.0-alpha.9",
|
|
65
|
+
"@loaders.gl/schema": "4.4.0-alpha.9",
|
|
66
|
+
"@loaders.gl/schema-utils": "4.4.0-alpha.9",
|
|
67
|
+
"@loaders.gl/wkt": "4.4.0-alpha.9",
|
|
68
|
+
"@loaders.gl/worker-utils": "4.4.0-alpha.9",
|
|
66
69
|
"@math.gl/polygon": "^4.1.0",
|
|
67
70
|
"apache-arrow": ">= 16.1.0"
|
|
68
71
|
},
|
|
69
72
|
"peerDependencies": {
|
|
70
73
|
"@loaders.gl/core": "4.4.0-alpha.1"
|
|
71
74
|
},
|
|
72
|
-
"gitHead": "
|
|
75
|
+
"gitHead": "e9e6710379718c7663e97eba868c76e15de4cb84"
|
|
73
76
|
}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import * as arrow from 'apache-arrow';
|
|
6
6
|
import {AnyArrayType, VECTOR_TYPES} from '../types';
|
|
7
|
+
import {ensureArrayBuffer} from '@loaders.gl/loader-utils';
|
|
7
8
|
|
|
8
9
|
export type ColumnarTable = {
|
|
9
10
|
name: string;
|
|
@@ -26,7 +27,7 @@ export function encodeArrowSync(data: ColumnarTable): ArrayBuffer {
|
|
|
26
27
|
}
|
|
27
28
|
const table = new arrow.Table(vectors);
|
|
28
29
|
const arrowBuffer = arrow.tableToIPC(table);
|
|
29
|
-
return arrowBuffer;
|
|
30
|
+
return ensureArrayBuffer(arrowBuffer);
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
/**
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import * as arrow from 'apache-arrow';
|
|
6
6
|
import {AnyArrayType, VECTOR_TYPES} from '../types';
|
|
7
|
+
import {ensureArrayBuffer} from '@loaders.gl/loader-utils';
|
|
7
8
|
|
|
8
9
|
export type ColumnarTable = {
|
|
9
10
|
name: string;
|
|
@@ -26,7 +27,7 @@ export function encodeGeoArrowSync(data: ColumnarTable): ArrayBuffer {
|
|
|
26
27
|
}
|
|
27
28
|
const table = new arrow.Table(vectors);
|
|
28
29
|
const arrowBuffer = arrow.tableToIPC(table);
|
|
29
|
-
return arrowBuffer;
|
|
30
|
+
return ensureArrayBuffer(arrowBuffer);
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
/**
|
|
@@ -6,6 +6,7 @@ import * as arrow from 'apache-arrow';
|
|
|
6
6
|
import type {Table, ArrowTableBatch} from '@loaders.gl/schema';
|
|
7
7
|
import {ArrowLoaderOptions} from '../../exports/arrow-loader';
|
|
8
8
|
import {convertArrowToTable} from '@loaders.gl/schema-utils';
|
|
9
|
+
import {toArrayBufferIterator} from '@loaders.gl/loader-utils';
|
|
9
10
|
|
|
10
11
|
/** Parses arrow to a loaders.gl table. Defaults to `arrow-table` */
|
|
11
12
|
export function parseArrowSync(arrayBuffer, options?: {shape?: Table['shape']}): Table {
|
|
@@ -15,7 +16,9 @@ export function parseArrowSync(arrayBuffer, options?: {shape?: Table['shape']}):
|
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
export function parseArrowInBatches(
|
|
18
|
-
asyncIterator:
|
|
19
|
+
asyncIterator:
|
|
20
|
+
| AsyncIterable<ArrayBufferLike | ArrayBufferView>
|
|
21
|
+
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
19
22
|
options?: ArrowLoaderOptions
|
|
20
23
|
): AsyncIterable<ArrowTableBatch> {
|
|
21
24
|
// Creates the appropriate arrow.RecordBatchReader subclasses from the input
|
|
@@ -38,7 +41,7 @@ export function parseArrowInBatches(
|
|
|
38
41
|
|
|
39
42
|
async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {
|
|
40
43
|
// @ts-ignore
|
|
41
|
-
const readers = arrow.RecordBatchReader.readAll(asyncIterator);
|
|
44
|
+
const readers = arrow.RecordBatchReader.readAll(toArrayBufferIterator(asyncIterator));
|
|
42
45
|
for await (const reader of readers) {
|
|
43
46
|
for await (const recordBatch of reader) {
|
|
44
47
|
// use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
|