@loaders.gl/arrow 4.1.0-alpha.2 → 4.1.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts +1 -0
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js +1 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +5 -2
- package/dist/arrow-writer.js +1 -1
- package/dist/arrow-writer.js.map +1 -1
- package/dist/dist.dev.js +45 -25
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +1 -1
- package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -1
- package/dist/geoarrow/get-arrow-bounds.js.map +1 -1
- package/dist/geoarrow-loader.d.ts.map +1 -1
- package/dist/geoarrow-loader.js.map +1 -1
- package/dist/geoarrow-writer.js +1 -1
- package/dist/geoarrow-writer.js.map +1 -1
- package/dist/index.cjs +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js.map +1 -1
- package/dist/lib/arrow-table.d.ts.map +1 -1
- package/dist/lib/arrow-table.js.map +1 -1
- package/dist/lib/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encode-arrow.js.map +1 -1
- package/dist/lib/encode-geoarrow.d.ts.map +1 -1
- package/dist/lib/encode-geoarrow.js.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.d.ts +2 -1
- package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.js +8 -1
- package/dist/parsers/parse-arrow-in-batches.js.map +1 -1
- package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-sync.js.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.js.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/convert-arrow-schema.d.ts.map +1 -1
- package/dist/schema/convert-arrow-schema.js.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.js.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.js.map +1 -1
- package/dist/tables/convert-columnar-to-row-table.d.ts.map +1 -1
- package/dist/tables/convert-columnar-to-row-table.js.map +1 -1
- package/dist/tables/convert-table-to-arrow.d.ts.map +1 -1
- package/dist/tables/convert-table-to-arrow.js.map +1 -1
- package/dist/triangulate-on-worker.d.ts.map +1 -1
- package/dist/triangulate-on-worker.js +1 -1
- package/dist/triangulate-on-worker.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/workers/arrow-worker.js.map +1 -1
- package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
- package/dist/workers/triangulation-worker-node.js.map +1 -1
- package/dist/workers/triangulation-worker.js.map +1 -1
- package/package.json +6 -6
- package/src/arrow-loader.ts +3 -1
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +2 -1
- package/src/geoarrow/convert-geoarrow-to-geojson-geometry.ts +2 -1
- package/src/geoarrow/get-arrow-bounds.ts +2 -1
- package/src/geoarrow-loader.ts +2 -1
- package/src/index.ts +2 -1
- package/src/lib/arrow-table-batch.ts +2 -1
- package/src/lib/arrow-table.ts +2 -1
- package/src/lib/encode-arrow.ts +2 -1
- package/src/lib/encode-geoarrow.ts +2 -1
- package/src/parsers/parse-arrow-in-batches.ts +9 -2
- package/src/parsers/parse-arrow-sync.ts +2 -1
- package/src/parsers/parse-geoarrow-in-batches.ts +2 -1
- package/src/parsers/parse-geoarrow-sync.ts +2 -1
- package/src/schema/arrow-type-utils.ts +2 -1
- package/src/schema/convert-arrow-schema.ts +2 -1
- package/src/tables/convert-arrow-to-columnar-table.ts +2 -1
- package/src/tables/convert-arrow-to-geojson-table.ts +2 -1
- package/src/tables/convert-columnar-to-row-table.ts +2 -1
- package/src/tables/convert-table-to-arrow.ts +2 -1
- package/src/triangulate-on-worker.ts +2 -1
- package/src/types.ts +2 -1
- package/src/workers/arrow-worker.ts +2 -1
- package/src/workers/triangulation-worker-node.ts +2 -1
- package/src/workers/triangulation-worker.ts +2 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"triangulation-worker.js","names":["arrow","createWorker","getTriangleIndices","getBinaryGeometriesFromArrow","data","options","arguments","length","undefined","input","operation","triangulateBatch","parseGeoArrowBatch","Error","console","error","triangleIndices","polygonIndices","primitivePolygonIndices","flatCoordinateArray","nDim","binaryDataFromGeoArrow","chunkData","chunkIndex","chunkOffset","geometryEncoding","calculateMeanCenters","triangle","arrowData","Data","type","offset","nullCount","buffers","children","dictionary","geometryColumn","makeVector"],"sources":["../../src/workers/triangulation-worker.ts"],"sourcesContent":["// loaders.gl
|
|
1
|
+
{"version":3,"file":"triangulation-worker.js","names":["arrow","createWorker","getTriangleIndices","getBinaryGeometriesFromArrow","data","options","arguments","length","undefined","input","operation","triangulateBatch","parseGeoArrowBatch","Error","console","error","triangleIndices","polygonIndices","primitivePolygonIndices","flatCoordinateArray","nDim","binaryDataFromGeoArrow","chunkData","chunkIndex","chunkOffset","geometryEncoding","calculateMeanCenters","triangle","arrowData","Data","type","offset","nullCount","buffers","children","dictionary","geometryColumn","makeVector"],"sources":["../../src/workers/triangulation-worker.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport {createWorker} from '@loaders.gl/worker-utils';\nimport {\n getTriangleIndices,\n getBinaryGeometriesFromArrow,\n BinaryDataFromGeoArrow\n} from '../geoarrow/convert-geoarrow-to-binary-geometry';\nimport type {\n TriangulationWorkerInput,\n TriangulateInput,\n TriangulateResult,\n ParseGeoArrowInput,\n ParseGeoArrowResult\n} from '../triangulate-on-worker';\n\ncreateWorker(async (data, options = {}) => {\n const input = data as TriangulationWorkerInput;\n const operation = input?.operation;\n switch (operation) {\n case 'test':\n return input;\n case 'triangulate':\n return triangulateBatch(data);\n case 'parse-geoarrow':\n return parseGeoArrowBatch(data);\n default:\n throw new Error(\n `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`\n );\n }\n});\n\nfunction triangulateBatch(data: TriangulateInput): TriangulateResult {\n // Parse any WKT/WKB geometries\n // Build binary geometries\n // Call earcut and triangulate\n console.error('TriangulationWorker: tessellating batch', data);\n const triangleIndices = getTriangleIndices(\n data.polygonIndices,\n data.primitivePolygonIndices,\n data.flatCoordinateArray,\n data.nDim\n );\n return {...data, ...(triangleIndices ? {triangleIndices} : {})};\n}\n\n/**\n * Reading the arrow file into memory is very fast. Parsing the geoarrow column is slow, and blocking the main thread.\n * To address this issue, we can move the parsing job from main thread to parallel web workers.\n * Each web worker will parse one chunk/batch of geoarrow column, and return binary geometries to main thread.\n * The app on the main thread will render the binary geometries and the parsing will not block the main thread.\n *\n * @param data\n * @returns\n */\nfunction parseGeoArrowBatch(data: ParseGeoArrowInput): ParseGeoArrowResult {\n let binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null = null;\n const {chunkData, chunkIndex, chunkOffset, geometryEncoding, calculateMeanCenters, triangle} =\n data;\n // rebuild chunkData that is only for geoarrow column\n const arrowData = new arrow.Data(\n chunkData.type,\n chunkData.offset,\n chunkData.length,\n chunkData.nullCount,\n chunkData.buffers,\n chunkData.children,\n chunkData.dictionary\n );\n // rebuild geometry column with chunkData\n const geometryColumn = arrow.makeVector(arrowData);\n if (geometryColumn) {\n // NOTE: for a rebuild arrow.Vector, there is only one chunk, so chunkIndex is always 0\n const options = {calculateMeanCenters, triangle, chunkIndex: 0, chunkOffset};\n binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(\n geometryColumn,\n geometryEncoding,\n options\n );\n // NOTE: here binaryGeometry will be copied to main thread\n return {\n binaryDataFromGeoArrow,\n chunkIndex\n };\n }\n return {\n binaryDataFromGeoArrow,\n chunkIndex\n };\n}\n"],"mappings":"AAIA,OAAO,KAAKA,KAAK,MAAM,cAAc;AACrC,SAAQC,YAAY,QAAO,0BAA0B;AAAC,SAEpDC,kBAAkB,EAClBC,4BAA4B;AAW9BF,YAAY,CAAC,gBAAOG,IAAI,EAAmB;EAAA,IAAjBC,OAAO,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EACpC,MAAMG,KAAK,GAAGL,IAAgC;EAC9C,MAAMM,SAAS,GAAGD,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEC,SAAS;EAClC,QAAQA,SAAS;IACf,KAAK,MAAM;MACT,OAAOD,KAAK;IACd,KAAK,aAAa;MAChB,OAAOE,gBAAgB,CAACP,IAAI,CAAC;IAC/B,KAAK,gBAAgB;MACnB,OAAOQ,kBAAkB,CAACR,IAAI,CAAC;IACjC;MACE,MAAM,IAAIS,KAAK,CACZ,8CAA6CH,SAAU,0BAC1D,CAAC;EACL;AACF,CAAC,CAAC;AAEF,SAASC,gBAAgBA,CAACP,IAAsB,EAAqB;EAInEU,OAAO,CAACC,KAAK,CAAC,yCAAyC,EAAEX,IAAI,CAAC;EAC9D,MAAMY,eAAe,GAAGd,kBAAkB,CACxCE,IAAI,CAACa,cAAc,EACnBb,IAAI,CAACc,uBAAuB,EAC5Bd,IAAI,CAACe,mBAAmB,EACxBf,IAAI,CAACgB,IACP,CAAC;EACD,OAAO;IAAC,GAAGhB,IAAI;IAAE,IAAIY,eAAe,GAAG;MAACA;IAAe,CAAC,GAAG,CAAC,CAAC;EAAC,CAAC;AACjE;AAWA,SAASJ,kBAAkBA,CAACR,IAAwB,EAAuB;EACzE,IAAIiB,sBAAqD,GAAG,IAAI;EAChE,MAAM;IAACC,SAAS;IAAEC,UAAU;IAAEC,WAAW;IAAEC,gBAAgB;IAAEC,oBAAoB;IAAEC;EAAQ,CAAC,GAC1FvB,IAAI;EAEN,MAAMwB,SAAS,GAAG,IAAI5B,KAAK,CAAC6B,IAAI,CAC9BP,SAAS,CAACQ,IAAI,EACdR,SAAS,CAACS,MAAM,EAChBT,SAAS,CAACf,MAAM,EAChBe,SAAS,CAACU,SAAS,EACnBV,SAAS,CAACW,OAAO,EACjBX,SAAS,CAACY,QAAQ,EAClBZ,SAAS,CAACa,UACZ,CAAC;EAED,MAAMC,cAAc,GAAGpC,KAAK,CAACqC,UAAU,CAACT,SAAS,CAAC;EAClD,IAAIQ,cAAc,EAAE;IAElB,MAAM/B,OAAO,GAAG;MAACqB,oBAAoB;MAAEC,QAAQ;MAAEJ,UAAU,EAAE,CAAC;MAAEC;IAAW,CAAC;IAC5EH,sBAAsB,GAAGlB,4BAA4B,CACnDiC,cAAc,EACdX,gBAAgB,EAChBpB,OACF,CAAC;IAED,OAAO;MACLgB,sBAAsB;MACtBE;IACF,CAAC;EACH;EACA,OAAO;IACLF,sBAAsB;IACtBE;EACF,CAAC;AACH"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/arrow",
|
|
3
|
-
"version": "4.1.0-alpha.
|
|
3
|
+
"version": "4.1.0-alpha.4",
|
|
4
4
|
"description": "Simple columnar table loader for the Apache Arrow format",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -52,12 +52,12 @@
|
|
|
52
52
|
"build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@loaders.gl/gis": "4.1.0-alpha.
|
|
56
|
-
"@loaders.gl/loader-utils": "4.1.0-alpha.
|
|
57
|
-
"@loaders.gl/schema": "4.1.0-alpha.
|
|
58
|
-
"@loaders.gl/wkt": "4.1.0-alpha.
|
|
55
|
+
"@loaders.gl/gis": "4.1.0-alpha.4",
|
|
56
|
+
"@loaders.gl/loader-utils": "4.1.0-alpha.4",
|
|
57
|
+
"@loaders.gl/schema": "4.1.0-alpha.4",
|
|
58
|
+
"@loaders.gl/wkt": "4.1.0-alpha.4",
|
|
59
59
|
"@math.gl/polygon": "4.0.0",
|
|
60
60
|
"apache-arrow": "^13.0.0"
|
|
61
61
|
},
|
|
62
|
-
"gitHead": "
|
|
62
|
+
"gitHead": "b18ba1d63be704fd021e4470e8ab84175621e62d"
|
|
63
63
|
}
|
package/src/arrow-loader.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
// loaders.gl
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
2
3
|
// Copyright (c) vis.gl contributors
|
|
3
4
|
|
|
4
5
|
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
@@ -19,6 +20,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
|
19
20
|
export type ArrowLoaderOptions = LoaderOptions & {
|
|
20
21
|
arrow?: {
|
|
21
22
|
shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
|
|
23
|
+
batchDebounceMs?: number;
|
|
22
24
|
};
|
|
23
25
|
};
|
|
24
26
|
|
package/src/geoarrow-loader.ts
CHANGED
package/src/index.ts
CHANGED
package/src/lib/arrow-table.ts
CHANGED
package/src/lib/encode-arrow.ts
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
// loaders.gl
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
2
3
|
// Copyright (c) vis.gl contributors
|
|
3
4
|
|
|
4
5
|
import type {ArrowTableBatch} from '../lib/arrow-table';
|
|
5
6
|
import * as arrow from 'apache-arrow';
|
|
7
|
+
import {ArrowLoaderOptions} from '../arrow-loader';
|
|
6
8
|
// import {isIterable} from '@loaders.gl/core';
|
|
7
9
|
|
|
8
10
|
/**
|
|
9
11
|
*/
|
|
10
12
|
export function parseArrowInBatches(
|
|
11
|
-
asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer
|
|
13
|
+
asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
|
|
14
|
+
options?: ArrowLoaderOptions
|
|
12
15
|
): AsyncIterable<ArrowTableBatch> {
|
|
13
16
|
// Creates the appropriate arrow.RecordBatchReader subclasses from the input
|
|
14
17
|
// This will also close the underlying source in case of early termination or errors
|
|
@@ -33,6 +36,10 @@ export function parseArrowInBatches(
|
|
|
33
36
|
const readers = arrow.RecordBatchReader.readAll(asyncIterator);
|
|
34
37
|
for await (const reader of readers) {
|
|
35
38
|
for await (const recordBatch of reader) {
|
|
39
|
+
// use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
|
|
40
|
+
if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
|
|
41
|
+
await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
|
|
42
|
+
}
|
|
36
43
|
const arrowTabledBatch: ArrowTableBatch = {
|
|
37
44
|
shape: 'arrow-table',
|
|
38
45
|
batchType: 'data',
|
package/src/types.ts
CHANGED