@loaders.gl/arrow 4.1.0-alpha.1 → 4.1.0-alpha.11

package/src/triangulate-on-worker.ts

@@ -1,19 +1,53 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import * as arrow from 'apache-arrow';
 import type {WorkerOptions} from '@loaders.gl/worker-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
+import {BinaryDataFromGeoArrow, GeoArrowEncoding} from '@loaders.gl/arrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export type TriangulationWorkerInput = TriangulateInput | {operation: 'test'; data: any};
-export type TriangulationWorkerOutput = TriangulateResult | {operation: 'test'; data: any};
+export type TriangulationWorkerInput =
+  | ({operation: 'triangulate'} & TriangulateInput)
+  | ParseGeoArrowInput
+  | {operation: 'test'; data: any};
+
+export type TriangulationWorkerOutput =
+  | ({operation: 'triangulate'} & TriangulateResult)
+  | ({operation: 'parse-geoarrow'} & ParseGeoArrowResult)
+  | {operation: 'test'; data: any};
+
+type GeoArrowChunkData = {
+  type: arrow.DataType;
+  offset: number;
+  length: number;
+  nullCount: number;
+  buffers: any;
+  children: arrow.Data[];
+  dictionary?: arrow.Vector;
+};
+
+export type ParseGeoArrowInput = {
+  operation: 'parse-geoarrow';
+  chunkData: GeoArrowChunkData;
+  chunkIndex: number;
+  chunkOffset: number;
+  geometryEncoding: GeoArrowEncoding;
+  calculateMeanCenters: boolean;
+  triangle: boolean;
+};
+
+export type ParseGeoArrowResult = {
+  chunkIndex: number;
+  binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null;
+};
 
 /** Input data for operation: 'triangulate' */
 export type TriangulateInput = {
-  operation: 'triangulate';
   polygonIndices: Uint16Array;
   primitivePolygonIndices: Int32Array;
   flatCoordinateArray: Float64Array;
@@ -37,11 +71,21 @@ export const TriangulationWorker = {
 };
 
 /**
- * Provide type safety
+ * Triangulate a set of polygons on worker, type safe API
  */
 export function triangulateOnWorker(
-  data: TriangulationWorkerInput,
+  data: TriangulateInput,
+  options: WorkerOptions = {}
+): Promise<TriangulateResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'triangulate'}, options);
+}
+
+/**
+ * Parse GeoArrow geometry colum on worker, type safe API
+ */
+export function parseGeoArrowOnWorker(
+  data: ParseGeoArrowInput,
   options: WorkerOptions = {}
-): Promise<TriangulationWorkerOutput> {
-  return processOnWorker(TriangulationWorker, data, options);
+): Promise<ParseGeoArrowResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'parse-geoarrow'}, options);
 }

package/src/types.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;

package/src/workers/arrow-worker.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';

package/src/workers/hard-clone.ts

@@ -0,0 +1,162 @@
+import * as arrow from 'apache-arrow';
+import type {Buffers} from 'apache-arrow/data';
+
+type TypedArray =
+  | Uint8Array
+  | Uint8ClampedArray
+  | Uint16Array
+  | Uint32Array
+  | Int8Array
+  | Int16Array
+  | Int32Array
+  | Float32Array
+  | Float64Array;
+
+/**
+ * Clone an Arrow JS Data or Vector, detaching from an existing ArrayBuffer if
+ * it is shared with other.
+ *
+ * The purpose of this function is to enable transferring a `Data` instance,
+ * e.g. to a web worker, without neutering any other data.
+ *
+ * Any internal buffers that are a slice of a larger `ArrayBuffer` (i.e. where
+ * the typed array's `byteOffset` is not `0` and where its `byteLength` does not
+ * match its `array.buffer.byteLength`) are copied into new `ArrayBuffers`.
+ *
+ * If `force` is `true`, always clone internal buffers, even if not shared. If
+ * the default, `false`, any internal buffers that are **not** a slice of a
+ * larger `ArrayBuffer` will not be copied.
+ */
+export function hardClone<T extends arrow.DataType>(
+  input: arrow.Data<T>,
+  force?: boolean
+): arrow.Data<T>;
+export function hardClone<T extends arrow.DataType>(
+  input: arrow.Vector<T>,
+  force?: boolean
+): arrow.Vector<T>;
+
+export function hardClone<T extends arrow.DataType>(
+  data: arrow.Data<T> | arrow.Vector<T>,
+  force: boolean = false
+): arrow.Data<T> | arrow.Vector<T> {
+  // Check if `data` is an arrow.Vector
+  if ('data' in data) {
+    return new arrow.Vector(data.data.map((data) => hardClone(data, force)));
+  }
+
+  // Clone each of the children, recursively
+  const clonedChildren: arrow.Data[] = [];
+  for (const childData of data.children) {
+    clonedChildren.push(hardClone(childData, force));
+  }
+
+  // Clone the dictionary if there is one
+  let clonedDictionary: arrow.Vector | undefined;
+  if (data.dictionary !== undefined) {
+    clonedDictionary = hardClone(data.dictionary, force);
+  }
+
+  // Buffers can have up to four entries. Each of these can be `undefined` for
+  // one or more array types.
+  //
+  // - OFFSET: value offsets for variable size list types
+  // - DATA: the underlying data
+  // - VALIDITY: the null buffer. This may be empty or undefined if all elements
+  //   are non-null/valid.
+  // - TYPE: type ids for a union type.
+  const clonedBuffers: Buffers<T> = {
+    [arrow.BufferType.OFFSET]: cloneBuffer(data.buffers[arrow.BufferType.OFFSET], force),
+    [arrow.BufferType.DATA]: cloneBuffer(data.buffers[arrow.BufferType.DATA], force),
+    [arrow.BufferType.VALIDITY]: cloneBuffer(data.buffers[arrow.BufferType.VALIDITY], force),
+    [arrow.BufferType.TYPE]: cloneBuffer(data.buffers[arrow.BufferType.TYPE], force)
+  };
+
+  // Note: the data.offset is passed on so that a sliced Data instance will not
+  // be "un-sliced". However keep in mind that this means we're cloning the
+  // _original backing buffer_, not only the portion of the Data that was
+  // sliced.
+  return new arrow.Data(
+    data.type,
+    data.offset,
+    data.length,
+    // @ts-expect-error _nullCount is protected. We're using it here to mimic
+    // `Data.clone`
+    data._nullCount,
+    clonedBuffers,
+    clonedChildren,
+    clonedDictionary
+  );
+}
+
+/**
+ * Test whether an arrow.Data instance is a slice of a larger `ArrayBuffer`.
+ */
+export function isShared<T extends arrow.DataType>(data: arrow.Data<T> | arrow.Vector<T>): boolean {
+  // Loop over arrow.Vector
+  if ('data' in data) {
+    return data.data.some((data) => isShared(data));
+  }
+
+  // Check child data
+  for (const childData of data.children) {
+    if (isShared(childData)) {
+      return true;
+    }
+  }
+
+  // Check dictionary
+  if (data.dictionary !== undefined) {
+    if (isShared(data.dictionary)) {
+      return true;
+    }
+  }
+
+  const bufferTypes = [
+    arrow.BufferType.OFFSET,
+    arrow.BufferType.DATA,
+    arrow.BufferType.VALIDITY,
+    arrow.BufferType.TYPE
+  ];
+  for (const bufferType of bufferTypes) {
+    if (data.buffers[bufferType] !== undefined && isTypedArraySliced(data.buffers[bufferType])) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Returns true if the current typed array is a partial slice on a larger
+ * ArrayBuffer
+ */
+function isTypedArraySliced(arr: TypedArray): boolean {
+  return !(arr.byteOffset === 0 && arr.byteLength === arr.buffer.byteLength);
+}
+
+/**
+ * If a slice of a larger ArrayBuffer, clone to a fresh `ArrayBuffer`.
+ *
+ * If `force` is `true`, always clone the array, even if not shared.
+ */
+function cloneBuffer<A extends TypedArray | undefined>(arr: A, force: boolean): A {
+  // Not all buffer types are defined for every type of Arrow array. E.g.
+  // `arrow.BufferType.TYPE` is only defined for the Union type.
+  if (arr === undefined) {
+    return arr;
+  }
+
+  // The current array is not a part of a larger ArrayBuffer, don't clone it
+  if (!force && !isTypedArraySliced(arr)) {
+    return arr;
+  }
+
+  // Note: TypedArray.slice() **copies** into a new ArrayBuffer
+
+  // @ts-expect-error 'Uint8Array' is assignable to the constraint of type 'A',
+  // but 'A' could be instantiated with a different subtype of constraint
+  // 'TypedArray'
+  // We know from arr.slice that it will always return the same
+  return arr.slice();
+}

package/src/workers/triangulation-worker-node.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import './triangulation-worker';

package/src/workers/triangulation-worker.ts

@@ -1,12 +1,20 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import * as arrow from 'apache-arrow';
 import {createWorker} from '@loaders.gl/worker-utils';
-import {getTriangleIndices} from '../geoarrow/convert-geoarrow-to-binary-geometry';
+import {
+  getTriangleIndices,
+  getBinaryGeometriesFromArrow,
+  BinaryDataFromGeoArrow
+} from '../geoarrow/convert-geoarrow-to-binary-geometry';
 import type {
   TriangulationWorkerInput,
   TriangulateInput,
-  TriangulateResult
+  TriangulateResult,
+  ParseGeoArrowInput,
+  ParseGeoArrowResult
 } from '../triangulate-on-worker';
 
 createWorker(async (data, options = {}) => {
@@ -17,6 +25,8 @@ createWorker(async (data, options = {}) => {
       return input;
     case 'triangulate':
       return triangulateBatch(data);
+    case 'parse-geoarrow':
+      return parseGeoArrowBatch(data);
     default:
       throw new Error(
         `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`
@@ -28,7 +38,6 @@ function triangulateBatch(data: TriangulateInput): TriangulateResult {
   // Parse any WKT/WKB geometries
   // Build binary geometries
   // Call earcut and triangulate
-  console.error('TriangulationWorker: tessellating batch', data);
   const triangleIndices = getTriangleIndices(
     data.polygonIndices,
     data.primitivePolygonIndices,
@@ -37,3 +46,48 @@ function triangulateBatch(data: TriangulateInput): TriangulateResult {
   );
   return {...data, ...(triangleIndices ? {triangleIndices} : {})};
 }
+
+/**
+ * Reading the arrow file into memory is very fast. Parsing the geoarrow column is slow, and blocking the main thread.
+ * To address this issue, we can move the parsing job from main thread to parallel web workers.
+ * Each web worker will parse one chunk/batch of geoarrow column, and return binary geometries to main thread.
+ * The app on the main thread will render the binary geometries and the parsing will not block the main thread.
+ *
+ * @param data
+ * @returns
+ */
+function parseGeoArrowBatch(data: ParseGeoArrowInput): ParseGeoArrowResult {
+  let binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null = null;
+  const {chunkData, chunkIndex, chunkOffset, geometryEncoding, calculateMeanCenters, triangle} =
+    data;
+  // rebuild chunkData that is only for geoarrow column
+  const arrowData = new arrow.Data(
+    chunkData.type,
+    chunkData.offset,
+    chunkData.length,
+    chunkData.nullCount,
+    chunkData.buffers,
+    chunkData.children,
+    chunkData.dictionary
+  );
+  // rebuild geometry column with chunkData
+  const geometryColumn = arrow.makeVector(arrowData);
+  if (geometryColumn) {
+    // NOTE: for a rebuild arrow.Vector, there is only one chunk, so chunkIndex is always 0
+    const options = {calculateMeanCenters, triangle, chunkIndex: 0, chunkOffset};
+    binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(
+      geometryColumn,
+      geometryEncoding,
+      options
+    );
+    // NOTE: here binaryGeometry will be copied to main thread
+    return {
+      binaryDataFromGeoArrow,
+      chunkIndex
+    };
+  }
+  return {
+    binaryDataFromGeoArrow,
+    chunkIndex
+  };
+}