@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/schema-utils/lib/arrow/split-arrow-buffers.ts

@@ -0,0 +1,235 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {Buffers} from 'apache-arrow/data';
+
+/** Arrow JS typed array used in Arrow internal buffer slots. */
+type ArrowTypedArray =
+  | Uint8Array
+  | Uint8ClampedArray
+  | Uint16Array
+  | Uint32Array
+  | Int8Array
+  | Int16Array
+  | Int32Array
+  | Float32Array
+  | Float64Array
+  | BigInt64Array
+  | BigUint64Array;
+
+/** Arrow JS object shapes supported by splitArrowBuffers. */
+export type SplitArrowBuffersInput =
+  | arrow.Table
+  | arrow.RecordBatch
+  | arrow.Vector
+  | arrow.Data<arrow.DataType>;
+
+/** Options for splitArrowBuffers. */
+export type SplitArrowBuffersOptions = {
+  /**
+   * Buffer copy mode.
+   * - 'none': do not copy any typed arrays.
+   * - 'sliced': copy only typed arrays that view a larger ArrayBuffer.
+   * - 'all': copy every Arrow internal typed array.
+   */
+  copy?: 'none' | 'sliced' | 'all';
+};
+
+/**
+ * Rebuilds an Arrow object so sliced internal typed-array buffers are split into standalone
+ * ArrayBuffers, while whole-buffer typed arrays are reused by default.
+ * @param input Arrow table, record batch, vector, or data node to rebuild.
+ * @param options Buffer splitting options.
+ * @returns A real Arrow object of the same kind with transfer-safe internal buffers.
+ */
+export function splitArrowBuffers<T extends arrow.TypeMap>(
+  input: arrow.Table<T>,
+  options?: SplitArrowBuffersOptions
+): arrow.Table<T>;
+export function splitArrowBuffers<T extends arrow.TypeMap>(
+  input: arrow.RecordBatch<T>,
+  options?: SplitArrowBuffersOptions
+): arrow.RecordBatch<T>;
+export function splitArrowBuffers<T extends arrow.DataType>(
+  input: arrow.Vector<T>,
+  options?: SplitArrowBuffersOptions
+): arrow.Vector<T>;
+export function splitArrowBuffers<T extends arrow.DataType>(
+  input: arrow.Data<T>,
+  options?: SplitArrowBuffersOptions
+): arrow.Data<T>;
+export function splitArrowBuffers<T extends SplitArrowBuffersInput>(
+  input: T,
+  options: SplitArrowBuffersOptions = {}
+): T {
+  if (isArrowTable(input)) {
+    return splitArrowTableBuffersInternal(input, options) as T;
+  }
+
+  if (isArrowRecordBatch(input)) {
+    return splitArrowRecordBatchBuffers(input, options) as T;
+  }
+
+  if (isArrowVector(input)) {
+    return splitArrowVectorBuffers(input, options) as T;
+  }
+
+  return splitArrowDataBuffers(input, options) as T;
+}
+
+/**
+ * Rebuilds an Arrow table so sliced internal typed-array buffers are split into standalone
+ * ArrayBuffers, while whole-buffer typed arrays are reused by default.
+ * @param table Arrow table to rebuild without mutating the original table.
+ * @param options Buffer splitting options.
+ * @returns A real Arrow table with transfer-safe internal buffers.
+ */
+export function splitArrowTableBuffers<T extends arrow.TypeMap>(
+  table: arrow.Table<T>,
+  options?: SplitArrowBuffersOptions
+): arrow.Table<T> {
+  return splitArrowTableBuffersInternal(table, options ?? {});
+}
+
+/**
+ * Tests whether a value is an Arrow table.
+ * @param input Value to test.
+ * @returns True when the value is an Arrow table.
+ */
+function isArrowTable(input: SplitArrowBuffersInput): input is arrow.Table {
+  return 'batches' in input;
+}
+
+/**
+ * Tests whether a value is an Arrow record batch.
+ * @param input Value to test.
+ * @returns True when the value is an Arrow record batch.
+ */
+function isArrowRecordBatch(input: SplitArrowBuffersInput): input is arrow.RecordBatch {
+  return 'schema' in input && 'data' in input && !Array.isArray(input.data);
+}
+
+/**
+ * Tests whether a value is an Arrow vector.
+ * @param input Value to test.
+ * @returns True when the value is an Arrow vector.
+ */
+function isArrowVector(input: SplitArrowBuffersInput): input is arrow.Vector {
+  return 'data' in input && Array.isArray(input.data);
+}
+
+/**
+ * Rebuilds an Arrow table by rebuilding each record batch.
+ * @param table Arrow table to rebuild.
+ * @param options Buffer splitting options.
+ * @returns Rebuilt Arrow table.
+ */
+function splitArrowTableBuffersInternal<T extends arrow.TypeMap>(
+  table: arrow.Table<T>,
+  options: SplitArrowBuffersOptions
+): arrow.Table<T> {
+  const recordBatches = table.batches.map(recordBatch =>
+    splitArrowRecordBatchBuffers(recordBatch, options)
+  );
+  return new arrow.Table(table.schema, recordBatches);
+}
+
+/**
+ * Rebuilds an Arrow record batch by rebuilding its data node.
+ * @param recordBatch Arrow record batch to rebuild.
+ * @param options Buffer splitting options.
+ * @returns Rebuilt Arrow record batch.
+ */
+function splitArrowRecordBatchBuffers<T extends arrow.TypeMap>(
+  recordBatch: arrow.RecordBatch<T>,
+  options: SplitArrowBuffersOptions
+): arrow.RecordBatch<T> {
+  return new arrow.RecordBatch(
+    recordBatch.schema,
+    splitArrowDataBuffers(recordBatch.data, options)
+  );
+}
+
+/**
+ * Recursively rebuilds Arrow Data and isolates requested internal typed-array buffers.
+ * @param data Arrow data node to rebuild.
+ * @param options Buffer splitting options.
+ * @returns Rebuilt Arrow data node.
+ */
+function splitArrowDataBuffers<T extends arrow.DataType>(
+  data: arrow.Data<T>,
+  options: SplitArrowBuffersOptions
+): arrow.Data<T> {
+  const children = data.children.map(childData => splitArrowDataBuffers(childData, options));
+  const dictionary = data.dictionary
+    ? splitArrowVectorBuffers(data.dictionary, options)
+    : undefined;
+
+  const buffers: Partial<Buffers<T>> = {
+    [arrow.BufferType.OFFSET]: splitArrowBuffer(data.buffers[arrow.BufferType.OFFSET], options),
+    [arrow.BufferType.DATA]: splitArrowBuffer(data.buffers[arrow.BufferType.DATA], options),
+    [arrow.BufferType.VALIDITY]: splitArrowBuffer(data.buffers[arrow.BufferType.VALIDITY], options),
+    [arrow.BufferType.TYPE]: splitArrowBuffer(data.buffers[arrow.BufferType.TYPE], options)
+  };
+
+  return new arrow.Data(
+    data.type,
+    data.offset,
+    data.length,
+    // @ts-expect-error _nullCount is protected. Reuse it to preserve lazy null-count state.
+    data._nullCount,
+    buffers,
+    children,
+    dictionary
+  );
+}
+
+/**
+ * Rebuilds an Arrow vector by rebuilding each underlying Data chunk.
+ * @param vector Arrow vector to rebuild.
+ * @param options Buffer splitting options.
+ * @returns Rebuilt Arrow vector.
+ */
+function splitArrowVectorBuffers<T extends arrow.DataType>(
+  vector: arrow.Vector<T>,
+  options: SplitArrowBuffersOptions
+): arrow.Vector<T> {
+  return new arrow.Vector(vector.data.map(data => splitArrowDataBuffers(data, options)));
+}
+
+/**
+ * Copies a typed-array view when requested or when it does not span its full backing buffer.
+ * @param array Typed array from an Arrow buffer slot.
+ * @param options Buffer splitting options.
+ * @returns The original typed array or a copied typed array with a standalone buffer.
+ */
+function splitArrowBuffer<T>(array: T, options: SplitArrowBuffersOptions): T {
+  if (!isArrowTypedArray(array)) {
+    return array;
+  }
+
+  if (options.copy === 'none') {
+    return array;
+  }
+
+  if (
+    options.copy !== 'all' &&
+    array.byteOffset === 0 &&
+    array.byteLength === array.buffer.byteLength
+  ) {
+    return array;
+  }
+
+  return array.slice() as T;
+}
+
+/**
+ * Tests whether a value is an Arrow typed array instead of a DataView.
+ * @param value Value to test.
+ * @returns True when the value is one of the Arrow internal typed-array buffer types.
+ */
+function isArrowTypedArray(value: unknown): value is ArrowTypedArray {
+  return ArrayBuffer.isView(value) && !(value instanceof DataView);
+}

package/src/schema-utils/lib/arrow-utils/arrow-fixed-size-list-utils.ts

@@ -0,0 +1,78 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  BigTypedArray,
+  DataType,
+  getFloat16Storage,
+  isNativeFloat16Array
+} from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray} from '../schema/data-type';
+import {deserializeArrowType} from '../schema/convert-arrow-schema';
+
+export function isFixedSizeList(vector: arrow.Vector): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListVector(
+  typedArray: BigTypedArray,
+  stride: number,
+  logicalType?: DataType
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride, logicalType);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: BigTypedArray,
+  stride: number,
+  logicalType?: DataType
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride, logicalType);
+  const nestedType = listType.children[0].type;
+  const storage =
+    logicalType === 'float16' && isNativeFloat16Array(typedArray)
+      ? getFloat16Storage(typedArray)
+      : typedArray;
+  const buffers: Partial<Record<arrow.BufferType, any>> = {
+    // valueOffsets: undefined,
+    [arrow.BufferType.DATA]: storage // values
+    // nullBitmap: undefined,
+    // typeIds: undefined
+  };
+
+  // Note: The contiguous array of data is held by the nested "primitive type" column
+  const nestedData = new arrow.Data(nestedType, 0, storage.length, 0, buffers);
+
+  // Wrapped in a FixedSizeList column that provides a "strided" view of the data
+  const data = new arrow.Data<arrow.FixedSizeList>(
+    listType,
+    0,
+    storage.length / stride,
+    0,
+    undefined,
+    [nestedData]
+  );
+
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(
+  typedArray: BigTypedArray,
+  stride: number,
+  logicalType?: DataType
+): arrow.FixedSizeList {
+  const type = logicalType || getDataTypeFromArray(typedArray).type;
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType, false));
+  return listType;
+}

package/src/schema-utils/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.ts

@@ -0,0 +1,48 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/core/types';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray} from '../schema/data-type';
+import {deserializeArrowType} from '../schema/convert-arrow-schema';
+
+export function isListFixedSizeList(
+  vector: arrow.Vector
+): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getListFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getListFixedSizeListVector(
+  indexes: Uint32Array,
+  typedArray: TypedArray,
+  stride: number
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride);
+  const data = new arrow.Data<arrow.FixedSizeList>(listType, 0, typedArray.length / stride, 0, [
+    typedArray
+  ]);
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(typedArray: TypedArray, stride: number): arrow.FixedSizeList {
+  const {type} = getDataTypeFromArray(typedArray);
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType));
+  return listType;
+}

package/src/schema-utils/lib/converters/convert.ts

@@ -0,0 +1,142 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ConversionContext, Converter} from './types';
+
+type ConversionStep<Shape extends string = string> = {
+  shape: Shape;
+  converter: Converter<Shape>;
+};
+
+/**
+ * Converts a value to the requested target shape using the supplied converters.
+ *
+ * The dispatcher does not own any conversion logic. It detects an initial source shape,
+ * finds the shortest supported path through the supplied converter set, and then executes
+ * each direct conversion step in order.
+ */
+export function convert<Shape extends string>(
+  input: unknown,
+  targetShape: Shape,
+  converters: Converter<Shape> | Converter<Shape>[],
+  options?: unknown
+): unknown {
+  const normalizedConverters = Array.isArray(converters) ? converters : [converters];
+  const sourceShape = detectSourceShape(input, normalizedConverters);
+
+  if (sourceShape === targetShape) {
+    return input;
+  }
+
+  const steps = findConversionPath(sourceShape, targetShape, normalizedConverters);
+  if (!steps) {
+    throw new Error(
+      `No conversion path found from "${sourceShape}" to "${targetShape}" using converters: ${normalizedConverters
+        .map(converter => converter.id)
+        .join(', ')}`
+    );
+  }
+
+  let currentValue = input;
+  let currentShape = sourceShape;
+  const path = [sourceShape, ...steps.map(step => step.shape)];
+
+  for (let stepIndex = 0; stepIndex < steps.length; stepIndex++) {
+    const step = steps[stepIndex];
+    const context: ConversionContext<Shape> = {
+      sourceShape,
+      targetShape,
+      currentShape,
+      path,
+      converters: normalizedConverters,
+      stepIndex
+    };
+    currentValue = step.converter.convert(currentValue, step.shape, options, context);
+    currentShape = step.shape;
+  }
+
+  return currentValue;
+}
+
+function detectSourceShape<Shape extends string>(
+  input: unknown,
+  converters: Converter<Shape>[]
+): Shape {
+  const detectedShapes = new Set<Shape>();
+
+  const fallbackShape = getShapeFromInput(input);
+  if (fallbackShape !== null) {
+    detectedShapes.add(fallbackShape as Shape);
+  }
+
+  for (const converter of converters) {
+    const detectedShape = converter.detectInputShape?.(input);
+    if (detectedShape) {
+      detectedShapes.add(detectedShape);
+    }
+  }
+
+  if (detectedShapes.size === 0) {
+    throw new Error('Could not detect source shape for conversion input');
+  }
+
+  if (detectedShapes.size > 1) {
+    throw new Error(
+      `Ambiguous source shape for conversion input: ${Array.from(detectedShapes).join(', ')}`
+    );
+  }
+
+  return Array.from(detectedShapes)[0];
+}
+
+function getShapeFromInput(input: unknown): string | null {
+  return typeof input === 'object' &&
+    input !== null &&
+    'shape' in input &&
+    typeof input.shape === 'string'
+    ? input.shape
+    : null;
+}
+
+function findConversionPath<Shape extends string>(
+  sourceShape: Shape,
+  targetShape: Shape,
+  converters: Converter<Shape>[]
+): ConversionStep<Shape>[] | null {
+  const queue: Array<{shape: Shape; path: ConversionStep<Shape>[]}> = [
+    {shape: sourceShape, path: []}
+  ];
+  const visitedShapes = new Set<Shape>([sourceShape]);
+
+  while (queue.length > 0) {
+    const candidate = queue.shift()!;
+
+    for (const converter of converters) {
+      if (!converter.from.includes(candidate.shape)) {
+        continue;
+      }
+
+      for (const nextShape of converter.to) {
+        if (
+          !converter.canConvert?.(candidate.shape, nextShape) &&
+          converter.canConvert !== undefined
+        ) {
+          continue;
+        }
+
+        const nextPath = [...candidate.path, {shape: nextShape, converter}];
+        if (nextShape === targetShape) {
+          return nextPath;
+        }
+
+        if (!visitedShapes.has(nextShape)) {
+          visitedShapes.add(nextShape);
+          queue.push({shape: nextShape, path: nextPath});
+        }
+      }
+    }
+  }
+
+  return null;
+}

package/src/schema-utils/lib/converters/converters.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {convert} from './convert';
+export type {Converter, ConversionContext} from './types';

package/src/schema-utils/lib/converters/types.ts

@@ -0,0 +1,44 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Context passed to converter steps while executing a conversion path.
+ */
+export type ConversionContext<Shape extends string = string> = {
+  /** Shape detected on the original input value. */
+  sourceShape: Shape;
+  /** Final shape requested by the caller. */
+  targetShape: Shape;
+  /** Shape of the value being passed into the current converter step. */
+  currentShape: Shape;
+  /** Planned shape path, including the source and final target shapes. */
+  path: Shape[];
+  /** Converter set available to the dispatcher for this request. */
+  converters: Converter<Shape>[];
+  /** Zero-based index of the current step in the conversion path. */
+  stepIndex: number;
+};
+
+/**
+ * A direct converter between one or more source and target shapes.
+ */
+export type Converter<Shape extends string = string, Options = unknown> = {
+  /** Stable converter identifier for debugging and error messages. */
+  id: string;
+  /** Source shapes accepted by this converter. */
+  from: readonly Shape[];
+  /** Target shapes produced by this converter. */
+  to: readonly Shape[];
+  /** Optional runtime shape detector for raw inputs that do not expose a `shape` discriminator. */
+  detectInputShape?: (input: unknown) => Shape | null;
+  /** Optional guard to prune unsupported source/target pairs within the declared shape sets. */
+  canConvert?: (sourceShape: Shape, targetShape: Shape) => boolean;
+  /** Performs one direct conversion from the current shape to the requested target shape. */
+  convert: (
+    input: unknown,
+    targetShape: Shape,
+    options?: Options,
+    context?: ConversionContext<Shape>
+  ) => unknown;
+};