@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.14

package/src/schema-utils/lib/mesh/convert-mesh-to-table.ts

@@ -0,0 +1,311 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {getFloat16Storage, isNativeFloat16Array} from '@loaders.gl/core/schema';
+import type {
+  BigTypedArray,
+  Mesh,
+  MeshAttribute,
+  MeshArrowTable,
+  MeshTable,
+  ArrowTable,
+  ColumnarTable,
+  Schema
+} from '@loaders.gl/core/schema';
+import {indexedMeshArrowSchema, meshArrowSchema} from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+import {getFixedSizeListVector} from '../arrow-utils/arrow-fixed-size-list-utils';
+import {
+  deserializeArrowField,
+  deserializeArrowMetadata,
+  serializeArrowSchema
+} from '../schema/convert-arrow-schema';
+
+/** Arrow column inputs, including exact 64-bit integers that are not GPU attributes. */
+export type MeshArrowAttributes = Record<
+  string,
+  Omit<MeshAttribute, 'value'> & {
+    /** Typed column values, retaining 64-bit integer precision. */
+    value: BigTypedArray;
+  }
+>;
+
+const MESH_ARROW_ATTRIBUTE_ORDER = ['POSITION'];
+
+/** Options for constructing a MeshArrowTable directly from typed mesh attribute arrays. */
+export type MeshArrowTableOptions = {
+  /** Schema describing the mesh attributes and metadata. */
+  schema?: Schema;
+  /** Mesh primitive topology represented by the table rows. */
+  topology?:
+    | 'point-list'
+    | 'line-list'
+    | 'line-loop'
+    | 'line-strip'
+    | 'triangle-list'
+    | 'triangle-strip'
+    | 'triangle-fan';
+  /** Numeric draw mode associated with the mesh topology. */
+  mode?: number;
+  /** Optional mesh bounding box metadata. */
+  boundingBox?: [number[], number[]];
+  /** Optional top-level primitive indices accessor for indexed meshes. */
+  indices?: MeshAttribute;
+};
+
+/** Convert a mesh to a columnar table. */
+export function convertMeshToTable(mesh: Mesh, shape: 'columnar-table'): MeshTable;
+/** Convert a mesh to an Apache Arrow table. */
+export function convertMeshToTable(mesh: Mesh, shape: 'arrow-table'): MeshArrowTable;
+
+/**
+ * Convert a mesh to a specific table shape.
+ * @param mesh Mesh to convert.
+ * @param shape Target table shape.
+ * @returns Mesh data in the requested table shape.
+ */
+export function convertMeshToTable(
+  mesh: Mesh,
+  shape: 'columnar-table' | 'arrow-table'
+): Mesh | ColumnarTable | ArrowTable | MeshArrowTable {
+  switch (shape) {
+    case 'columnar-table':
+      return convertMeshToColumnarTable(mesh);
+    case 'arrow-table':
+      return convertMeshToArrowTable(mesh);
+    default:
+      throw new Error(shape);
+  }
+}
+
+/**
+ * Convert a loaders.gl Mesh to a columnar table.
+ * @param mesh Mesh to convert.
+ * @returns Mesh data as a columnar table.
+ */
+export function convertMeshToColumnarTable(mesh: Mesh): MeshTable {
+  const columns = {};
+  const hasIndices = hasMeshIndices(mesh);
+
+  for (const [columnName, attribute] of Object.entries(mesh.attributes)) {
+    columns[columnName] = attribute.value;
+  }
+
+  return {
+    shape: 'columnar-table',
+    schema: mesh.schema,
+    data: columns,
+    topology: mesh.topology,
+    indices: hasIndices ? mesh.indices : undefined
+  };
+}
+
+/**
+ * Convert a loaders.gl Mesh to an Apache Arrow table.
+ * @param mesh Mesh to convert.
+ * @param batchSize Reserved for future chunked Arrow conversion.
+ * @returns Mesh data as an Apache Arrow table wrapper.
+ */
+export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): MeshArrowTable {
+  return makeMeshArrowTable(mesh.attributes, {
+    schema: mesh.schema,
+    topology: mesh.topology,
+    mode: mesh.mode,
+    boundingBox: mesh.header?.boundingBox,
+    indices: hasMeshIndices(mesh) ? mesh.indices : undefined
+  });
+}
+
+/**
+ * Create a MeshArrowTable directly from mesh attribute typed arrays.
+ * @param attributes Mesh attributes to expose as Arrow columns.
+ * @param options Mesh table metadata and optional schema.
+ * @returns Mesh data as an Apache Arrow table wrapper.
+ */
+export function makeMeshArrowTable(
+  attributes: MeshArrowAttributes,
+  options: MeshArrowTableOptions = {}
+): MeshArrowTable {
+  const fields: arrow.Field[] = [];
+  const columns: {[columnName: string]: arrow.Vector} = {};
+  const attributeNames = getOrderedAttributeNames(attributes);
+
+  for (const attributeName of attributeNames) {
+    const attribute = attributes[attributeName];
+    const {value, size = 1} = attribute;
+    const column = getAttributeArrowVector(value, size, attribute.componentType);
+
+    columns[attributeName] = column;
+    fields.push(
+      getAttributeArrowField(options.schema, attributeName, column, attribute.componentType)
+    );
+
+    if (attributeName === 'POSITION' && options.indices?.value?.length) {
+      const indicesField = indexedMeshArrowSchema.fields.find(field => field.name === 'indices')!;
+      columns.indices = getIndicesVector(options.indices.value, column.length, indicesField.type);
+      fields.push(indicesField);
+    }
+  }
+
+  const arrowSchema = new arrow.Schema(fields, getMeshArrowMetadata(options));
+  // Arrow's column-object constructor synthesizes an incompatible batch schema for empty
+  // fixed-size-list vectors. A schema-only table preserves the declared mesh fields instead.
+  const table =
+    columns[attributeNames[0]]?.length === 0
+      ? new arrow.Table(arrowSchema)
+      : new arrow.Table(arrowSchema, columns);
+  const schema = serializeArrowSchema(table.schema);
+
+  return {
+    shape: 'arrow-table',
+    schema,
+    data: table,
+    topology: options.topology || 'point-list',
+    indices: options.indices
+  };
+}
+
+/** Return an Arrow vector for a mesh attribute. */
+function getAttributeArrowVector(
+  value: BigTypedArray,
+  size: number,
+  componentType?: MeshAttribute['componentType']
+): arrow.Vector {
+  if (componentType === 'float16') {
+    const storage = isNativeFloat16Array(value) ? getFloat16Storage(value) : value;
+    if (size === 1) {
+      const data = new arrow.Data(new arrow.Float16(), 0, storage.length, 0, {
+        [arrow.BufferType.DATA]: storage as Uint16Array
+      });
+      return new arrow.Vector([data]);
+    }
+    return getFixedSizeListVector(value, size, componentType);
+  }
+  return size === 1 ? arrow.makeVector(value as any) : getFixedSizeListVector(value, size);
+}
+
+/** Return mesh attribute names with predefined Mesh Arrow fields first. */
+function getOrderedAttributeNames(attributes: MeshArrowAttributes): string[] {
+  const attributeNames = Object.keys(attributes);
+  const orderedAttributeNames = MESH_ARROW_ATTRIBUTE_ORDER.filter(
+    attributeName => attributeName in attributes
+  );
+  const remainingAttributeNames = attributeNames.filter(
+    attributeName => !MESH_ARROW_ATTRIBUTE_ORDER.includes(attributeName)
+  );
+
+  return [...orderedAttributeNames, ...remainingAttributeNames];
+}
+
+/** Return true when a mesh has a non-empty top-level index accessor. */
+function hasMeshIndices(mesh: Mesh): mesh is Mesh & {indices: MeshAttribute} {
+  return Boolean(mesh.indices?.value?.length);
+}
+
+/** Return the Arrow field for a mesh attribute column. */
+function getAttributeArrowField(
+  schema: Schema | undefined,
+  attributeName: string,
+  column: arrow.Vector,
+  componentType?: MeshAttribute['componentType']
+): arrow.Field {
+  if (attributeName === 'POSITION' && isMeshPositionColumn(column)) {
+    const canonicalField = meshArrowSchema.fields[0];
+    const suppliedField = schema?.fields.find(schemaField => schemaField.name === attributeName);
+    if (!suppliedField?.metadata) {
+      return canonicalField;
+    }
+    const metadata = new Map(canonicalField.metadata);
+    for (const [key, value] of Object.entries(suppliedField.metadata)) {
+      metadata.set(key, value);
+    }
+    return new arrow.Field(
+      canonicalField.name,
+      canonicalField.type,
+      canonicalField.nullable,
+      metadata
+    );
+  }
+
+  const field = schema?.fields.find(schemaField => schemaField.name === attributeName);
+  if (componentType === 'float16') {
+    const metadata = new Map(Object.entries(field?.metadata || {}));
+    metadata.set('componentType', 'float16');
+    return new arrow.Field(attributeName, column.type, false, metadata);
+  }
+  if (field && field.type.toString() !== column.type.toString()) {
+    const deserializedField = deserializeArrowField(field);
+    return new arrow.Field(
+      attributeName,
+      column.type,
+      deserializedField.nullable,
+      deserializedField.metadata
+    );
+  }
+  return field ? deserializeArrowField(field) : new arrow.Field(attributeName, column.type, false);
+}
+
+/** Return true when an Arrow column matches the predefined Mesh POSITION field. */
+function isMeshPositionColumn(column: arrow.Vector): boolean {
+  return (
+    column.type instanceof arrow.FixedSizeList &&
+    column.type.listSize === 3 &&
+    column.type.children[0].type instanceof arrow.Float32
+  );
+}
+
+/** Return an IndexedMesh indices column with the full index list stored in row 0. */
+function getIndicesVector(
+  indices: MeshAttribute['value'],
+  vertexCount: number,
+  type: arrow.DataType
+): arrow.Vector {
+  const indicesType = type as arrow.List<arrow.Uint32>;
+  const values = indices instanceof Uint32Array ? indices : Uint32Array.from(indices);
+  const valueOffsets = new Int32Array(vertexCount + 1);
+  if (vertexCount > 0) {
+    valueOffsets.fill(values.length, 1);
+  }
+
+  const nullBitmap = new Uint8Array(Math.ceil(vertexCount / 8));
+  if (vertexCount > 0) {
+    nullBitmap[0] = 1;
+  }
+
+  const valuesData = new arrow.Data<arrow.Uint32>(
+    indicesType.children[0].type,
+    0,
+    values.length,
+    0,
+    {[arrow.BufferType.DATA]: values}
+  );
+  const indicesData = new arrow.Data<arrow.List<arrow.Uint32>>(
+    indicesType,
+    0,
+    vertexCount,
+    Math.max(0, vertexCount - 1),
+    {
+      [arrow.BufferType.OFFSET]: valueOffsets,
+      [arrow.BufferType.VALIDITY]: nullBitmap
+    },
+    [valuesData]
+  );
+
+  return new arrow.Vector([indicesData]);
+}
+
+/** Return Arrow schema metadata for mesh-level properties. */
+function getMeshArrowMetadata(options: MeshArrowTableOptions): Map<string, string> {
+  const metadata = {...options.schema?.metadata};
+  if (options.topology) {
+    metadata.topology ||= options.topology;
+  }
+  if (Number.isFinite(options.mode)) {
+    metadata.mode ||= String(options.mode);
+  }
+  if (options.boundingBox) {
+    metadata.boundingBox ||= JSON.stringify(options.boundingBox);
+  }
+  return deserializeArrowMetadata(metadata);
+}

package/src/schema-utils/lib/mesh/convert-table-to-mesh.ts

@@ -0,0 +1,257 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {createFloat16Array, getFloat16Value, setFloat16Value} from '@loaders.gl/core/schema';
+import type {Mesh, MeshAttribute, ColumnarTable, ArrowTable, Schema} from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+import {getFixedSizeListSize} from '../arrow-utils/arrow-fixed-size-list-utils';
+import {serializeArrowSchema} from '../schema/convert-arrow-schema';
+import {getMeshBoundingBox} from './mesh-utils';
+
+/**
+ * Convert a table to a mesh.
+ * @param table Table to convert.
+ * @returns Mesh reconstructed from the table.
+ */
+export function convertTableToMesh(table: ColumnarTable | ArrowTable): Mesh {
+  switch (table.shape) {
+    // case 'columnar-table':
+    //   return convertColumnarTableToMesh(table);
+    case 'arrow-table':
+      return convertArrowTableToMesh(table);
+    default:
+      throw new Error(table.shape);
+  }
+}
+
+/**
+ * Convert an Apache Arrow mesh table to a loaders.gl Mesh.
+ * @param table Arrow table wrapper to convert.
+ * @returns Mesh reconstructed from the Arrow table.
+ */
+export function convertArrowTableToMesh(table: ArrowTable): Mesh {
+  const arrowTable = table.data;
+
+  const schema = serializeArrowSchema(arrowTable.schema);
+  const fields = schema.fields;
+
+  const attributes: Mesh['attributes'] = {};
+  const columns: {name: string; data: readonly arrow.Data[]}[] = [];
+  for (const field of fields) {
+    const {name} = field;
+    if (name === 'indices') {
+      continue;
+    }
+    const attributeData = arrowTable.getChild(name)!;
+    const size = getFixedSizeListSize(attributeData);
+    const typedArray = getAttributeTypedArray(attributeData, size);
+    const metadata = getMeshAttributeMetadata(field.metadata);
+    const value =
+      metadata.componentType === 'float16' ? restoreFloat16Array(typedArray) : typedArray;
+    // Preserve exact physical-storage aliases across column copies without merging equal values.
+    const shared = columns.find(
+      column =>
+        column.data.length === attributeData.data.length &&
+        attributes[column.name].value.constructor === value.constructor &&
+        column.data.every((chunk, index) => shareArrowStorage(chunk, attributeData.data[index]))
+    );
+    attributes[name] = {
+      value: shared ? attributes[shared.name].value : value,
+      size,
+      ...metadata
+    };
+    if (!shared) columns.push({name, data: attributeData.data});
+  }
+
+  fixMetadata(schema);
+  const topology = schema.metadata.topology as any;
+  const mode = getMeshMode(schema);
+  const indices = getIndices(arrowTable);
+  const boundingBox =
+    attributes.POSITION?.transform?.type === 'quantization'
+      ? getMeshBoundingBoxFromSchema(schema) || getMeshBoundingBox(attributes)
+      : getMeshBoundingBox(attributes);
+
+  return {
+    schema,
+    attributes,
+    topology,
+    mode,
+    indices,
+    header: {
+      vertexCount: arrowTable.numRows,
+      boundingBox: boundingBox ?? undefined
+    }
+  };
+}
+
+/** Restores a serialized logical bounding box when one is available. */
+function getMeshBoundingBoxFromSchema(
+  schema: Schema
+): [[number, number, number], [number, number, number]] | null {
+  const serializedBoundingBox = schema.metadata?.boundingBox;
+  if (!serializedBoundingBox) {
+    return null;
+  }
+  try {
+    const boundingBox = JSON.parse(serializedBoundingBox);
+    if (
+      Array.isArray(boundingBox) &&
+      boundingBox.length === 2 &&
+      boundingBox.every(
+        (corner: unknown) =>
+          Array.isArray(corner) &&
+          corner.length === 3 &&
+          corner.every(value => typeof value === 'number')
+      )
+    ) {
+      return boundingBox as [[number, number, number], [number, number, number]];
+    }
+  } catch {
+    // Ignore malformed optional metadata and recompute from decoded values.
+  }
+  return null;
+}
+
+/** Restores Mesh attribute properties serialized into Arrow field metadata. */
+function getMeshAttributeMetadata(metadata: Record<string, string> = {}): Partial<MeshAttribute> {
+  const result: Partial<MeshAttribute> = {};
+  if (metadata.byteOffset !== undefined) {
+    result.byteOffset = Number(metadata.byteOffset);
+  }
+  if (metadata.byteStride !== undefined) {
+    result.byteStride = Number(metadata.byteStride);
+  }
+  if (metadata.normalized !== undefined) {
+    result.normalized = metadata.normalized === 'true';
+  }
+  if (metadata.componentType === 'float16') {
+    result.componentType = 'float16';
+  }
+  if (metadata['loaders.gl.transform']) {
+    try {
+      const transform = JSON.parse(metadata['loaders.gl.transform']);
+      if (isMeshAttributeTransform(transform)) {
+        result.transform = transform;
+      }
+    } catch {
+      // Ignore malformed optional metadata and preserve the decoded values.
+    }
+  }
+  return result;
+}
+
+/** Validates the compact transform representation stored in Arrow metadata. */
+function isMeshAttributeTransform(value: unknown): value is MeshAttribute['transform'] {
+  if (!value || typeof value !== 'object' || !('type' in value)) {
+    return false;
+  }
+  const transform = value as Record<string, unknown>;
+  if (transform.type === 'octahedron') {
+    return (
+      Number.isInteger(transform.bits) &&
+      (transform.bits as number) >= 1 &&
+      (transform.bits as number) <= 30
+    );
+  }
+  return (
+    transform.type === 'quantization' &&
+    Number.isInteger(transform.bits) &&
+    (transform.bits as number) >= 1 &&
+    (transform.bits as number) <= 30 &&
+    Array.isArray(transform.origin) &&
+    transform.origin.every((component: unknown) => typeof component === 'number') &&
+    typeof transform.range === 'number'
+  );
+}
+
+/** Ensure required mesh metadata defaults are present on a schema. */
+function fixMetadata(schema: Schema) {
+  schema.metadata ||= {};
+  schema.metadata.topology ||= 'point-list';
+  schema.metadata.mode ||= '0';
+}
+
+/** Return a typed attribute array from an Arrow attribute column. */
+function getAttributeTypedArray(attributeData: arrow.Vector, size: number): any {
+  if (!(attributeData.type instanceof arrow.FixedSizeList)) {
+    return attributeData.toArray();
+  }
+
+  const values = attributeData.data[0].children[0].values;
+  const TypedArrayConstructor = values.constructor as any;
+  const typedArray = new TypedArrayConstructor(attributeData.length * size);
+  let targetOffset = 0;
+
+  for (const data of attributeData.data) {
+    const child = data.children[0];
+    const sourceLength = data.length * size;
+    const sourceOffset = child.offset || data.offset * size;
+    // Arrow chunks can either slice child values eagerly or retain offsets into the source buffer.
+    const sourceStart = sourceOffset + sourceLength <= child.values.length ? sourceOffset : 0;
+    const source = child.values.subarray(sourceStart, sourceStart + sourceLength);
+    typedArray.set(source, targetOffset);
+    targetOffset += sourceLength;
+  }
+
+  return typedArray;
+}
+
+/** Restore native Float16 storage when the runtime supports it. */
+function restoreFloat16Array(typedArray: any): Float16Array | Uint16Array {
+  const result = createFloat16Array(typedArray.length);
+  for (let index = 0; index < typedArray.length; index++) {
+    setFloat16Value(result, index, getFloat16Value(typedArray, index));
+  }
+  return result;
+}
+
+/** Return the mesh drawing mode stored in Arrow schema metadata. */
+function getMeshMode(schema: Schema): number {
+  const mode = Number(schema.metadata?.mode);
+  return Number.isFinite(mode) ? mode : 0;
+}
+
+/** Return top-level mesh indices from the predefined IndexedMesh Arrow column. */
+function getIndices(arrowTable: arrow.Table): Mesh['indices'] | undefined {
+  const indicesColumn = arrowTable.getChild('indices');
+  if (!indicesColumn?.isValid(0)) {
+    return undefined;
+  }
+
+  const indicesData = indicesColumn.data[0];
+  const valueOffsets = indicesData.valueOffsets;
+  const start = valueOffsets[indicesData.offset];
+  const end = valueOffsets[indicesData.offset + 1];
+  const values = indicesData.children[0].values.subarray(start, end);
+
+  return {value: values, size: 1};
+}
+
+/** Checks complete Arrow chunk regions, including child buffers and validity/offset metadata. */
+function shareArrowStorage(left: arrow.Data, right: arrow.Data): boolean {
+  const rightBuffers = Object.values(right.buffers);
+  return (
+    left.type.toString() === right.type.toString() &&
+    left.offset === right.offset &&
+    left.length === right.length &&
+    left.nullCount === right.nullCount &&
+    left.dictionary === right.dictionary &&
+    Object.values(left.buffers).every((buffer, index) => {
+      const other = rightBuffers[index];
+      return (
+        buffer === other ||
+        Boolean(
+          buffer &&
+            other &&
+            buffer.buffer === other.buffer &&
+            buffer.byteOffset === other.byteOffset &&
+            buffer.byteLength === other.byteLength
+        )
+      );
+    }) &&
+    left.children.length === right.children.length &&
+    left.children.every((child, index) => shareArrowStorage(child, right.children[index]))
+  );
+}

package/src/schema-utils/lib/mesh/deduce-mesh-schema.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType, MeshAttribute, MeshAttributes, Schema, Field} from '@loaders.gl/core/schema';
+import {getDataTypeFromTypedArray} from '../schema/data-type';
+
+/**
+ * Create a schema for mesh attributes data
+ * @param attributes
+ * @param metadata
+ * @returns
+ */
+export function deduceMeshSchema(
+  attributes: MeshAttributes,
+  metadata: Record<string, string> = {}
+): Schema {
+  const fields = deduceMeshFields(attributes);
+  return {fields, metadata};
+}
+
+/**
+ * Create arrow-like schema field for mesh attribute
+ * @param attributeName
+ * @param attribute
+ * @param optionalMetadata
+ * @returns
+ */
+export function deduceMeshField(
+  name: string,
+  attribute: MeshAttribute,
+  optionalMetadata?: Record<string, string>
+): Field {
+  const type = attribute.componentType || getDataTypeFromTypedArray(attribute.value);
+  const metadata = optionalMetadata ? optionalMetadata : makeMeshAttributeMetadata(attribute);
+  const dataType: DataType =
+    attribute.size === 1
+      ? type
+      : {type: 'fixed-size-list', listSize: attribute.size, children: [{name: 'value', type}]};
+  return {
+    name,
+    type: dataType,
+    nullable: false,
+    metadata
+  };
+}
+
+/**
+ * Create fields array for mesh attributes
+ * @param attributes
+ * @returns
+ */
+function deduceMeshFields(attributes: MeshAttributes): Field[] {
+  const fields: Field[] = [];
+  for (const attributeName in attributes) {
+    const attribute: MeshAttribute = attributes[attributeName];
+    fields.push(deduceMeshField(attributeName, attribute));
+  }
+  return fields;
+}
+
+/**
+ * Make metadata by mesh attribute properties
+ * @param attribute
+ * @returns
+ */
+export function makeMeshAttributeMetadata(attribute: MeshAttribute): Record<string, string> {
+  const result: Record<string, string> = {};
+  if ('byteOffset' in attribute) {
+    result.byteOffset = attribute.byteOffset!.toString(10);
+  }
+  if ('byteStride' in attribute) {
+    result.byteStride = attribute.byteStride!.toString(10);
+  }
+  if ('normalized' in attribute) {
+    result.normalized = attribute.normalized!.toString();
+  }
+  if (attribute.componentType) {
+    result.componentType = attribute.componentType;
+  }
+  if (attribute.transform) {
+    result['loaders.gl.transform'] = JSON.stringify(attribute.transform);
+  }
+  return result;
+}