@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/index.ts

@@ -0,0 +1,134 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Schema
+export {getArrayTypeFromDataType} from './lib/schema/data-type';
+
+// Schema utils
+export {
+  convertArrowToSchema,
+  convertSchemaToArrow,
+  // DETAILED FUNCTIONS
+  serializeArrowSchema,
+  deserializeArrowSchema,
+  serializeArrowMetadata,
+  deserializeArrowMetadata,
+  serializeArrowField,
+  deserializeArrowField,
+  serializeArrowType,
+  deserializeArrowType
+} from './lib/schema/convert-arrow-schema';
+export {getDataTypeFromArray} from './lib/schema/data-type';
+
+// TABLE CATEGORY UTILS
+
+export {deduceTableSchema} from './lib/schema/deduce-table-schema';
+export {makeTableFromData} from './lib/table/tables/make-table';
+export {makeTableFromBatches} from './lib/table/batches/make-table-from-batches';
+
+export {convertTable} from './lib/table/tables/convert-table';
+export {convertToObjectRow, convertToArrayRow} from './lib/table/tables/row-utils';
+export {convertArrowToTable, convertTableToArrow} from './lib/table/tables/convert-arrow-table';
+
+export {
+  makeTableBatchIterator,
+  makeBatchFromTable
+} from './lib/table/batches/make-table-batch-iterator';
+export {
+  makeArrowTableBatchIterator,
+  makeArrowRecordBatchIterator
+} from './lib/table/batches/make-arrow-batch-iterator';
+export {convertBatch, convertBatches} from './lib/table/batches/convert-batches';
+
+export {
+  isArrayRowTable,
+  isObjectRowTable,
+  isColumnarTable,
+  isGeoJSONTable,
+  isArrowTable
+} from './lib/table/tables/table-types';
+
+export {
+  isTable,
+  getTableLength,
+  getTableNumCols,
+  getTableCell,
+  getTableCellAt,
+  getTableRowShape,
+  getTableColumnIndex,
+  getTableColumnName,
+  getTableRowAsObject,
+  getTableRowAsArray,
+  makeRowIterator,
+  makeArrayRowIterator,
+  makeObjectRowIterator
+} from './lib/table/tables/table-accessors';
+
+// Table batch builders
+
+export {TableBatchBuilder} from './lib/table/batch-builder/table-batch-builder';
+export type {TableBatchAggregator} from './lib/table/batch-builder/table-batch-aggregator';
+export {RowTableBatchAggregator} from './lib/table/batch-builder/row-table-batch-aggregator';
+export {ColumnarTableBatchAggregator} from './lib/table/batch-builder/columnar-table-batch-aggregator';
+
+export {ArrowLikeTable} from './lib/table/arrow-api/arrow-like-table';
+
+// MESH CATEGORY
+
+export {getMeshSize, getMeshBoundingBox} from './lib/mesh/mesh-utils';
+export {convertMeshToTable} from './lib/mesh/convert-mesh-to-table';
+export {convertTableToMesh} from './lib/mesh/convert-table-to-mesh';
+export {
+  deduceMeshSchema,
+  deduceMeshField,
+  makeMeshAttributeMetadata
+} from './lib/mesh/deduce-mesh-schema';
+
+// SCHEMA
+export {
+  Schema as ArrowLikeSchema,
+  Field as ArrowLikeField,
+  DataType as ArrowLikeDataType
+  // Null,
+  // Binary,
+  // Bool,
+  // Int,
+  // Int8,
+  // Int16,
+  // Int32,
+  // Int64,
+  // Uint8,
+  // Uint16,
+  // Uint32,
+  // Uint64,
+  // Float,
+  // Float16,
+  // Float32,
+  // Float64,
+  // Utf8,
+  // Date,
+  // DateDay,
+  // DateMillisecond,
+  // Time,
+  // TimeMillisecond,
+  // TimeSecond,
+  // Timestamp,
+  // TimestampSecond,
+  // TimestampMillisecond,
+  // TimestampMicrosecond,
+  // TimestampNanosecond,
+  // Interval,
+  // IntervalDayTime,
+  // IntervalYearMonth,
+  // FixedSizeList,
+  // Struct
+} from './lib/table/arrow-api/index';
+
+// EXPERIMENTAL APIs
+export {ArrowTableBuilder} from './lib/table/batch-builder/arrow-table-builder';
+
+// Schema utils
+export {getTypeInfo} from './lib/table/arrow-api/get-type-info';
+
+export {default as AsyncQueue} from './lib/utils/async-queue';

package/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts

@@ -0,0 +1,63 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray} from '../schema/data-type';
+import {deserializeArrowType} from '../schema/convert-arrow-schema';
+
+export function isFixedSizeList(vector: arrow.Vector): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListVector(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride);
+  const nestedType = listType.children[0].type;
+  const buffers: Partial<Record<arrow.BufferType, any>> = {
+    // valueOffsets: undefined,
+    [arrow.BufferType.DATA]: typedArray // values
+    // nullBitmap: undefined,
+    // typeIds: undefined
+  };
+
+  // Note: The contiguous array of data is held by the nested "primitive type" column
+  const nestedData = new arrow.Data(nestedType, 0, typedArray.length, 0, buffers);
+
+  // Wrapped in a FixedSizeList column that provides a "strided" view of the data
+  const data = new arrow.Data<arrow.FixedSizeList>(
+    listType,
+    0,
+    typedArray.length / stride,
+    0,
+    undefined,
+    [nestedData]
+  );
+
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(typedArray: TypedArray, stride: number): arrow.FixedSizeList {
+  const {type} = getDataTypeFromArray(typedArray);
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType));
+  return listType;
+}

package/src/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.ts

@@ -0,0 +1,48 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray} from '../schema/data-type';
+import {deserializeArrowType} from '../schema/convert-arrow-schema';
+
+export function isListFixedSizeList(
+  vector: arrow.Vector
+): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getListFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getListFixedSizeListVector(
+  indexes: Uint32Array,
+  typedArray: TypedArray,
+  stride: number
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride);
+  const data = new arrow.Data<arrow.FixedSizeList>(listType, 0, typedArray.length / stride, 0, [
+    typedArray
+  ]);
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(typedArray: TypedArray, stride: number): arrow.FixedSizeList {
+  const {type} = getDataTypeFromArray(typedArray);
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType));
+  return listType;
+}

package/src/lib/mesh/convert-mesh-to-table.ts

@@ -0,0 +1,77 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Mesh, ArrowTable, ColumnarTable} from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+import {getFixedSizeListData} from '../arrow-utils/arrow-fixed-size-list-utils';
+import {deserializeArrowSchema} from '../schema/convert-arrow-schema';
+
+export function convertMeshToTable(mesh: Mesh, shape: 'columnar-table'): ColumnarTable;
+export function convertMeshToTable(mesh: Mesh, shape: 'arrow-table'): ArrowTable;
+
+/**
+ * Convert a mesh to a specific shape
+ */
+export function convertMeshToTable(
+  mesh: Mesh,
+  shape: 'columnar-table' | 'arrow-table'
+): Mesh | ColumnarTable | ArrowTable {
+  switch (shape) {
+    case 'columnar-table':
+      return convertMeshToColumnarTable(mesh);
+    case 'arrow-table':
+      return convertMeshToArrowTable(mesh);
+    default:
+      throw new Error(shape);
+  }
+}
+
+/**
+ * Convert a loaders.gl Mesh to a Columnar Table
+ * @param mesh
+ * @returns
+ */
+export function convertMeshToColumnarTable(mesh: Mesh): ColumnarTable {
+  const columns = {};
+
+  for (const [columnName, attribute] of Object.entries(mesh.attributes)) {
+    columns[columnName] = attribute.value;
+  }
+
+  return {
+    shape: 'columnar-table',
+    schema: mesh.schema,
+    data: columns
+  };
+}
+
+/**
+ * * Convert a loaders.gl Mesh to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ */
+export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): ArrowTable {
+  const {schema} = mesh;
+  const arrowSchema = deserializeArrowSchema(schema);
+
+  const arrowDatas: arrow.Data[] = [];
+  for (const attributeKey in mesh.attributes) {
+    const attribute = mesh.attributes[attributeKey];
+    const {value, size = 1} = attribute;
+
+    const listData = getFixedSizeListData(value, size);
+    arrowDatas.push(listData);
+    // fields.push(field);
+  }
+
+  const structField = new arrow.Struct(arrowSchema.fields);
+  const length = arrowDatas[0].length;
+  const structData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
+  const recordBatch = new arrow.RecordBatch(arrowSchema, structData);
+  const table = new arrow.Table([recordBatch]);
+
+  return {shape: 'arrow-table', schema, data: table};
+}

package/src/lib/mesh/convert-table-to-mesh.ts

@@ -0,0 +1,49 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Mesh, ColumnarTable, ArrowTable, Schema} from '@loaders.gl/schema';
+import {getFixedSizeListSize} from '../arrow-utils/arrow-fixed-size-list-utils';
+import {serializeArrowSchema} from '../schema/convert-arrow-schema';
+// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
+
+/**
+ * Convert a mesh to a specific shape
+ */
+export function convertTableToMesh(table: ColumnarTable | ArrowTable): Mesh {
+  switch (table.shape) {
+    // case 'columnar-table':
+    //   return convertColumnarTableToMesh(table);
+    case 'arrow-table':
+      return convertArrowTableToMesh(table);
+    default:
+      throw new Error(table.shape);
+  }
+}
+
+export function convertArrowTableToMesh(table: ArrowTable): Mesh {
+  const arrowTable = table.data;
+
+  const schema = serializeArrowSchema(arrowTable.schema);
+  const fields = schema.fields;
+
+  const attributes: Mesh['attributes'] = {};
+  for (const field of fields) {
+    const {name} = field;
+    const attributeData = arrowTable.getChild(name)!;
+    const size = getFixedSizeListSize(attributeData);
+    const typedArray = attributeData?.toArray();
+    attributes[name] = {value: typedArray, size};
+  }
+
+  fixMetadata(schema);
+  const topology = schema.metadata.topology as any;
+
+  return {schema, attributes, topology, mode: 0};
+}
+
+function fixMetadata(schema: Schema) {
+  schema.metadata ||= {};
+  schema.metadata.topology ||= 'point-list';
+  schema.metadata.mode ||= '0';
+}

package/src/lib/mesh/deduce-mesh-schema.ts

@@ -0,0 +1,75 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {MeshAttribute, MeshAttributes, Schema, Field} from '@loaders.gl/schema';
+import {getDataTypeFromTypedArray} from '../schema/data-type';
+
+/**
+ * Create a schema for mesh attributes data
+ * @param attributes
+ * @param metadata
+ * @returns
+ */
+export function deduceMeshSchema(
+  attributes: MeshAttributes,
+  metadata: Record<string, string> = {}
+): Schema {
+  const fields = deduceMeshFields(attributes);
+  return {fields, metadata};
+}
+
+/**
+ * Create arrow-like schema field for mesh attribute
+ * @param attributeName
+ * @param attribute
+ * @param optionalMetadata
+ * @returns
+ */
+export function deduceMeshField(
+  name: string,
+  attribute: MeshAttribute,
+  optionalMetadata?: Record<string, string>
+): Field {
+  const type = getDataTypeFromTypedArray(attribute.value);
+  const metadata = optionalMetadata ? optionalMetadata : makeMeshAttributeMetadata(attribute);
+  return {
+    name,
+    type: {type: 'fixed-size-list', listSize: attribute.size, children: [{name: 'value', type}]},
+    nullable: false,
+    metadata
+  };
+}
+
+/**
+ * Create fields array for mesh attributes
+ * @param attributes
+ * @returns
+ */
+function deduceMeshFields(attributes: MeshAttributes): Field[] {
+  const fields: Field[] = [];
+  for (const attributeName in attributes) {
+    const attribute: MeshAttribute = attributes[attributeName];
+    fields.push(deduceMeshField(attributeName, attribute));
+  }
+  return fields;
+}
+
+/**
+ * Make metadata by mesh attribute properties
+ * @param attribute
+ * @returns
+ */
+export function makeMeshAttributeMetadata(attribute: MeshAttribute): Record<string, string> {
+  const result: Record<string, string> = {};
+  if ('byteOffset' in attribute) {
+    result.byteOffset = attribute.byteOffset!.toString(10);
+  }
+  if ('byteStride' in attribute) {
+    result.byteStride = attribute.byteStride!.toString(10);
+  }
+  if ('normalized' in attribute) {
+    result.normalized = attribute.normalized!.toString();
+  }
+  return result;
+}

package/src/lib/mesh/mesh-utils.ts

@@ -0,0 +1,64 @@
+// Mesh category utilities
+// TODO - move to mesh category module, or to math.gl/geometry module
+import {MeshAttributes} from '@loaders.gl/schema';
+import {TypedArray} from '../../types/types';
+
+type TypedArrays = {[key: string]: TypedArray};
+
+/**
+ * Holds an axis aligned bounding box
+ * TODO - make sure AxisAlignedBoundingBox in math.gl/culling understands this format (or change this format)
+ */
+type BoundingBox = [[number, number, number], [number, number, number]];
+
+/**
+ * Get number of vertices in mesh
+ * @param attributes
+ */
+export function getMeshSize(attributes: TypedArrays): number {
+  let size = 0;
+  for (const attributeName in attributes) {
+    const attribute = attributes[attributeName];
+    if (ArrayBuffer.isView(attribute)) {
+      // @ts-ignore DataView doesn't have BYTES_PER_ELEMENT
+      size += attribute.byteLength * attribute.BYTES_PER_ELEMENT;
+    }
+  }
+  return size;
+}
+
+/**
+ * Get the (axis aligned) bounding box of a mesh
+ * @param attributes
+ * @returns array of two vectors representing the axis aligned bounding box
+ */
+// eslint-disable-next-line complexity
+export function getMeshBoundingBox(attributes: MeshAttributes): BoundingBox {
+  let minX = Infinity;
+  let minY = Infinity;
+  let minZ = Infinity;
+  let maxX = -Infinity;
+  let maxY = -Infinity;
+  let maxZ = -Infinity;
+
+  const positions = attributes.POSITION ? attributes.POSITION.value : [];
+  const len = positions && positions.length;
+
+  for (let i = 0; i < len; i += 3) {
+    const x = positions[i];
+    const y = positions[i + 1];
+    const z = positions[i + 2];
+
+    minX = x < minX ? x : minX;
+    minY = y < minY ? y : minY;
+    minZ = z < minZ ? z : minZ;
+
+    maxX = x > maxX ? x : maxX;
+    maxY = y > maxY ? y : maxY;
+    maxZ = z > maxZ ? z : maxZ;
+  }
+  return [
+    [minX, minY, minZ],
+    [maxX, maxY, maxZ]
+  ];
+}