@loaders.gl/arrow 4.3.2 → 4.4.0-alpha.1

package/src/lib/geoarrow/get-arrow-bounds.ts

@@ -1,41 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-/**
- *  Update bounds from geoarrow sample data
- *
- * @param flatCoords the flattend coordinates array from one chunk of geoarrow column
- * @param nDim the number of dimensions of the coordinates
- * @param bounds the bounds to be updated
- * @param sampleSize how many samples to be used to update the bounds, default is 1000 per chunk
- * @returns the updated bounds
- */
-export function updateBoundsFromGeoArrowSamples(
-  flatCoords: Float64Array,
-  nDim: number,
-  bounds: [number, number, number, number],
-  sampleSize: number = 100
-): [number, number, number, number] {
-  const numberOfFeatures = flatCoords.length / nDim;
-  const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
-
-  const newBounds: [number, number, number, number] = [...bounds];
-  for (let i = 0; i < numberOfFeatures; i += sampleStep) {
-    const lng = flatCoords[i * nDim];
-    const lat = flatCoords[i * nDim + 1];
-    if (lng < newBounds[0]) {
-      newBounds[0] = lng;
-    }
-    if (lat < newBounds[1]) {
-      newBounds[1] = lat;
-    }
-    if (lng > newBounds[2]) {
-      newBounds[2] = lng;
-    }
-    if (lat > newBounds[3]) {
-      newBounds[3] = lat;
-    }
-  }
-  return newBounds;
-}

package/src/lib/tables/convert-arrow-schema.ts

@@ -1,301 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-
-/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
-export function convertArrowToSchema(arrowSchema: arrow.Schema): Schema {
-  return serializeArrowSchema(arrowSchema);
-}
-
-/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
-export function convertSchemaToArrow(schema: Schema): arrow.Schema {
-  return deserializeArrowSchema(schema);
-}
-
-/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
-export function serializeArrowSchema(arrowSchema: arrow.Schema): Schema {
-  return {
-    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
-    metadata: serializeArrowMetadata(arrowSchema.metadata)
-  };
-}
-
-/** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
-export function deserializeArrowSchema(schema: Schema): arrow.Schema {
-  return new arrow.Schema(
-    schema.fields.map((field) => deserializeArrowField(field)),
-    deserializeArrowMetadata(schema.metadata)
-  );
-}
-
-/** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
-export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
-  return Object.fromEntries(arrowMetadata);
-}
-
-/** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
-export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
-  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
-}
-
-/** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
-export function serializeArrowField(field: arrow.Field): Field {
-  return {
-    name: field.name,
-    type: serializeArrowType(field.type),
-    nullable: field.nullable,
-    metadata: serializeArrowMetadata(field.metadata)
-  };
-}
-
-/** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
-export function deserializeArrowField(field: Field): arrow.Field {
-  return new arrow.Field(
-    field.name,
-    deserializeArrowType(field.type),
-    field.nullable,
-    deserializeArrowMetadata(field.metadata)
-  );
-}
-
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-// eslint-disable-next-line complexity
-export function serializeArrowType(arrowType: arrow.DataType): DataType {
-  switch (arrowType.constructor) {
-    case arrow.Null:
-      return 'null';
-    case arrow.Binary:
-      return 'binary';
-    case arrow.Bool:
-      return 'bool';
-    case arrow.Int:
-      const intType = arrowType as arrow.Int;
-      return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
-    case arrow.Int8:
-      return 'int8';
-    case arrow.Int16:
-      return 'int16';
-    case arrow.Int32:
-      return 'int32';
-    case arrow.Int64:
-      return 'int64';
-    case arrow.Uint8:
-      return 'uint8';
-    case arrow.Uint16:
-      return 'uint16';
-    case arrow.Uint32:
-      return 'uint32';
-    case arrow.Uint64:
-      return 'uint64';
-    case arrow.Float:
-      const precision = (arrowType as arrow.Float).precision;
-      // return `float(precision + 1) * 16`;
-      switch (precision) {
-        case arrow.Precision.HALF:
-          return 'float16';
-        case arrow.Precision.SINGLE:
-          return 'float32';
-        case arrow.Precision.DOUBLE:
-          return 'float64';
-        default:
-          return 'float16';
-      }
-    case arrow.Float16:
-      return 'float16';
-    case arrow.Float32:
-      return 'float32';
-    case arrow.Float64:
-      return 'float64';
-    case arrow.Utf8:
-      return 'utf8';
-    case arrow.Decimal:
-      const decimal = arrowType as arrow.Decimal;
-      return {
-        type: 'decimal',
-        bitWidth: decimal.bitWidth,
-        precision: decimal.precision,
-        scale: decimal.scale
-      };
-    case arrow.Date_:
-      const dateUnit = (arrowType as arrow.Date_).unit;
-      return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
-    case arrow.DateDay:
-      return 'date-day';
-    case arrow.DateMillisecond:
-      return 'date-millisecond';
-    case arrow.Time:
-      const timeUnit = (arrowType as arrow.Time).unit;
-      switch (timeUnit) {
-        case arrow.TimeUnit.SECOND:
-          return 'time-second';
-        case arrow.TimeUnit.MILLISECOND:
-          return 'time-millisecond';
-        case arrow.TimeUnit.MICROSECOND:
-          return 'time-microsecond';
-        case arrow.TimeUnit.NANOSECOND:
-          return 'time-nanosecond';
-        default:
-          return 'time-second';
-      }
-    case arrow.TimeMillisecond:
-      return 'time-millisecond';
-    case arrow.TimeSecond:
-      return 'time-second';
-    case arrow.TimeMicrosecond:
-      return 'time-microsecond';
-    case arrow.TimeNanosecond:
-      return 'time-nanosecond';
-    case arrow.Timestamp:
-      const timeStampUnit = (arrowType as arrow.Timestamp).unit;
-      switch (timeStampUnit) {
-        case arrow.TimeUnit.SECOND:
-          return 'timestamp-second';
-        case arrow.TimeUnit.MILLISECOND:
-          return 'timestamp-millisecond';
-        case arrow.TimeUnit.MICROSECOND:
-          return 'timestamp-microsecond';
-        case arrow.TimeUnit.NANOSECOND:
-          return 'timestamp-nanosecond';
-        default:
-          return 'timestamp-second';
-      }
-    case arrow.TimestampSecond:
-      return 'timestamp-second';
-    case arrow.TimestampMillisecond:
-      return 'timestamp-millisecond';
-    case arrow.TimestampMicrosecond:
-      return 'timestamp-microsecond';
-    case arrow.TimestampNanosecond:
-      return 'timestamp-nanosecond';
-    case arrow.Interval:
-      const intervalUnit = (arrowType as arrow.Interval).unit;
-      switch (intervalUnit) {
-        case arrow.IntervalUnit.DAY_TIME:
-          return 'interval-daytime';
-        case arrow.IntervalUnit.YEAR_MONTH:
-          return 'interval-yearmonth';
-        default:
-          return 'interval-daytime';
-      }
-    case arrow.IntervalDayTime:
-      return 'interval-daytime';
-    case arrow.IntervalYearMonth:
-      return 'interval-yearmonth';
-    case arrow.Map_:
-      const mapType = arrowType as arrow.Map_;
-      return {
-        type: 'map',
-        keysSorted: mapType.keysSorted,
-        children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
-      };
-    case arrow.List:
-      const listType = arrowType as arrow.List;
-      const listField = listType.valueField;
-      return {
-        type: 'list',
-        children: [serializeArrowField(listField)]
-      };
-    case arrow.FixedSizeList:
-      const fixedSizeList = arrowType as arrow.FixedSizeList;
-      return {
-        type: 'fixed-size-list',
-        listSize: fixedSizeList.listSize,
-        children: [serializeArrowField(fixedSizeList.children[0])]
-      };
-    case arrow.Struct:
-      const structType = arrowType as arrow.Struct;
-      return {
-        type: 'struct',
-        children: structType.children.map((arrowField) => serializeArrowField(arrowField))
-      };
-    default:
-      throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
-  }
-}
-
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-// eslint-disable-next-line complexity
-export function deserializeArrowType(dataType: DataType): arrow.DataType {
-  if (typeof dataType === 'object') {
-    switch (dataType.type) {
-      case 'decimal':
-        return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
-      case 'map':
-        let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
-        return new arrow.Map_(children as any, dataType.keysSorted);
-      case 'list':
-        const field = deserializeArrowField(dataType.children[0]);
-        return new arrow.List(field);
-      case 'fixed-size-list':
-        const child = deserializeArrowField(dataType.children[0]);
-        return new arrow.FixedSizeList(dataType.listSize, child);
-      case 'struct':
-        children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
-        return new arrow.Struct(children);
-      default:
-        throw new Error('array type not supported');
-    }
-  }
-
-  switch (dataType) {
-    case 'null':
-      return new arrow.Null();
-    case 'binary':
-      return new arrow.Binary();
-    case 'bool':
-      return new arrow.Bool();
-    case 'int8':
-      return new arrow.Int8();
-    case 'int16':
-      return new arrow.Int16();
-    case 'int32':
-      return new arrow.Int32();
-    case 'int64':
-      return new arrow.Int64();
-    case 'uint8':
-      return new arrow.Uint8();
-    case 'uint16':
-      return new arrow.Uint16();
-    case 'uint32':
-      return new arrow.Uint32();
-    case 'uint64':
-      return new arrow.Uint64();
-    case 'float16':
-      return new arrow.Float16();
-    case 'float32':
-      return new arrow.Float32();
-    case 'float64':
-      return new arrow.Float64();
-    case 'utf8':
-      return new arrow.Utf8();
-    case 'date-day':
-      return new arrow.DateDay();
-    case 'date-millisecond':
-      return new arrow.DateMillisecond();
-    case 'time-second':
-      return new arrow.TimeSecond();
-    case 'time-millisecond':
-      return new arrow.TimeMillisecond();
-    case 'time-microsecond':
-      return new arrow.TimeMicrosecond();
-    case 'time-nanosecond':
-      return new arrow.TimeNanosecond();
-    case 'timestamp-second':
-      return new arrow.TimestampSecond();
-    case 'timestamp-millisecond':
-      return new arrow.TimestampMillisecond();
-    case 'timestamp-microsecond':
-      return new arrow.TimestampMicrosecond();
-    case 'timestamp-nanosecond':
-      return new arrow.TimestampNanosecond();
-    case 'interval-daytime':
-      return new arrow.IntervalDayTime();
-    case 'interval-yearmonth':
-      return new arrow.IntervalYearMonth();
-    default:
-      throw new Error('array type not supported');
-  }
-}

package/src/lib/tables/convert-arrow-to-table.ts

@@ -1,144 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import * as arrow from 'apache-arrow';
-import type {
-  Table,
-  ArrayRowTable,
-  ColumnarTable,
-  ObjectRowTable,
-  GeoJSONTable,
-  Feature
-} from '@loaders.gl/schema';
-import type {ArrowTable} from '../../schema/arrow-table-type';
-
-import {convertTable} from '@loaders.gl/schema';
-import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
-import {convertArrowToSchema} from '../tables/convert-arrow-schema';
-import {parseGeometryFromArrow} from '../geoarrow/convert-geoarrow-to-geojson-geometry';
-
-/**
- * Convert an Apache Arrow table to a loaders.gl Table
- * @note Currently does not convert schema
- */
-export function convertArrowToTable(arrow: arrow.Table, shape: 'arrow-table'): ArrowTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: 'object-row-table'): ObjectRowTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: 'array-row-table'): ArrayRowTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: 'geojson-table'): GeoJSONTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
-export function convertArrowToTable(arrow: arrow.Table, shape: Table['shape']): Table;
-
-/**
- * Convert an Apache Arrow table to a loaders.gl Table
- * @note Currently does not convert schema
- */
-export function convertArrowToTable(arrowTable: arrow.Table, shape: Table['shape']): Table {
-  switch (shape) {
-    case 'arrow-table':
-      return convertArrowToArrowTable(arrowTable);
-    case 'array-row-table':
-      return convertArrowToArrayRowTable(arrowTable);
-    case 'object-row-table':
-      return convertArrowToObjectRowTable(arrowTable);
-    case 'columnar-table':
-      return convertArrowToColumnarTable(arrowTable);
-    case 'geojson-table':
-      return convertArrowToGeoJSONTable(arrowTable);
-    default:
-      throw new Error(shape);
-  }
-}
-
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-function convertArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
-  return {
-    shape: 'arrow-table',
-    schema: convertArrowToSchema(arrowTable.schema),
-    data: arrowTable
-  };
-}
-
-function convertArrowToArrayRowTable(arrowTable: arrow.Table): Table {
-  const columnarTable = convertArrowToColumnarTable(arrowTable);
-  return convertTable(columnarTable, 'array-row-table');
-}
-
-function convertArrowToObjectRowTable(arrowTable: arrow.Table): Table {
-  const columnarTable = convertArrowToColumnarTable(arrowTable);
-  return convertTable(columnarTable, 'object-row-table');
-}
-
-/**
- * Convert an Apache Arrow table to a ColumnarTable
- * @note Currently does not convert schema
- */
-function convertArrowToColumnarTable(arrowTable: arrow.Table): ColumnarTable {
-  // TODO - avoid calling `getColumn` on columns we are not interested in?
-  // Add options object?
-
-  const columns: ColumnarTable['data'] = {};
-
-  for (const field of arrowTable.schema.fields) {
-    // This (is intended to) coalesce all record batches into a single typed array
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn?.toArray();
-    columns[field.name] = values;
-  }
-
-  const schema = convertArrowToSchema(arrowTable.schema);
-
-  return {
-    shape: 'columnar-table',
-    schema,
-    data: columns
-  };
-}
-
-/**
- * Convert an Apache Arrow table to a GeoJSONTable
- * @note Currently does not convert schema
- */
-function convertArrowToGeoJSONTable(arrowTable: arrow.Table): GeoJSONTable {
-  const schema = convertArrowToSchema(arrowTable.schema);
-  const geometryColumns = getGeometryColumnsFromSchema(schema);
-
-  // get encoding from geometryColumns['geometry']
-  const encoding = geometryColumns.geometry.encoding;
-
-  const features: Feature[] = [];
-
-  // Remove geometry columns
-  const propertyColumnNames = arrowTable.schema.fields
-    .map((field) => field.name)
-    // TODO - this deletes all geometry columns
-    .filter((name) => !(name in geometryColumns));
-  const propertiesTable = arrowTable.select(propertyColumnNames);
-
-  const arrowGeometryColumn = arrowTable.getChild('geometry');
-
-  for (let row = 0; row < arrowTable.numRows; row++) {
-    // get the geometry value from arrow geometry column
-    // Note that type can vary
-    const arrowGeometry = arrowGeometryColumn?.get(row);
-    // parse arrow geometry to geojson feature
-    const feature = parseGeometryFromArrow(arrowGeometry, encoding);
-    if (feature) {
-      const properties = propertiesTable.get(row)?.toJSON() || {};
-      features.push({type: 'Feature', geometry: feature, properties});
-    }
-  }
-
-  return {
-    shape: 'geojson-table',
-    type: 'FeatureCollection',
-    schema,
-    features
-  };
-}

package/src/lib/tables/convert-table-to-arrow.ts

@@ -1,72 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import * as arrow from 'apache-arrow';
-import {Table, getTableLength, getTableNumCols, getTableCellAt} from '@loaders.gl/schema';
-import {deserializeArrowSchema} from '../tables/convert-arrow-schema';
-
-/**
- * * Convert a loaders.gl Table to an Apache Arrow Table
- * @param mesh
- * @param metadata
- * @param batchSize
- * @returns
- */
-export function convertTableToArrow(table: Table, options?: {batchSize?: number}): arrow.Table {
-  switch (table.shape) {
-    case 'arrow-table':
-      return table.data as arrow.Table;
-
-    case 'columnar-table':
-    // TODO - optimized implementation is possible
-    // return convertColumnarTableToArrow(table, options);
-
-    // fall through
-
-    default:
-      const arrowBatchIterator = makeTableToArrowBatchesIterator(table, options);
-      return new arrow.Table(arrowBatchIterator);
-  }
-}
-
-export function* makeTableToArrowBatchesIterator(
-  table: Table,
-  options?: {batchSize?: number}
-): IterableIterator<arrow.RecordBatch> {
-  const arrowSchema = deserializeArrowSchema(table.schema!);
-
-  const length = getTableLength(table);
-  const numColumns = getTableNumCols(table);
-  const batchSize = options?.batchSize || length;
-
-  const builders = arrowSchema?.fields.map((arrowField) => arrow.makeBuilder(arrowField));
-  const structField = new arrow.Struct(arrowSchema.fields);
-
-  let batchLength = 0;
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
-      const value = getTableCellAt(table, rowIndex, columnIndex);
-
-      const builder = builders[columnIndex];
-      builder.append(value);
-      batchLength++;
-
-      if (batchLength >= batchSize) {
-        const datas = builders.map((builder) => builder.flush());
-        const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
-        yield new arrow.RecordBatch(arrowSchema, structData);
-        batchLength = 0;
-      }
-    }
-  }
-
-  if (batchLength > 0) {
-    const datas = builders.map((builder) => builder.flush());
-    const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
-    yield new arrow.RecordBatch(arrowSchema, structData);
-    batchLength = 0;
-  }
-
-  builders.map((builder) => builder.finish());
-}

package/src/schema/arrow-table-batch.ts

@@ -1,82 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
-import type {ArrowTableBatch} from './arrow-table-type';
-import * as arrow from 'apache-arrow';
-
-/** @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency */
-export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
-  arrowSchema: arrow.Schema | null;
-
-  constructor(schema, options) {
-    super(schema, options);
-    this.arrowSchema = null;
-  }
-
-  getBatch(): ArrowTableBatch | null {
-    const batch = super.getBatch();
-    if (batch) {
-      // Get the arrow schema
-      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-
-      // Get arrow format vectors
-      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-
-      // Create the record batch
-      const recordBatch = new arrow.RecordBatch(
-        this.arrowSchema,
-        arrow.makeData({
-          type: new arrow.Struct(this.arrowSchema.fields),
-          children: arrowVectors.map(({data}) => data[0])
-        })
-      );
-
-      return {
-        shape: 'arrow-table',
-        batchType: 'data',
-        data: new arrow.Table([recordBatch]),
-        length: batch.length
-      };
-    }
-
-    return null;
-  }
-}
-
-// Convert from a simple loaders.gl schema to an Arrow schema
-function getArrowSchema(schema): arrow.Schema {
-  const arrowFields: arrow.Field[] = [];
-  for (const key in schema) {
-    const field = schema[key];
-    if (field.type === Float32Array) {
-      // TODO - just store the original field as metadata?
-      const metadata = new Map(); // field;
-      // arrow: new arrow.Field(name, nullable, metadata)
-      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
-      arrowFields.push(arrowField);
-    }
-  }
-  if (arrowFields.length === 0) {
-    throw new Error('No arrow convertible fields');
-  }
-
-  return new arrow.Schema(arrowFields);
-}
-
-// Convert from simple loaders.gl arrays to arrow vectors
-function getArrowVectors(arrowSchema, data): arrow.Vector[] {
-  const arrowVectors: any[] = [];
-  for (const field of arrowSchema.fields) {
-    const vector = data[field.name];
-    if (vector instanceof Float32Array) {
-      const arrowVector = arrow.makeVector(vector);
-      arrowVectors.push(arrowVector);
-    }
-  }
-  if (arrowSchema.fields.length !== arrowVectors.length) {
-    throw new Error('Some columns not arrow convertible');
-  }
-  return arrowVectors;
-}

package/src/schema/arrow-table-type.ts

@@ -1,30 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Batch, Schema} from '@loaders.gl/schema';
-import type * as arrow from 'apache-arrow';
-
-/**
- * A table organized as an Apache Arrow table
- * @note This is a variant of the type from loaders.gl/schema
- * @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency
- */
-export type ArrowTable = {
-  shape: 'arrow-table';
-  schema?: Schema;
-  data: arrow.Table;
-};
-
-/**
- * Batch for a table organized as an Apache Arrow table
- * @note This is a variant of the type from loaders.gl/schema
- * @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency
- */
-export type ArrowTableBatch = Batch & {
-  shape: 'arrow-table';
-  schemaType?: 'explicit' | 'deduced';
-  schema?: Schema;
-  data: arrow.Table; // ApacheRecordBatch;
-  length: number;
-};