@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator-builders.ts.disabled

@@ -0,0 +1,190 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, ColumnarTableBatch, ArrowTableBatch} from '@loaders.gl/core/schema';
+import {TableBatchAggregator} from './table-batch-aggregator';
+import {convertSchemaToArrow} from '../../schema/convert-arrow-schema';
+import * as arrow from 'apache-arrow';
+
+/** @note Should be part of @loaders.gl/core/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends TableBatchAggregator {
+
+  constructor(schema: Schema, options) {
+    super(schema, options);
+    this.arrowSchema = null;
+  }
+
+  getBatch(): ArrowTableBatch | null {
+    const batch = super.getBatch();
+    if (batch) {
+      // Get the arrow schema
+      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+
+      // Get arrow format vectors
+      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+
+      // Create the record batch
+      const recordBatch = new arrow.RecordBatch(
+        this.arrowSchema,
+        arrow.makeData({
+          type: new arrow.Struct(this.arrowSchema.fields),
+          children: arrowVectors.map(({data}) => data[0])
+        })
+      );
+
+      return {
+        shape: 'arrow-table',
+        batchType: 'data',
+        data: new arrow.Table([recordBatch]),
+        length: batch.length
+      };
+    }
+
+    return null;
+  }
+}
+
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema): arrow.Schema {
+  const arrowFields: arrow.Field[] = [];
+  for (const key in schema) {
+    const field = schema[key];
+    if (field.type === Float32Array) {
+      // TODO - just store the original field as metadata?
+      const metadata = new Map(); // field;
+      // arrow: new arrow.Field(name, nullable, metadata)
+      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+      arrowFields.push(arrowField);
+    }
+  }
+  if (arrowFields.length === 0) {
+    throw new Error('No arrow convertible fields');
+  }
+
+  return new arrow.Schema(arrowFields);
+}
+
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data): arrow.Vector[] {
+  const arrowVectors: any[] = [];
+  for (const field of arrowSchema.fields) {
+    const vector = data[field.name];
+    if (vector instanceof Float32Array) {
+      const arrowVector = arrow.makeVector(vector);
+      arrowVectors.push(arrowVector);
+    }
+  }
+  if (arrowSchema.fields.length !== arrowVectors.length) {
+    throw new Error('Some columns not arrow convertible');
+  }
+  return arrowVectors;
+}
+
+type ColumnarTableBatchOptions = {};
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class ArrowBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  length: number = 0;
+  allocated: number = 0;
+  columns: {[columnName: string]: any[]} = {};
+
+  arrowSchema: arrow.Schema | null;
+
+  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
+    this.schema = schema;
+    this.arrowSchema = convertSchemaToArrow(schema);
+    this._reallocateColumns();
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[]) {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    let i = 0;
+    // TODO what if no csv header, columns not populated?
+    for (const fieldName in this.columns) {
+      this.columns[fieldName][this.length] = row[i++];
+    }
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}): void {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    for (const fieldName in row) {
+      this.columns[fieldName][this.length] = row[fieldName];
+    }
+    this.length++;
+  }
+
+  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
+    this._pruneColumns();
+    const columns = Array.isArray(this.schema) ? this.columns : {};
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    // columns should match schema format
+    if (!Array.isArray(this.schema)) {
+      for (const fieldName in this.schema) {
+        const field = this.schema[fieldName];
+        columns[field.name] = this.columns[field.index];
+      }
+    }
+
+    this.columns = {};
+
+    const batch: ColumnarTableBatch = {
+      shape: 'columnar-table',
+      batchType: 'data',
+      data: columns,
+      schema: this.schema,
+      length: this.length
+    };
+
+    return batch;
+  }
+
+  // HELPERS
+
+  _reallocateColumns() {
+    if (this.length < this.allocated) {
+      return;
+    }
+
+    // @ts-ignore TODO
+    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
+    this.columns = {};
+
+    for (const fieldName in this.schema) {
+      const field = this.schema[fieldName];
+      const ArrayType = field.type || Float32Array;
+      const oldColumn = this.columns[field.index];
+
+      if (oldColumn && ArrayBuffer.isView(oldColumn)) {
+        // Copy the old data to the new array
+        const typedArray = new ArrayType(this.allocated);
+        typedArray.set(oldColumn);
+        this.columns[field.index] = typedArray;
+      } else if (oldColumn) {
+        // Plain array
+        oldColumn.length = this.allocated;
+        this.columns[field.index] = oldColumn;
+      } else {
+        // Create new
+        this.columns[field.index] = new ArrayType(this.allocated);
+      }
+    }
+  }
+
+  _pruneColumns() {
+    for (const [columnName, column] of Object.entries(this.columns)) {
+      this.columns[columnName] = column.slice(0, this.length);
+    }
+  }
+}

package/src/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTableBatch} from '@loaders.gl/core/schema';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
+import * as arrow from 'apache-arrow';
+
+/** @note Should be part of @loaders.gl/core/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+  arrowSchema: arrow.Schema | null;
+
+  constructor(schema, options) {
+    super(schema, options);
+    this.arrowSchema = null;
+  }
+
+  getBatch(): ArrowTableBatch | null {
+    const batch = super.getBatch();
+    if (batch) {
+      // Get the arrow schema
+      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+
+      // Get arrow format vectors
+      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+
+      // Create the record batch
+      const recordBatch = new arrow.RecordBatch(
+        this.arrowSchema,
+        arrow.makeData({
+          type: new arrow.Struct(this.arrowSchema.fields),
+          children: arrowVectors.map(({data}) => data[0])
+        })
+      );
+
+      return {
+        shape: 'arrow-table',
+        batchType: 'data',
+        data: new arrow.Table([recordBatch]),
+        length: batch.length
+      };
+    }
+
+    return null;
+  }
+}
+
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema): arrow.Schema {
+  const arrowFields: arrow.Field[] = [];
+  const fields = Array.isArray(schema.fields) ? schema.fields : Object.values(schema);
+  for (const field of fields as any[]) {
+    if (field.type === 'float32' || field.type === Float32Array) {
+      // TODO - just store the original field as metadata?
+      const metadata = new Map(); // field;
+      // arrow: new arrow.Field(name, nullable, metadata)
+      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+      arrowFields.push(arrowField);
+    }
+  }
+  if (arrowFields.length === 0) {
+    throw new Error('No arrow convertible fields');
+  }
+  if (arrowFields.length !== fields.length) {
+    throw new Error('Some schema fields are not arrow convertible');
+  }
+
+  return new arrow.Schema(arrowFields);
+}
+
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data): arrow.Vector[] {
+  const arrowVectors: any[] = [];
+  for (const field of arrowSchema.fields) {
+    const vector = data[field.name];
+    if (vector instanceof Float32Array) {
+      const arrowVector = arrow.makeVector(vector as any);
+      arrowVectors.push(arrowVector);
+    }
+  }
+  if (arrowSchema.fields.length !== arrowVectors.length) {
+    throw new Error('Some columns not arrow convertible');
+  }
+  return arrowVectors;
+}

package/src/schema-utils/lib/table/batch-builder/arrow-table-builder.ts

@@ -0,0 +1,123 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, ArrowTable, ArrowTableBatch} from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+import {
+  convertArrowToSchema,
+  convertSchemaToArrow,
+  type ArrowSchemaConversionOptions
+} from '../../schema/convert-arrow-schema';
+
+/** Options for constructing Arrow tables from loaders.gl rows. */
+export type ArrowTableBuilderOptions = ArrowSchemaConversionOptions;
+
+/** Builds an arrow table or batches */
+export class ArrowTableBuilder {
+  schema: Schema;
+  arrowSchema: arrow.Schema;
+  arrowBuilders: arrow.Builder[];
+  length: number;
+
+  constructor(schema: Schema, options?: ArrowTableBuilderOptions) {
+    this.arrowSchema = convertSchemaToArrow(schema, options);
+    this.schema =
+      options?.viewTypes && options.viewTypes !== 'never'
+        ? convertArrowToSchema(this.arrowSchema)
+        : schema;
+    this.arrowBuilders = this.arrowSchema.fields.map(field =>
+      arrow.makeBuilder({type: field.type, nullValues: [null]})
+    );
+    this.length = 0;
+  }
+
+  addObjectRow(row: {[key: string]: any}) {
+    for (let i = 0; i < this.arrowBuilders.length; i++) {
+      const columnName = this.schema.fields[i].name;
+      const value = row[columnName];
+      // if (this.schema.fields[i].type.toString() === 'bool') {
+      //   debugger;
+      // }
+      this.arrowBuilders[i].append(value);
+    }
+    this.length++;
+  }
+
+  addArrayRow(row: any[]) {
+    for (let i = 0; i < this.arrowBuilders.length; i++) {
+      this.arrowBuilders[i].append(row[i]);
+    }
+    this.length++;
+  }
+
+  /** Makes sure that a first batch with schema is sent even if no rows */
+  firstBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    // If there is data, a batch will be sent later
+    if (arrowRecordBatch.numCols !== 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Flush the current batch if conditions are right */
+  flushBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    if (arrowRecordBatch.numCols === 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Get a last batch if any data is left */
+  finishBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    this.arrowBuilders.forEach(builder => builder.finish());
+    if (arrowRecordBatch.numCols === 0 && arrowRecordBatch.numRows === 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Return a table with all the accumulated data */
+  finishTable(): ArrowTable {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    this.arrowBuilders.forEach(builder => builder.finish());
+    return {
+      shape: 'arrow-table',
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Extract a record batch flushing the currently accumulated data in the builders */
+  _getArrowRecordBatch(): arrow.RecordBatch {
+    const {arrowBuilders, arrowSchema} = this;
+    const arrowDatas = arrowBuilders.map(builder => builder.flush());
+    const length = arrowDatas[0]?.length || this.length;
+    this.length = 0;
+    const structField = new arrow.Struct(arrowSchema.fields);
+    const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
+    const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);
+    return arrowRecordBatch;
+  }
+}

package/src/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.ts

@@ -0,0 +1,82 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/core/schema';
+import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class BaseTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  options: TableBatchOptions;
+
+  shape?: 'array-row-table' | 'object-row-table';
+  length: number = 0;
+  rows: any[] | null = null;
+  cursor: number = 0;
+  private _headers: string[] = [];
+
+  constructor(schema: Schema, options: TableBatchOptions) {
+    this.options = options;
+    this.schema = schema;
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    if (!Array.isArray(schema)) {
+      this._headers = [];
+      for (const key in schema) {
+        this._headers[schema[key].index] = schema[key].name;
+      }
+    }
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[], cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    this.shape = 'array-row-table';
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    this.shape = 'object-row-table';
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  getBatch(): TableBatch | null {
+    let rows = this.rows;
+    if (!rows) {
+      return null;
+    }
+
+    rows = rows.slice(0, this.length);
+    this.rows = null;
+
+    const batch: TableBatch = {
+      shape: this.shape || 'array-row-table',
+      batchType: 'data',
+      data: rows,
+      length: this.length,
+      schema: this.schema,
+      cursor: this.cursor
+    };
+
+    return batch;
+  }
+}

package/src/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.ts

@@ -0,0 +1,104 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  Schema,
+  ColumnarTableBatch,
+  ArrowTableBatch,
+  TypedArray
+} from '@loaders.gl/core/schema';
+import {isTypedArray} from '@math.gl/core/types';
+import {getArrayTypeFromDataType} from '../../schema/data-type';
+import {TableBatchAggregator} from './table-batch-aggregator';
+type ColumnarTableBatchOptions = {};
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class ColumnarTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  length: number = 0;
+  allocated: number = 0;
+  columns: Record<string, TypedArray | Array<any>> = {};
+
+  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
+    this.schema = schema;
+    this._reallocateColumns();
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[]) {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    let i = 0;
+    // TODO what if no csv header, columns not populated?
+    for (const fieldName in this.columns) {
+      this.columns[fieldName][this.length] = row[i++];
+    }
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}): void {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    for (const fieldName in row) {
+      this.columns[fieldName][this.length] = row[fieldName];
+    }
+    this.length++;
+  }
+
+  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
+    this._pruneColumns();
+
+    const batch: ColumnarTableBatch = {
+      shape: 'columnar-table',
+      batchType: 'data',
+      data: this.columns,
+      schema: this.schema,
+      length: this.length
+    };
+
+    return batch;
+  }
+
+  // HELPERS
+
+  _reallocateColumns() {
+    if (this.length < this.allocated) {
+      return;
+    }
+
+    // @ts-ignore TODO
+    this.allocated = this.allocated > 0 ? this.allocated * 2 : DEFAULT_ROW_COUNT;
+    this.columns = {};
+
+    for (const field of this.schema.fields) {
+      const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+      const oldColumn = this.columns[field.name];
+
+      if (!oldColumn) {
+        // Create new
+        this.columns[field.name] = new ArrayType(this.allocated);
+      } else if (Array.isArray(oldColumn)) {
+        // Plain array, just increase its size
+        oldColumn.length = this.allocated;
+      } else if (isTypedArray(oldColumn)) {
+        const typedArray = new ArrayType(this.allocated) as TypedArray;
+        // Copy the old data to the new array
+        typedArray.set(oldColumn);
+        this.columns[field.name] = typedArray;
+      } else {
+        throw new Error('Invalid column type');
+      }
+    }
+  }
+
+  _pruneColumns() {
+    for (const [columnName, column] of Object.entries(this.columns)) {
+      this.columns[columnName] = column.slice(0, this.length);
+    }
+  }
+}

package/src/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.ts

@@ -0,0 +1,108 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/core/schema';
+import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
+import {
+  convertToArrayRow,
+  convertToObjectRow,
+  inferHeadersFromArrayRow,
+  inferHeadersFromObjectRow
+} from '../tables/row-utils';
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class RowTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema | null;
+  options: TableBatchOptions;
+
+  length: number = 0;
+  objectRows: {[columnName: string]: unknown}[] | null = null;
+  arrayRows: unknown[][] | null = null;
+  cursor: number = 0;
+  private _headers: string[] | null = null;
+
+  constructor(schema: Schema | null, options: TableBatchOptions) {
+    this.options = options;
+    this.schema = schema;
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    if (schema) {
+      this._headers = [];
+      for (let i = 0; i < schema.fields.length; i++) {
+        this._headers[i] = schema.fields[i].name;
+      }
+    }
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[], cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    // TODO - infer schema at a higher level, instead of hacking headers here?
+    this._headers ||= inferHeadersFromArrayRow(row);
+
+    // eslint-disable-next-line default-case
+    switch (this.options.shape) {
+      case 'object-row-table':
+        const rowObject = convertToObjectRow(row, this._headers);
+        this.addObjectRow(rowObject, cursor);
+        break;
+      case 'array-row-table':
+        this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
+        this.arrayRows[this.length] = row;
+        this.length++;
+        break;
+    }
+  }
+
+  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    // TODO - infer schema at a higher level, instead of hacking headers here?
+    this._headers ||= inferHeadersFromObjectRow(row);
+
+    // eslint-disable-next-line default-case
+    switch (this.options.shape) {
+      case 'array-row-table':
+        const rowArray = convertToArrayRow(row, this._headers);
+        this.addArrayRow(rowArray, cursor);
+        break;
+      case 'object-row-table':
+        this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
+        this.objectRows[this.length] = row;
+        this.length++;
+        break;
+    }
+  }
+
+  getBatch(): TableBatch | null {
+    let rows = this.arrayRows || this.objectRows;
+    if (!rows) {
+      return null;
+    }
+
+    rows = rows.slice(0, this.length);
+    this.arrayRows = null;
+    this.objectRows = null;
+
+    return {
+      shape: this.options.shape,
+      batchType: 'data',
+      data: rows,
+      length: this.length,
+      // @ts-expect-error we should infer a schema
+      schema: this.schema,
+      cursor: this.cursor
+    };
+  }
+}