@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/lib/table/batch-builder/columnar-table-batch-aggregator.ts

@@ -0,0 +1,99 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, ColumnarTableBatch, ArrowTableBatch, TypedArray} from '@loaders.gl/schema';
+import {isTypedArray} from '@math.gl/types';
+import {getArrayTypeFromDataType} from '../../schema/data-type';
+import {TableBatchAggregator} from './table-batch-aggregator';
+type ColumnarTableBatchOptions = {};
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class ColumnarTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  length: number = 0;
+  allocated: number = 0;
+  columns: Record<string, TypedArray | Array<any>> = {};
+
+  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
+    this.schema = schema;
+    this._reallocateColumns();
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[]) {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    let i = 0;
+    // TODO what if no csv header, columns not populated?
+    for (const fieldName in this.columns) {
+      this.columns[fieldName][this.length] = row[i++];
+    }
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}): void {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    for (const fieldName in row) {
+      this.columns[fieldName][this.length] = row[fieldName];
+    }
+    this.length++;
+  }
+
+  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
+    this._pruneColumns();
+
+    const batch: ColumnarTableBatch = {
+      shape: 'columnar-table',
+      batchType: 'data',
+      data: this.columns,
+      schema: this.schema,
+      length: this.length
+    };
+
+    return batch;
+  }
+
+  // HELPERS
+
+  _reallocateColumns() {
+    if (this.length < this.allocated) {
+      return;
+    }
+
+    // @ts-ignore TODO
+    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
+    this.columns = {};
+
+    for (const field of this.schema.fields) {
+      const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+      const oldColumn = this.columns[field.name];
+
+      if (!oldColumn) {
+        // Create new
+        this.columns[field.name] = new ArrayType(this.allocated);
+      } else if (Array.isArray(oldColumn)) {
+        // Plain array, just increase its size
+        oldColumn.length = this.allocated;
+      } else if (isTypedArray(oldColumn)) {
+        const typedArray = new ArrayType(this.allocated) as TypedArray;
+        // Copy the old data to the new array
+        typedArray.set(oldColumn);
+        this.columns[field.name] = typedArray;
+      } else {
+        throw new Error('Invalid column type');
+      }
+    }
+  }
+
+  _pruneColumns() {
+    for (const [columnName, column] of Object.entries(this.columns)) {
+      this.columns[columnName] = column.slice(0, this.length);
+    }
+  }
+}

package/src/lib/table/batch-builder/row-table-batch-aggregator.ts

@@ -0,0 +1,108 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/schema';
+import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
+import {
+  convertToArrayRow,
+  convertToObjectRow,
+  inferHeadersFromArrayRow,
+  inferHeadersFromObjectRow
+} from '../tables/row-utils';
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class RowTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema | null;
+  options: TableBatchOptions;
+
+  length: number = 0;
+  objectRows: {[columnName: string]: unknown}[] | null = null;
+  arrayRows: unknown[][] | null = null;
+  cursor: number = 0;
+  private _headers: string[] | null = null;
+
+  constructor(schema: Schema | null, options: TableBatchOptions) {
+    this.options = options;
+    this.schema = schema;
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    if (schema) {
+      this._headers = [];
+      for (let i = 0; i < schema.fields.length; i++) {
+        this._headers[i] = schema.fields[i].name;
+      }
+    }
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[], cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    // TODO - infer schema at a higher level, instead of hacking headers here?
+    this._headers ||= inferHeadersFromArrayRow(row);
+
+    // eslint-disable-next-line default-case
+    switch (this.options.shape) {
+      case 'object-row-table':
+        const rowObject = convertToObjectRow(row, this._headers);
+        this.addObjectRow(rowObject, cursor);
+        break;
+      case 'array-row-table':
+        this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
+        this.arrayRows[this.length] = row;
+        this.length++;
+        break;
+    }
+  }
+
+  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    // TODO - infer schema at a higher level, instead of hacking headers here?
+    this._headers ||= inferHeadersFromObjectRow(row);
+
+    // eslint-disable-next-line default-case
+    switch (this.options.shape) {
+      case 'array-row-table':
+        const rowArray = convertToArrayRow(row, this._headers);
+        this.addArrayRow(rowArray, cursor);
+        break;
+      case 'object-row-table':
+        this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
+        this.objectRows[this.length] = row;
+        this.length++;
+        break;
+    }
+  }
+
+  getBatch(): TableBatch | null {
+    let rows = this.arrayRows || this.objectRows;
+    if (!rows) {
+      return null;
+    }
+
+    rows = rows.slice(0, this.length);
+    this.arrayRows = null;
+    this.objectRows = null;
+
+    return {
+      shape: this.options.shape,
+      batchType: 'data',
+      data: rows,
+      length: this.length,
+      // @ts-expect-error we should infer a schema
+      schema: this.schema,
+      cursor: this.cursor
+    };
+  }
+}

package/src/lib/table/batch-builder/table-batch-aggregator.ts

@@ -0,0 +1,28 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/schema';
+
+export interface TableBatchOptions {
+  batchSize: number | string;
+  [key: string]: any;
+}
+
+export interface TableBatchConstructor {
+  new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
+}
+
+/**
+ * TableBatchBuilder delegates batch building to this interface
+ */
+export interface TableBatchAggregator {
+  /** Number of rows */
+  rowCount(): number;
+  /** Add one row */
+  addArrayRow(row: any[]): void;
+  /** Add one row */
+  addObjectRow(row: {[columnName: string]: any}): void;
+  /** return a batch object */
+  getBatch(): TableBatch | null;
+}

package/src/lib/table/batch-builder/table-batch-builder.ts

@@ -0,0 +1,185 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/schema';
+import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
+import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
+import {RowTableBatchAggregator} from './row-table-batch-aggregator';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
+import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';
+
+// TODO define interface instead
+type TableBatchBuilderOptions = {
+  shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
+  batchSize?: number | 'auto';
+  batchDebounceMs?: number;
+  limit?: number;
+  _limitMB?: number;
+};
+
+type GetBatchOptions = {
+  bytesUsed?: number;
+  [key: string]: any;
+};
+
+const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
+  shape: undefined!,
+  batchSize: 'auto',
+  batchDebounceMs: 0,
+  limit: 0,
+  _limitMB: 0
+};
+
+/** Incrementally builds batches from a stream of rows */
+export class TableBatchBuilder {
+  schema: Schema;
+  options: Required<TableBatchBuilderOptions>;
+
+  private aggregator: TableBatchAggregator | null = null;
+  private batchCount: number = 0;
+  private bytesUsed: number = 0;
+  private isChunkComplete: boolean = false;
+  private lastBatchEmittedMs: number = Date.now();
+  private totalLength: number = 0;
+  private totalBytes: number = 0;
+  private rowBytes: number = 0;
+
+  static ArrowBatch?: TableBatchConstructor;
+
+  constructor(schema: Schema, options?: TableBatchBuilderOptions) {
+    this.schema = schema;
+    this.options = {...DEFAULT_OPTIONS, ...options};
+  }
+
+  limitReached(): boolean {
+    if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
+      return true;
+    }
+    if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
+      return true;
+    }
+    return false;
+  }
+
+  /** @deprecated Use addArrayRow or addObjectRow */
+  addRow(row: any[] | {[columnName: string]: any}): void {
+    if (this.limitReached()) {
+      return;
+    }
+    this.totalLength++;
+    this.rowBytes = this.rowBytes || this._estimateRowMB(row);
+    this.totalBytes += this.rowBytes;
+    if (Array.isArray(row)) {
+      this.addArrayRow(row);
+    } else {
+      this.addObjectRow(row);
+    }
+  }
+
+  /** Add one row to the batch */
+  protected addArrayRow(row: any[]) {
+    if (!this.aggregator) {
+      const TableBatchType = this._getTableBatchType();
+      this.aggregator = new TableBatchType(this.schema, this.options);
+    }
+    this.aggregator.addArrayRow(row);
+  }
+
+  /** Add one row to the batch */
+  protected addObjectRow(row: {[columnName: string]: any}): void {
+    if (!this.aggregator) {
+      const TableBatchType = this._getTableBatchType();
+      this.aggregator = new TableBatchType(this.schema, this.options);
+    }
+    this.aggregator.addObjectRow(row);
+  }
+
+  /** Mark an incoming raw memory chunk has completed */
+  chunkComplete(chunk: ArrayBuffer | string): void {
+    if (chunk instanceof ArrayBuffer) {
+      this.bytesUsed += chunk.byteLength;
+    }
+    if (typeof chunk === 'string') {
+      this.bytesUsed += chunk.length;
+    }
+    this.isChunkComplete = true;
+  }
+
+  getFullBatch(options?: GetBatchOptions): TableBatch | null {
+    return this._isFull() ? this._getBatch(options) : null;
+  }
+
+  getFinalBatch(options?: GetBatchOptions): TableBatch | null {
+    return this._getBatch(options);
+  }
+
+  // INTERNAL
+
+  _estimateRowMB(row: any[] | object): number {
+    return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
+  }
+
+  private _isFull(): boolean {
+    // No batch, not ready
+    if (!this.aggregator || this.aggregator.rowCount() === 0) {
+      return false;
+    }
+
+    // if batchSize === 'auto' we wait for chunk to complete
+    // if batchSize === number, ensure we have enough rows
+    if (this.options.batchSize === 'auto') {
+      if (!this.isChunkComplete) {
+        return false;
+      }
+    } else if (this.options.batchSize > this.aggregator.rowCount()) {
+      return false;
+    }
+
+    // Debounce batches
+    if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
+      return false;
+    }
+
+    // Emit batch
+    this.isChunkComplete = false;
+    this.lastBatchEmittedMs = Date.now();
+    return true;
+  }
+
+  /**
+   * bytesUsed can be set via chunkComplete or via getBatch*
+   */
+  private _getBatch(options?: GetBatchOptions): TableBatch | null {
+    if (!this.aggregator) {
+      return null;
+    }
+
+    // TODO - this can overly increment bytes used?
+    if (options?.bytesUsed) {
+      this.bytesUsed = options.bytesUsed;
+    }
+    const normalizedBatch = this.aggregator.getBatch() as TableBatch;
+    normalizedBatch.count = this.batchCount;
+    normalizedBatch.bytesUsed = this.bytesUsed;
+    Object.assign(normalizedBatch, options);
+
+    this.batchCount++;
+    this.aggregator = null;
+    return normalizedBatch;
+  }
+
+  private _getTableBatchType(): TableBatchConstructor {
+    switch (this.options.shape) {
+      case 'array-row-table':
+      case 'object-row-table':
+        return RowTableBatchAggregator;
+      case 'columnar-table':
+        return ColumnarTableBatchAggregator;
+      case 'arrow-table':
+        return ArrowTableBatchAggregator;
+      default:
+        return BaseTableBatchAggregator;
+    }
+  }
+}

package/src/lib/table/batches/convert-batches.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  TableBatch,
+  ArrayRowTableBatch,
+  ObjectRowTableBatch,
+  ColumnarTableBatch,
+  ArrowTableBatch
+} from '@loaders.gl/schema';
+import {convertTable} from '../tables/convert-table';
+
+export function convertBatch(batches: TableBatch, shape: 'object-row-table'): ObjectRowTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'array-row-table'): ArrayRowTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'columnar-table'): ColumnarTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'arrow-table'): ArrowTableBatch;
+
+/** Convert a table batch to a different shape */
+export function convertBatch(
+  batch: TableBatch,
+  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
+): TableBatch {
+  switch (shape) {
+    case 'object-row-table':
+      return {...batch, ...convertTable(batch, 'object-row-table')};
+    case 'array-row-table':
+      return {...batch, ...convertTable(batch, 'array-row-table')};
+    case 'columnar-table':
+      return {...batch, ...convertTable(batch, 'columnar-table')};
+    case 'arrow-table':
+      return {...batch, ...convertTable(batch, 'arrow-table')};
+    default:
+      throw new Error(shape);
+  }
+}
+
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'object-row-table'
+): AsyncIterableIterator<ObjectRowTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'array-row-table'
+): AsyncIterableIterator<ArrayRowTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'columnar-table'
+): AsyncIterableIterator<ColumnarTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'arrow-table'
+): AsyncIterableIterator<ArrowTableBatch>;
+
+/**
+ * Convert batches to a different shape
+ * @param table
+ * @param shape
+ * @returns
+ */
+export async function* convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
+): AsyncIterableIterator<TableBatch> {
+  for await (const batch of batches) {
+    switch (shape) {
+      case 'object-row-table':
+        yield convertBatch(batch, 'object-row-table');
+        break;
+      case 'array-row-table':
+        yield convertBatch(batch, 'array-row-table');
+        break;
+      case 'columnar-table':
+        yield convertBatch(batch, 'columnar-table');
+        break;
+      case 'arrow-table':
+        yield convertBatch(batch, 'arrow-table');
+        break;
+      default:
+        throw new Error(shape);
+    }
+  }
+}

package/src/lib/table/batches/make-arrow-batch-iterator.ts

@@ -0,0 +1,75 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {Table, ArrowTableBatch} from '@loaders.gl/schema';
+
+import {convertSchemaToArrow} from '../../schema/convert-arrow-schema';
+import {getTableLength, getTableNumCols, getTableCellAt} from '../tables/table-accessors';
+
+/**
+ * Returns an iterator that yields a single table as a sequence of ArrowTable batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowTableBatchIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<ArrowTableBatch> {
+  for (const batch of makeArrowRecordBatchIterator(table, options)) {
+    const arrowTable = new arrow.Table([batch]);
+    yield {
+      ...batch,
+      shape: 'arrow-table',
+      schema: table.schema,
+      batchType: 'data',
+      length: arrowTable.numRows,
+      data: arrowTable
+    };
+  }
+}
+
+/**
+ * Returns an iterator that yields a single table as a sequence of arrow.RecordBatch batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowRecordBatchIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<arrow.RecordBatch> {
+  const arrowSchema = convertSchemaToArrow(table.schema!);
+
+  const length = getTableLength(table);
+  const numColumns = getTableNumCols(table);
+  const batchSize = options?.batchSize || length;
+
+  const builders = arrowSchema?.fields.map((arrowField) => arrow.makeBuilder(arrowField));
+  const structField = new arrow.Struct(arrowSchema.fields);
+
+  let batchLength = 0;
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+      const value = getTableCellAt(table, rowIndex, columnIndex);
+
+      const builder = builders[columnIndex];
+      builder.append(value);
+      batchLength++;
+
+      if (batchLength >= batchSize) {
+        const datas = builders.map((builder) => builder.flush());
+        const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+        yield new arrow.RecordBatch(arrowSchema, structData);
+        batchLength = 0;
+      }
+    }
+  }
+
+  if (batchLength > 0) {
+    const datas = builders.map((builder) => builder.flush());
+    const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+    yield new arrow.RecordBatch(arrowSchema, structData);
+    batchLength = 0;
+  }
+
+  builders.map((builder) => builder.finish());
+}

package/src/lib/table/batches/make-table-batch-iterator.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {TableBatch, Table} from '@loaders.gl/schema';
+import {getTableLength} from '../tables/table-accessors';
+
+/**
+ * Returns an iterator that yields the contents of a table as a sequence of batches.
+ * @todo Currently only a single batch is yielded.
+ * @note All batches will have the same shape and schema as the original table.
+ * @returns
+ */
+export function* makeTableBatchIterator(table: Table): IterableIterator<TableBatch> {
+  yield makeBatchFromTable(table);
+}
+
+/**
+ * Returns a table packaged as a single table batch
+ * @note The batch will have the same shape and schema as the original table.
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+export function makeBatchFromTable(table: Table): TableBatch {
+  return {...table, length: getTableLength(table), batchType: 'data'};
+}

package/src/lib/table/batches/make-table-from-batches.ts

@@ -0,0 +1,84 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  Schema,
+  TableBatch,
+  Table,
+  ObjectRowTable,
+  ArrayRowTable,
+  Feature
+} from '@loaders.gl/schema';
+import {getTableLength} from '../tables/table-accessors';
+
+/**
+ * Assembles all batches from an async iterator into a single table.
+ * @note All batches must have the same shape and schema
+ * @param batchIterator
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+// eslint-disable-next-line complexity
+export async function makeTableFromBatches(
+  batchIterator: AsyncIterable<TableBatch> | Iterable<TableBatch>
+): Promise<Table | null> {
+  let arrayRows: ArrayRowTable['data'];
+  let objectRows: ObjectRowTable['data'];
+  let features: Feature[];
+  let shape: Table['shape'] | null = null;
+  let schema: Schema | undefined;
+
+  for await (const batch of batchIterator) {
+    shape = shape || batch.shape;
+    schema = schema || batch.schema;
+
+    switch (batch.shape) {
+      case 'array-row-table':
+        arrayRows = arrayRows! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.data[rowIndex];
+          arrayRows.push(row);
+        }
+        break;
+
+      case 'object-row-table':
+        objectRows = objectRows! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.data[rowIndex];
+          objectRows.push(row);
+        }
+        break;
+
+      case 'geojson-table':
+        features = features! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.features[rowIndex];
+          features.push(row);
+        }
+        break;
+
+      case 'columnar-table':
+      case 'arrow-table':
+      default:
+        throw new Error('shape');
+    }
+  }
+
+  if (!shape) {
+    return null;
+  }
+
+  switch (shape) {
+    case 'array-row-table':
+      return {shape: 'array-row-table', data: arrayRows!, schema};
+
+    case 'object-row-table':
+      return {shape: 'object-row-table', data: objectRows!, schema};
+
+    case 'geojson-table':
+      return {shape: 'geojson-table', type: 'FeatureCollection', features: features!, schema};
+
+    default:
+      return null;
+  }
+}