@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/schema-utils/lib/table/batch-builder/table-batch-aggregator.ts

@@ -0,0 +1,28 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/core/schema';
+
+export interface TableBatchOptions {
+  batchSize: number | string;
+  [key: string]: any;
+}
+
+export interface TableBatchConstructor {
+  new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
+}
+
+/**
+ * TableBatchBuilder delegates batch building to this interface
+ */
+export interface TableBatchAggregator {
+  /** Number of rows */
+  rowCount(): number;
+  /** Add one row */
+  addArrayRow(row: any[]): void;
+  /** Add one row */
+  addObjectRow(row: {[columnName: string]: any}): void;
+  /** return a batch object */
+  getBatch(): TableBatch | null;
+}

package/src/schema-utils/lib/table/batch-builder/table-batch-builder.ts

@@ -0,0 +1,185 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/core/schema';
+import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
+import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
+import {RowTableBatchAggregator} from './row-table-batch-aggregator';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
+import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';
+
+// TODO define interface instead
+type TableBatchBuilderOptions = {
+  shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
+  batchSize?: number | 'auto';
+  batchDebounceMs?: number;
+  limit?: number;
+  _limitMB?: number;
+};
+
+type GetBatchOptions = {
+  bytesUsed?: number;
+  [key: string]: any;
+};
+
+const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
+  shape: undefined!,
+  batchSize: 'auto',
+  batchDebounceMs: 0,
+  limit: 0,
+  _limitMB: 0
+};
+
+/** Incrementally builds batches from a stream of rows */
+export class TableBatchBuilder {
+  schema: Schema;
+  options: Required<TableBatchBuilderOptions>;
+
+  private aggregator: TableBatchAggregator | null = null;
+  private batchCount: number = 0;
+  private bytesUsed: number = 0;
+  private isChunkComplete: boolean = false;
+  private lastBatchEmittedMs: number = Date.now();
+  private totalLength: number = 0;
+  private totalBytes: number = 0;
+  private rowBytes: number = 0;
+
+  static ArrowBatch?: TableBatchConstructor;
+
+  constructor(schema: Schema, options?: TableBatchBuilderOptions) {
+    this.schema = schema;
+    this.options = {...DEFAULT_OPTIONS, ...options};
+  }
+
+  limitReached(): boolean {
+    if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
+      return true;
+    }
+    if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
+      return true;
+    }
+    return false;
+  }
+
+  /** @deprecated Use addArrayRow or addObjectRow */
+  addRow(row: any[] | {[columnName: string]: any}): void {
+    if (this.limitReached()) {
+      return;
+    }
+    this.totalLength++;
+    this.rowBytes = this.rowBytes || this._estimateRowMB(row);
+    this.totalBytes += this.rowBytes;
+    if (Array.isArray(row)) {
+      this.addArrayRow(row);
+    } else {
+      this.addObjectRow(row);
+    }
+  }
+
+  /** Add one row to the batch */
+  protected addArrayRow(row: any[]) {
+    if (!this.aggregator) {
+      const TableBatchType = this._getTableBatchType();
+      this.aggregator = new TableBatchType(this.schema, this.options);
+    }
+    this.aggregator.addArrayRow(row);
+  }
+
+  /** Add one row to the batch */
+  protected addObjectRow(row: {[columnName: string]: any}): void {
+    if (!this.aggregator) {
+      const TableBatchType = this._getTableBatchType();
+      this.aggregator = new TableBatchType(this.schema, this.options);
+    }
+    this.aggregator.addObjectRow(row);
+  }
+
+  /** Mark an incoming raw memory chunk has completed */
+  chunkComplete(chunk: ArrayBuffer | string): void {
+    if (chunk instanceof ArrayBuffer) {
+      this.bytesUsed += chunk.byteLength;
+    }
+    if (typeof chunk === 'string') {
+      this.bytesUsed += chunk.length;
+    }
+    this.isChunkComplete = true;
+  }
+
+  getFullBatch(options?: GetBatchOptions): TableBatch | null {
+    return this._isFull() ? this._getBatch(options) : null;
+  }
+
+  getFinalBatch(options?: GetBatchOptions): TableBatch | null {
+    return this._getBatch(options);
+  }
+
+  // INTERNAL
+
+  _estimateRowMB(row: any[] | object): number {
+    return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
+  }
+
+  private _isFull(): boolean {
+    // No batch, not ready
+    if (!this.aggregator || this.aggregator.rowCount() === 0) {
+      return false;
+    }
+
+    // if batchSize === 'auto' we wait for chunk to complete
+    // if batchSize === number, ensure we have enough rows
+    if (this.options.batchSize === 'auto') {
+      if (!this.isChunkComplete) {
+        return false;
+      }
+    } else if (this.options.batchSize > this.aggregator.rowCount()) {
+      return false;
+    }
+
+    // Debounce batches
+    if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
+      return false;
+    }
+
+    // Emit batch
+    this.isChunkComplete = false;
+    this.lastBatchEmittedMs = Date.now();
+    return true;
+  }
+
+  /**
+   * bytesUsed can be set via chunkComplete or via getBatch*
+   */
+  private _getBatch(options?: GetBatchOptions): TableBatch | null {
+    if (!this.aggregator) {
+      return null;
+    }
+
+    // TODO - this can overly increment bytes used?
+    if (options?.bytesUsed) {
+      this.bytesUsed = options.bytesUsed;
+    }
+    const normalizedBatch = this.aggregator.getBatch() as TableBatch;
+    normalizedBatch.count = this.batchCount;
+    normalizedBatch.bytesUsed = this.bytesUsed;
+    Object.assign(normalizedBatch, options);
+
+    this.batchCount++;
+    this.aggregator = null;
+    return normalizedBatch;
+  }
+
+  private _getTableBatchType(): TableBatchConstructor {
+    switch (this.options.shape) {
+      case 'array-row-table':
+      case 'object-row-table':
+        return RowTableBatchAggregator;
+      case 'columnar-table':
+        return ColumnarTableBatchAggregator;
+      case 'arrow-table':
+        return ArrowTableBatchAggregator;
+      default:
+        return BaseTableBatchAggregator;
+    }
+  }
+}

package/src/schema-utils/lib/table/batches/convert-batches.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  TableBatch,
+  ArrayRowTableBatch,
+  ObjectRowTableBatch,
+  ColumnarTableBatch,
+  ArrowTableBatch
+} from '@loaders.gl/core/schema';
+import {convertTable} from '../tables/convert-table';
+
+export function convertBatch(batches: TableBatch, shape: 'object-row-table'): ObjectRowTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'array-row-table'): ArrayRowTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'columnar-table'): ColumnarTableBatch;
+export function convertBatch(batches: TableBatch, shape: 'arrow-table'): ArrowTableBatch;
+
+/** Convert a table batch to a different shape */
+export function convertBatch(
+  batch: TableBatch,
+  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
+): TableBatch {
+  switch (shape) {
+    case 'object-row-table':
+      return {...batch, ...convertTable(batch, 'object-row-table')};
+    case 'array-row-table':
+      return {...batch, ...convertTable(batch, 'array-row-table')};
+    case 'columnar-table':
+      return {...batch, ...convertTable(batch, 'columnar-table')};
+    case 'arrow-table':
+      return {...batch, ...convertTable(batch, 'arrow-table')};
+    default:
+      throw new Error(shape);
+  }
+}
+
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'object-row-table'
+): AsyncIterableIterator<ObjectRowTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'array-row-table'
+): AsyncIterableIterator<ArrayRowTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'columnar-table'
+): AsyncIterableIterator<ColumnarTableBatch>;
+export function convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'arrow-table'
+): AsyncIterableIterator<ArrowTableBatch>;
+
+/**
+ * Convert batches to a different shape
+ * @param table
+ * @param shape
+ * @returns
+ */
+export async function* convertBatches(
+  batches: Iterable<TableBatch> | AsyncIterable<TableBatch>,
+  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
+): AsyncIterableIterator<TableBatch> {
+  for await (const batch of batches) {
+    switch (shape) {
+      case 'object-row-table':
+        yield convertBatch(batch, 'object-row-table');
+        break;
+      case 'array-row-table':
+        yield convertBatch(batch, 'array-row-table');
+        break;
+      case 'columnar-table':
+        yield convertBatch(batch, 'columnar-table');
+        break;
+      case 'arrow-table':
+        yield convertBatch(batch, 'arrow-table');
+        break;
+      default:
+        throw new Error(shape);
+    }
+  }
+}

package/src/schema-utils/lib/table/batches/make-arrow-batch-iterator.ts

@@ -0,0 +1,81 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {Table, ArrowTableBatch} from '@loaders.gl/core/schema';
+
+import {convertSchemaToArrow} from '../../schema/convert-arrow-schema';
+import {getTableLength, getTableNumCols, getTableCellAt} from '../tables/table-accessors';
+
+/**
+ * Returns an iterator that yields a single table as a sequence of ArrowTable batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowTableBatchIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<ArrowTableBatch> {
+  for (const batch of makeArrowRecordBatchIterator(table, options)) {
+    const arrowTable = new arrow.Table([batch]);
+    yield {
+      ...batch,
+      shape: 'arrow-table',
+      schema: table.schema,
+      batchType: 'data',
+      length: arrowTable.numRows,
+      data: arrowTable
+    };
+  }
+}
+
+/**
+ * Returns an iterator that yields a single table as a sequence of arrow.RecordBatch batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowRecordBatchIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<arrow.RecordBatch> {
+  const arrowSchema = convertSchemaToArrow(table.schema!);
+
+  const length = getTableLength(table);
+  const numColumns = getTableNumCols(table);
+  const batchSize = options?.batchSize || length;
+
+  const builders = arrowSchema?.fields.map(arrowField =>
+    arrow.makeBuilder({
+      type: arrowField.type,
+      nullValues: [null, undefined]
+    })
+  );
+  const structField = new arrow.Struct(arrowSchema.fields);
+
+  let batchLength = 0;
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+      const value = getTableCellAt(table, rowIndex, columnIndex);
+
+      const builder = builders[columnIndex];
+      builder.append(value);
+    }
+
+    batchLength++;
+
+    if (batchLength >= batchSize) {
+      const datas = builders.map(builder => builder.flush());
+      const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+      yield new arrow.RecordBatch(arrowSchema, structData);
+      batchLength = 0;
+    }
+  }
+
+  if (batchLength > 0) {
+    const datas = builders.map(builder => builder.flush());
+    const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+    yield new arrow.RecordBatch(arrowSchema, structData);
+    batchLength = 0;
+  }
+
+  builders.map(builder => builder.finish());
+}

package/src/schema-utils/lib/table/batches/make-table-batch-iterator.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {TableBatch, Table} from '@loaders.gl/core/schema';
+import {getTableLength} from '../tables/table-accessors';
+
+/**
+ * Returns an iterator that yields the contents of a table as a sequence of batches.
+ * @todo Currently only a single batch is yielded.
+ * @note All batches will have the same shape and schema as the original table.
+ * @returns
+ */
+export function* makeTableBatchIterator(table: Table): IterableIterator<TableBatch> {
+  yield makeBatchFromTable(table);
+}
+
+/**
+ * Returns a table packaged as a single table batch
+ * @note The batch will have the same shape and schema as the original table.
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+export function makeBatchFromTable(table: Table): TableBatch {
+  return {...table, length: getTableLength(table), batchType: 'data'};
+}

package/src/schema-utils/lib/table/batches/make-table-from-batches.ts

@@ -0,0 +1,84 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  Schema,
+  TableBatch,
+  Table,
+  ObjectRowTable,
+  ArrayRowTable,
+  Feature
+} from '@loaders.gl/core/schema';
+import {getTableLength} from '../tables/table-accessors';
+
+/**
+ * Assembles all batches from an async iterator into a single table.
+ * @note All batches must have the same shape and schema
+ * @param batchIterator
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+// eslint-disable-next-line complexity
+export async function makeTableFromBatches(
+  batchIterator: AsyncIterable<TableBatch> | Iterable<TableBatch>
+): Promise<Table | null> {
+  let arrayRows: ArrayRowTable['data'];
+  let objectRows: ObjectRowTable['data'];
+  let features: Feature[];
+  let shape: Table['shape'] | null = null;
+  let schema: Schema | undefined;
+
+  for await (const batch of batchIterator) {
+    shape = shape || batch.shape;
+    schema = schema || batch.schema;
+
+    switch (batch.shape) {
+      case 'array-row-table':
+        arrayRows = arrayRows! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.data[rowIndex];
+          arrayRows.push(row);
+        }
+        break;
+
+      case 'object-row-table':
+        objectRows = objectRows! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.data[rowIndex];
+          objectRows.push(row);
+        }
+        break;
+
+      case 'geojson-table':
+        features = features! || [];
+        for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+          const row = batch.features[rowIndex];
+          features.push(row);
+        }
+        break;
+
+      case 'columnar-table':
+      case 'arrow-table':
+      default:
+        throw new Error('shape');
+    }
+  }
+
+  if (!shape) {
+    return null;
+  }
+
+  switch (shape) {
+    case 'array-row-table':
+      return {shape: 'array-row-table', data: arrayRows!, schema};
+
+    case 'object-row-table':
+      return {shape: 'object-row-table', data: objectRows!, schema};
+
+    case 'geojson-table':
+      return {shape: 'geojson-table', type: 'FeatureCollection', features: features!, schema};
+
+    default:
+      return null;
+  }
+}

package/src/schema-utils/lib/table/query-arrow-table.ts

@@ -0,0 +1,130 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTable} from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+
+/** Portable options for applying a residual query to an Arrow table. */
+export type ArrowTableQueryOptions<PredicateT = unknown> = Readonly<{
+  /** Predicate evaluated against all source columns before projection. */
+  predicate?: PredicateT;
+  /** Output columns in caller-specified order. Defaults to all source columns. */
+  columns?: readonly string[];
+  /** Maximum number of rows retained after filtering. */
+  limit?: number;
+}>;
+
+/** Callback used by Arrow table query adapters to evaluate a portable predicate. */
+export type ArrowTableRowFilter<PredicateT = unknown> = (
+  predicate: PredicateT,
+  columns: Record<string, unknown[]>,
+  rowCount: number
+) => readonly number[];
+
+/**
+ * Applies residual filtering, projection, and limiting to a materialized Arrow table.
+ *
+ * The Arrow-specific vector construction lives here so format adapters only provide their
+ * predicate evaluator. Predicate columns are materialized independently from the projection,
+ * allowing WHERE columns that are not returned to the caller.
+ */
+export function queryArrowTable<PredicateT = unknown>(
+  table: ArrowTable,
+  options: ArrowTableQueryOptions<PredicateT> = {},
+  filterRows?: ArrowTableRowFilter<PredicateT>
+): ArrowTable {
+  const availableColumns = table.data.schema.fields.map(field => field.name);
+  const selectedColumns = options.columns ? [...options.columns] : availableColumns;
+  for (const columnName of selectedColumns) {
+    if (!availableColumns.includes(columnName)) {
+      throw new Error(`Arrow table query could not read column "${columnName}".`);
+    }
+  }
+
+  let rowIndices: readonly number[] | undefined;
+  if (options.predicate !== undefined) {
+    if (!filterRows) throw new Error('Arrow table query requires a predicate evaluator.');
+    const columns = Object.fromEntries(
+      availableColumns.map(name => [name, [...(table.data.getChild(name) || [])]])
+    );
+    rowIndices = filterRows(options.predicate, columns, table.data.numRows);
+  }
+
+  const limitedRowIndices = rowIndices
+    ? rowIndices.slice(0, options.limit ?? Number.POSITIVE_INFINITY)
+    : undefined;
+  const data = limitedRowIndices
+    ? createArrowTableFromRowIndices(table.data, selectedColumns, limitedRowIndices)
+    : table.data.select(selectedColumns).slice(0, options.limit ?? Number.POSITIVE_INFINITY);
+  const schema = table.schema
+    ? {
+        ...table.schema,
+        fields: selectedColumns.flatMap(
+          columnName => table.schema?.fields.filter(field => field.name === columnName) || []
+        )
+      }
+    : undefined;
+  return schema ? {shape: 'arrow-table', schema, data} : {shape: 'arrow-table', data};
+}
+
+/** Selects rows from an Arrow table while preserving the requested column order. */
+export function selectArrowTableRows(
+  table: ArrowTable,
+  rowIndices: readonly number[],
+  columns?: readonly string[],
+  limit?: number
+): ArrowTable {
+  const availableColumns = table.data.schema.fields.map(field => field.name);
+  const selectedColumns = columns ? [...columns] : availableColumns;
+  for (const columnName of selectedColumns) {
+    if (!availableColumns.includes(columnName)) {
+      throw new Error(`Arrow table query could not read column "${columnName}".`);
+    }
+  }
+  const limitedRowIndices = rowIndices.slice(0, limit ?? Number.POSITIVE_INFINITY);
+  const data = createArrowTableFromRowIndices(table.data, selectedColumns, limitedRowIndices);
+  const schema = table.schema
+    ? {
+        ...table.schema,
+        fields: selectedColumns.flatMap(
+          columnName => table.schema?.fields.filter(field => field.name === columnName) || []
+        )
+      }
+    : undefined;
+  return schema ? {shape: 'arrow-table', schema, data} : {shape: 'arrow-table', data};
+}
+
+function createArrowTableFromRowIndices(
+  sourceData: arrow.Table,
+  selectedColumns: readonly string[],
+  rowIndices: readonly number[]
+): arrow.Table {
+  const schema = new arrow.Schema(
+    selectedColumns.map(
+      columnName => sourceData.schema.fields.find(field => field.name === columnName)!
+    )
+  );
+  const vectors = Object.fromEntries(
+    selectedColumns.map(columnName => {
+      const sourceVector = sourceData.getChild(columnName);
+      return [
+        columnName,
+        arrow.vectorFromArray(
+          rowIndices.map(rowIndex => sourceVector?.get(rowIndex)),
+          sourceVector?.type
+        )
+      ];
+    })
+  );
+  const children = selectedColumns.map(columnName => vectors[columnName].data[0]);
+  const data = new arrow.Data(
+    new arrow.Struct(schema.fields),
+    0,
+    rowIndices.length,
+    0,
+    undefined,
+    children
+  );
+  return new arrow.Table(schema, new arrow.RecordBatch(schema, data));
+}