@loaders.gl/schema 4.3.2 → 4.4.0-alpha.1

package/src/lib/table/batches/base-table-batch-aggregator.ts

@@ -1,83 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Schema} from '../../../types/schema';
-import type {TableBatch} from '../../../types/category-table';
-import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
-
-const DEFAULT_ROW_COUNT = 100;
-
-export class BaseTableBatchAggregator implements TableBatchAggregator {
-  schema: Schema;
-  options: TableBatchOptions;
-
-  shape?: 'array-row-table' | 'object-row-table';
-  length: number = 0;
-  rows: any[] | null = null;
-  cursor: number = 0;
-  private _headers: string[] = [];
-
-  constructor(schema: Schema, options: TableBatchOptions) {
-    this.options = options;
-    this.schema = schema;
-
-    // schema is an array if there're no headers
-    // object if there are headers
-    if (!Array.isArray(schema)) {
-      this._headers = [];
-      for (const key in schema) {
-        this._headers[schema[key].index] = schema[key].name;
-      }
-    }
-  }
-
-  rowCount(): number {
-    return this.length;
-  }
-
-  addArrayRow(row: any[], cursor?: number): void {
-    if (Number.isFinite(cursor)) {
-      this.cursor = cursor as number;
-    }
-
-    this.shape = 'array-row-table';
-
-    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
-    this.rows[this.length] = row;
-    this.length++;
-  }
-
-  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
-    if (Number.isFinite(cursor)) {
-      this.cursor = cursor as number;
-    }
-
-    this.shape = 'object-row-table';
-
-    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
-    this.rows[this.length] = row;
-    this.length++;
-  }
-
-  getBatch(): TableBatch | null {
-    let rows = this.rows;
-    if (!rows) {
-      return null;
-    }
-
-    rows = rows.slice(0, this.length);
-    this.rows = null;
-
-    const batch: TableBatch = {
-      shape: this.shape || 'array-row-table',
-      batchType: 'data',
-      data: rows,
-      length: this.length,
-      schema: this.schema,
-      cursor: this.cursor
-    };
-
-    return batch;
-  }
-}

package/src/lib/table/batches/columnar-table-batch-aggregator.ts

@@ -1,112 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Schema} from '../../../types/schema';
-import type {ColumnarTableBatch, ArrowTableBatch} from '../../../types/category-table';
-import {TableBatchAggregator} from './table-batch-aggregator';
-
-type ColumnarTableBatchOptions = {};
-
-const DEFAULT_ROW_COUNT = 100;
-
-export class ColumnarTableBatchAggregator implements TableBatchAggregator {
-  schema: Schema;
-  length: number = 0;
-  allocated: number = 0;
-  columns: {[columnName: string]: any[]} = {};
-
-  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
-    this.schema = schema;
-    this._reallocateColumns();
-  }
-
-  rowCount(): number {
-    return this.length;
-  }
-
-  addArrayRow(row: any[]) {
-    // If user keeps pushing rows beyond batch size, reallocate
-    this._reallocateColumns();
-    let i = 0;
-    // TODO what if no csv header, columns not populated?
-    for (const fieldName in this.columns) {
-      this.columns[fieldName][this.length] = row[i++];
-    }
-    this.length++;
-  }
-
-  addObjectRow(row: {[columnName: string]: any}): void {
-    // If user keeps pushing rows beyond batch size, reallocate
-    this._reallocateColumns();
-    for (const fieldName in row) {
-      this.columns[fieldName][this.length] = row[fieldName];
-    }
-    this.length++;
-  }
-
-  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
-    this._pruneColumns();
-    const columns = Array.isArray(this.schema) ? this.columns : {};
-
-    // schema is an array if there're no headers
-    // object if there are headers
-    // columns should match schema format
-    if (!Array.isArray(this.schema)) {
-      for (const fieldName in this.schema) {
-        const field = this.schema[fieldName];
-        columns[field.name] = this.columns[field.index];
-      }
-    }
-
-    this.columns = {};
-
-    const batch: ColumnarTableBatch = {
-      shape: 'columnar-table',
-      batchType: 'data',
-      data: columns,
-      schema: this.schema,
-      length: this.length
-    };
-
-    return batch;
-  }
-
-  // HELPERS
-
-  _reallocateColumns() {
-    if (this.length < this.allocated) {
-      return;
-    }
-
-    // @ts-ignore TODO
-    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
-    this.columns = {};
-
-    for (const fieldName in this.schema) {
-      const field = this.schema[fieldName];
-      const ArrayType = field.type || Float32Array;
-      const oldColumn = this.columns[field.index];
-
-      if (oldColumn && ArrayBuffer.isView(oldColumn)) {
-        // Copy the old data to the new array
-        const typedArray = new ArrayType(this.allocated);
-        typedArray.set(oldColumn);
-        this.columns[field.index] = typedArray;
-      } else if (oldColumn) {
-        // Plain array
-        oldColumn.length = this.allocated;
-        this.columns[field.index] = oldColumn;
-      } else {
-        // Create new
-        this.columns[field.index] = new ArrayType(this.allocated);
-      }
-    }
-  }
-
-  _pruneColumns() {
-    for (const [columnName, column] of Object.entries(this.columns)) {
-      this.columns[columnName] = column.slice(0, this.length);
-    }
-  }
-}

package/src/lib/table/batches/row-table-batch-aggregator.ts

@@ -1,109 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Schema} from '../../../types/schema';
-import type {TableBatch} from '../../../types/category-table';
-import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
-import {
-  convertToArrayRow,
-  convertToObjectRow,
-  inferHeadersFromArrayRow,
-  inferHeadersFromObjectRow
-} from '../simple-table/row-utils';
-
-const DEFAULT_ROW_COUNT = 100;
-
-export class RowTableBatchAggregator implements TableBatchAggregator {
-  schema: Schema | null;
-  options: TableBatchOptions;
-
-  length: number = 0;
-  objectRows: {[columnName: string]: unknown}[] | null = null;
-  arrayRows: unknown[][] | null = null;
-  cursor: number = 0;
-  private _headers: string[] | null = null;
-
-  constructor(schema: Schema | null, options: TableBatchOptions) {
-    this.options = options;
-    this.schema = schema;
-
-    // schema is an array if there're no headers
-    // object if there are headers
-    if (schema) {
-      this._headers = [];
-      for (const key in schema) {
-        this._headers[schema[key].index] = schema[key].name;
-      }
-    }
-  }
-
-  rowCount(): number {
-    return this.length;
-  }
-
-  addArrayRow(row: any[], cursor?: number): void {
-    if (Number.isFinite(cursor)) {
-      this.cursor = cursor as number;
-    }
-
-    // TODO - infer schema at a higher level, instead of hacking headers here?
-    this._headers ||= inferHeadersFromArrayRow(row);
-
-    // eslint-disable-next-line default-case
-    switch (this.options.shape) {
-      case 'object-row-table':
-        const rowObject = convertToObjectRow(row, this._headers);
-        this.addObjectRow(rowObject, cursor);
-        break;
-      case 'array-row-table':
-        this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
-        this.arrayRows[this.length] = row;
-        this.length++;
-        break;
-    }
-  }
-
-  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
-    if (Number.isFinite(cursor)) {
-      this.cursor = cursor as number;
-    }
-
-    // TODO - infer schema at a higher level, instead of hacking headers here?
-    this._headers ||= inferHeadersFromObjectRow(row);
-
-    // eslint-disable-next-line default-case
-    switch (this.options.shape) {
-      case 'array-row-table':
-        const rowArray = convertToArrayRow(row, this._headers);
-        this.addArrayRow(rowArray, cursor);
-        break;
-      case 'object-row-table':
-        this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
-        this.objectRows[this.length] = row;
-        this.length++;
-        break;
-    }
-  }
-
-  getBatch(): TableBatch | null {
-    let rows = this.arrayRows || this.objectRows;
-    if (!rows) {
-      return null;
-    }
-
-    rows = rows.slice(0, this.length);
-    this.arrayRows = null;
-    this.objectRows = null;
-
-    return {
-      shape: this.options.shape,
-      batchType: 'data',
-      data: rows,
-      length: this.length,
-      // @ts-expect-error we should infer a schema
-      schema: this.schema,
-      cursor: this.cursor
-    };
-  }
-}

package/src/lib/table/batches/table-batch-aggregator.ts

@@ -1,29 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Schema} from '../../../types/schema';
-import type {TableBatch} from '../../../types/category-table';
-
-export interface TableBatchOptions {
-  batchSize: number | string;
-  [key: string]: any;
-}
-
-export interface TableBatchConstructor {
-  new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
-}
-
-/**
- * TableBatchBuilder delegates batch building to this interface
- */
-export interface TableBatchAggregator {
-  /** Number of rows */
-  rowCount(): number;
-  /** Add one row */
-  addArrayRow(row: any[]): void;
-  /** Add one row */
-  addObjectRow(row: {[columnName: string]: any}): void;
-  /** return a batch object */
-  getBatch(): TableBatch | null;
-}

package/src/lib/table/batches/table-batch-builder.ts

@@ -1,190 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Schema} from '../../../types/schema';
-import type {TableBatch} from '../../../types/category-table';
-import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
-import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
-import {RowTableBatchAggregator} from './row-table-batch-aggregator';
-import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
-
-// TODO define interface instead
-type TableBatchBuilderOptions = {
-  shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
-  batchSize?: number | 'auto';
-  batchDebounceMs?: number;
-  limit?: number;
-  _limitMB?: number;
-};
-
-type GetBatchOptions = {
-  bytesUsed?: number;
-  [key: string]: any;
-};
-
-const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
-  shape: undefined!,
-  batchSize: 'auto',
-  batchDebounceMs: 0,
-  limit: 0,
-  _limitMB: 0
-};
-
-const ERR_MESSAGE = 'TableBatchBuilder';
-
-/** Incrementally builds batches from a stream of rows */
-export class TableBatchBuilder {
-  schema: Schema;
-  options: Required<TableBatchBuilderOptions>;
-
-  private aggregator: TableBatchAggregator | null = null;
-  private batchCount: number = 0;
-  private bytesUsed: number = 0;
-  private isChunkComplete: boolean = false;
-  private lastBatchEmittedMs: number = Date.now();
-  private totalLength: number = 0;
-  private totalBytes: number = 0;
-  private rowBytes: number = 0;
-
-  static ArrowBatch?: TableBatchConstructor;
-
-  constructor(schema: Schema, options?: TableBatchBuilderOptions) {
-    this.schema = schema;
-    this.options = {...DEFAULT_OPTIONS, ...options};
-  }
-
-  limitReached(): boolean {
-    if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
-      return true;
-    }
-    if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
-      return true;
-    }
-    return false;
-  }
-
-  /** @deprecated Use addArrayRow or addObjectRow */
-  addRow(row: any[] | {[columnName: string]: any}): void {
-    if (this.limitReached()) {
-      return;
-    }
-    this.totalLength++;
-    this.rowBytes = this.rowBytes || this._estimateRowMB(row);
-    this.totalBytes += this.rowBytes;
-    if (Array.isArray(row)) {
-      this.addArrayRow(row);
-    } else {
-      this.addObjectRow(row);
-    }
-  }
-
-  /** Add one row to the batch */
-  protected addArrayRow(row: any[]) {
-    if (!this.aggregator) {
-      const TableBatchType = this._getTableBatchType();
-      this.aggregator = new TableBatchType(this.schema, this.options);
-    }
-    this.aggregator.addArrayRow(row);
-  }
-
-  /** Add one row to the batch */
-  protected addObjectRow(row: {[columnName: string]: any}): void {
-    if (!this.aggregator) {
-      const TableBatchType = this._getTableBatchType();
-      this.aggregator = new TableBatchType(this.schema, this.options);
-    }
-    this.aggregator.addObjectRow(row);
-  }
-
-  /** Mark an incoming raw memory chunk has completed */
-  chunkComplete(chunk: ArrayBuffer | string): void {
-    if (chunk instanceof ArrayBuffer) {
-      this.bytesUsed += chunk.byteLength;
-    }
-    if (typeof chunk === 'string') {
-      this.bytesUsed += chunk.length;
-    }
-    this.isChunkComplete = true;
-  }
-
-  getFullBatch(options?: GetBatchOptions): TableBatch | null {
-    return this._isFull() ? this._getBatch(options) : null;
-  }
-
-  getFinalBatch(options?: GetBatchOptions): TableBatch | null {
-    return this._getBatch(options);
-  }
-
-  // INTERNAL
-
-  _estimateRowMB(row: any[] | object): number {
-    return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
-  }
-
-  private _isFull(): boolean {
-    // No batch, not ready
-    if (!this.aggregator || this.aggregator.rowCount() === 0) {
-      return false;
-    }
-
-    // if batchSize === 'auto' we wait for chunk to complete
-    // if batchSize === number, ensure we have enough rows
-    if (this.options.batchSize === 'auto') {
-      if (!this.isChunkComplete) {
-        return false;
-      }
-    } else if (this.options.batchSize > this.aggregator.rowCount()) {
-      return false;
-    }
-
-    // Debounce batches
-    if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
-      return false;
-    }
-
-    // Emit batch
-    this.isChunkComplete = false;
-    this.lastBatchEmittedMs = Date.now();
-    return true;
-  }
-
-  /**
-   * bytesUsed can be set via chunkComplete or via getBatch*
-   */
-  private _getBatch(options?: GetBatchOptions): TableBatch | null {
-    if (!this.aggregator) {
-      return null;
-    }
-
-    // TODO - this can overly increment bytes used?
-    if (options?.bytesUsed) {
-      this.bytesUsed = options.bytesUsed;
-    }
-    const normalizedBatch = this.aggregator.getBatch() as TableBatch;
-    normalizedBatch.count = this.batchCount;
-    normalizedBatch.bytesUsed = this.bytesUsed;
-    Object.assign(normalizedBatch, options);
-
-    this.batchCount++;
-    this.aggregator = null;
-    return normalizedBatch;
-  }
-
-  private _getTableBatchType(): TableBatchConstructor {
-    switch (this.options.shape) {
-      case 'array-row-table':
-      case 'object-row-table':
-        return RowTableBatchAggregator;
-      case 'columnar-table':
-        return ColumnarTableBatchAggregator;
-      case 'arrow-table':
-        if (!TableBatchBuilder.ArrowBatch) {
-          throw new Error(ERR_MESSAGE);
-        }
-        return TableBatchBuilder.ArrowBatch;
-      default:
-        return BaseTableBatchAggregator;
-    }
-  }
-}

package/src/lib/table/simple-table/convert-table.ts

@@ -1,149 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import {
-  getTableCell,
-  getTableLength,
-  getTableRowAsArray,
-  getTableRowAsObject
-} from './table-accessors';
-import {
-  Table,
-  ArrayRowTable,
-  ObjectRowTable,
-  ColumnarTable,
-  ArrowTable
-} from '../../../types/category-table';
-import {deduceTableSchema} from './table-schema';
-import {makeColumnFromField} from './table-column';
-
-export function convertTable(table: Table, shape: 'object-row-table'): ObjectRowTable;
-export function convertTable(table: Table, shape: 'array-row-table'): ArrayRowTable;
-export function convertTable(table: Table, shape: 'columnar-table'): ColumnarTable;
-export function convertTable(table: Table, shape: 'arrow-table'): ArrowTable;
-
-/**
- * Convert a table to a different shape
- * @param table
- * @param shape
- * @returns
- */
-export function convertTable(
-  table: Table,
-  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
-) {
-  switch (shape) {
-    case 'object-row-table':
-      return makeObjectRowTable(table);
-    case 'array-row-table':
-      return makeArrayRowTable(table);
-    case 'columnar-table':
-      return makeColumnarTable(table);
-    case 'arrow-table':
-      return makeArrowTable(table);
-    default:
-      throw new Error(shape);
-  }
-}
-
-/**
- * Convert a table to apache arrow format
- * @note this depends on the `@loaders.gl/arrow module being imported
- */
-export function makeArrowTable(table: Table): Table {
-  const _makeArrowTable = globalThis.__loaders?._makeArrowTable;
-  if (!_makeArrowTable) {
-    throw new Error('');
-  }
-  return _makeArrowTable(table);
-}
-
-/** Convert any simple table into columnar format */
-export function makeColumnarTable(table: Table): ColumnarTable {
-  // TODO - should schema really be optional?
-  const schema = table.schema || deduceTableSchema(table);
-  const fields = table.schema?.fields || [];
-
-  if (table.shape === 'columnar-table') {
-    return {...table, schema};
-  }
-
-  const length = getTableLength(table);
-
-  const columns: {[column: string]: ArrayLike<unknown>} = {};
-  for (const field of fields) {
-    const column = makeColumnFromField(field, length);
-    columns[field.name] = column;
-    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-      column[rowIndex] = getTableCell(table, rowIndex, field.name);
-    }
-  }
-
-  return {
-    shape: 'columnar-table',
-    schema,
-    data: columns
-  };
-}
-
-/** Convert any table into array row format */
-export function makeArrayRowTable(table: Table): ArrayRowTable {
-  if (table.shape === 'array-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<unknown[]>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsArray(table, rowIndex);
-  }
-  return {
-    shape: 'array-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-/** Convert any table into object row format */
-export function makeObjectRowTable(table: Table): ObjectRowTable {
-  if (table.shape === 'object-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<{[key: string]: unknown}>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsObject(table, rowIndex);
-  }
-  return {
-    shape: 'object-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-/**
- *
- * @note - should be part of schema module
- *
-export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-
-  const rowFormatTable: {}[] = [];
-
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
-  }
-
-  return {
-    shape: 'object-row-table',
-    schema: columnarTable.schema,
-    data: rowFormatTable
-  };
-}
-*/