@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/lib/table/tables/table-accessors.ts

@@ -0,0 +1,417 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* eslint-disable no-else-return */
+
+import type {Table, ArrayRowTable, ObjectRowTable} from '@loaders.gl/schema';
+
+export function isTable(table: any): table is Table {
+  const shape = typeof table === 'object' && table?.shape;
+  switch (shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return Array.isArray(table.data);
+
+    case 'geojson-table':
+      return Array.isArray(table.features);
+
+    case 'columnar-table':
+      return table.data && typeof table.data === 'object';
+
+    case 'arrow-table':
+      return Boolean(table?.data?.numRows !== undefined);
+
+    default:
+      return false;
+  }
+}
+
+/**
+ * Returns the length of the table (i.e. the number of rows)
+ */
+export function getTableLength(table: Table): number {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return table.data.length;
+
+    case 'geojson-table':
+      return table.features.length;
+
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      return arrowTable.numRows;
+
+    case 'columnar-table':
+      for (const column of Object.values(table.data)) {
+        return column.length || 0;
+      }
+      return 0;
+    default:
+      throw new Error('table');
+  }
+}
+
+/**
+ * Returns the number of columns in the table
+ * @throws Fails to deduce number of columns if the table has no schema and is empty
+ */
+export function getTableNumCols(table: Table): number {
+  if (table.schema) {
+    return table.schema.fields.length;
+  }
+  if (getTableLength(table) === 0) {
+    throw new Error('empty table');
+  }
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[0].length;
+    case 'object-row-table':
+      return Object.keys(table.data[0]).length;
+    case 'geojson-table':
+      return Object.keys(table.features[0]).length;
+    case 'columnar-table':
+      return Object.keys(table.data).length;
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      return arrowTable.numCols;
+    default:
+      throw new Error('table');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCell(table: Table, rowIndex: number, columnName: string): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      const columnIndex = getTableColumnIndex(table, columnName);
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+      return table.data[rowIndex][columnName];
+
+    case 'geojson-table':
+      return table.features[rowIndex][columnName];
+
+    case 'columnar-table':
+      const column = table.data[columnName];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      const arrowColumnIndex = arrowTable.schema.fields.findIndex(
+        (field) => field.name === columnName
+      );
+      return arrowTable.getChildAt(arrowColumnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCellAt(table: Table, rowIndex: number, columnIndex: number): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+      const columnName1 = getTableColumnName(table, columnIndex);
+      return table.data[rowIndex][columnName1];
+
+    case 'geojson-table':
+      const columnName2 = getTableColumnName(table, columnIndex);
+      return table.features[rowIndex][columnName2];
+
+    case 'columnar-table':
+      const columnName3 = getTableColumnName(table, columnIndex);
+      const column = table.data[columnName3];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      return arrowTable.getChildAt(columnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Deduce the table row shape */
+export function getTableRowShape(table: Table): 'array-row-table' | 'object-row-table' {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return table.shape;
+
+    case 'geojson-table':
+      // TODO - this is not correct, geojson-table is not a row table
+      return 'object-row-table';
+
+    case 'columnar-table':
+    default:
+      throw new Error('Not a row table');
+  }
+}
+
+/** Get the index of a named table column. Requires the table to have a schema */
+export function getTableColumnIndex(table: Table, columnName: string): number {
+  const columnIndex = table.schema?.fields.findIndex((field) => field.name === columnName);
+  if (columnIndex === undefined) {
+    throw new Error(columnName);
+  }
+  return columnIndex;
+}
+
+/** Get the name of a table column by index. Requires the table to have a schema */
+export function getTableColumnName(table: Table, columnIndex: number): string {
+  const columnName = table.schema?.fields[columnIndex]?.name;
+  if (!columnName) {
+    throw new Error(`${columnIndex}`);
+  }
+  return columnName;
+}
+
+/**
+ * Returns one row of the table in object format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsObject(
+  table: Table,
+  rowIndex: number,
+  target?: {[columnName: string]: unknown},
+  copy?: 'copy'
+): {[columnName: string]: unknown} {
+  switch (table.shape) {
+    case 'object-row-table':
+      return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
+
+    case 'array-row-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] = table.data[rowIndex][i];
+        }
+        return objectRow;
+      }
+      throw new Error('no schema');
+
+    case 'geojson-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        // TODO - should lift properties to top level
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] = table.features[rowIndex][i];
+        }
+        return objectRow;
+      }
+      throw new Error('no schema');
+
+    case 'columnar-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] =
+            table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return objectRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (const [name, column] of Object.entries(table.data)) {
+          objectRow[name] = column[rowIndex];
+        }
+        return objectRow;
+      }
+
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      const objectRow: {[columnName: string]: unknown} = target || {};
+      const row = arrowTable.get(rowIndex);
+      const schema = arrowTable.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        objectRow[schema.fields[i].name] = row?.[schema.fields[i].name];
+      }
+      return objectRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/**
+ * Returns one row of the table in array format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance.
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsArray(
+  table: Table,
+  rowIndex: number,
+  target?: unknown[],
+  copy?: 'copy'
+): unknown[] {
+  switch (table.shape) {
+    case 'array-row-table':
+      return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
+
+    case 'object-row-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[rowIndex][table.schema.fields[i].name];
+        }
+        return arrayRow;
+      }
+      // Warning: just slap on the values, this risks mismatches between rows
+      return Object.values(table.data[rowIndex]);
+
+    case 'geojson-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        // TODO - should lift properties to top level
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.features[rowIndex][table.schema.fields[i].name];
+        }
+        return arrayRow;
+      }
+      // Warning: just slap on the values, this risks mismatches between rows
+      return Object.values(table.features[rowIndex]);
+
+    case 'columnar-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return arrayRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const arrayRow: unknown[] = target || [];
+        let i = 0;
+        for (const column of Object.values(table.data)) {
+          arrayRow[i] = column[rowIndex];
+          i++;
+        }
+        return arrayRow;
+      }
+
+    case 'arrow-table':
+      const arrowTable = table.data as any;
+      const arrayRow: unknown[] = target || [];
+      const row = arrowTable.get(rowIndex);
+      const schema = arrowTable.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        arrayRow[i] = row?.[schema.fields[i].name];
+      }
+      return arrayRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/** Convert any table into array row format */
+export function makeArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeColumnarTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+// Row Iterators
+
+/**
+ * Iterate over table rows
+ * @param table
+ * @param shape
+ */
+export function* makeRowIterator(
+  table: Table,
+  shape: 'object-row-table' | 'array-row-table'
+): Iterable<unknown[] | {[key: string]: unknown}> {
+  switch (shape) {
+    case 'array-row-table':
+      yield* makeArrayRowIterator(table);
+      break;
+    case 'object-row-table':
+      yield* makeObjectRowIterator(table);
+      break;
+
+    default:
+      throw new Error(`Unknown row type ${shape}`);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding array rows
+ * @param table
+ * @param shape
+ */
+export function* makeArrayRowIterator(table: Table, target: unknown[] = []): Iterable<unknown[]> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsArray(table, rowIndex, target);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding object rows
+ * @param table
+ * @param shape
+ */
+export function* makeObjectRowIterator(
+  table: Table,
+  target: {[key: string]: unknown} = {}
+): Iterable<{[key: string]: unknown}> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsObject(table, rowIndex, target);
+  }
+}

package/src/lib/table/tables/table-column.ts

@@ -0,0 +1,49 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// import type {TypedArray,} from '../../../types/types';
+import type {Field} from '@loaders.gl/schema';
+import {getArrayTypeFromDataType} from '../../schema/data-type';
+
+export interface ArrayType<T = unknown> {
+  readonly length: number;
+  [n: number]: T;
+}
+
+export function makeColumnFromField(field: Field, length: number): ArrayType {
+  const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+  return new ArrayType(length);
+}
+
+/*
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function deduceSchema(rows) {
+  const row = rows[0];
+
+  const schema = {};
+  let i = 0;
+  for (const columnName in row) {
+    const value = row[columnName];
+    switch (typeof value) {
+      case 'number':
+      case 'boolean':
+        // TODO - booleans could be handled differently...
+        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+        break;
+
+      case 'object':
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+        break;
+
+      case 'string':
+      default:
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+      // We currently only handle numeric rows
+      // TODO we could offer a function to map strings to numbers?
+    }
+    i++;
+  }
+  return schema;
+}
+*/

package/src/lib/table/tables/table-types.ts

@@ -0,0 +1,37 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  Table,
+  ObjectRowTable,
+  ArrayRowTable,
+  ColumnarTable,
+  GeoJSONTable,
+  ArrowTable
+} from '@loaders.gl/schema';
+
+/** Checks if a table is of array row layout */
+export function isArrayRowTable(table: Table): table is ArrayRowTable {
+  return table.shape === 'array-row-table';
+}
+
+/** Checks if a table is of array row layout */
+export function isObjectRowTable(table: Table): table is ObjectRowTable {
+  return table.shape === 'object-row-table';
+}
+
+/** Checks if a table is of columnar layout */
+export function isColumnarTable(table: Table): table is ColumnarTable {
+  return table.shape === 'columnar-table';
+}
+
+/** Checks if a table is of GeoJSON format */
+export function isGeoJSONTable(table: Table): table is GeoJSONTable {
+  return table.shape === 'geojson-table';
+}
+
+/** Checks if table wraps an Apache Arrow table */
+export function isArrowTable(table: Table): table is ArrowTable {
+  return table.shape === 'arrow-table';
+}

package/src/lib/utils/assert.ts

@@ -0,0 +1,8 @@
+// Replacement for the external assert method to reduce bundle size
+// Note: We don't use the second "message" argument in calling code,
+// so no need to support it here
+export function assert(condition: boolean, message?: string) {
+  if (!condition) {
+    throw new Error(message || 'loader assertion failed.');
+  }
+}

package/src/lib/utils/async-queue.ts

@@ -0,0 +1,104 @@
+// From https://github.com/rauschma/async-iter-demo/tree/master/src under MIT license
+// http://2ality.com/2016/10/asynchronous-iteration.html
+
+class ArrayQueue<T> extends Array<T> {
+  enqueue(value: T) {
+    // Add at the end
+    return this.push(value);
+  }
+  dequeue(): T {
+    // Remove first element
+    return this.shift() as T;
+  }
+}
+
+export default class AsyncQueue<T> {
+  private _values: ArrayQueue<T | Error>;
+  private _settlers: ArrayQueue<{resolve: (value: any) => void; reject: (reason?: any) => void}>;
+  private _closed: boolean;
+
+  constructor() {
+    // enqueues > dequeues
+    this._values = new ArrayQueue<T>();
+    // dequeues > enqueues
+    this._settlers = new ArrayQueue<{
+      resolve: (value: any) => void;
+      reject: (reason?: any) => void;
+    }>();
+    this._closed = false;
+  }
+
+  close(): void {
+    while (this._settlers.length > 0) {
+      this._settlers.dequeue().resolve({done: true});
+    }
+    this._closed = true;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return this;
+  }
+
+  enqueue(value: T | Error): void {
+    if (this._closed) {
+      throw new Error('Closed');
+    }
+
+    if (this._settlers.length > 0) {
+      if (this._values.length > 0) {
+        throw new Error('Illegal internal state');
+      }
+      const settler = this._settlers.dequeue();
+      if (value instanceof Error) {
+        settler.reject(value);
+      } else {
+        settler.resolve({value});
+      }
+    } else {
+      this._values.enqueue(value);
+    }
+  }
+
+  /**
+   * @returns a Promise for an IteratorResult
+   */
+  next(): Promise<any> {
+    if (this._values.length > 0) {
+      const value = this._values.dequeue();
+      if (value instanceof Error) {
+        return Promise.reject(value);
+      }
+      return Promise.resolve({value});
+    }
+
+    if (this._closed) {
+      if (this._settlers.length > 0) {
+        throw new Error('Illegal internal state');
+      }
+      return Promise.resolve({done: true});
+    }
+    // Wait for new values to be enqueued
+    return new Promise((resolve, reject) => {
+      this._settlers.enqueue({resolve, reject});
+    });
+  }
+}
+
+/**
+ * @returns a Promise for an Array with the elements in `asyncIterable`
+ */
+export async function takeAsync(
+  asyncIterable: AsyncIterable<any>,
+  count = Infinity
+): Promise<any[]> {
+  const result: Array<any> = [];
+  const iterator = asyncIterable[Symbol.asyncIterator]();
+  while (result.length < count) {
+    const {value, done} = await iterator.next();
+    if (done) {
+      break;
+    }
+    result.push(value);
+  }
+  return result;
+}

package/src/types/types.ts

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Any typed array */
+export type TypedArray =
+  | Int8Array
+  | Uint8Array
+  | Int16Array
+  | Uint16Array
+  | Int32Array
+  | Uint32Array
+  | Uint8ClampedArray
+  | Float32Array
+  | Float64Array;
+
+export type BigTypedArray = TypedArray | BigInt64Array | BigUint64Array;
+
+export type TypedArrayConstructor =
+  | Int8ArrayConstructor
+  | Uint8ArrayConstructor
+  | Int16ArrayConstructor
+  | Uint16ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Float32ArrayConstructor
+  | Float64ArrayConstructor;
+
+export type BigTypedArrayConstructor =
+  | TypedArrayConstructor
+  | BigInt64ArrayConstructor
+  | BigUint64ArrayConstructor;
+
+/** Any numeric array: typed array or `number[]` */
+export type NumberArray = number[] | TypedArray;
+
+export type NumericArray = number[] | TypedArray;
+
+export interface ArrayType<T = unknown> {
+  readonly length: number;
+  [n: number]: T;
+}
+
+/** Any array: typed array or js array (`any[]`) */
+export type AnyArray = any[] | TypedArray;