@loaders.gl/schema 4.3.2 → 4.4.0-alpha.2

package/src/lib/table/simple-table/table-accessors.ts

@@ -1,417 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-/* eslint-disable no-else-return */
-
-import {Table, ArrayRowTable, ObjectRowTable} from '../../../types/category-table';
-
-export function isTable(table: any): table is Table {
-  const shape = typeof table === 'object' && table?.shape;
-  switch (shape) {
-    case 'array-row-table':
-    case 'object-row-table':
-      return Array.isArray(table.data);
-
-    case 'geojson-table':
-      return Array.isArray(table.features);
-
-    case 'columnar-table':
-      return table.data && typeof table.data === 'object';
-
-    case 'arrow-table':
-      return Boolean(table?.data?.numRows !== undefined);
-
-    default:
-      return false;
-  }
-}
-
-/**
- * Returns the length of the table (i.e. the number of rows)
- */
-export function getTableLength(table: Table): number {
-  switch (table.shape) {
-    case 'array-row-table':
-    case 'object-row-table':
-      return table.data.length;
-
-    case 'geojson-table':
-      return table.features.length;
-
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      return arrowTable.numRows;
-
-    case 'columnar-table':
-      for (const column of Object.values(table.data)) {
-        return column.length || 0;
-      }
-      return 0;
-    default:
-      throw new Error('table');
-  }
-}
-
-/**
- * Returns the number of columns in the table
- * @throws Fails to deduce number of columns if the table has no schema and is empty
- */
-export function getTableNumCols(table: Table): number {
-  if (table.schema) {
-    return table.schema.fields.length;
-  }
-  if (getTableLength(table) === 0) {
-    throw new Error('empty table');
-  }
-  switch (table.shape) {
-    case 'array-row-table':
-      return table.data[0].length;
-    case 'object-row-table':
-      return Object.keys(table.data[0]).length;
-    case 'geojson-table':
-      return Object.keys(table.features[0]).length;
-    case 'columnar-table':
-      return Object.keys(table.data).length;
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      return arrowTable.numCols;
-    default:
-      throw new Error('table');
-  }
-}
-
-/** Get a table cell value at row index and column name */
-export function getTableCell(table: Table, rowIndex: number, columnName: string): unknown {
-  switch (table.shape) {
-    case 'array-row-table':
-      const columnIndex = getTableColumnIndex(table, columnName);
-      return table.data[rowIndex][columnIndex];
-
-    case 'object-row-table':
-      return table.data[rowIndex][columnName];
-
-    case 'geojson-table':
-      return table.features[rowIndex][columnName];
-
-    case 'columnar-table':
-      const column = table.data[columnName];
-      return column[rowIndex];
-
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      const arrowColumnIndex = arrowTable.schema.fields.findIndex(
-        (field) => field.name === columnName
-      );
-      return arrowTable.getChildAt(arrowColumnIndex)?.get(rowIndex);
-
-    default:
-      throw new Error('todo');
-  }
-}
-
-/** Get a table cell value at row index and column name */
-export function getTableCellAt(table: Table, rowIndex: number, columnIndex: number): unknown {
-  switch (table.shape) {
-    case 'array-row-table':
-      return table.data[rowIndex][columnIndex];
-
-    case 'object-row-table':
-      const columnName1 = getTableColumnName(table, columnIndex);
-      return table.data[rowIndex][columnName1];
-
-    case 'geojson-table':
-      const columnName2 = getTableColumnName(table, columnIndex);
-      return table.features[rowIndex][columnName2];
-
-    case 'columnar-table':
-      const columnName3 = getTableColumnName(table, columnIndex);
-      const column = table.data[columnName3];
-      return column[rowIndex];
-
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      return arrowTable.getChildAt(columnIndex)?.get(rowIndex);
-
-    default:
-      throw new Error('todo');
-  }
-}
-
-/** Deduce the table row shape */
-export function getTableRowShape(table: Table): 'array-row-table' | 'object-row-table' {
-  switch (table.shape) {
-    case 'array-row-table':
-    case 'object-row-table':
-      return table.shape;
-
-    case 'geojson-table':
-      // TODO - this is not correct, geojson-table is not a row table
-      return 'object-row-table';
-
-    case 'columnar-table':
-    default:
-      throw new Error('Not a row table');
-  }
-}
-
-/** Get the index of a named table column. Requires the table to have a schema */
-export function getTableColumnIndex(table: Table, columnName: string): number {
-  const columnIndex = table.schema?.fields.findIndex((field) => field.name === columnName);
-  if (columnIndex === undefined) {
-    throw new Error(columnName);
-  }
-  return columnIndex;
-}
-
-/** Get the name of a table column by index. Requires the table to have a schema */
-export function getTableColumnName(table: Table, columnIndex: number): string {
-  const columnName = table.schema?.fields[columnIndex]?.name;
-  if (!columnName) {
-    throw new Error(`${columnIndex}`);
-  }
-  return columnName;
-}
-
-/**
- * Returns one row of the table in object format.
- * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance
- * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
- */
-// eslint-disable-next-line complexity
-export function getTableRowAsObject(
-  table: Table,
-  rowIndex: number,
-  target?: {[columnName: string]: unknown},
-  copy?: 'copy'
-): {[columnName: string]: unknown} {
-  switch (table.shape) {
-    case 'object-row-table':
-      return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
-
-    case 'array-row-table':
-      if (table.schema) {
-        const objectRow: {[columnName: string]: unknown} = target || {};
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          objectRow[table.schema.fields[i].name] = table.data[rowIndex][i];
-        }
-        return objectRow;
-      }
-      throw new Error('no schema');
-
-    case 'geojson-table':
-      if (table.schema) {
-        const objectRow: {[columnName: string]: unknown} = target || {};
-        // TODO - should lift properties to top level
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          objectRow[table.schema.fields[i].name] = table.features[rowIndex][i];
-        }
-        return objectRow;
-      }
-      throw new Error('no schema');
-
-    case 'columnar-table':
-      if (table.schema) {
-        const objectRow: {[columnName: string]: unknown} = target || {};
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          objectRow[table.schema.fields[i].name] =
-            table.data[table.schema.fields[i].name][rowIndex];
-        }
-        return objectRow;
-      } else {
-        // eslint-disable-line no-else-return
-        const objectRow: {[columnName: string]: unknown} = target || {};
-        for (const [name, column] of Object.entries(table.data)) {
-          objectRow[name] = column[rowIndex];
-        }
-        return objectRow;
-      }
-
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      const objectRow: {[columnName: string]: unknown} = target || {};
-      const row = arrowTable.get(rowIndex);
-      const schema = arrowTable.schema;
-      for (let i = 0; i < schema.fields.length; i++) {
-        objectRow[schema.fields[i].name] = row?.[schema.fields[i].name];
-      }
-      return objectRow;
-
-    default:
-      throw new Error('shape');
-  }
-}
-
-/**
- * Returns one row of the table in array format.
- * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance.
- * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
- */
-// eslint-disable-next-line complexity
-export function getTableRowAsArray(
-  table: Table,
-  rowIndex: number,
-  target?: unknown[],
-  copy?: 'copy'
-): unknown[] {
-  switch (table.shape) {
-    case 'array-row-table':
-      return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
-
-    case 'object-row-table':
-      if (table.schema) {
-        const arrayRow: unknown[] = target || [];
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          arrayRow[i] = table.data[rowIndex][table.schema.fields[i].name];
-        }
-        return arrayRow;
-      }
-      // Warning: just slap on the values, this risks mismatches between rows
-      return Object.values(table.data[rowIndex]);
-
-    case 'geojson-table':
-      if (table.schema) {
-        const arrayRow: unknown[] = target || [];
-        // TODO - should lift properties to top level
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          arrayRow[i] = table.features[rowIndex][table.schema.fields[i].name];
-        }
-        return arrayRow;
-      }
-      // Warning: just slap on the values, this risks mismatches between rows
-      return Object.values(table.features[rowIndex]);
-
-    case 'columnar-table':
-      if (table.schema) {
-        const arrayRow: unknown[] = target || [];
-        for (let i = 0; i < table.schema.fields.length; i++) {
-          arrayRow[i] = table.data[table.schema.fields[i].name][rowIndex];
-        }
-        return arrayRow;
-      } else {
-        // eslint-disable-line no-else-return
-        const arrayRow: unknown[] = target || [];
-        let i = 0;
-        for (const column of Object.values(table.data)) {
-          arrayRow[i] = column[rowIndex];
-          i++;
-        }
-        return arrayRow;
-      }
-
-    case 'arrow-table':
-      const arrowTable = table.data as any;
-      const arrayRow: unknown[] = target || [];
-      const row = arrowTable.get(rowIndex);
-      const schema = arrowTable.schema;
-      for (let i = 0; i < schema.fields.length; i++) {
-        arrayRow[i] = row?.[schema.fields[i].name];
-      }
-      return arrayRow;
-
-    default:
-      throw new Error('shape');
-  }
-}
-
-/** Convert any table into array row format */
-export function makeArrayRowTable(table: Table): ArrayRowTable {
-  if (table.shape === 'array-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<unknown[]>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsArray(table, rowIndex);
-  }
-  return {
-    shape: 'array-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-/** Convert any table into object row format */
-export function makeObjectRowTable(table: Table): ObjectRowTable {
-  if (table.shape === 'object-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<{[key: string]: unknown}>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsObject(table, rowIndex);
-  }
-  return {
-    shape: 'object-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-/** Convert any table into object row format */
-export function makeColumnarTable(table: Table): ObjectRowTable {
-  if (table.shape === 'object-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<{[key: string]: unknown}>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsObject(table, rowIndex);
-  }
-  return {
-    shape: 'object-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-// Row Iterators
-
-/**
- * Iterate over table rows
- * @param table
- * @param shape
- */
-export function* makeRowIterator(
-  table: Table,
-  shape: 'object-row-table' | 'array-row-table'
-): Iterable<unknown[] | {[key: string]: unknown}> {
-  switch (shape) {
-    case 'array-row-table':
-      yield* makeArrayRowIterator(table);
-      break;
-    case 'object-row-table':
-      yield* makeObjectRowIterator(table);
-      break;
-
-    default:
-      throw new Error(`Unknown row type ${shape}`);
-  }
-}
-
-/**
- * Streaming processing: Iterate over table, yielding array rows
- * @param table
- * @param shape
- */
-export function* makeArrayRowIterator(table: Table, target: unknown[] = []): Iterable<unknown[]> {
-  const length = getTableLength(table);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    yield getTableRowAsArray(table, rowIndex, target);
-  }
-}
-
-/**
- * Streaming processing: Iterate over table, yielding object rows
- * @param table
- * @param shape
- */
-export function* makeObjectRowIterator(
-  table: Table,
-  target: {[key: string]: unknown} = {}
-): Iterable<{[key: string]: unknown}> {
-  const length = getTableLength(table);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    yield getTableRowAsObject(table, rowIndex, target);
-  }
-}

package/src/lib/table/simple-table/table-column.ts

@@ -1,49 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-// import type {TypedArray,} from '../../../types/types';
-import {Field} from '../../../types/schema';
-import {getArrayTypeFromDataType} from './data-type';
-
-export interface ArrayType<T = unknown> {
-  readonly length: number;
-  [n: number]: T;
-}
-
-export function makeColumnFromField(field: Field, length: number): ArrayType {
-  const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
-  return new ArrayType(length);
-}
-
-/*
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function deduceSchema(rows) {
-  const row = rows[0];
-
-  const schema = {};
-  let i = 0;
-  for (const columnName in row) {
-    const value = row[columnName];
-    switch (typeof value) {
-      case 'number':
-      case 'boolean':
-        // TODO - booleans could be handled differently...
-        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
-        break;
-
-      case 'object':
-        schema[columnName] = {name: String(columnName), index: i, type: Array};
-        break;
-
-      case 'string':
-      default:
-        schema[columnName] = {name: String(columnName), index: i, type: Array};
-      // We currently only handle numeric rows
-      // TODO we could offer a function to map strings to numbers?
-    }
-    i++;
-  }
-  return schema;
-}
-*/

package/src/lib/table/simple-table/table-schema.ts

@@ -1,121 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-// Type deduction
-import {Schema, Field} from '../../../types/schema';
-import {ArrayType} from '../../../types/types';
-import {Table} from '../../../types/category-table';
-import {getDataTypeFromArray, getDataTypeFromValue} from './data-type';
-
-/**
- * SCHEMA SUPPORT - AUTODEDUCTION
- * @param {*} table
- * @param {*} schema
- * @returns
- */
-export function deduceTableSchema(table: Table): Schema {
-  switch (table.shape) {
-    case 'array-row-table':
-    case 'object-row-table':
-      return deduceSchemaFromRows(table.data);
-
-    case 'geojson-table':
-      return deduceSchemaFromGeoJSON(table.features);
-
-    case 'columnar-table':
-      return deduceSchemaFromColumns(table.data);
-
-    case 'arrow-table':
-    default:
-      throw new Error('Deduce schema');
-  }
-}
-
-export function deduceSchema(
-  data: unknown[][] | {[key: string]: unknown}[] | {[key: string]: unknown[]}
-): Schema {
-  return Array.isArray(data) ? deduceSchemaFromRows(data) : deduceSchemaFromColumns(data);
-}
-
-/** Given an object with columnar arrays, try to deduce a schema */
-function deduceSchemaFromColumns(columnarTable: {[key: string]: ArrayType}): Schema {
-  const fields: Field[] = [];
-  for (const [columnName, column] of Object.entries(columnarTable)) {
-    const field = deduceFieldFromColumn(column, columnName);
-    fields.push(field);
-  }
-  return {fields, metadata: {}};
-}
-
-/** Given an array of rows, try to deduce a schema */
-function deduceSchemaFromRows(rowTable: unknown[][] | {[key: string]: unknown}[]): Schema {
-  if (!rowTable.length) {
-    throw new Error('deduce from empty table');
-  }
-  const fields: Field[] = [];
-  const row0 = rowTable[0];
-  // TODO - fields can be nullable, false detection...
-  // Could look at additional rows if nulls in first row
-  // TODO - if array, column names will be numbers
-  for (const [columnName, value] of Object.entries(row0)) {
-    fields.push(deduceFieldFromValue(value, columnName));
-  }
-
-  return {fields, metadata: {}};
-}
-
-/** Given a GeoJSON, try to deduce a schema */
-function deduceSchemaFromGeoJSON(features: {properties: Record<string, number> | null}[]): Schema {
-  if (!features.length) {
-    throw new Error('deduce from empty table');
-  }
-  const fields: Field[] = [];
-  const row0 = features[0].properties || {};
-  // TODO - fields can be nullable, false detection...
-  // Could look at additional rows if nulls in first row
-  // TODO - if array, column names will be numbers
-  for (const [columnName, value] of Object.entries(row0)) {
-    fields.push(deduceFieldFromValue(value, columnName));
-  }
-
-  return {fields, metadata: {}};
-}
-
-/** Given a column (i.e. array), attempt to deduce an appropriate `Field` */
-function deduceFieldFromColumn(column: ArrayType, name: string): Field {
-  if (ArrayBuffer.isView(column)) {
-    const type = getDataTypeFromArray(column);
-    return {
-      name,
-      type: type.type || 'null',
-      nullable: type.nullable
-      // metadata: {}
-    };
-  }
-
-  if (Array.isArray(column) && column.length > 0) {
-    const value = column[0];
-    const type = getDataTypeFromValue(value);
-    // TODO - support nested schemas?
-    return {
-      name,
-      type,
-      nullable: true
-      // metadata: {},
-    };
-  }
-
-  throw new Error('empty table');
-}
-
-/** Given a value, attempt to deduce an appropriate `Field` */
-function deduceFieldFromValue(value: unknown, name: string): Field {
-  const type = getDataTypeFromValue(value);
-  return {
-    name,
-    type,
-    nullable: true
-    // metadata: {}
-  };
-}

package/src/lib/utils/assert.ts

@@ -1,8 +0,0 @@
-// Replacement for the external assert method to reduce bundle size
-// Note: We don't use the second "message" argument in calling code,
-// so no need to support it here
-export function assert(condition: boolean, message?: string) {
-  if (!condition) {
-    throw new Error(message || 'loader assertion failed.');
-  }
-}

package/src/lib/utils/async-queue.ts

@@ -1,104 +0,0 @@
-// From https://github.com/rauschma/async-iter-demo/tree/master/src under MIT license
-// http://2ality.com/2016/10/asynchronous-iteration.html
-
-class ArrayQueue<T> extends Array<T> {
-  enqueue(value: T) {
-    // Add at the end
-    return this.push(value);
-  }
-  dequeue(): T {
-    // Remove first element
-    return this.shift() as T;
-  }
-}
-
-export default class AsyncQueue<T> {
-  private _values: ArrayQueue<T | Error>;
-  private _settlers: ArrayQueue<{resolve: (value: any) => void; reject: (reason?: any) => void}>;
-  private _closed: boolean;
-
-  constructor() {
-    // enqueues > dequeues
-    this._values = new ArrayQueue<T>();
-    // dequeues > enqueues
-    this._settlers = new ArrayQueue<{
-      resolve: (value: any) => void;
-      reject: (reason?: any) => void;
-    }>();
-    this._closed = false;
-  }
-
-  close(): void {
-    while (this._settlers.length > 0) {
-      this._settlers.dequeue().resolve({done: true});
-    }
-    this._closed = true;
-  }
-
-  [Symbol.asyncIterator](): AsyncIterator<T> {
-    return this;
-  }
-
-  enqueue(value: T | Error): void {
-    if (this._closed) {
-      throw new Error('Closed');
-    }
-
-    if (this._settlers.length > 0) {
-      if (this._values.length > 0) {
-        throw new Error('Illegal internal state');
-      }
-      const settler = this._settlers.dequeue();
-      if (value instanceof Error) {
-        settler.reject(value);
-      } else {
-        settler.resolve({value});
-      }
-    } else {
-      this._values.enqueue(value);
-    }
-  }
-
-  /**
-   * @returns a Promise for an IteratorResult
-   */
-  next(): Promise<any> {
-    if (this._values.length > 0) {
-      const value = this._values.dequeue();
-      if (value instanceof Error) {
-        return Promise.reject(value);
-      }
-      return Promise.resolve({value});
-    }
-
-    if (this._closed) {
-      if (this._settlers.length > 0) {
-        throw new Error('Illegal internal state');
-      }
-      return Promise.resolve({done: true});
-    }
-    // Wait for new values to be enqueued
-    return new Promise((resolve, reject) => {
-      this._settlers.enqueue({resolve, reject});
-    });
-  }
-}
-
-/**
- * @returns a Promise for an Array with the elements in `asyncIterable`
- */
-export async function takeAsync(
-  asyncIterable: AsyncIterable<any>,
-  count = Infinity
-): Promise<any[]> {
-  const result: Array<any> = [];
-  const iterator = asyncIterable[Symbol.asyncIterator]();
-  while (result.length < count) {
-    const {value, done} = await iterator.next();
-    if (done) {
-      break;
-    }
-    result.push(value);
-  }
-  return result;
-}