@loaders.gl/schema 4.0.0-alpha.5 → 4.0.0-alpha.6

package/src/lib/table/simple-table/convert-table.ts

@@ -0,0 +1,72 @@
+// loaders.gl, MIT license
+import {
+  getTableCell,
+  getTableLength,
+  getTableRowAsArray,
+  getTableRowAsObject
+} from './table-accessors';
+import {Table, ArrayRowTable, ObjectRowTable, ColumnarTable} from '../../../types/category-table';
+import {deduceTableSchema} from './table-schema';
+import {makeColumnFromField} from './table-column';
+
+/** Convert any simple table into columnar format */
+export function makeColumnarTable(table: Table): ColumnarTable {
+  // TODO - should schema really be optional?
+  const schema = table.schema || deduceTableSchema(table);
+  const fields = table.schema?.fields || [];
+
+  if (table.shape === 'columnar-table') {
+    return {...table, schema};
+  }
+
+  const length = getTableLength(table);
+
+  const columns: {[column: string]: ArrayLike<unknown>} = {};
+  for (const field of fields) {
+    const column = makeColumnFromField(field, length);
+    columns[field.name] = column;
+    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+      column[rowIndex] = getTableCell(table, rowIndex, field.name);
+    }
+  }
+
+  return {
+    shape: 'columnar-table',
+    schema,
+    data: columns
+  };
+}
+
+/** Convert any table into array row format */
+export function makeArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}

package/src/lib/table/simple-table/data-type.ts

@@ -0,0 +1,102 @@
+// loaders.gl, MIT license
+
+import {DataType} from '../../../types/schema';
+import {TypedArray, TypedArrayConstructor, ArrayType} from '../../../types/types';
+
+/** Deduce column types from values */
+export function getDataTypeFromValue(
+  value: unknown,
+  defaultNumberType: 'float32' = 'float32'
+): DataType {
+  if (value instanceof Date) {
+    return 'date-millisecond';
+  }
+  if (value instanceof Number) {
+    return defaultNumberType;
+  }
+  if (typeof value === 'string') {
+    return 'utf8';
+  }
+  if (value === null || value === 'undefined') {
+    return 'null';
+  }
+  return 'null';
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromArray(array: ArrayType): {type: DataType; nullable: boolean} {
+  let type = getDataTypeFromTypedArray(array as TypedArray);
+  if (type !== 'null') {
+    return {type, nullable: false};
+  }
+  if (array.length > 0) {
+    type = getDataTypeFromValue(array[0]);
+    return {type, nullable: true};
+  }
+
+  return {type: 'null', nullable: true};
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromTypedArray(array: TypedArray): DataType {
+  switch (array.constructor) {
+    case Int8Array:
+      return 'int8';
+    case Uint8Array:
+    case Uint8ClampedArray:
+      return 'uint8';
+    case Int16Array:
+      return 'int16';
+    case Uint16Array:
+      return 'uint16';
+    case Int32Array:
+      return 'int32';
+    case Uint32Array:
+      return 'uint32';
+    case Float32Array:
+      return 'float32';
+    case Float64Array:
+      return 'float64';
+    default:
+      return 'null';
+  }
+}
+
+export function getArrayTypeFromDataType(
+  type: DataType,
+  nullable: boolean | undefined
+): TypedArrayConstructor | ArrayConstructor {
+  if (!nullable) {
+    switch (type) {
+      case 'int8':
+        return Int8Array;
+      case 'uint8':
+        return Uint8Array;
+      case 'int16':
+        return Int16Array;
+      case 'uint16':
+        return Uint16Array;
+      case 'int32':
+        return Int32Array;
+      case 'uint32':
+        return Uint32Array;
+      case 'float32':
+        return Float32Array;
+      case 'float64':
+        return Float64Array;
+      default:
+        break;
+    }
+  }
+
+  // if (typeof BigInt64Array !== 'undefined') {
+  //   TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
+  //   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
+  // }
+
+  return Array;
+}

package/src/lib/table/simple-table/make-table.ts

@@ -0,0 +1,108 @@
+// loaders.gl, MIT license
+import {Table, ArrayRowTable, ObjectRowTable, ColumnarTable} from '../../../types/category-table';
+import {deduceTableSchema} from './table-schema';
+
+/**
+ * Makes a typed table from data.
+ * @throws Row tables must contain at least one row. Columnar tables must contain empty arrays
+ */
+export function makeTableFromData(data: unknown[][]): ArrayRowTable;
+export function makeTableFromData(data: {[column: string]: unknown}[]): ObjectRowTable;
+export function makeTableFromData(data: {[column: string]: ArrayLike<unknown>}): ColumnarTable;
+export function makeTableFromData(data: unknown): Table {
+  let table: Table;
+  switch (getTableShapeFromData(data)) {
+    case 'array-row-table':
+      table = {shape: 'array-row-table', data: data as unknown[][]};
+      break;
+    case 'object-row-table':
+      table = {shape: 'object-row-table', data: data as {[key: string]: unknown}[]};
+      break;
+    case 'columnar-table':
+      table = {shape: 'columnar-table', data: data as {[column: string]: ArrayLike<unknown>}};
+      break;
+    default:
+      throw new Error('table');
+  }
+  const schema = deduceTableSchema(table);
+  return {...table, schema};
+}
+
+/** Helper function to get shape of data */
+function getTableShapeFromData(data) {
+  if (Array.isArray(data)) {
+    if (data.length === 0) {
+      throw new Error('cannot deduce type of empty table');
+    }
+
+    // Deduce the table shape from the first row
+    const firstRow = data[0];
+
+    if (Array.isArray(firstRow)) {
+      return 'array-row-table';
+    }
+
+    if (firstRow && typeof firstRow === 'object') {
+      return 'object-row-table';
+    }
+  }
+
+  if (data && typeof data === 'object') {
+    return 'columnar-table';
+  }
+
+  throw new Error('invalid table');
+}
+
+/** Convert any table into object row format *
+export function makeColumnarTable(table: Table): ColumnarTable {
+  if (table.shape === 'columnar-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'columnar-table',
+    schema: table.schema,
+    data
+  };
+}
+
+
+/** Convert any table into array row format *
+export function makeArrayRowTable(table: TableLike): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format *
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+*/

package/src/lib/table/simple-table/table-accessors.ts

@@ -0,0 +1,360 @@
+// loaders.gl, MIT license
+
+/* eslint-disable no-else-return */
+
+import {Table, ArrayRowTable, ObjectRowTable} from '../../../types/category-table';
+
+/**
+ * Returns the length of the table (i.e. the number of rows)
+ */
+export function getTableLength(table: Table): number {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return table.data.length;
+
+    case 'arrow-table':
+      return table.data.numRows;
+
+    case 'columnar-table':
+      for (const column of Object.values(table.data)) {
+        return column.length || 0;
+      }
+      return 0;
+    default:
+      throw new Error('table');
+  }
+}
+
+/**
+ * Returns the number of columns in the table
+ * @throws Fails to deduce number of columns if the table has no schema and is empty
+ */
+export function getTableNumCols(table: Table): number {
+  if (table.schema) {
+    return table.schema.fields.length;
+  }
+  if (getTableLength(table) === 0) {
+    throw new Error('empty table');
+  }
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[0].length;
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return Object.keys(table.data[0]).length;
+
+    case 'columnar-table':
+      return Object.keys(table.data).length;
+
+    case 'arrow-table':
+      return table.data.numCols;
+    default:
+      throw new Error('table');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCell(table: Table, rowIndex: number, columnName: string): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      const columnIndex = getTableColumnIndex(table, columnName);
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return table.data[rowIndex][columnName];
+
+    case 'columnar-table':
+      const column = table.data[columnName];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      const arrowColumnIndex = table.data.schema.fields.findIndex(
+        (field) => field.name === columnName
+      );
+      return table.data.getChildAt(arrowColumnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCellAt(table: Table, rowIndex: number, columnIndex: number): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      let columnName = getTableColumnName(table, columnIndex);
+      return table.data[rowIndex][columnName];
+
+    case 'columnar-table':
+      columnName = getTableColumnName(table, columnIndex);
+      const column = table.data[columnName];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      return table.data.getChildAt(columnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Deduce the table row shape */
+export function getTableRowShape(table: Table): 'array-row-table' | 'object-row-table' {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return table.shape;
+
+    case 'geojson-row-table':
+      return 'object-row-table';
+
+    case 'columnar-table':
+    default:
+      throw new Error('Not a row table');
+  }
+}
+
+/** Get the index of a named table column. Requires the table to have a schema */
+export function getTableColumnIndex(table: Table, columnName: string): number {
+  const columnIndex = table.schema?.fields.findIndex((field) => field.name === columnName);
+  if (columnIndex === undefined) {
+    throw new Error(columnName);
+  }
+  return columnIndex;
+}
+
+/** Get the name of a table column by index. Requires the table to have a schema */
+export function getTableColumnName(table: Table, columnIndex: number): string {
+  const columnName = table.schema?.fields[columnIndex]?.name;
+  if (!columnName) {
+    throw new Error(`${columnIndex}`);
+  }
+  return columnName;
+}
+
+/**
+ * Returns one row of the table in object format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsObject(
+  table: Table,
+  rowIndex: number,
+  target?: {[columnName: string]: unknown},
+  copy?: 'copy'
+): {[columnName: string]: unknown} {
+  switch (table.shape) {
+    case 'object-row-table':
+      return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
+
+    case 'array-row-table':
+    case 'geojson-row-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] = table.data[rowIndex][i];
+        }
+        return objectRow;
+      }
+      throw new Error('no schema');
+
+    case 'columnar-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] =
+            table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return objectRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (const [name, column] of Object.entries(table.data)) {
+          objectRow[name] = column[rowIndex];
+        }
+        return objectRow;
+      }
+
+    case 'arrow-table':
+      const objectRow: {[columnName: string]: unknown} = target || {};
+      const row = table.data.get(rowIndex);
+      const schema = table.data.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        objectRow[schema.fields[i].name] = row?.[schema.fields[i].name];
+      }
+      return objectRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/**
+ * Returns one row of the table in array format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance.
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsArray(
+  table: Table,
+  rowIndex: number,
+  target?: unknown[],
+  copy?: 'copy'
+): unknown[] {
+  switch (table.shape) {
+    case 'array-row-table':
+      return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[rowIndex][table.schema.fields[i].name];
+        }
+        return arrayRow;
+      }
+      // Warning: just slap on the values, this risks mismatches between rows
+      return Object.values(table.data[rowIndex]);
+
+    case 'columnar-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return arrayRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const arrayRow: unknown[] = target || [];
+        let i = 0;
+        for (const column of Object.values(table.data)) {
+          arrayRow[i] = column[rowIndex];
+          i++;
+        }
+        return arrayRow;
+      }
+
+    case 'arrow-table':
+      const arrayRow: unknown[] = target || [];
+      const row = table.data.get(rowIndex);
+      const schema = table.data.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        arrayRow[i] = row?.[schema.fields[i].name];
+      }
+      return arrayRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/** Convert any table into array row format */
+export function makeArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeColumnarTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+// Row Iterators
+
+/**
+ * Iterate over table rows
+ * @param table
+ * @param shape
+ */
+export function* makeRowIterator(
+  table: Table,
+  shape: 'object-row-table' | 'array-row-table'
+): Iterable<unknown[] | {[key: string]: unknown}> {
+  switch (shape) {
+    case 'array-row-table':
+      yield* makeArrayRowIterator(table);
+      break;
+    case 'object-row-table':
+      yield* makeObjectRowIterator(table);
+      break;
+
+    default:
+      throw new Error(`Unknown row type ${shape}`);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding array rows
+ * @param table
+ * @param shape
+ */
+export function* makeArrayRowIterator(table: Table, target: unknown[] = []): Iterable<unknown[]> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsArray(table, rowIndex, target);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding object rows
+ * @param table
+ * @param shape
+ */
+export function* makeObjectRowIterator(
+  table: Table,
+  target: {[key: string]: unknown} = {}
+): Iterable<{[key: string]: unknown}> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsObject(table, rowIndex, target);
+  }
+}

package/src/lib/table/simple-table/table-column.ts

@@ -0,0 +1,47 @@
+// loaders.gl, MIT license
+
+// import type {TypedArray,} from '../../../types/types';
+import {Field} from '../../../types/schema';
+import {getArrayTypeFromDataType} from './data-type';
+
+export interface ArrayType<T = unknown> {
+  readonly length: number;
+  [n: number]: T;
+}
+
+export function makeColumnFromField(field: Field, length: number): ArrayType {
+  const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+  return new ArrayType(length);
+}
+
+/*
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function deduceSchema(rows) {
+  const row = rows[0];
+
+  const schema = {};
+  let i = 0;
+  for (const columnName in row) {
+    const value = row[columnName];
+    switch (typeof value) {
+      case 'number':
+      case 'boolean':
+        // TODO - booleans could be handled differently...
+        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+        break;
+
+      case 'object':
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+        break;
+
+      case 'string':
+      default:
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+      // We currently only handle numeric rows
+      // TODO we could offer a function to map strings to numbers?
+    }
+    i++;
+  }
+  return schema;
+}
+*/