@loaders.gl/schema 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/lib/table/simple-table/make-table.ts

@@ -0,0 +1,108 @@
+// loaders.gl, MIT license
+import {Table, ArrayRowTable, ObjectRowTable, ColumnarTable} from '../../../types/category-table';
+import {deduceTableSchema} from './table-schema';
+
+/**
+ * Makes a typed table from data.
+ * @throws Row tables must contain at least one row. Columnar tables must contain empty arrays
+ */
+export function makeTableFromData(data: unknown[][]): ArrayRowTable;
+export function makeTableFromData(data: {[column: string]: unknown}[]): ObjectRowTable;
+export function makeTableFromData(data: {[column: string]: ArrayLike<unknown>}): ColumnarTable;
+export function makeTableFromData(data: unknown): Table {
+  let table: Table;
+  switch (getTableShapeFromData(data)) {
+    case 'array-row-table':
+      table = {shape: 'array-row-table', data: data as unknown[][]};
+      break;
+    case 'object-row-table':
+      table = {shape: 'object-row-table', data: data as {[key: string]: unknown}[]};
+      break;
+    case 'columnar-table':
+      table = {shape: 'columnar-table', data: data as {[column: string]: ArrayLike<unknown>}};
+      break;
+    default:
+      throw new Error('table');
+  }
+  const schema = deduceTableSchema(table);
+  return {...table, schema};
+}
+
+/** Helper function to get shape of data */
+function getTableShapeFromData(data) {
+  if (Array.isArray(data)) {
+    if (data.length === 0) {
+      throw new Error('cannot deduce type of empty table');
+    }
+
+    // Deduce the table shape from the first row
+    const firstRow = data[0];
+
+    if (Array.isArray(firstRow)) {
+      return 'array-row-table';
+    }
+
+    if (firstRow && typeof firstRow === 'object') {
+      return 'object-row-table';
+    }
+  }
+
+  if (data && typeof data === 'object') {
+    return 'columnar-table';
+  }
+
+  throw new Error('invalid table');
+}
+
+/** Convert any table into object row format *
+export function makeColumnarTable(table: Table): ColumnarTable {
+  if (table.shape === 'columnar-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'columnar-table',
+    schema: table.schema,
+    data
+  };
+}
+
+
+/** Convert any table into array row format *
+export function makeArrayRowTable(table: TableLike): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format *
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+*/

package/src/lib/table/simple-table/table-accessors.ts

@@ -0,0 +1,360 @@
+// loaders.gl, MIT license
+
+/* eslint-disable no-else-return */
+
+import {Table, ArrayRowTable, ObjectRowTable} from '../../../types/category-table';
+
+/**
+ * Returns the length of the table (i.e. the number of rows)
+ */
+export function getTableLength(table: Table): number {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return table.data.length;
+
+    case 'arrow-table':
+      return table.data.numRows;
+
+    case 'columnar-table':
+      for (const column of Object.values(table.data)) {
+        return column.length || 0;
+      }
+      return 0;
+    default:
+      throw new Error('table');
+  }
+}
+
+/**
+ * Returns the number of columns in the table
+ * @throws Fails to deduce number of columns if the table has no schema and is empty
+ */
+export function getTableNumCols(table: Table): number {
+  if (table.schema) {
+    return table.schema.fields.length;
+  }
+  if (getTableLength(table) === 0) {
+    throw new Error('empty table');
+  }
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[0].length;
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return Object.keys(table.data[0]).length;
+
+    case 'columnar-table':
+      return Object.keys(table.data).length;
+
+    case 'arrow-table':
+      return table.data.numCols;
+    default:
+      throw new Error('table');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCell(table: Table, rowIndex: number, columnName: string): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      const columnIndex = getTableColumnIndex(table, columnName);
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      return table.data[rowIndex][columnName];
+
+    case 'columnar-table':
+      const column = table.data[columnName];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      const arrowColumnIndex = table.data.schema.fields.findIndex(
+        (field) => field.name === columnName
+      );
+      return table.data.getChildAt(arrowColumnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Get a table cell value at row index and column name */
+export function getTableCellAt(table: Table, rowIndex: number, columnIndex: number): unknown {
+  switch (table.shape) {
+    case 'array-row-table':
+      return table.data[rowIndex][columnIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      let columnName = getTableColumnName(table, columnIndex);
+      return table.data[rowIndex][columnName];
+
+    case 'columnar-table':
+      columnName = getTableColumnName(table, columnIndex);
+      const column = table.data[columnName];
+      return column[rowIndex];
+
+    case 'arrow-table':
+      return table.data.getChildAt(columnIndex)?.get(rowIndex);
+
+    default:
+      throw new Error('todo');
+  }
+}
+
+/** Deduce the table row shape */
+export function getTableRowShape(table: Table): 'array-row-table' | 'object-row-table' {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return table.shape;
+
+    case 'geojson-row-table':
+      return 'object-row-table';
+
+    case 'columnar-table':
+    default:
+      throw new Error('Not a row table');
+  }
+}
+
+/** Get the index of a named table column. Requires the table to have a schema */
+export function getTableColumnIndex(table: Table, columnName: string): number {
+  const columnIndex = table.schema?.fields.findIndex((field) => field.name === columnName);
+  if (columnIndex === undefined) {
+    throw new Error(columnName);
+  }
+  return columnIndex;
+}
+
+/** Get the name of a table column by index. Requires the table to have a schema */
+export function getTableColumnName(table: Table, columnIndex: number): string {
+  const columnName = table.schema?.fields[columnIndex]?.name;
+  if (!columnName) {
+    throw new Error(`${columnIndex}`);
+  }
+  return columnName;
+}
+
+/**
+ * Returns one row of the table in object format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsObject(
+  table: Table,
+  rowIndex: number,
+  target?: {[columnName: string]: unknown},
+  copy?: 'copy'
+): {[columnName: string]: unknown} {
+  switch (table.shape) {
+    case 'object-row-table':
+      return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
+
+    case 'array-row-table':
+    case 'geojson-row-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] = table.data[rowIndex][i];
+        }
+        return objectRow;
+      }
+      throw new Error('no schema');
+
+    case 'columnar-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] =
+            table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return objectRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        for (const [name, column] of Object.entries(table.data)) {
+          objectRow[name] = column[rowIndex];
+        }
+        return objectRow;
+      }
+
+    case 'arrow-table':
+      const objectRow: {[columnName: string]: unknown} = target || {};
+      const row = table.data.get(rowIndex);
+      const schema = table.data.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        objectRow[schema.fields[i].name] = row?.[schema.fields[i].name];
+      }
+      return objectRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/**
+ * Returns one row of the table in array format.
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance.
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
+ */
+// eslint-disable-next-line complexity
+export function getTableRowAsArray(
+  table: Table,
+  rowIndex: number,
+  target?: unknown[],
+  copy?: 'copy'
+): unknown[] {
+  switch (table.shape) {
+    case 'array-row-table':
+      return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
+
+    case 'object-row-table':
+    case 'geojson-row-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[rowIndex][table.schema.fields[i].name];
+        }
+        return arrayRow;
+      }
+      // Warning: just slap on the values, this risks mismatches between rows
+      return Object.values(table.data[rowIndex]);
+
+    case 'columnar-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.data[table.schema.fields[i].name][rowIndex];
+        }
+        return arrayRow;
+      } else {
+        // eslint-disable-line no-else-return
+        const arrayRow: unknown[] = target || [];
+        let i = 0;
+        for (const column of Object.values(table.data)) {
+          arrayRow[i] = column[rowIndex];
+          i++;
+        }
+        return arrayRow;
+      }
+
+    case 'arrow-table':
+      const arrayRow: unknown[] = target || [];
+      const row = table.data.get(rowIndex);
+      const schema = table.data.schema;
+      for (let i = 0; i < schema.fields.length; i++) {
+        arrayRow[i] = row?.[schema.fields[i].name];
+      }
+      return arrayRow;
+
+    default:
+      throw new Error('shape');
+  }
+}
+
+/** Convert any table into array row format */
+export function makeArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeColumnarTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+// Row Iterators
+
+/**
+ * Iterate over table rows
+ * @param table
+ * @param shape
+ */
+export function* makeRowIterator(
+  table: Table,
+  shape: 'object-row-table' | 'array-row-table'
+): Iterable<unknown[] | {[key: string]: unknown}> {
+  switch (shape) {
+    case 'array-row-table':
+      yield* makeArrayRowIterator(table);
+      break;
+    case 'object-row-table':
+      yield* makeObjectRowIterator(table);
+      break;
+
+    default:
+      throw new Error(`Unknown row type ${shape}`);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding array rows
+ * @param table
+ * @param shape
+ */
+export function* makeArrayRowIterator(table: Table, target: unknown[] = []): Iterable<unknown[]> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsArray(table, rowIndex, target);
+  }
+}
+
+/**
+ * Streaming processing: Iterate over table, yielding object rows
+ * @param table
+ * @param shape
+ */
+export function* makeObjectRowIterator(
+  table: Table,
+  target: {[key: string]: unknown} = {}
+): Iterable<{[key: string]: unknown}> {
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    yield getTableRowAsObject(table, rowIndex, target);
+  }
+}

package/src/lib/table/simple-table/table-column.ts

@@ -0,0 +1,47 @@
+// loaders.gl, MIT license
+
+// import type {TypedArray,} from '../../../types/types';
+import {Field} from '../../../types/schema';
+import {getArrayTypeFromDataType} from './data-type';
+
+export interface ArrayType<T = unknown> {
+  readonly length: number;
+  [n: number]: T;
+}
+
+export function makeColumnFromField(field: Field, length: number): ArrayType {
+  const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+  return new ArrayType(length);
+}
+
+/*
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function deduceSchema(rows) {
+  const row = rows[0];
+
+  const schema = {};
+  let i = 0;
+  for (const columnName in row) {
+    const value = row[columnName];
+    switch (typeof value) {
+      case 'number':
+      case 'boolean':
+        // TODO - booleans could be handled differently...
+        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+        break;
+
+      case 'object':
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+        break;
+
+      case 'string':
+      default:
+        schema[columnName] = {name: String(columnName), index: i, type: Array};
+      // We currently only handle numeric rows
+      // TODO we could offer a function to map strings to numbers?
+    }
+    i++;
+  }
+  return schema;
+}
+*/

package/src/lib/table/simple-table/table-schema.ts

@@ -0,0 +1,97 @@
+// Type deduction
+import {Schema, Field} from '../../../types/schema';
+import {ArrayType} from '../../../types/types';
+import {Table} from '../../../types/category-table';
+import {getDataTypeFromArray, getDataTypeFromValue} from './data-type';
+
+/**
+ * SCHEMA SUPPORT - AUTODEDUCTION
+ * @param {*} table
+ * @param {*} schema
+ * @returns
+ */
+export function deduceTableSchema(table: Table): Schema {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return deduceSchemaFromRows(table.data);
+
+    case 'columnar-table':
+      return deduceSchemaFromColumns(table.data);
+
+    case 'arrow-table':
+    default:
+      throw new Error('Deduce schema');
+  }
+}
+
+export function deduceSchema(
+  data: unknown[][] | {[key: string]: unknown}[] | {[key: string]: unknown[]}
+): Schema {
+  return Array.isArray(data) ? deduceSchemaFromRows(data) : deduceSchemaFromColumns(data);
+}
+
+/** Given an object with columnar arrays, try to deduce a schema */
+function deduceSchemaFromColumns(columnarTable: {[key: string]: ArrayType}): Schema {
+  const fields: Field[] = [];
+  for (const [columnName, column] of Object.entries(columnarTable)) {
+    const field = deduceFieldFromColumn(column, columnName);
+    fields.push(field);
+  }
+  return {fields, metadata: {}};
+}
+
+/** Given an array of rows, try to deduce a schema */
+function deduceSchemaFromRows(rowTable: unknown[][] | {[key: string]: unknown}[]): Schema {
+  if (!rowTable.length) {
+    throw new Error('deduce from empty table');
+  }
+  const fields: Field[] = [];
+  const row0 = rowTable[0];
+  // TODO - fields can be nullable, false detection...
+  // Could look at additional rows if nulls in first row
+  // TODO - if array, column names will be numbers
+  for (const [columnName, value] of Object.entries(row0)) {
+    fields.push(deduceFieldFromValue(value, columnName));
+  }
+
+  return {fields, metadata: {}};
+}
+
+/** Given a column (i.e. array), attempt to deduce an appropriate `Field` */
+function deduceFieldFromColumn(column: ArrayType, name: string): Field {
+  if (ArrayBuffer.isView(column)) {
+    const type = getDataTypeFromArray(column);
+    return {
+      name,
+      type: type.type || 'null',
+      nullable: type.nullable
+      // metadata: {}
+    };
+  }
+
+  if (Array.isArray(column) && column.length > 0) {
+    const value = column[0];
+    const type = getDataTypeFromValue(value);
+    // TODO - support nested schemas?
+    return {
+      name,
+      type,
+      nullable: true
+      // metadata: {},
+    };
+  }
+
+  throw new Error('empty table');
+}
+
+/** Given a value, attempt to deduce an appropriate `Field` */
+function deduceFieldFromValue(value: unknown, name: string): Field {
+  const type = getDataTypeFromValue(value);
+  return {
+    name,
+    type,
+    nullable: true
+    // metadata: {}
+  };
+}