@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/schema-utils/lib/table/tables/convert-arrow-table.ts

@@ -0,0 +1,198 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {
+  Table,
+  ArrayRowTable,
+  ColumnarTable,
+  ObjectRowTable,
+  GeoJSONTable,
+  ArrowTable,
+  Feature
+} from '@loaders.gl/core/schema';
+
+import {convertTable} from './convert-table';
+import {convertArrowToSchema} from '../../schema/convert-arrow-schema';
+import {makeArrowRecordBatchIterator} from '../batches/make-arrow-batch-iterator';
+
+/**
+ * * Convert a loaders.gl Table to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ */
+export function convertTableToArrow(table: Table, options?: {batchSize?: number}): arrow.Table {
+  switch (table.shape) {
+    case 'arrow-table':
+      return table.data;
+
+    case 'columnar-table':
+    // TODO - optimized implementation is possible
+    // return convertColumnarTableToArrow(table, options);
+
+    // fall through
+
+    default:
+      const arrowBatchIterator = makeArrowRecordBatchIterator(table, options);
+      return new arrow.Table(arrowBatchIterator);
+  }
+}
+
+/**
+ * Convert an Apache Arrow table to a loaders.gl Table
+ * @note Currently does not convert schema
+ */
+export function convertArrowToTable(arrow: arrow.Table, shape: 'arrow-table'): ArrowTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: 'object-row-table'): ObjectRowTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: 'array-row-table'): ArrayRowTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: 'geojson-table'): GeoJSONTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export function convertArrowToTable(arrow: arrow.Table, shape: Table['shape']): Table;
+
+/**
+ * Convert an Apache Arrow table to a loaders.gl Table
+ * @note Currently does not convert schema
+ */
+export function convertArrowToTable(arrowTable: arrow.Table, shape: Table['shape']): Table {
+  switch (shape) {
+    case 'arrow-table':
+      return convertArrowToArrowTable(arrowTable);
+    case 'array-row-table':
+      return convertArrowToArrayRowTable(arrowTable);
+    case 'object-row-table':
+      return convertArrowToObjectRowTable(arrowTable);
+    case 'columnar-table':
+      return convertArrowToColumnarTable(arrowTable);
+    case 'geojson-table':
+      return convertArrowToGeoJSONTable(arrowTable);
+    default:
+      throw new Error(shape);
+  }
+}
+
+/**
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
+ * From this additional conversions are available.
+ * @param arrowTable
+ * @returns
+ */
+function convertArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
+  return {
+    shape: 'arrow-table',
+    schema: convertArrowToSchema(arrowTable.schema),
+    data: arrowTable
+  };
+}
+
+function convertArrowToArrayRowTable(arrowTable: arrow.Table): Table {
+  const columnarTable = convertArrowToColumnarTable(arrowTable);
+  return convertTable(columnarTable, 'array-row-table');
+}
+
+function convertArrowToObjectRowTable(arrowTable: arrow.Table): Table {
+  const columnarTable = convertArrowToColumnarTable(arrowTable);
+  return convertTable(columnarTable, 'object-row-table');
+}
+
+/**
+ * Convert an Apache Arrow table to a ColumnarTable
+ * @note Currently does not convert schema
+ */
+function convertArrowToColumnarTable(arrowTable: arrow.Table): ColumnarTable {
+  // TODO - avoid calling `getColumn` on columns we are not interested in?
+  // Add options object?
+
+  const columns: ColumnarTable['data'] = {};
+
+  for (const field of arrowTable.schema.fields) {
+    // This (is intended to) coalesce all record batches into a single typed array
+    const arrowColumn = arrowTable.getChild(field.name);
+    const values = arrowColumn?.toArray();
+    columns[field.name] = values;
+  }
+
+  const schema = convertArrowToSchema(arrowTable.schema);
+
+  return {
+    shape: 'columnar-table',
+    schema,
+    data: columns
+  };
+}
+
+/**
+ * Convert an Apache Arrow table to a GeoJSONTable
+ * @note Currently does not convert schema
+ */
+function convertArrowToGeoJSONTable(arrowTable: arrow.Table): GeoJSONTable {
+  const schema = convertArrowToSchema(arrowTable.schema);
+
+  const features: Feature[] = [];
+
+  for (let row = 0; row < arrowTable.numRows; row++) {
+    // parse arrow geometry to geojson feature
+    const properties = arrowTable.get(row)?.toJSON() || {};
+    features.push({type: 'Feature', geometry: null!, properties});
+  }
+
+  return {
+    shape: 'geojson-table',
+    type: 'FeatureCollection',
+    schema,
+    features
+  };
+}
+
+// /**
+//  * Wrap an apache arrow table in a loaders.gl table wrapper.
+//  * From this additional conversions are available.
+//  * @param arrowTable
+//  * @returns
+//  */
+// function convertArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
+//   return {
+//     shape: 'arrow-table',
+//     schema: convertArrowToSchema(arrowTable.schema),
+//     data: arrowTable
+//   };
+// }
+
+// function convertArrowToArrayRowTable(arrowTable: arrow.Table): Table {
+//   const columnarTable = convertArrowToColumnarTable(arrowTable);
+//   return convertTable(columnarTable, 'array-row-table');
+// }
+
+// function convertArrowToObjectRowTable(arrowTable: arrow.Table): Table {
+//   const columnarTable = convertArrowToColumnarTable(arrowTable);
+//   return convertTable(columnarTable, 'object-row-table');
+// }
+
+// /**
+//  * Convert an Apache Arrow table to a ColumnarTable
+//  * @note Currently does not convert schema
+//  */
+// function convertArrowToColumnarTable(arrowTable: arrow.Table): ColumnarTable {
+//   // TODO - avoid calling `getColumn` on columns we are not interested in?
+//   // Add options object?
+
+//   const columns: ColumnarTable['data'] = {};
+
+//   for (const field of arrowTable.schema.fields) {
+//     // This (is intended to) coalesce all record batches into a single typed array
+//     const arrowColumn = arrowTable.getChild(field.name);
+//     const values = arrowColumn?.toArray();
+//     columns[field.name] = values;
+//   }
+
+//   const schema = convertArrowToSchema(arrowTable.schema);
+
+//   return {
+//     shape: 'columnar-table',
+//     schema,
+//     data: columns
+//   };
+// }

package/src/schema-utils/lib/table/tables/convert-table.ts

@@ -0,0 +1,152 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  Table,
+  ArrayRowTable,
+  ObjectRowTable,
+  ColumnarTable,
+  ArrowTable
+} from '@loaders.gl/core/schema';
+import {
+  getTableCell,
+  getTableLength,
+  getTableRowAsArray,
+  getTableRowAsObject
+} from './table-accessors';
+import {deduceTableSchema} from '../../schema/deduce-table-schema';
+import {makeColumnFromField} from './table-column';
+import {convertTableToArrow} from './convert-arrow-table';
+
+export function convertTable(table: Table, shape: 'object-row-table'): ObjectRowTable;
+export function convertTable(table: Table, shape: 'array-row-table'): ArrayRowTable;
+export function convertTable(table: Table, shape: 'columnar-table'): ColumnarTable;
+export function convertTable(table: Table, shape: 'arrow-table'): ArrowTable;
+
+/**
+ * Convert a table to a different shape
+ * @param table
+ * @param shape
+ * @returns
+ */
+export function convertTable(
+  table: Table,
+  shape: 'object-row-table' | 'array-row-table' | 'columnar-table' | 'arrow-table'
+) {
+  switch (shape) {
+    case 'object-row-table':
+      return convertToObjectRowTable(table);
+    case 'array-row-table':
+      return convertToArrayRowTable(table);
+    case 'columnar-table':
+      return convertToColumnarTable(table);
+    case 'arrow-table':
+      return convertToArrowTable(table);
+    default:
+      throw new Error(shape);
+  }
+}
+
+/** Convert any simple table into columnar format */
+export function convertToColumnarTable(table: Table): ColumnarTable {
+  // TODO - should schema really be optional?
+  const schema = table.schema || deduceTableSchema(table);
+  const fields = schema.fields;
+
+  if (table.shape === 'columnar-table') {
+    return {...table, schema};
+  }
+
+  const length = getTableLength(table);
+
+  const columns: {[column: string]: ArrayLike<unknown>} = {};
+  for (const field of fields) {
+    const column = makeColumnFromField(field, length);
+    columns[field.name] = column;
+    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+      column[rowIndex] = getTableCell(table, rowIndex, field.name);
+    }
+  }
+
+  return {
+    shape: 'columnar-table',
+    schema,
+    data: columns
+  };
+}
+
+/** Convert any table into array row format */
+export function convertToArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function convertToObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/**
+ * Convert a table to apache arrow format
+ * @note this depends on the `@loaders.gl/arrow module being imported
+ */
+export function convertToArrowTable(table: Table): ArrowTable {
+  const schema = table.schema || deduceTableSchema(table);
+  const arrowTable = convertTableToArrow({...table, schema});
+  return {
+    shape: 'arrow-table',
+    schema,
+    data: arrowTable
+  };
+}
+
+/**
+ *
+ * @note - should be part of schema module
+ *
+export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
+  const tableKeys = Object.keys(columnarTable);
+  const tableRowsCount = columnarTable[tableKeys[0]].length;
+
+  const rowFormatTable: {}[] = [];
+
+  for (let index = 0; index < tableRowsCount; index++) {
+    const tableItem = {};
+    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      const fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+
+  return {
+    shape: 'object-row-table',
+    schema: columnarTable.schema,
+    data: rowFormatTable
+  };
+}
+*/

package/src/schema-utils/lib/table/tables/make-table.ts

@@ -0,0 +1,127 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {
+  Table,
+  ArrayRowTable,
+  ObjectRowTable,
+  ColumnarTable,
+  ArrowTable
+} from '@loaders.gl/core/schema';
+import {deduceTableSchema} from '../../schema/deduce-table-schema';
+
+/**
+ * Makes a typed table from data.
+ * @throws Row tables must contain at least one row. Columnar tables must contain empty arrays
+ */
+export function makeTableFromData(data: unknown[][]): ArrayRowTable;
+export function makeTableFromData(data: {[column: string]: unknown}[]): ObjectRowTable;
+export function makeTableFromData(data: {[column: string]: ArrayLike<unknown>}): ColumnarTable;
+export function makeTableFromData(data: arrow.Table): ArrowTable;
+
+export function makeTableFromData(data: unknown): Table {
+  let table: Table;
+  switch (getTableShapeFromData(data)) {
+    case 'array-row-table':
+      table = {shape: 'array-row-table', data: data as unknown[][]};
+      break;
+    case 'object-row-table':
+      table = {shape: 'object-row-table', data: data as {[key: string]: unknown}[]};
+      break;
+    case 'columnar-table':
+      table = {shape: 'columnar-table', data: data as {[column: string]: ArrayLike<unknown>}};
+      break;
+    case 'arrow-table':
+      table = {shape: 'arrow-table', data: data as arrow.Table};
+      break;
+    default:
+      throw new Error('table');
+  }
+  const schema = deduceTableSchema(table);
+  return {...table, schema};
+}
+
+/** Helper function to get shape of data */
+function getTableShapeFromData(data) {
+  if (data instanceof arrow.Table) {
+    return 'arrow-table';
+  }
+
+  if (Array.isArray(data)) {
+    if (data.length === 0) {
+      throw new Error('cannot deduce type of empty table');
+    }
+
+    // Deduce the table shape from the first row
+    const firstRow = data[0];
+
+    if (Array.isArray(firstRow)) {
+      return 'array-row-table';
+    }
+
+    if (firstRow && typeof firstRow === 'object') {
+      return 'object-row-table';
+    }
+  }
+
+  if (data && typeof data === 'object') {
+    return 'columnar-table';
+  }
+
+  throw new Error('invalid table');
+}
+
+/** Convert any table into object row format *
+export function makeColumnarTable(table: Table): ColumnarTable {
+  if (table.shape === 'columnar-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'columnar-table',
+    schema: table.schema,
+    data
+  };
+}
+
+
+/** Convert any table into array row format *
+export function makeArrayRowTable(table: TableLike): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format *
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}
+*/

package/src/schema-utils/lib/table/tables/row-utils.ts

@@ -0,0 +1,59 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Convert an object row to an array row */
+export function convertToObjectRow(
+  arrayRow: unknown[],
+  headers: string[] | null
+): {[columnName: string]: unknown} {
+  if (!arrayRow) {
+    throw new Error('null row');
+  }
+  const objectRow: {[columnName: string]: unknown} = {};
+  if (headers) {
+    for (let i = 0; i < headers.length; i++) {
+      objectRow[headers[i]] = arrayRow[i];
+    }
+  } else {
+    for (let i = 0; i < arrayRow.length; i++) {
+      const columnName = `column-${i}`;
+      objectRow[columnName] = arrayRow[i];
+    }
+  }
+  return objectRow;
+}
+
+/** Convert an object row to an array row */
+export function convertToArrayRow(
+  objectRow: {[columnName: string]: unknown},
+  headers: string[] | null
+): unknown[] {
+  if (!objectRow) {
+    throw new Error('null row');
+  }
+
+  if (headers) {
+    const arrayRow = new Array(headers.length);
+    for (let i = 0; i < headers.length; i++) {
+      arrayRow[i] = objectRow[headers[i]];
+    }
+    return arrayRow;
+  }
+  return Object.values(objectRow);
+}
+
+/** Get headers from a sample array row */
+export function inferHeadersFromArrayRow(arrayRow: unknown[]) {
+  const headers: string[] = [];
+  for (let i = 0; i < arrayRow.length; i++) {
+    const columnName = `column-${i}`;
+    headers.push(columnName);
+  }
+  return headers;
+}
+
+/** Get headers from a smaple object row */
+export function inferHeadersFromObjectRow(row: {[columnName: string]: unknown}) {
+  return Object.keys(row);
+}