@loaders.gl/schema 4.0.0-alpha.4 → 4.0.0-alpha.6

package/src/lib/table/arrow/convert-table-to-arrow.ts

@@ -0,0 +1,59 @@
+// loaders.gl, MIT license
+
+// import {
+//   Table as ApacheArrowTable,
+//   Schema as ApacheArrowSchema,
+//   RecordBatch,
+//   FixedSizeList,
+//   Field,
+//   Data,
+//   FixedSizeListVector
+// } from 'apache-arrow/Arrow.dom';
+// import {AbstractVector} from 'apache-arrow/vector';
+
+// import {Table} from '../../types/category-table';
+// import {getArrowType, getArrowVector} from './arrow-type-utils';
+// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
+// import {getTableLength, getTableNumCols} from './table-accessors';
+
+/**
+ * * Convert a loaders.gl Mesh to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ *
+export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
+  const vectors: AbstractVector[] = [];
+  const fields: Field[] = [];
+
+  const length = getTableLength(table);
+  const batchSize = options?.batchSize || length;
+
+  const recordBatches: RecordBatch[] = [];
+  for (let i = 0; i < length; i += batchSize) {
+    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
+      const field_ = table.schema.fields[columnIndex];
+      const column = getTableColumnAt(table, columnIndex, i, batchSize);
+      const type = getArrowType(column);
+      const vector = getArrowVector(column);
+      const listType = new FixedSizeList(size, new Field('value', type));
+      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
+      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
+      const listVector = new FixedSizeListVector(data);
+      vectors.push(listVector);
+      fields.push(field);
+    }
+
+  for (const attributeKey in table.columns) {
+    const attribute = mesh.attributes[attributeKey];
+    const {value, size = 1} = attribute;
+  }
+
+  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
+  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
+  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
+
+  return apacheArrowTable;
+}
+*/

package/src/lib/{schema/impl/field.ts → table/arrow-api/arrow-like-field.ts}

@@ -1,4 +1,6 @@
-import {DataType} from './type';
+// loaders.gl, MIT license
+
+import {DataType} from './arrow-like-type';
 
 /**
  * ArrowJS `Field` API-compatible class for row-based tables
@@ -6,7 +8,7 @@ import {DataType} from './type';
  * A field holds name, nullable, and metadata information about a table "column"
  * A Schema is essentially a list of fields
  */
-export default class Field {
+export class ArrowLikeField {
   name: string;
   type: DataType;
   nullable: boolean;
@@ -28,8 +30,8 @@ export default class Field {
     return this.type && this.type.typeId;
   }
 
-  clone(): Field {
-    return new Field(this.name, this.type, this.nullable, this.metadata);
+  clone(): ArrowLikeField {
+    return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
   }
 
   compareTo(other: this): boolean {

package/src/lib/table/arrow-api/arrow-like-schema.ts

@@ -0,0 +1,99 @@
+// loaders.gl, MIT license
+
+import {SchemaMetadata, Field} from '../../../types/schema';
+import {ArrowLikeField} from './arrow-like-field';
+
+export class ArrowLikeSchema {
+  fields: ArrowLikeField[];
+  metadata: Map<string, string>;
+
+  constructor(
+    fields: ArrowLikeField[] | Field[],
+    metadata: SchemaMetadata | Map<string, string> = new Map<string, string>()
+  ) {
+    // checkNames(fields);
+    // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
+    this.fields = fields.map(
+      (field) => new ArrowLikeField(field.name, field.type, field.nullable, field.metadata)
+    );
+    this.metadata =
+      metadata instanceof Map ? metadata : new Map<string, string>(Object.entries(metadata));
+  }
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: ArrowLikeSchema): boolean {
+    if (this.metadata !== other.metadata) {
+      return false;
+    }
+    if (this.fields.length !== other.fields.length) {
+      return false;
+    }
+    for (let i = 0; i < this.fields.length; ++i) {
+      if (!this.fields[i].compareTo(other.fields[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  select(...columnNames: string[]): ArrowLikeSchema {
+    // Ensure column names reference valid fields
+    const nameMap = Object.create(null);
+    for (const name of columnNames) {
+      nameMap[name] = true;
+    }
+    const selectedFields = this.fields.filter((field) => nameMap[field.name]);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  selectAt(...columnIndices: number[]): ArrowLikeSchema {
+    // Ensure column indices reference valid fields
+    const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  assign(schemaOrFields: ArrowLikeSchema | ArrowLikeField[]): ArrowLikeSchema {
+    let fields: ArrowLikeField[];
+    let metadata = this.metadata;
+
+    if (schemaOrFields instanceof ArrowLikeSchema) {
+      const otherArrowLikeSchema = schemaOrFields;
+      fields = otherArrowLikeSchema.fields;
+      metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
+    } else {
+      fields = schemaOrFields;
+    }
+
+    // Create a merged list of fields, overwrite fields in place, new fields at end
+    const fieldMap: {[key: string]: ArrowLikeField} = Object.create(null);
+
+    for (const field of this.fields) {
+      fieldMap[field.name] = field;
+    }
+
+    for (const field of fields) {
+      fieldMap[field.name] = field;
+    }
+
+    const mergedFields = Object.values(fieldMap);
+
+    return new ArrowLikeSchema(mergedFields, metadata);
+  }
+}
+
+// Warn if any duplicated field names
+// function checkNames(fields: Field[]): void {
+//   const usedNames: Record<string, boolean> = {};
+//   for (const field of fields) {
+//     if (usedNames[field.name]) {
+//       // eslint-disable-next-line
+//       console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
+//     }
+//     usedNames[field.name] = true;
+//   }
+// }
+
+function mergeMaps<T>(m1: T, m2: T): T {
+  // @ts-ignore
+  return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
+}

package/src/lib/table/arrow-api/arrow-like-table.ts

@@ -0,0 +1,83 @@
+// loaders.gl, MIT license
+
+import {Table} from '../../../types/category-table';
+
+import {ArrowLikeSchema} from './arrow-like-schema';
+
+import {deduceTableSchema} from '../simple-table/table-schema';
+
+import {
+  getTableCell,
+  getTableLength,
+  getTableNumCols
+  // getTableCell,
+  // getTableRowShape,
+  // getTableColumnIndex,
+  // getTableColumnName,
+  // getTableRowAsObject,
+  // getTableRowAsArray
+} from '../simple-table/table-accessors';
+
+class ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  constructor(table: Table, columnName: string) {
+    this.table = table;
+    this.columnName = columnName;
+  }
+
+  get(rowIndex: number): unknown {
+    return getTableCell(this.table, rowIndex, this.columnName);
+  }
+
+  toArray(): ArrayLike<unknown> {
+    switch (this.table.shape) {
+      case 'arrow-table':
+        return this.table.data.getChild(this.columnName)?.toArray();
+      case 'columnar-table':
+        return this.table.data[this.columnName];
+      default:
+        throw new Error(this.table.shape);
+    }
+  }
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export class ArrowLikeTable {
+  schema: ArrowLikeSchema;
+  table: Table;
+
+  constructor(table: Table) {
+    const schema = table.schema || deduceTableSchema(table);
+    this.schema = new ArrowLikeSchema(schema.fields, schema.metadata);
+    this.table = {...table, schema};
+  }
+
+  // get schema() {
+  //   return this.table.schema;
+  // }
+
+  get data() {
+    return this.table.data;
+  }
+
+  get numCols(): number {
+    return getTableNumCols(this.table);
+  }
+
+  get length(): number {
+    return getTableLength(this.table);
+  }
+
+  getChild(columnName: string): ArrowLikeVector {
+    return new ArrowLikeVector(this.table, columnName);
+  }
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}

package/src/lib/{schema/impl/type.ts → table/arrow-api/arrow-like-type.ts}

@@ -3,7 +3,7 @@
 
 import {Type} from './enum';
 
-import Field from './field';
+import {ArrowLikeField as Field} from './arrow-like-field';
 
 export {Type} from './enum';
 
@@ -24,6 +24,7 @@ export type TypedArray = TypedIntArray | TypedFloatArray;
 
 export type AnyArrayType = Array<any> | TypedIntArray | TypedFloatArray;
 
+/** ArrowLike DataType class */
 export class DataType {
   static isNull(x: any): boolean {
     return x && x.typeId === Type.Null;

package/src/lib/{arrow → table/arrow-api}/get-type-info.ts

@@ -1,5 +1,6 @@
-import {Type} from '../schema';
-import {AnyArray} from '../../types';
+// loaders.gl, MIT license
+
+import {Type} from './arrow-like-type';
 
 /**
  * Gets type information from an Arrow type object or "mock" Arrow type object
@@ -7,7 +8,7 @@ import {AnyArray} from '../../types';
  */
 export function getTypeInfo(arrowTypeLike: any): {
   typeId: Type;
-  ArrayType: AnyArray;
+  ArrayType: ArrayLike<unknown>;
   typeName: string;
   typeEnumName?: string;
   precision?: number;

package/src/lib/table/arrow-api/index.ts

@@ -0,0 +1,6 @@
+// loaders.gl, MIT license
+
+export {ArrowLikeField as Field} from './arrow-like-field';
+export {ArrowLikeSchema as Schema} from './arrow-like-schema';
+export {ArrowLikeTable as Table} from './arrow-like-table';
+export * from './arrow-like-type';

package/src/lib/{batches → table/batches}/base-table-batch-aggregator.ts

@@ -1,10 +1,12 @@
-import type {Schema} from '../schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class RowTableBatchAggregator implements TableBatchAggregator {
+export class BaseTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   options: TableBatchOptions;
 

package/src/lib/{batches → table/batches}/columnar-table-batch-aggregator.ts

@@ -1,12 +1,14 @@
-import type {Schema} from '../schema';
-import type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {ColumnarTableBatch, ArrowTableBatch} from '../../../types/category-table';
 import {TableBatchAggregator} from './table-batch-aggregator';
 
 type ColumnarTableBatchOptions = {};
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class ColumnarTableBatchAggregator implements TableBatchAggregator {
+export class ColumnarTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   length: number = 0;
   allocated: number = 0;

package/src/lib/{batches → table/batches}/row-table-batch-aggregator.ts

@@ -1,12 +1,14 @@
-import type {Schema} from '../schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 // import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
-import {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';
+import {convertToArrayRow, convertToObjectRow} from '../simple-table/row-utils';
 import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class RowTableBatchAggregator implements TableBatchAggregator {
+export class RowTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   options: TableBatchOptions;
 

package/src/lib/{batches → table/batches}/table-batch-aggregator.ts

@@ -1,5 +1,7 @@
-import type {Schema} from '../schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 
 export interface TableBatchOptions {
   batchSize: number | string;

package/src/lib/{batches → table/batches}/table-batch-builder.ts

@@ -1,9 +1,11 @@
-import type {Schema} from '../schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
-import BaseTableBatchAggregator from './base-table-batch-aggregator';
-import RowTableBatchAggregator from './row-table-batch-aggregator';
-import ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';
+import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
+import {RowTableBatchAggregator} from './row-table-batch-aggregator';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
 
 // TODO define interface instead
 type TableBatchBuilderOptions = {
@@ -30,7 +32,7 @@ const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
 const ERR_MESSAGE = 'TableBatchBuilder';
 
 /** Incrementally builds batches from a stream of rows */
-export default class TableBatchBuilder {
+export class TableBatchBuilder {
   schema: Schema;
   options: Required<TableBatchBuilderOptions>;
 

package/src/lib/table/simple-table/convert-table.ts

@@ -0,0 +1,72 @@
+// loaders.gl, MIT license
+import {
+  getTableCell,
+  getTableLength,
+  getTableRowAsArray,
+  getTableRowAsObject
+} from './table-accessors';
+import {Table, ArrayRowTable, ObjectRowTable, ColumnarTable} from '../../../types/category-table';
+import {deduceTableSchema} from './table-schema';
+import {makeColumnFromField} from './table-column';
+
+/** Convert any simple table into columnar format */
+export function makeColumnarTable(table: Table): ColumnarTable {
+  // TODO - should schema really be optional?
+  const schema = table.schema || deduceTableSchema(table);
+  const fields = table.schema?.fields || [];
+
+  if (table.shape === 'columnar-table') {
+    return {...table, schema};
+  }
+
+  const length = getTableLength(table);
+
+  const columns: {[column: string]: ArrayLike<unknown>} = {};
+  for (const field of fields) {
+    const column = makeColumnFromField(field, length);
+    columns[field.name] = column;
+    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+      column[rowIndex] = getTableCell(table, rowIndex, field.name);
+    }
+  }
+
+  return {
+    shape: 'columnar-table',
+    schema,
+    data: columns
+  };
+}
+
+/** Convert any table into array row format */
+export function makeArrayRowTable(table: Table): ArrayRowTable {
+  if (table.shape === 'array-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<unknown[]>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsArray(table, rowIndex);
+  }
+  return {
+    shape: 'array-row-table',
+    schema: table.schema,
+    data
+  };
+}
+
+/** Convert any table into object row format */
+export function makeObjectRowTable(table: Table): ObjectRowTable {
+  if (table.shape === 'object-row-table') {
+    return table;
+  }
+  const length = getTableLength(table);
+  const data = new Array<{[key: string]: unknown}>(length);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    data[rowIndex] = getTableRowAsObject(table, rowIndex);
+  }
+  return {
+    shape: 'object-row-table',
+    schema: table.schema,
+    data
+  };
+}

package/src/lib/table/simple-table/data-type.ts

@@ -0,0 +1,102 @@
+// loaders.gl, MIT license
+
+import {DataType} from '../../../types/schema';
+import {TypedArray, TypedArrayConstructor, ArrayType} from '../../../types/types';
+
+/** Deduce column types from values */
+export function getDataTypeFromValue(
+  value: unknown,
+  defaultNumberType: 'float32' = 'float32'
+): DataType {
+  if (value instanceof Date) {
+    return 'date-millisecond';
+  }
+  if (value instanceof Number) {
+    return defaultNumberType;
+  }
+  if (typeof value === 'string') {
+    return 'utf8';
+  }
+  if (value === null || value === 'undefined') {
+    return 'null';
+  }
+  return 'null';
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromArray(array: ArrayType): {type: DataType; nullable: boolean} {
+  let type = getDataTypeFromTypedArray(array as TypedArray);
+  if (type !== 'null') {
+    return {type, nullable: false};
+  }
+  if (array.length > 0) {
+    type = getDataTypeFromValue(array[0]);
+    return {type, nullable: true};
+  }
+
+  return {type: 'null', nullable: true};
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromTypedArray(array: TypedArray): DataType {
+  switch (array.constructor) {
+    case Int8Array:
+      return 'int8';
+    case Uint8Array:
+    case Uint8ClampedArray:
+      return 'uint8';
+    case Int16Array:
+      return 'int16';
+    case Uint16Array:
+      return 'uint16';
+    case Int32Array:
+      return 'int32';
+    case Uint32Array:
+      return 'uint32';
+    case Float32Array:
+      return 'float32';
+    case Float64Array:
+      return 'float64';
+    default:
+      return 'null';
+  }
+}
+
+export function getArrayTypeFromDataType(
+  type: DataType,
+  nullable: boolean | undefined
+): TypedArrayConstructor | ArrayConstructor {
+  if (!nullable) {
+    switch (type) {
+      case 'int8':
+        return Int8Array;
+      case 'uint8':
+        return Uint8Array;
+      case 'int16':
+        return Int16Array;
+      case 'uint16':
+        return Uint16Array;
+      case 'int32':
+        return Int32Array;
+      case 'uint32':
+        return Uint32Array;
+      case 'float32':
+        return Float32Array;
+      case 'float64':
+        return Float64Array;
+      default:
+        break;
+    }
+  }
+
+  // if (typeof BigInt64Array !== 'undefined') {
+  //   TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
+  //   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
+  // }
+
+  return Array;
+}