@loaders.gl/schema 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/lib/table/arrow/convert-schema-arrow.ts

@@ -0,0 +1,232 @@
+// loaders.gl, MIT license
+
+import type {DataType, Field, Schema, SchemaMetadata} from '../../../types/schema';
+import {
+  Field as ArrowField,
+  Schema as ArrowSchema,
+  DataType as ArrowDataType,
+  Null,
+  Binary,
+  Bool,
+  // Int,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  // Float,
+  Float16,
+  Float32,
+  Float64,
+  Utf8,
+  // Date,
+  DateDay,
+  DateMillisecond,
+  // Time,
+  TimeMillisecond,
+  TimeSecond,
+  // Timestamp,
+  TimestampSecond,
+  TimestampMillisecond,
+  TimestampMicrosecond,
+  TimestampNanosecond,
+  // Interval,
+  IntervalDayTime,
+  IntervalYearMonth,
+  FixedSizeList,
+  Struct
+} from 'apache-arrow/Arrow.dom';
+
+export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
+  return {
+    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
+    metadata: serializeArrowMetadata(arrowSchema.metadata)
+  };
+}
+
+export function deserializeArrowSchema(schema: Schema): ArrowSchema {
+  return new ArrowSchema(
+    schema.fields.map((field) => deserializeArrowField(field)),
+    deserializeArrowMetadata(schema.metadata)
+  );
+}
+
+export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
+  return Object.fromEntries(arrowMetadata);
+}
+
+export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
+  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
+}
+
+export function serializeArrowField(field: ArrowField): Field {
+  return {
+    name: field.name,
+    type: serializeArrowType(field.type),
+    nullable: field.nullable,
+    metadata: serializeArrowMetadata(field.metadata)
+  };
+}
+
+export function deserializeArrowField(field: Field): ArrowField {
+  return new ArrowField(
+    field.name,
+    deserializeArrowType(field.type),
+    field.nullable,
+    deserializeArrowMetadata(field.metadata)
+  );
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function serializeArrowType(arrowType: ArrowDataType): DataType {
+  switch (arrowType.constructor) {
+    case Null:
+      return 'null';
+    case Binary:
+      return 'binary';
+    case Bool:
+      return 'bool';
+    // case Int: return 'int';
+    case Int8:
+      return 'int8';
+    case Int16:
+      return 'int16';
+    case Int32:
+      return 'int32';
+    case Int64:
+      return 'int64';
+    case Uint8:
+      return 'uint8';
+    case Uint16:
+      return 'uint16';
+    case Uint32:
+      return 'uint32';
+    case Uint64:
+      return 'uint64';
+    // case Float: return 'float';
+    case Float16:
+      return 'float16';
+    case Float32:
+      return 'float32';
+    case Float64:
+      return 'float64';
+    case Utf8:
+      return 'utf8';
+    // case Date: return 'date';
+    case DateDay:
+      return 'date-day';
+    case DateMillisecond:
+      return 'date-millisecond';
+    // case Time: return 'time';
+    case TimeMillisecond:
+      return 'time-millisecond';
+    case TimeSecond:
+      return 'time-second';
+    // case Timestamp: return 'timestamp';
+    case TimestampSecond:
+      return 'timestamp-second';
+    case TimestampMillisecond:
+      return 'timestamp-millisecond';
+    case TimestampMicrosecond:
+      return 'timestamp-microsecond';
+    case TimestampNanosecond:
+      return 'timestamp-nanosecond';
+    // case Interval: return 'interval';
+    case IntervalDayTime:
+      return 'interval-daytime';
+    case IntervalYearMonth:
+      return 'interval-yearmonth';
+    case FixedSizeList:
+      return {
+        type: 'fixed-size-list',
+        listSize: (arrowType as FixedSizeList).listSize,
+        children: [serializeArrowField((arrowType as FixedSizeList).children[0])]
+      };
+    // case Struct:
+    //   return {type: 'struct', children: (arrowType as Struct).children};
+    default:
+      throw new Error('array type not supported');
+  }
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function deserializeArrowType(dataType: DataType): ArrowDataType {
+  if (typeof dataType === 'object') {
+    switch (dataType.type) {
+      case 'fixed-size-list':
+        const child = deserializeArrowField(dataType.children[0]);
+        return new FixedSizeList(dataType.listSize, child);
+      case 'struct':
+        const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new Struct(children);
+      default:
+        throw new Error('array type not supported');
+    }
+  }
+
+  switch (dataType) {
+    case 'null':
+      return new Null();
+    case 'binary':
+      return new Binary();
+    case 'bool':
+      return new Bool();
+    // case 'int': return new Int();
+    case 'int8':
+      return new Int8();
+    case 'int16':
+      return new Int16();
+    case 'int32':
+      return new Int32();
+    case 'int64':
+      return new Int64();
+    case 'uint8':
+      return new Uint8();
+    case 'uint16':
+      return new Uint16();
+    case 'uint32':
+      return new Uint32();
+    case 'uint64':
+      return new Uint64();
+    // case 'float': return new Float();
+    case 'float16':
+      return new Float16();
+    case 'float32':
+      return new Float32();
+    case 'float64':
+      return new Float64();
+    case 'utf8':
+      return new Utf8();
+    // case 'date': return new Date();
+    case 'date-day':
+      return new DateDay();
+    case 'date-millisecond':
+      return new DateMillisecond();
+    // case 'time': return new Time();
+    case 'time-millisecond':
+      return new TimeMillisecond();
+    case 'time-second':
+      return new TimeSecond();
+    // case 'timestamp': return new Timestamp();
+    case 'timestamp-second':
+      return new TimestampSecond();
+    case 'timestamp-millisecond':
+      return new TimestampMillisecond();
+    case 'timestamp-microsecond':
+      return new TimestampMicrosecond();
+    case 'timestamp-nanosecond':
+      return new TimestampNanosecond();
+    // case 'interval': return new Interval();
+    case 'interval-daytime':
+      return new IntervalDayTime();
+    case 'interval-yearmonth':
+      return new IntervalYearMonth();
+    default:
+      throw new Error('array type not supported');
+  }
+}

package/src/lib/table/arrow/convert-table-to-arrow.ts

@@ -0,0 +1,59 @@
+// loaders.gl, MIT license
+
+// import {
+//   Table as ApacheArrowTable,
+//   Schema as ApacheArrowSchema,
+//   RecordBatch,
+//   FixedSizeList,
+//   Field,
+//   Data,
+//   FixedSizeListVector
+// } from 'apache-arrow/Arrow.dom';
+// import {AbstractVector} from 'apache-arrow/vector';
+
+// import {Table} from '../../types/category-table';
+// import {getArrowType, getArrowVector} from './arrow-type-utils';
+// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
+// import {getTableLength, getTableNumCols} from './table-accessors';
+
+/**
+ * * Convert a loaders.gl Mesh to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ *
+export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
+  const vectors: AbstractVector[] = [];
+  const fields: Field[] = [];
+
+  const length = getTableLength(table);
+  const batchSize = options?.batchSize || length;
+
+  const recordBatches: RecordBatch[] = [];
+  for (let i = 0; i < length; i += batchSize) {
+    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
+      const field_ = table.schema.fields[columnIndex];
+      const column = getTableColumnAt(table, columnIndex, i, batchSize);
+      const type = getArrowType(column);
+      const vector = getArrowVector(column);
+      const listType = new FixedSizeList(size, new Field('value', type));
+      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
+      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
+      const listVector = new FixedSizeListVector(data);
+      vectors.push(listVector);
+      fields.push(field);
+    }
+
+  for (const attributeKey in table.columns) {
+    const attribute = mesh.attributes[attributeKey];
+    const {value, size = 1} = attribute;
+  }
+
+  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
+  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
+  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
+
+  return apacheArrowTable;
+}
+*/

package/src/lib/{schema/impl/field.ts → table/arrow-api/arrow-like-field.ts}

@@ -1,4 +1,6 @@
-import {DataType} from './type';
+// loaders.gl, MIT license
+
+import {DataType} from './arrow-like-type';
 
 /**
  * ArrowJS `Field` API-compatible class for row-based tables
@@ -6,7 +8,7 @@ import {DataType} from './type';
  * A field holds name, nullable, and metadata information about a table "column"
  * A Schema is essentially a list of fields
  */
-export default class Field {
+export class ArrowLikeField {
   name: string;
   type: DataType;
   nullable: boolean;
@@ -28,8 +30,8 @@ export default class Field {
     return this.type && this.type.typeId;
   }
 
-  clone(): Field {
-    return new Field(this.name, this.type, this.nullable, this.metadata);
+  clone(): ArrowLikeField {
+    return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
   }
 
   compareTo(other: this): boolean {

package/src/lib/table/arrow-api/arrow-like-schema.ts

@@ -0,0 +1,99 @@
+// loaders.gl, MIT license
+
+import {SchemaMetadata, Field} from '../../../types/schema';
+import {ArrowLikeField} from './arrow-like-field';
+
+export class ArrowLikeSchema {
+  fields: ArrowLikeField[];
+  metadata: Map<string, string>;
+
+  constructor(
+    fields: ArrowLikeField[] | Field[],
+    metadata: SchemaMetadata | Map<string, string> = new Map<string, string>()
+  ) {
+    // checkNames(fields);
+    // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
+    this.fields = fields.map(
+      (field) => new ArrowLikeField(field.name, field.type, field.nullable, field.metadata)
+    );
+    this.metadata =
+      metadata instanceof Map ? metadata : new Map<string, string>(Object.entries(metadata));
+  }
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: ArrowLikeSchema): boolean {
+    if (this.metadata !== other.metadata) {
+      return false;
+    }
+    if (this.fields.length !== other.fields.length) {
+      return false;
+    }
+    for (let i = 0; i < this.fields.length; ++i) {
+      if (!this.fields[i].compareTo(other.fields[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  select(...columnNames: string[]): ArrowLikeSchema {
+    // Ensure column names reference valid fields
+    const nameMap = Object.create(null);
+    for (const name of columnNames) {
+      nameMap[name] = true;
+    }
+    const selectedFields = this.fields.filter((field) => nameMap[field.name]);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  selectAt(...columnIndices: number[]): ArrowLikeSchema {
+    // Ensure column indices reference valid fields
+    const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  assign(schemaOrFields: ArrowLikeSchema | ArrowLikeField[]): ArrowLikeSchema {
+    let fields: ArrowLikeField[];
+    let metadata = this.metadata;
+
+    if (schemaOrFields instanceof ArrowLikeSchema) {
+      const otherArrowLikeSchema = schemaOrFields;
+      fields = otherArrowLikeSchema.fields;
+      metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
+    } else {
+      fields = schemaOrFields;
+    }
+
+    // Create a merged list of fields, overwrite fields in place, new fields at end
+    const fieldMap: {[key: string]: ArrowLikeField} = Object.create(null);
+
+    for (const field of this.fields) {
+      fieldMap[field.name] = field;
+    }
+
+    for (const field of fields) {
+      fieldMap[field.name] = field;
+    }
+
+    const mergedFields = Object.values(fieldMap);
+
+    return new ArrowLikeSchema(mergedFields, metadata);
+  }
+}
+
+// Warn if any duplicated field names
+// function checkNames(fields: Field[]): void {
+//   const usedNames: Record<string, boolean> = {};
+//   for (const field of fields) {
+//     if (usedNames[field.name]) {
+//       // eslint-disable-next-line
+//       console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
+//     }
+//     usedNames[field.name] = true;
+//   }
+// }
+
+function mergeMaps<T>(m1: T, m2: T): T {
+  // @ts-ignore
+  return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
+}

package/src/lib/table/arrow-api/arrow-like-table.ts

@@ -0,0 +1,83 @@
+// loaders.gl, MIT license
+
+import {Table} from '../../../types/category-table';
+
+import {ArrowLikeSchema} from './arrow-like-schema';
+
+import {deduceTableSchema} from '../simple-table/table-schema';
+
+import {
+  getTableCell,
+  getTableLength,
+  getTableNumCols
+  // getTableCell,
+  // getTableRowShape,
+  // getTableColumnIndex,
+  // getTableColumnName,
+  // getTableRowAsObject,
+  // getTableRowAsArray
+} from '../simple-table/table-accessors';
+
+class ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  constructor(table: Table, columnName: string) {
+    this.table = table;
+    this.columnName = columnName;
+  }
+
+  get(rowIndex: number): unknown {
+    return getTableCell(this.table, rowIndex, this.columnName);
+  }
+
+  toArray(): ArrayLike<unknown> {
+    switch (this.table.shape) {
+      case 'arrow-table':
+        return this.table.data.getChild(this.columnName)?.toArray();
+      case 'columnar-table':
+        return this.table.data[this.columnName];
+      default:
+        throw new Error(this.table.shape);
+    }
+  }
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export class ArrowLikeTable {
+  schema: ArrowLikeSchema;
+  table: Table;
+
+  constructor(table: Table) {
+    const schema = table.schema || deduceTableSchema(table);
+    this.schema = new ArrowLikeSchema(schema.fields, schema.metadata);
+    this.table = {...table, schema};
+  }
+
+  // get schema() {
+  //   return this.table.schema;
+  // }
+
+  get data() {
+    return this.table.data;
+  }
+
+  get numCols(): number {
+    return getTableNumCols(this.table);
+  }
+
+  get length(): number {
+    return getTableLength(this.table);
+  }
+
+  getChild(columnName: string): ArrowLikeVector {
+    return new ArrowLikeVector(this.table, columnName);
+  }
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}

package/src/lib/{schema/impl/type.ts → table/arrow-api/arrow-like-type.ts}

@@ -3,7 +3,7 @@
 
 import {Type} from './enum';
 
-import Field from './field';
+import {ArrowLikeField as Field} from './arrow-like-field';
 
 export {Type} from './enum';
 
@@ -24,6 +24,7 @@ export type TypedArray = TypedIntArray | TypedFloatArray;
 
 export type AnyArrayType = Array<any> | TypedIntArray | TypedFloatArray;
 
+/** ArrowLike DataType class */
 export class DataType {
   static isNull(x: any): boolean {
     return x && x.typeId === Type.Null;

package/src/lib/{arrow → table/arrow-api}/get-type-info.ts

@@ -1,5 +1,6 @@
-import {Type} from '../schema/schema';
-import {AnyArray} from '../../types';
+// loaders.gl, MIT license
+
+import {Type} from './arrow-like-type';
 
 /**
  * Gets type information from an Arrow type object or "mock" Arrow type object
@@ -7,7 +8,7 @@ import {AnyArray} from '../../types';
  */
 export function getTypeInfo(arrowTypeLike: any): {
   typeId: Type;
-  ArrayType: AnyArray;
+  ArrayType: ArrayLike<unknown>;
   typeName: string;
   typeEnumName?: string;
   precision?: number;

package/src/lib/table/arrow-api/index.ts

@@ -0,0 +1,6 @@
+// loaders.gl, MIT license
+
+export {ArrowLikeField as Field} from './arrow-like-field';
+export {ArrowLikeSchema as Schema} from './arrow-like-schema';
+export {ArrowLikeTable as Table} from './arrow-like-table';
+export * from './arrow-like-type';

package/src/lib/{batches → table/batches}/base-table-batch-aggregator.ts

@@ -1,10 +1,12 @@
-import type {Schema} from '../schema/schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class RowTableBatchAggregator implements TableBatchAggregator {
+export class BaseTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   options: TableBatchOptions;
 

package/src/lib/{batches → table/batches}/columnar-table-batch-aggregator.ts

@@ -1,12 +1,14 @@
-import type {Schema} from '../schema/schema';
-import type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {ColumnarTableBatch, ArrowTableBatch} from '../../../types/category-table';
 import {TableBatchAggregator} from './table-batch-aggregator';
 
 type ColumnarTableBatchOptions = {};
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class ColumnarTableBatchAggregator implements TableBatchAggregator {
+export class ColumnarTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   length: number = 0;
   allocated: number = 0;

package/src/lib/{batches → table/batches}/row-table-batch-aggregator.ts

@@ -1,12 +1,14 @@
-import type {Schema} from '../schema/schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 // import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
-import {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';
+import {convertToArrayRow, convertToObjectRow} from '../simple-table/row-utils';
 import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
 
 const DEFAULT_ROW_COUNT = 100;
 
-export default class RowTableBatchAggregator implements TableBatchAggregator {
+export class RowTableBatchAggregator implements TableBatchAggregator {
   schema: Schema;
   options: TableBatchOptions;
 

package/src/lib/{batches → table/batches}/table-batch-aggregator.ts

@@ -1,5 +1,7 @@
-import type {Schema} from '../schema/schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 
 export interface TableBatchOptions {
   batchSize: number | string;

package/src/lib/{batches → table/batches}/table-batch-builder.ts

@@ -1,9 +1,11 @@
-import type {Schema} from '../schema/schema';
-import type {TableBatch} from '../../category/table/table-types';
+// loaders.gl, MIT license
+
+import type {Schema} from '../../../types/schema';
+import type {TableBatch} from '../../../types/category-table';
 import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
-import BaseTableBatchAggregator from './base-table-batch-aggregator';
-import RowTableBatchAggregator from './row-table-batch-aggregator';
-import ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';
+import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
+import {RowTableBatchAggregator} from './row-table-batch-aggregator';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
 
 // TODO define interface instead
 type TableBatchBuilderOptions = {
@@ -30,7 +32,7 @@ const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
 const ERR_MESSAGE = 'TableBatchBuilder';
 
 /** Incrementally builds batches from a stream of rows */
-export default class TableBatchBuilder {
+export class TableBatchBuilder {
   schema: Schema;
   options: Required<TableBatchBuilderOptions>;