@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/lib/schema/convert-arrow-schema.ts

@@ -0,0 +1,301 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+
+/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
+export function convertArrowToSchema(arrowSchema: arrow.Schema): Schema {
+  return serializeArrowSchema(arrowSchema);
+}
+
+/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
+export function convertSchemaToArrow(schema: Schema): arrow.Schema {
+  return deserializeArrowSchema(schema);
+}
+
+/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
+export function serializeArrowSchema(arrowSchema: arrow.Schema): Schema {
+  return {
+    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
+    metadata: serializeArrowMetadata(arrowSchema.metadata)
+  };
+}
+
+/** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
+export function deserializeArrowSchema(schema: Schema): arrow.Schema {
+  return new arrow.Schema(
+    schema.fields.map((field) => deserializeArrowField(field)),
+    deserializeArrowMetadata(schema.metadata)
+  );
+}
+
+/** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
+export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
+  return Object.fromEntries(arrowMetadata);
+}
+
+/** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
+export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
+  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
+}
+
+/** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
+export function serializeArrowField(field: arrow.Field): Field {
+  return {
+    name: field.name,
+    type: serializeArrowType(field.type),
+    nullable: field.nullable,
+    metadata: serializeArrowMetadata(field.metadata)
+  };
+}
+
+/** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
+export function deserializeArrowField(field: Field): arrow.Field {
+  return new arrow.Field(
+    field.name,
+    deserializeArrowType(field.type),
+    field.nullable,
+    deserializeArrowMetadata(field.metadata)
+  );
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function serializeArrowType(arrowType: arrow.DataType): DataType {
+  switch (arrowType.constructor) {
+    case arrow.Null:
+      return 'null';
+    case arrow.Binary:
+      return 'binary';
+    case arrow.Bool:
+      return 'bool';
+    case arrow.Int:
+      const intType = arrowType as arrow.Int;
+      return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
+    case arrow.Int8:
+      return 'int8';
+    case arrow.Int16:
+      return 'int16';
+    case arrow.Int32:
+      return 'int32';
+    case arrow.Int64:
+      return 'int64';
+    case arrow.Uint8:
+      return 'uint8';
+    case arrow.Uint16:
+      return 'uint16';
+    case arrow.Uint32:
+      return 'uint32';
+    case arrow.Uint64:
+      return 'uint64';
+    case arrow.Float:
+      const precision = (arrowType as arrow.Float).precision;
+      // return `float(precision + 1) * 16`;
+      switch (precision) {
+        case arrow.Precision.HALF:
+          return 'float16';
+        case arrow.Precision.SINGLE:
+          return 'float32';
+        case arrow.Precision.DOUBLE:
+          return 'float64';
+        default:
+          return 'float16';
+      }
+    case arrow.Float16:
+      return 'float16';
+    case arrow.Float32:
+      return 'float32';
+    case arrow.Float64:
+      return 'float64';
+    case arrow.Utf8:
+      return 'utf8';
+    case arrow.Decimal:
+      const decimal = arrowType as arrow.Decimal;
+      return {
+        type: 'decimal',
+        bitWidth: decimal.bitWidth,
+        precision: decimal.precision,
+        scale: decimal.scale
+      };
+    case arrow.Date_:
+      const dateUnit = (arrowType as arrow.Date_).unit;
+      return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
+    case arrow.DateDay:
+      return 'date-day';
+    case arrow.DateMillisecond:
+      return 'date-millisecond';
+    case arrow.Time:
+      const timeUnit = (arrowType as arrow.Time).unit;
+      switch (timeUnit) {
+        case arrow.TimeUnit.SECOND:
+          return 'time-second';
+        case arrow.TimeUnit.MILLISECOND:
+          return 'time-millisecond';
+        case arrow.TimeUnit.MICROSECOND:
+          return 'time-microsecond';
+        case arrow.TimeUnit.NANOSECOND:
+          return 'time-nanosecond';
+        default:
+          return 'time-second';
+      }
+    case arrow.TimeMillisecond:
+      return 'time-millisecond';
+    case arrow.TimeSecond:
+      return 'time-second';
+    case arrow.TimeMicrosecond:
+      return 'time-microsecond';
+    case arrow.TimeNanosecond:
+      return 'time-nanosecond';
+    case arrow.Timestamp:
+      const timeStampUnit = (arrowType as arrow.Timestamp).unit;
+      switch (timeStampUnit) {
+        case arrow.TimeUnit.SECOND:
+          return 'timestamp-second';
+        case arrow.TimeUnit.MILLISECOND:
+          return 'timestamp-millisecond';
+        case arrow.TimeUnit.MICROSECOND:
+          return 'timestamp-microsecond';
+        case arrow.TimeUnit.NANOSECOND:
+          return 'timestamp-nanosecond';
+        default:
+          return 'timestamp-second';
+      }
+    case arrow.TimestampSecond:
+      return 'timestamp-second';
+    case arrow.TimestampMillisecond:
+      return 'timestamp-millisecond';
+    case arrow.TimestampMicrosecond:
+      return 'timestamp-microsecond';
+    case arrow.TimestampNanosecond:
+      return 'timestamp-nanosecond';
+    case arrow.Interval:
+      const intervalUnit = (arrowType as arrow.Interval).unit;
+      switch (intervalUnit) {
+        case arrow.IntervalUnit.DAY_TIME:
+          return 'interval-daytime';
+        case arrow.IntervalUnit.YEAR_MONTH:
+          return 'interval-yearmonth';
+        default:
+          return 'interval-daytime';
+      }
+    case arrow.IntervalDayTime:
+      return 'interval-daytime';
+    case arrow.IntervalYearMonth:
+      return 'interval-yearmonth';
+    case arrow.Map_:
+      const mapType = arrowType as arrow.Map_;
+      return {
+        type: 'map',
+        keysSorted: mapType.keysSorted,
+        children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
+      };
+    case arrow.List:
+      const listType = arrowType as arrow.List;
+      const listField = listType.valueField;
+      return {
+        type: 'list',
+        children: [serializeArrowField(listField)]
+      };
+    case arrow.FixedSizeList:
+      const fixedSizeList = arrowType as arrow.FixedSizeList;
+      return {
+        type: 'fixed-size-list',
+        listSize: fixedSizeList.listSize,
+        children: [serializeArrowField(fixedSizeList.children[0])]
+      };
+    case arrow.Struct:
+      const structType = arrowType as arrow.Struct;
+      return {
+        type: 'struct',
+        children: structType.children.map((arrowField) => serializeArrowField(arrowField))
+      };
+    default:
+      throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
+  }
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function deserializeArrowType(dataType: DataType): arrow.DataType {
+  if (typeof dataType === 'object') {
+    switch (dataType.type) {
+      case 'decimal':
+        return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
+      case 'map':
+        let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new arrow.Map_(children as any, dataType.keysSorted);
+      case 'list':
+        const field = deserializeArrowField(dataType.children[0]);
+        return new arrow.List(field);
+      case 'fixed-size-list':
+        const child = deserializeArrowField(dataType.children[0]);
+        return new arrow.FixedSizeList(dataType.listSize, child);
+      case 'struct':
+        children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new arrow.Struct(children);
+      default:
+        throw new Error('array type not supported');
+    }
+  }
+
+  switch (dataType) {
+    case 'null':
+      return new arrow.Null();
+    case 'binary':
+      return new arrow.Binary();
+    case 'bool':
+      return new arrow.Bool();
+    case 'int8':
+      return new arrow.Int8();
+    case 'int16':
+      return new arrow.Int16();
+    case 'int32':
+      return new arrow.Int32();
+    case 'int64':
+      return new arrow.Int64();
+    case 'uint8':
+      return new arrow.Uint8();
+    case 'uint16':
+      return new arrow.Uint16();
+    case 'uint32':
+      return new arrow.Uint32();
+    case 'uint64':
+      return new arrow.Uint64();
+    case 'float16':
+      return new arrow.Float16();
+    case 'float32':
+      return new arrow.Float32();
+    case 'float64':
+      return new arrow.Float64();
+    case 'utf8':
+      return new arrow.Utf8();
+    case 'date-day':
+      return new arrow.DateDay();
+    case 'date-millisecond':
+      return new arrow.DateMillisecond();
+    case 'time-second':
+      return new arrow.TimeSecond();
+    case 'time-millisecond':
+      return new arrow.TimeMillisecond();
+    case 'time-microsecond':
+      return new arrow.TimeMicrosecond();
+    case 'time-nanosecond':
+      return new arrow.TimeNanosecond();
+    case 'timestamp-second':
+      return new arrow.TimestampSecond();
+    case 'timestamp-millisecond':
+      return new arrow.TimestampMillisecond();
+    case 'timestamp-microsecond':
+      return new arrow.TimestampMicrosecond();
+    case 'timestamp-nanosecond':
+      return new arrow.TimestampNanosecond();
+    case 'interval-daytime':
+      return new arrow.IntervalDayTime();
+    case 'interval-yearmonth':
+      return new arrow.IntervalYearMonth();
+    default:
+      throw new Error('array type not supported');
+  }
+}

package/src/lib/schema/data-type.ts

@@ -0,0 +1,103 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType, TypedArray, TypedArrayConstructor, ArrayType} from '@loaders.gl/schema';
+
+/** Deduce column types from values */
+export function getDataTypeFromValue(
+  value: unknown,
+  defaultNumberType: 'float32' = 'float32'
+): DataType {
+  if (value instanceof Date) {
+    return 'date-millisecond';
+  }
+  if (value instanceof Number) {
+    return defaultNumberType;
+  }
+  if (typeof value === 'string') {
+    return 'utf8';
+  }
+  if (value === null || value === 'undefined') {
+    return 'null';
+  }
+  return 'null';
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromArray(array: ArrayType): {type: DataType; nullable: boolean} {
+  let type = getDataTypeFromTypedArray(array as TypedArray);
+  if (type !== 'null') {
+    return {type, nullable: false};
+  }
+  if (array.length > 0) {
+    type = getDataTypeFromValue(array[0]);
+    return {type, nullable: true};
+  }
+
+  return {type: 'null', nullable: true};
+}
+
+/**
+ * Deduces a simple data type "descriptor from a typed array instance
+ */
+export function getDataTypeFromTypedArray(array: TypedArray): DataType {
+  switch (array.constructor) {
+    case Int8Array:
+      return 'int8';
+    case Uint8Array:
+    case Uint8ClampedArray:
+      return 'uint8';
+    case Int16Array:
+      return 'int16';
+    case Uint16Array:
+      return 'uint16';
+    case Int32Array:
+      return 'int32';
+    case Uint32Array:
+      return 'uint32';
+    case Float32Array:
+      return 'float32';
+    case Float64Array:
+      return 'float64';
+    default:
+      return 'null';
+  }
+}
+
+export function getArrayTypeFromDataType(
+  type: DataType,
+  nullable: boolean | undefined
+): TypedArrayConstructor | ArrayConstructor {
+  if (!nullable) {
+    switch (type) {
+      case 'int8':
+        return Int8Array;
+      case 'uint8':
+        return Uint8Array;
+      case 'int16':
+        return Int16Array;
+      case 'uint16':
+        return Uint16Array;
+      case 'int32':
+        return Int32Array;
+      case 'uint32':
+        return Uint32Array;
+      case 'float32':
+        return Float32Array;
+      case 'float64':
+        return Float64Array;
+      default:
+        break;
+    }
+  }
+
+  // if (typeof BigInt64Array !== 'undefined') {
+  //   TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
+  //   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
+  // }
+
+  return Array;
+}

package/src/lib/schema/deduce-table-schema.ts

@@ -0,0 +1,119 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Type deduction
+import type {Schema, Field, ArrayType, Table} from '@loaders.gl/schema';
+import {getDataTypeFromArray, getDataTypeFromValue} from './data-type';
+
+/**
+ * SCHEMA SUPPORT - AUTODEDUCTION
+ * @param {*} table
+ * @param {*} schema
+ * @returns
+ */
+export function deduceTableSchema(table: Table): Schema {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return deduceSchemaFromRows(table.data);
+
+    case 'geojson-table':
+      return deduceSchemaFromGeoJSON(table.features);
+
+    case 'columnar-table':
+      return deduceSchemaFromColumns(table.data);
+
+    case 'arrow-table':
+    default:
+      throw new Error('Deduce schema');
+  }
+}
+
+export function deduceSchema(
+  data: unknown[][] | {[key: string]: unknown}[] | {[key: string]: unknown[]}
+): Schema {
+  return Array.isArray(data) ? deduceSchemaFromRows(data) : deduceSchemaFromColumns(data);
+}
+
+/** Given an object with columnar arrays, try to deduce a schema */
+function deduceSchemaFromColumns(columnarTable: {[key: string]: ArrayType}): Schema {
+  const fields: Field[] = [];
+  for (const [columnName, column] of Object.entries(columnarTable)) {
+    const field = deduceFieldFromColumn(column, columnName);
+    fields.push(field);
+  }
+  return {fields, metadata: {}};
+}
+
+/** Given an array of rows, try to deduce a schema */
+function deduceSchemaFromRows(rowTable: unknown[][] | {[key: string]: unknown}[]): Schema {
+  if (!rowTable.length) {
+    throw new Error('deduce from empty table');
+  }
+  const fields: Field[] = [];
+  const row0 = rowTable[0];
+  // TODO - fields can be nullable, false detection...
+  // Could look at additional rows if nulls in first row
+  // TODO - if array, column names will be numbers
+  for (const [columnName, value] of Object.entries(row0)) {
+    fields.push(deduceFieldFromValue(value, columnName));
+  }
+
+  return {fields, metadata: {}};
+}
+
+/** Given a GeoJSON, try to deduce a schema */
+function deduceSchemaFromGeoJSON(features: {properties: Record<string, number> | null}[]): Schema {
+  if (!features.length) {
+    throw new Error('deduce from empty table');
+  }
+  const fields: Field[] = [];
+  const row0 = features[0].properties || {};
+  // TODO - fields can be nullable, false detection...
+  // Could look at additional rows if nulls in first row
+  // TODO - if array, column names will be numbers
+  for (const [columnName, value] of Object.entries(row0)) {
+    fields.push(deduceFieldFromValue(value, columnName));
+  }
+
+  return {fields, metadata: {}};
+}
+
+/** Given a column (i.e. array), attempt to deduce an appropriate `Field` */
+function deduceFieldFromColumn(column: ArrayType, name: string): Field {
+  if (ArrayBuffer.isView(column)) {
+    const type = getDataTypeFromArray(column);
+    return {
+      name,
+      type: type.type || 'null',
+      nullable: type.nullable
+      // metadata: {}
+    };
+  }
+
+  if (Array.isArray(column) && column.length > 0) {
+    const value = column[0];
+    const type = getDataTypeFromValue(value);
+    // TODO - support nested schemas?
+    return {
+      name,
+      type,
+      nullable: true
+      // metadata: {},
+    };
+  }
+
+  throw new Error('empty table');
+}
+
+/** Given a value, attempt to deduce an appropriate `Field` */
+function deduceFieldFromValue(value: unknown, name: string): Field {
+  const type = getDataTypeFromValue(value);
+  return {
+    name,
+    type,
+    nullable: true
+    // metadata: {}
+  };
+}

package/src/lib/table/arrow-api/arrow-like-field.ts

@@ -0,0 +1,53 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType} from './arrow-like-type';
+
+/**
+ * ArrowJS `Field` API-compatible class for row-based tables
+ * https://loaders.gl/arrowjs/docs/api-reference/field
+ * A field holds name, nullable, and metadata information about a table "column"
+ * A Schema is essentially a list of fields
+ */
+export class ArrowLikeField {
+  name: string;
+  type: DataType;
+  nullable: boolean;
+  metadata: Map<string, string>;
+
+  constructor(
+    name: string,
+    type: DataType,
+    nullable = false,
+    metadata: Map<string, string> = new Map()
+  ) {
+    this.name = name;
+    this.type = type;
+    this.nullable = nullable;
+    this.metadata = metadata;
+  }
+
+  get typeId(): number {
+    return this.type && this.type.typeId;
+  }
+
+  clone(): ArrowLikeField {
+    return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
+  }
+
+  compareTo(other: this): boolean {
+    return (
+      this.name === other.name &&
+      this.type === other.type &&
+      this.nullable === other.nullable &&
+      this.metadata === other.metadata
+    );
+  }
+
+  toString(): string {
+    return `${JSON.stringify(this.type)}${this.nullable ? ', nullable' : ''}${
+      this.metadata ? `, metadata: ${JSON.stringify(this.metadata)}` : ''
+    }`;
+  }
+}

package/src/lib/table/arrow-api/arrow-like-interface.ts

@@ -0,0 +1,84 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType} from './arrow-like-type';
+
+/**
+ * ArrowJS `Field` API-compatible class for row-based tables
+ * https://loaders.gl/arrowjs/docs/api-reference/field
+ * A field holds name, nullable, and metadata information about a table "column"
+ * A Schema is essentially a list of fields
+ */
+export interface Field {
+  name: string;
+  type: DataType;
+  nullable: boolean;
+  metadata: Map<string, string>;
+
+  // constructor(
+  //   name: string,
+  //   type: DataType,
+  //   nullable?,
+  //   metadata?
+  // );
+
+  typeId(): number;
+
+  clone(): Field;
+
+  compareTo(other: this): boolean;
+
+  toString();
+}
+
+export interface Schema {
+  fields: Field[];
+  metadata: Map<string, string>;
+
+  // constructor(
+  //   fields: Field[]  ,
+  //   metadata: Map<string, string>
+  // )
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: Schema): boolean;
+
+  select(...columnNames: string[]): Schema;
+
+  selectAt(...columnIndices: number[]): Schema;
+
+  assign(schemaOrFields: Schema | Field[]): Schema;
+}
+
+export interface ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  // constructor(table: Table, columnName: string);
+
+  get(rowIndex: number): unknown;
+  toArray(): ArrayLike<unknown>;
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export interface Table {
+  schema: Schema;
+
+  // constructor(table: Table);
+
+  data: any;
+
+  numCols: number;
+
+  length(): number;
+
+  getChild(columnName: string): ArrowLikeVector;
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}