@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/lib/table/arrow-api/arrow-like-schema.ts

@@ -0,0 +1,101 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {SchemaMetadata, Field} from '@loaders.gl/schema';
+import {ArrowLikeField} from './arrow-like-field';
+
+export class ArrowLikeSchema {
+  fields: ArrowLikeField[];
+  metadata: Map<string, string>;
+
+  constructor(
+    fields: ArrowLikeField[] | Field[],
+    metadata: SchemaMetadata | Map<string, string> = new Map<string, string>()
+  ) {
+    // checkNames(fields);
+    // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
+    this.fields = fields.map(
+      (field) => new ArrowLikeField(field.name, field.type, field.nullable, field.metadata)
+    );
+    this.metadata =
+      metadata instanceof Map ? metadata : new Map<string, string>(Object.entries(metadata));
+  }
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: ArrowLikeSchema): boolean {
+    if (this.metadata !== other.metadata) {
+      return false;
+    }
+    if (this.fields.length !== other.fields.length) {
+      return false;
+    }
+    for (let i = 0; i < this.fields.length; ++i) {
+      if (!this.fields[i].compareTo(other.fields[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  select(...columnNames: string[]): ArrowLikeSchema {
+    // Ensure column names reference valid fields
+    const nameMap = Object.create(null);
+    for (const name of columnNames) {
+      nameMap[name] = true;
+    }
+    const selectedFields = this.fields.filter((field) => nameMap[field.name]);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  selectAt(...columnIndices: number[]): ArrowLikeSchema {
+    // Ensure column indices reference valid fields
+    const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  assign(schemaOrFields: ArrowLikeSchema | ArrowLikeField[]): ArrowLikeSchema {
+    let fields: ArrowLikeField[];
+    let metadata = this.metadata;
+
+    if (schemaOrFields instanceof ArrowLikeSchema) {
+      const otherArrowLikeSchema = schemaOrFields;
+      fields = otherArrowLikeSchema.fields;
+      metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
+    } else {
+      fields = schemaOrFields;
+    }
+
+    // Create a merged list of fields, overwrite fields in place, new fields at end
+    const fieldMap: {[key: string]: ArrowLikeField} = Object.create(null);
+
+    for (const field of this.fields) {
+      fieldMap[field.name] = field;
+    }
+
+    for (const field of fields) {
+      fieldMap[field.name] = field;
+    }
+
+    const mergedFields = Object.values(fieldMap);
+
+    return new ArrowLikeSchema(mergedFields, metadata);
+  }
+}
+
+// Warn if any duplicated field names
+// function checkNames(fields: Field[]): void {
+//   const usedNames: Record<string, boolean> = {};
+//   for (const field of fields) {
+//     if (usedNames[field.name]) {
+//       // eslint-disable-next-line
+//       console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
+//     }
+//     usedNames[field.name] = true;
+//   }
+// }
+
+function mergeMaps<T>(m1: T, m2: T): T {
+  // @ts-ignore
+  return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
+}

package/src/lib/table/arrow-api/arrow-like-table.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Table} from '@loaders.gl/schema';
+
+import {ArrowLikeSchema} from './arrow-like-schema';
+import {deduceTableSchema} from '../../schema/deduce-table-schema';
+
+import {
+  getTableCell,
+  getTableLength,
+  getTableNumCols
+  // getTableCell,
+  // getTableRowShape,
+  // getTableColumnIndex,
+  // getTableColumnName,
+  // getTableRowAsObject,
+  // getTableRowAsArray
+} from '../tables/table-accessors';
+
+class ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  constructor(table: Table, columnName: string) {
+    this.table = table;
+    this.columnName = columnName;
+  }
+
+  get(rowIndex: number): unknown {
+    return getTableCell(this.table, rowIndex, this.columnName);
+  }
+
+  toArray(): ArrayLike<unknown> {
+    switch (this.table.shape) {
+      case 'arrow-table':
+        const arrowTable = this.table.data as any;
+        return arrowTable.getChild(this.columnName)?.toArray();
+      case 'columnar-table':
+        return this.table.data[this.columnName];
+      default:
+        throw new Error(this.table.shape);
+    }
+  }
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export class ArrowLikeTable {
+  schema: ArrowLikeSchema;
+  table: Table;
+
+  constructor(table: Table) {
+    const schema = table.schema || deduceTableSchema(table);
+    this.schema = new ArrowLikeSchema(schema.fields, schema.metadata);
+    this.table = {...table, schema};
+  }
+
+  // get schema() {
+  //   return this.table.schema;
+  // }
+
+  get data() {
+    return this.table.shape === 'geojson-table' ? this.table.features : this.table.data;
+  }
+
+  get numCols(): number {
+    return getTableNumCols(this.table);
+  }
+
+  get length(): number {
+    return getTableLength(this.table);
+  }
+
+  getChild(columnName: string): ArrowLikeVector {
+    return new ArrowLikeVector(this.table, columnName);
+  }
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}

package/src/lib/table/arrow-api/arrow-like-type.ts

@@ -0,0 +1,502 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// This code is adapted from ArrowJS https://github.com/apache/arrow
+// under Apache license http://www.apache.org/licenses/LICENSE-2.0
+
+import {Type} from './enum';
+
+import {ArrowLikeField as Field} from './arrow-like-field';
+
+export {Type} from './enum';
+
+export type TypedIntArray =
+  | Int8Array
+  | Uint8Array
+  | Int16Array
+  | Uint16Array
+  | Int32Array
+  | Uint32Array
+  | Uint8ClampedArray;
+
+export type TypedFloatArray = Float32Array | Float64Array;
+
+export type TypedArray = TypedIntArray | TypedFloatArray;
+
+export type AnyArrayType = Array<any> | TypedIntArray | TypedFloatArray;
+
+/** ArrowLike DataType class */
+export class DataType {
+  static isNull(x: any): boolean {
+    return x && x.typeId === Type.Null;
+  }
+  static isInt(x: any): boolean {
+    return x && x.typeId === Type.Int;
+  }
+  static isFloat(x: any): boolean {
+    return x && x.typeId === Type.Float;
+  }
+  static isBinary(x: any): boolean {
+    return x && x.typeId === Type.Binary;
+  }
+  static isUtf8(x: any): boolean {
+    return x && x.typeId === Type.Utf8;
+  }
+  static isBool(x: any): boolean {
+    return x && x.typeId === Type.Bool;
+  }
+  static isDecimal(x: any): boolean {
+    return x && x.typeId === Type.Decimal;
+  }
+  static isDate(x: any): boolean {
+    return x && x.typeId === Type.Date;
+  }
+  static isTime(x: any): boolean {
+    return x && x.typeId === Type.Time;
+  }
+  static isTimestamp(x: any): boolean {
+    return x && x.typeId === Type.Timestamp;
+  }
+  static isInterval(x: any): boolean {
+    return x && x.typeId === Type.Interval;
+  }
+  static isList(x: any): boolean {
+    return x && x.typeId === Type.List;
+  }
+  static isStruct(x: any): boolean {
+    return x && x.typeId === Type.Struct;
+  }
+  static isUnion(x: any): boolean {
+    return x && x.typeId === Type.Union;
+  }
+  static isFixedSizeBinary(x: any): boolean {
+    return x && x.typeId === Type.FixedSizeBinary;
+  }
+  static isFixedSizeList(x: any): boolean {
+    return x && x.typeId === Type.FixedSizeList;
+  }
+  static isMap(x: any): boolean {
+    return x && x.typeId === Type.Map;
+  }
+  static isDictionary(x: any): boolean {
+    return x && x.typeId === Type.Dictionary;
+  }
+
+  get typeId(): Type {
+    return Type.NONE;
+  }
+
+  // get ArrayType(): AnyArrayType {
+  //   return Int8Array;
+  // }
+
+  // get ArrayType() { return Array; }
+  compareTo(other: DataType): boolean {
+    // TODO
+    return this === other; // comparer.visit(this, other);
+  }
+}
+
+// NULL
+
+export class Null extends DataType {
+  get typeId(): Type {
+    return Type.Null;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Null';
+  }
+  toString(): string {
+    return 'Null';
+  }
+}
+
+// BOOLEANS
+
+export class Bool extends DataType {
+  get typeId(): Type {
+    return Type.Bool;
+  }
+  // get ArrayType() {
+  //   return Uint8Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Bool';
+  }
+  toString(): string {
+    return 'Bool';
+  }
+}
+
+// INTS
+
+export class Int extends DataType {
+  readonly isSigned: boolean;
+  readonly bitWidth: number;
+  constructor(isSigned, bitWidth) {
+    super();
+    this.isSigned = isSigned;
+    this.bitWidth = bitWidth;
+  }
+  get typeId(): Type {
+    return Type.Int;
+  }
+  // get ArrayType() {
+  //   switch (this.bitWidth) {
+  //     case 8:
+  //       return this.isSigned ? Int8Array : Uint8Array;
+  //     case 16:
+  //       return this.isSigned ? Int16Array : Uint16Array;
+  //     case 32:
+  //       return this.isSigned ? Int32Array : Uint32Array;
+  //     case 64:
+  //       return this.isSigned ? Int32Array : Uint32Array;
+  //     default:
+  //       throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+  //   }
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Int';
+  }
+  toString(): string {
+    return `${this.isSigned ? 'I' : 'Ui'}nt${this.bitWidth}`;
+  }
+}
+
+export class Int8 extends Int {
+  constructor() {
+    super(true, 8);
+  }
+}
+export class Int16 extends Int {
+  constructor() {
+    super(true, 16);
+  }
+}
+export class Int32 extends Int {
+  constructor() {
+    super(true, 32);
+  }
+}
+export class Int64 extends Int {
+  constructor() {
+    super(true, 64);
+  }
+}
+export class Uint8 extends Int {
+  constructor() {
+    super(false, 8);
+  }
+}
+export class Uint16 extends Int {
+  constructor() {
+    super(false, 16);
+  }
+}
+export class Uint32 extends Int {
+  constructor() {
+    super(false, 32);
+  }
+}
+export class Uint64 extends Int {
+  constructor() {
+    super(false, 64);
+  }
+}
+
+// FLOATS
+
+const Precision = {
+  HALF: 16,
+  SINGLE: 32,
+  DOUBLE: 64
+};
+
+export class Float extends DataType {
+  readonly precision: number;
+  constructor(precision) {
+    super();
+    this.precision = precision;
+  }
+  get typeId(): Type {
+    return Type.Float;
+  }
+  // get ArrayType() {
+  //   switch (this.precision) {
+  //     case Precision.HALF:
+  //       return Uint16Array;
+  //     case Precision.SINGLE:
+  //       return Float32Array;
+  //     case Precision.DOUBLE:
+  //       return Float64Array;
+  //     default:
+  //       throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+  //   }
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Float';
+  }
+  toString(): string {
+    return `Float${this.precision}`;
+  }
+}
+
+export class Float16 extends Float {
+  constructor() {
+    super(Precision.HALF);
+  }
+}
+export class Float32 extends Float {
+  constructor() {
+    super(Precision.SINGLE);
+  }
+}
+export class Float64 extends Float {
+  constructor() {
+    super(Precision.DOUBLE);
+  }
+}
+
+export class Binary extends DataType {
+  constructor() {
+    super();
+  }
+  get typeId() {
+    return Type.Binary;
+  }
+  toString() {
+    return 'Binary';
+  }
+  get [Symbol.toStringTag]() {
+    return 'Binary';
+  }
+}
+
+// STRINGS
+
+export class Utf8 extends DataType {
+  get typeId(): Type {
+    return Type.Utf8;
+  }
+  // get ArrayType() {
+  //   return Uint8Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Utf8';
+  }
+  toString(): string {
+    return 'Utf8';
+  }
+}
+
+// DATES, TIMES AND INTERVALS
+
+const DateUnit = {
+  DAY: 0,
+  MILLISECOND: 1
+};
+
+export class Date extends DataType {
+  readonly unit: number;
+  constructor(unit) {
+    super();
+    this.unit = unit;
+  }
+  get typeId(): Type {
+    return Type.Date;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Date';
+  }
+  toString(): string {
+    return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`;
+  }
+}
+
+export class DateDay extends Date {
+  constructor() {
+    super(DateUnit.DAY);
+  }
+}
+export class DateMillisecond extends Date {
+  constructor() {
+    super(DateUnit.MILLISECOND);
+  }
+}
+
+const TimeUnit = {
+  SECOND: 1,
+  MILLISECOND: 1e3,
+  MICROSECOND: 1e6,
+  NANOSECOND: 1e9
+};
+
+export class Time extends DataType {
+  readonly unit: any;
+  readonly bitWidth: number;
+
+  constructor(unit: any, bitWidth: number) {
+    super();
+    this.unit = unit;
+    this.bitWidth = bitWidth;
+  }
+  get typeId(): Type {
+    return Type.Time;
+  }
+  toString(): string {
+    return `Time${this.bitWidth}<${TimeUnit[this.unit]}>`;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Time';
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+}
+
+export class TimeSecond extends Time {
+  constructor() {
+    super(TimeUnit.SECOND, 32);
+  }
+}
+export class TimeMillisecond extends Time {
+  constructor() {
+    super(TimeUnit.MILLISECOND, 32);
+  }
+}
+// export class TimeMicrosecond extends Time { constructor() { super(TimeUnit.MICROSECOND, 64); } }
+// export class TimeNanosecond extends Time { constructor() { super(TimeUnit.NANOSECOND, 64); } }
+
+export class Timestamp extends DataType {
+  readonly unit: any;
+  readonly timezone: any;
+
+  constructor(unit: any, timezone = null) {
+    super();
+    this.unit = unit;
+    this.timezone = timezone;
+  }
+  get typeId(): Type {
+    return Type.Timestamp;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Timestamp';
+  }
+  toString(): string {
+    return `Timestamp<${TimeUnit[this.unit]}${this.timezone ? `, ${this.timezone}` : ''}>`;
+  }
+}
+
+export class TimestampSecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.SECOND, timezone);
+  }
+}
+export class TimestampMillisecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.MILLISECOND, timezone);
+  }
+}
+export class TimestampMicrosecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.MICROSECOND, timezone);
+  }
+}
+export class TimestampNanosecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.NANOSECOND, timezone);
+  }
+}
+
+const IntervalUnit = {
+  DAY_TIME: 0,
+  YEAR_MONTH: 1
+};
+
+export class Interval extends DataType {
+  readonly unit: number;
+  constructor(unit: number) {
+    super();
+    this.unit = unit;
+  }
+  get typeId(): Type {
+    return Type.Interval;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Interval';
+  }
+  toString(): string {
+    return `Interval<${IntervalUnit[this.unit]}>`;
+  }
+}
+
+export class IntervalDayTime extends Interval {
+  constructor() {
+    super(IntervalUnit.DAY_TIME);
+  }
+}
+export class IntervalYearMonth extends Interval {
+  constructor() {
+    super(IntervalUnit.YEAR_MONTH);
+  }
+}
+
+export class FixedSizeList extends DataType {
+  readonly listSize: number;
+  readonly children: Field[];
+
+  constructor(listSize: number, child: Field) {
+    super();
+    this.listSize = listSize;
+    this.children = [child];
+  }
+  get typeId(): Type {
+    return Type.FixedSizeList;
+  }
+  get valueType() {
+    return this.children[0].type;
+  }
+  get valueField() {
+    return this.children[0];
+  }
+  // get ArrayType() {
+  //   return this.valueType.ArrayType;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'FixedSizeList';
+  }
+  toString(): string {
+    return `FixedSizeList[${this.listSize}]<${JSON.stringify(this.valueType)}>`;
+  }
+}
+
+export class Struct extends DataType {
+  public readonly children: Field[];
+
+  constructor(children: Field[]) {
+    super();
+    this.children = children;
+  }
+
+  public get typeId() {
+    return Type.Struct;
+  }
+  public toString() {
+    return `Struct<{${this.children
+      .map((f) => `${f.name}:${JSON.stringify(f.type)}`)
+      .join(', ')}}>`;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Struct';
+  }
+}