@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/schema-utils/lib/table/arrow-api/arrow-like-type.ts

@@ -0,0 +1,500 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// This code is adapted from ArrowJS https://github.com/apache/arrow
+// under Apache license http://www.apache.org/licenses/LICENSE-2.0
+
+import {Type} from './enum';
+
+import {ArrowLikeField as Field} from './arrow-like-field';
+
+export {Type} from './enum';
+
+export type TypedIntArray =
+  | Int8Array
+  | Uint8Array
+  | Int16Array
+  | Uint16Array
+  | Int32Array
+  | Uint32Array
+  | Uint8ClampedArray;
+
+export type TypedFloatArray = Float32Array | Float64Array;
+
+export type TypedArray = TypedIntArray | TypedFloatArray;
+
+export type AnyArrayType = Array<any> | TypedIntArray | TypedFloatArray;
+
+/** ArrowLike DataType class */
+export class DataType {
+  static isNull(x: any): boolean {
+    return x && x.typeId === Type.Null;
+  }
+  static isInt(x: any): boolean {
+    return x && x.typeId === Type.Int;
+  }
+  static isFloat(x: any): boolean {
+    return x && x.typeId === Type.Float;
+  }
+  static isBinary(x: any): boolean {
+    return x && x.typeId === Type.Binary;
+  }
+  static isUtf8(x: any): boolean {
+    return x && x.typeId === Type.Utf8;
+  }
+  static isBool(x: any): boolean {
+    return x && x.typeId === Type.Bool;
+  }
+  static isDecimal(x: any): boolean {
+    return x && x.typeId === Type.Decimal;
+  }
+  static isDate(x: any): boolean {
+    return x && x.typeId === Type.Date;
+  }
+  static isTime(x: any): boolean {
+    return x && x.typeId === Type.Time;
+  }
+  static isTimestamp(x: any): boolean {
+    return x && x.typeId === Type.Timestamp;
+  }
+  static isInterval(x: any): boolean {
+    return x && x.typeId === Type.Interval;
+  }
+  static isList(x: any): boolean {
+    return x && x.typeId === Type.List;
+  }
+  static isStruct(x: any): boolean {
+    return x && x.typeId === Type.Struct;
+  }
+  static isUnion(x: any): boolean {
+    return x && x.typeId === Type.Union;
+  }
+  static isFixedSizeBinary(x: any): boolean {
+    return x && x.typeId === Type.FixedSizeBinary;
+  }
+  static isFixedSizeList(x: any): boolean {
+    return x && x.typeId === Type.FixedSizeList;
+  }
+  static isMap(x: any): boolean {
+    return x && x.typeId === Type.Map;
+  }
+  static isDictionary(x: any): boolean {
+    return x && x.typeId === Type.Dictionary;
+  }
+
+  get typeId(): Type {
+    return Type.NONE;
+  }
+
+  // get ArrayType(): AnyArrayType {
+  //   return Int8Array;
+  // }
+
+  // get ArrayType() { return Array; }
+  compareTo(other: DataType): boolean {
+    // TODO
+    return this === other; // comparer.visit(this, other);
+  }
+}
+
+// NULL
+
+export class Null extends DataType {
+  get typeId(): Type {
+    return Type.Null;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Null';
+  }
+  toString(): string {
+    return 'Null';
+  }
+}
+
+// BOOLEANS
+
+export class Bool extends DataType {
+  get typeId(): Type {
+    return Type.Bool;
+  }
+  // get ArrayType() {
+  //   return Uint8Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Bool';
+  }
+  toString(): string {
+    return 'Bool';
+  }
+}
+
+// INTS
+
+export class Int extends DataType {
+  readonly isSigned: boolean;
+  readonly bitWidth: number;
+  constructor(isSigned, bitWidth) {
+    super();
+    this.isSigned = isSigned;
+    this.bitWidth = bitWidth;
+  }
+  get typeId(): Type {
+    return Type.Int;
+  }
+  // get ArrayType() {
+  //   switch (this.bitWidth) {
+  //     case 8:
+  //       return this.isSigned ? Int8Array : Uint8Array;
+  //     case 16:
+  //       return this.isSigned ? Int16Array : Uint16Array;
+  //     case 32:
+  //       return this.isSigned ? Int32Array : Uint32Array;
+  //     case 64:
+  //       return this.isSigned ? Int32Array : Uint32Array;
+  //     default:
+  //       throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+  //   }
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Int';
+  }
+  toString(): string {
+    return `${this.isSigned ? 'I' : 'Ui'}nt${this.bitWidth}`;
+  }
+}
+
+export class Int8 extends Int {
+  constructor() {
+    super(true, 8);
+  }
+}
+export class Int16 extends Int {
+  constructor() {
+    super(true, 16);
+  }
+}
+export class Int32 extends Int {
+  constructor() {
+    super(true, 32);
+  }
+}
+export class Int64 extends Int {
+  constructor() {
+    super(true, 64);
+  }
+}
+export class Uint8 extends Int {
+  constructor() {
+    super(false, 8);
+  }
+}
+export class Uint16 extends Int {
+  constructor() {
+    super(false, 16);
+  }
+}
+export class Uint32 extends Int {
+  constructor() {
+    super(false, 32);
+  }
+}
+export class Uint64 extends Int {
+  constructor() {
+    super(false, 64);
+  }
+}
+
+// FLOATS
+
+const Precision = {
+  HALF: 16,
+  SINGLE: 32,
+  DOUBLE: 64
+};
+
+export class Float extends DataType {
+  readonly precision: number;
+  constructor(precision) {
+    super();
+    this.precision = precision;
+  }
+  get typeId(): Type {
+    return Type.Float;
+  }
+  // get ArrayType() {
+  //   switch (this.precision) {
+  //     case Precision.HALF:
+  //       return Uint16Array;
+  //     case Precision.SINGLE:
+  //       return Float32Array;
+  //     case Precision.DOUBLE:
+  //       return Float64Array;
+  //     default:
+  //       throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+  //   }
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Float';
+  }
+  toString(): string {
+    return `Float${this.precision}`;
+  }
+}
+
+export class Float16 extends Float {
+  constructor() {
+    super(Precision.HALF);
+  }
+}
+export class Float32 extends Float {
+  constructor() {
+    super(Precision.SINGLE);
+  }
+}
+export class Float64 extends Float {
+  constructor() {
+    super(Precision.DOUBLE);
+  }
+}
+
+export class Binary extends DataType {
+  constructor() {
+    super();
+  }
+  get typeId() {
+    return Type.Binary;
+  }
+  toString() {
+    return 'Binary';
+  }
+  get [Symbol.toStringTag]() {
+    return 'Binary';
+  }
+}
+
+// STRINGS
+
+export class Utf8 extends DataType {
+  get typeId(): Type {
+    return Type.Utf8;
+  }
+  // get ArrayType() {
+  //   return Uint8Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Utf8';
+  }
+  toString(): string {
+    return 'Utf8';
+  }
+}
+
+// DATES, TIMES AND INTERVALS
+
+const DateUnit = {
+  DAY: 0,
+  MILLISECOND: 1
+};
+
+export class Date extends DataType {
+  readonly unit: number;
+  constructor(unit) {
+    super();
+    this.unit = unit;
+  }
+  get typeId(): Type {
+    return Type.Date;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Date';
+  }
+  toString(): string {
+    return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`;
+  }
+}
+
+export class DateDay extends Date {
+  constructor() {
+    super(DateUnit.DAY);
+  }
+}
+export class DateMillisecond extends Date {
+  constructor() {
+    super(DateUnit.MILLISECOND);
+  }
+}
+
+const TimeUnit = {
+  SECOND: 1,
+  MILLISECOND: 1e3,
+  MICROSECOND: 1e6,
+  NANOSECOND: 1e9
+};
+
+export class Time extends DataType {
+  readonly unit: any;
+  readonly bitWidth: number;
+
+  constructor(unit: any, bitWidth: number) {
+    super();
+    this.unit = unit;
+    this.bitWidth = bitWidth;
+  }
+  get typeId(): Type {
+    return Type.Time;
+  }
+  toString(): string {
+    return `Time${this.bitWidth}<${TimeUnit[this.unit]}>`;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Time';
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+}
+
+export class TimeSecond extends Time {
+  constructor() {
+    super(TimeUnit.SECOND, 32);
+  }
+}
+export class TimeMillisecond extends Time {
+  constructor() {
+    super(TimeUnit.MILLISECOND, 32);
+  }
+}
+// export class TimeMicrosecond extends Time { constructor() { super(TimeUnit.MICROSECOND, 64); } }
+// export class TimeNanosecond extends Time { constructor() { super(TimeUnit.NANOSECOND, 64); } }
+
+export class Timestamp extends DataType {
+  readonly unit: any;
+  readonly timezone: any;
+
+  constructor(unit: any, timezone = null) {
+    super();
+    this.unit = unit;
+    this.timezone = timezone;
+  }
+  get typeId(): Type {
+    return Type.Timestamp;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Timestamp';
+  }
+  toString(): string {
+    return `Timestamp<${TimeUnit[this.unit]}${this.timezone ? `, ${this.timezone}` : ''}>`;
+  }
+}
+
+export class TimestampSecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.SECOND, timezone);
+  }
+}
+export class TimestampMillisecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.MILLISECOND, timezone);
+  }
+}
+export class TimestampMicrosecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.MICROSECOND, timezone);
+  }
+}
+export class TimestampNanosecond extends Timestamp {
+  constructor(timezone = null) {
+    super(TimeUnit.NANOSECOND, timezone);
+  }
+}
+
+const IntervalUnit = {
+  DAY_TIME: 0,
+  YEAR_MONTH: 1
+};
+
+export class Interval extends DataType {
+  readonly unit: number;
+  constructor(unit: number) {
+    super();
+    this.unit = unit;
+  }
+  get typeId(): Type {
+    return Type.Interval;
+  }
+  // get ArrayType() {
+  //   return Int32Array;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'Interval';
+  }
+  toString(): string {
+    return `Interval<${IntervalUnit[this.unit]}>`;
+  }
+}
+
+export class IntervalDayTime extends Interval {
+  constructor() {
+    super(IntervalUnit.DAY_TIME);
+  }
+}
+export class IntervalYearMonth extends Interval {
+  constructor() {
+    super(IntervalUnit.YEAR_MONTH);
+  }
+}
+
+export class FixedSizeList extends DataType {
+  readonly listSize: number;
+  readonly children: Field[];
+
+  constructor(listSize: number, child: Field) {
+    super();
+    this.listSize = listSize;
+    this.children = [child];
+  }
+  get typeId(): Type {
+    return Type.FixedSizeList;
+  }
+  get valueType() {
+    return this.children[0].type;
+  }
+  get valueField() {
+    return this.children[0];
+  }
+  // get ArrayType() {
+  //   return this.valueType.ArrayType;
+  // }
+  get [Symbol.toStringTag](): string {
+    return 'FixedSizeList';
+  }
+  toString(): string {
+    return `FixedSizeList[${this.listSize}]<${JSON.stringify(this.valueType)}>`;
+  }
+}
+
+export class Struct extends DataType {
+  public readonly children: Field[];
+
+  constructor(children: Field[]) {
+    super();
+    this.children = children;
+  }
+
+  public get typeId() {
+    return Type.Struct;
+  }
+  public toString() {
+    return `Struct<{${this.children.map(f => `${f.name}:${JSON.stringify(f.type)}`).join(', ')}}>`;
+  }
+  get [Symbol.toStringTag](): string {
+    return 'Struct';
+  }
+}

package/src/schema-utils/lib/table/arrow-api/enum.ts

@@ -0,0 +1,99 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// This code is adapted from ArrowJS https://github.com/apache/arrow
+// under Apache license http://www.apache.org/licenses/LICENSE-2.0
+
+/**
+ * Main data type enumeration.
+ *
+ * Data types in this library are all *logical*. They can be expressed as
+ * either a primitive physical type (bytes or bits of some fixed size), a
+ * nested type consisting of other data types, or another data type (e.g. a
+ * timestamp encoded as an int64).
+ *
+ * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
+ * IPC payload.
+ *
+ * The rest of the values are specified here so TypeScript can narrow the type
+ * signatures further beyond the base Arrow Types. The Arrow DataTypes include
+ * metadata like `bitWidth` that impact the type signatures of the values we
+ * accept and return.
+ *
+ * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
+ * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
+ * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
+ * underlying `Int32Array`.
+ *
+ * Library consumers benefit by knowing the narrowest type, since we can ensure
+ * the types across all public methods are propagated, and never bail to `any`.
+ * These values are _never_ used at runtime, and they will _never_ be written
+ * to the flatbuffers metadata of serialized Arrow IPC payloads.
+ */
+export enum Type {
+  /** The default placeholder type */
+  NONE = 0,
+  /** A NULL type having no physical storage */
+  Null = 1,
+  /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
+  Int = 2,
+  /** 2, 4, or 8-byte floating point value */
+  Float = 3,
+  /** Variable-length bytes (no guarantee of UTF8-ness) */
+  Binary = 4,
+  /** UTF8 variable-length string as List<Char> */
+  Utf8 = 5,
+  /** Boolean as 1 bit, LSB bit-packed ordering */
+  Bool = 6,
+  /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
+  Decimal = 7,
+  /** int32_t days or int64_t milliseconds since the UNIX epoch */
+  Date = 8,
+  /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
+  Time = 9,
+  /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
+  Timestamp = 10,
+  /** YEAR_MONTH or DAY_TIME interval in SQL style */
+  Interval = 11,
+  /** A list of some logical data type */
+  List = 12,
+  /** Struct of logical types */
+  Struct = 13,
+  /** Union of logical types */
+  Union = 14,
+  /** Fixed-size binary. Each value occupies the same number of bytes */
+  FixedSizeBinary = 15,
+  /** Fixed-size list. Each value occupies the same number of bytes */
+  FixedSizeList = 16,
+  /** Map of named logical types */
+  Map = 17,
+
+  /** Dictionary aka Category type */
+  Dictionary = -1,
+  Int8 = -2,
+  Int16 = -3,
+  Int32 = -4,
+  Int64 = -5,
+  Uint8 = -6,
+  Uint16 = -7,
+  Uint32 = -8,
+  Uint64 = -9,
+  Float16 = -10,
+  Float32 = -11,
+  Float64 = -12,
+  DateDay = -13,
+  DateMillisecond = -14,
+  TimestampSecond = -15,
+  TimestampMillisecond = -16,
+  TimestampMicrosecond = -17,
+  TimestampNanosecond = -18,
+  TimeSecond = -19,
+  TimeMillisecond = -20,
+  TimeMicrosecond = -21,
+  TimeNanosecond = -22,
+  DenseUnion = -23,
+  SparseUnion = -24,
+  IntervalDayTime = -25,
+  IntervalYearMonth = -26
+}

package/src/schema-utils/lib/table/arrow-api/get-type-info.ts

@@ -0,0 +1,38 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Type} from './arrow-like-type';
+
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export function getTypeInfo(arrowTypeLike: any): {
+  typeId: Type;
+  ArrayType: ArrayLike<unknown>;
+  typeName: string;
+  typeEnumName?: string;
+  precision?: number;
+} {
+  return {
+    typeId: arrowTypeLike.typeId,
+    ArrayType: arrowTypeLike.ArrayType,
+    typeName: arrowTypeLike.toString(),
+    typeEnumName: getTypeKey(arrowTypeLike.typeId),
+    precision: arrowTypeLike.precision
+  };
+}
+
+let ReverseType: {[key: string]: string} | null = null;
+
+function getTypeKey(typeKey) {
+  if (!ReverseType) {
+    ReverseType = {};
+    for (const key in Type) {
+      ReverseType[Type[key]] = key;
+    }
+  }
+
+  return ReverseType[typeKey];
+}

package/src/schema-utils/lib/table/arrow-api/index.ts

@@ -0,0 +1,50 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {ArrowLikeField as Field} from './arrow-like-field';
+export {ArrowLikeSchema as Schema} from './arrow-like-schema';
+export {ArrowLikeTable as Table} from './arrow-like-table';
+export {
+  Type,
+  DataType,
+  Null,
+  Bool,
+  Int,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Float,
+  Float16,
+  Float32,
+  Float64,
+  Binary,
+  Utf8,
+  Date,
+  DateDay,
+  DateMillisecond,
+  Time,
+  TimeSecond,
+  TimeMillisecond,
+  Timestamp,
+  TimestampSecond,
+  TimestampMillisecond,
+  TimestampMicrosecond,
+  TimestampNanosecond,
+  Interval,
+  IntervalDayTime,
+  IntervalYearMonth,
+  FixedSizeList,
+  Struct
+} from './arrow-like-type';
+export type {
+  TypedIntArray,
+  TypedFloatArray,
+  TypedArray,
+  AnyArrayType
+} from './arrow-like-type';