@loaders.gl/schema-utils 4.4.0-alpha.1

package/src/lib/table/arrow-api/enum.ts

@@ -0,0 +1,99 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// This code is adapted from ArrowJS https://github.com/apache/arrow
+// under Apache license http://www.apache.org/licenses/LICENSE-2.0
+
+/**
+ * Main data type enumeration.
+ *
+ * Data types in this library are all *logical*. They can be expressed as
+ * either a primitive physical type (bytes or bits of some fixed size), a
+ * nested type consisting of other data types, or another data type (e.g. a
+ * timestamp encoded as an int64).
+ *
+ * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
+ * IPC payload.
+ *
+ * The rest of the values are specified here so TypeScript can narrow the type
+ * signatures further beyond the base Arrow Types. The Arrow DataTypes include
+ * metadata like `bitWidth` that impact the type signatures of the values we
+ * accept and return.
+ *
+ * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
+ * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
+ * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
+ * underlying `Int32Array`.
+ *
+ * Library consumers benefit by knowing the narrowest type, since we can ensure
+ * the types across all public methods are propagated, and never bail to `any`.
+ * These values are _never_ used at runtime, and they will _never_ be written
+ * to the flatbuffers metadata of serialized Arrow IPC payloads.
+ */
+export enum Type {
+  /** The default placeholder type */
+  NONE = 0,
+  /** A NULL type having no physical storage */
+  Null = 1,
+  /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
+  Int = 2,
+  /** 2, 4, or 8-byte floating point value */
+  Float = 3,
+  /** Variable-length bytes (no guarantee of UTF8-ness) */
+  Binary = 4,
+  /** UTF8 variable-length string as List<Char> */
+  Utf8 = 5,
+  /** Boolean as 1 bit, LSB bit-packed ordering */
+  Bool = 6,
+  /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
+  Decimal = 7,
+  /** int32_t days or int64_t milliseconds since the UNIX epoch */
+  Date = 8,
+  /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
+  Time = 9,
+  /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
+  Timestamp = 10,
+  /** YEAR_MONTH or DAY_TIME interval in SQL style */
+  Interval = 11,
+  /** A list of some logical data type */
+  List = 12,
+  /** Struct of logical types */
+  Struct = 13,
+  /** Union of logical types */
+  Union = 14,
+  /** Fixed-size binary. Each value occupies the same number of bytes */
+  FixedSizeBinary = 15,
+  /** Fixed-size list. Each value occupies the same number of bytes */
+  FixedSizeList = 16,
+  /** Map of named logical types */
+  Map = 17,
+
+  /** Dictionary aka Category type */
+  Dictionary = -1,
+  Int8 = -2,
+  Int16 = -3,
+  Int32 = -4,
+  Int64 = -5,
+  Uint8 = -6,
+  Uint16 = -7,
+  Uint32 = -8,
+  Uint64 = -9,
+  Float16 = -10,
+  Float32 = -11,
+  Float64 = -12,
+  DateDay = -13,
+  DateMillisecond = -14,
+  TimestampSecond = -15,
+  TimestampMillisecond = -16,
+  TimestampMicrosecond = -17,
+  TimestampNanosecond = -18,
+  TimeSecond = -19,
+  TimeMillisecond = -20,
+  TimeMicrosecond = -21,
+  TimeNanosecond = -22,
+  DenseUnion = -23,
+  SparseUnion = -24,
+  IntervalDayTime = -25,
+  IntervalYearMonth = -26
+}

package/src/lib/table/arrow-api/get-type-info.ts

@@ -0,0 +1,38 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Type} from './arrow-like-type';
+
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export function getTypeInfo(arrowTypeLike: any): {
+  typeId: Type;
+  ArrayType: ArrayLike<unknown>;
+  typeName: string;
+  typeEnumName?: string;
+  precision?: number;
+} {
+  return {
+    typeId: arrowTypeLike.typeId,
+    ArrayType: arrowTypeLike.ArrayType,
+    typeName: arrowTypeLike.toString(),
+    typeEnumName: getTypeKey(arrowTypeLike.typeId),
+    precision: arrowTypeLike.precision
+  };
+}
+
+let ReverseType: {[key: string]: string} | null = null;
+
+function getTypeKey(typeKey) {
+  if (!ReverseType) {
+    ReverseType = {};
+    for (const key in Type) {
+      ReverseType[Type[key]] = key;
+    }
+  }
+
+  return ReverseType[typeKey];
+}

package/src/lib/table/arrow-api/index.ts

@@ -0,0 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {ArrowLikeField as Field} from './arrow-like-field';
+export {ArrowLikeSchema as Schema} from './arrow-like-schema';
+export {ArrowLikeTable as Table} from './arrow-like-table';
+export * from './arrow-like-type';

package/src/lib/table/batch-builder/arrow-table-batch-aggregator-builders.ts.disabled

@@ -0,0 +1,190 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, ColumnarTableBatch, ArrowTableBatch} from '@loaders.gl/schema';
+import {TableBatchAggregator} from './table-batch-aggregator';
+import {convertSchemaToArrow} from '../../schema/convert-arrow-schema';
+import * as arrow from 'apache-arrow';
+
+/** @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends TableBatchAggregator {
+
+  constructor(schema: Schema, options) {
+    super(schema, options);
+    this.arrowSchema = null;
+  }
+
+  getBatch(): ArrowTableBatch | null {
+    const batch = super.getBatch();
+    if (batch) {
+      // Get the arrow schema
+      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+
+      // Get arrow format vectors
+      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+
+      // Create the record batch
+      const recordBatch = new arrow.RecordBatch(
+        this.arrowSchema,
+        arrow.makeData({
+          type: new arrow.Struct(this.arrowSchema.fields),
+          children: arrowVectors.map(({data}) => data[0])
+        })
+      );
+
+      return {
+        shape: 'arrow-table',
+        batchType: 'data',
+        data: new arrow.Table([recordBatch]),
+        length: batch.length
+      };
+    }
+
+    return null;
+  }
+}
+
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema): arrow.Schema {
+  const arrowFields: arrow.Field[] = [];
+  for (const key in schema) {
+    const field = schema[key];
+    if (field.type === Float32Array) {
+      // TODO - just store the original field as metadata?
+      const metadata = new Map(); // field;
+      // arrow: new arrow.Field(name, nullable, metadata)
+      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+      arrowFields.push(arrowField);
+    }
+  }
+  if (arrowFields.length === 0) {
+    throw new Error('No arrow convertible fields');
+  }
+
+  return new arrow.Schema(arrowFields);
+}
+
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data): arrow.Vector[] {
+  const arrowVectors: any[] = [];
+  for (const field of arrowSchema.fields) {
+    const vector = data[field.name];
+    if (vector instanceof Float32Array) {
+      const arrowVector = arrow.makeVector(vector);
+      arrowVectors.push(arrowVector);
+    }
+  }
+  if (arrowSchema.fields.length !== arrowVectors.length) {
+    throw new Error('Some columns not arrow convertible');
+  }
+  return arrowVectors;
+}
+
+type ColumnarTableBatchOptions = {};
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class ArrowBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  length: number = 0;
+  allocated: number = 0;
+  columns: {[columnName: string]: any[]} = {};
+
+  arrowSchema: arrow.Schema | null;
+
+  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
+    this.schema = schema;
+    this.arrowSchema = convertSchemaToArrow(schema);
+    this._reallocateColumns();
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[]) {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    let i = 0;
+    // TODO what if no csv header, columns not populated?
+    for (const fieldName in this.columns) {
+      this.columns[fieldName][this.length] = row[i++];
+    }
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}): void {
+    // If user keeps pushing rows beyond batch size, reallocate
+    this._reallocateColumns();
+    for (const fieldName in row) {
+      this.columns[fieldName][this.length] = row[fieldName];
+    }
+    this.length++;
+  }
+
+  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
+    this._pruneColumns();
+    const columns = Array.isArray(this.schema) ? this.columns : {};
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    // columns should match schema format
+    if (!Array.isArray(this.schema)) {
+      for (const fieldName in this.schema) {
+        const field = this.schema[fieldName];
+        columns[field.name] = this.columns[field.index];
+      }
+    }
+
+    this.columns = {};
+
+    const batch: ColumnarTableBatch = {
+      shape: 'columnar-table',
+      batchType: 'data',
+      data: columns,
+      schema: this.schema,
+      length: this.length
+    };
+
+    return batch;
+  }
+
+  // HELPERS
+
+  _reallocateColumns() {
+    if (this.length < this.allocated) {
+      return;
+    }
+
+    // @ts-ignore TODO
+    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
+    this.columns = {};
+
+    for (const fieldName in this.schema) {
+      const field = this.schema[fieldName];
+      const ArrayType = field.type || Float32Array;
+      const oldColumn = this.columns[field.index];
+
+      if (oldColumn && ArrayBuffer.isView(oldColumn)) {
+        // Copy the old data to the new array
+        const typedArray = new ArrayType(this.allocated);
+        typedArray.set(oldColumn);
+        this.columns[field.index] = typedArray;
+      } else if (oldColumn) {
+        // Plain array
+        oldColumn.length = this.allocated;
+        this.columns[field.index] = oldColumn;
+      } else {
+        // Create new
+        this.columns[field.index] = new ArrayType(this.allocated);
+      }
+    }
+  }
+
+  _pruneColumns() {
+    for (const [columnName, column] of Object.entries(this.columns)) {
+      this.columns[columnName] = column.slice(0, this.length);
+    }
+  }
+}

package/src/lib/table/batch-builder/arrow-table-batch-aggregator.ts

@@ -0,0 +1,82 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTableBatch} from '@loaders.gl/schema';
+import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
+import * as arrow from 'apache-arrow';
+
+/** @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+  arrowSchema: arrow.Schema | null;
+
+  constructor(schema, options) {
+    super(schema, options);
+    this.arrowSchema = null;
+  }
+
+  getBatch(): ArrowTableBatch | null {
+    const batch = super.getBatch();
+    if (batch) {
+      // Get the arrow schema
+      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+
+      // Get arrow format vectors
+      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+
+      // Create the record batch
+      const recordBatch = new arrow.RecordBatch(
+        this.arrowSchema,
+        arrow.makeData({
+          type: new arrow.Struct(this.arrowSchema.fields),
+          children: arrowVectors.map(({data}) => data[0])
+        })
+      );
+
+      return {
+        shape: 'arrow-table',
+        batchType: 'data',
+        data: new arrow.Table([recordBatch]),
+        length: batch.length
+      };
+    }
+
+    return null;
+  }
+}
+
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema): arrow.Schema {
+  const arrowFields: arrow.Field[] = [];
+  for (const key in schema) {
+    const field = schema[key];
+    if (field.type === Float32Array) {
+      // TODO - just store the original field as metadata?
+      const metadata = new Map(); // field;
+      // arrow: new arrow.Field(name, nullable, metadata)
+      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+      arrowFields.push(arrowField);
+    }
+  }
+  if (arrowFields.length === 0) {
+    throw new Error('No arrow convertible fields');
+  }
+
+  return new arrow.Schema(arrowFields);
+}
+
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data): arrow.Vector[] {
+  const arrowVectors: any[] = [];
+  for (const field of arrowSchema.fields) {
+    const vector = data[field.name];
+    if (vector instanceof Float32Array) {
+      const arrowVector = arrow.makeVector(vector);
+      arrowVectors.push(arrowVector);
+    }
+  }
+  if (arrowSchema.fields.length !== arrowVectors.length) {
+    throw new Error('Some columns not arrow convertible');
+  }
+  return arrowVectors;
+}

package/src/lib/table/batch-builder/arrow-table-builder.ts

@@ -0,0 +1,112 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+import {convertSchemaToArrow} from '@loaders.gl/schema-utils';
+
+/** Builds an arrow table or batches */
+export class ArrowTableBuilder {
+  schema: Schema;
+  arrowSchema: arrow.Schema;
+  arrowBuilders: arrow.Builder[];
+  length: number;
+
+  constructor(schema: Schema) {
+    this.schema = schema;
+    this.arrowSchema = convertSchemaToArrow(schema);
+    this.arrowBuilders = this.arrowSchema.fields.map((field) =>
+      arrow.makeBuilder({type: field.type, nullValues: [null]})
+    );
+    this.length = 0;
+  }
+
+  addObjectRow(row: {[key: string]: any}) {
+    for (let i = 0; i < this.arrowBuilders.length; i++) {
+      const columnName = this.schema.fields[i].name;
+      const value = row[columnName];
+      // if (this.schema.fields[i].type.toString() === 'bool') {
+      //   debugger;
+      // }
+      this.arrowBuilders[i].append(value);
+    }
+    this.length++;
+  }
+
+  addArrayRow(row: any[]) {
+    for (let i = 0; i < this.arrowBuilders.length; i++) {
+      this.arrowBuilders[i].append(row[i]);
+    }
+    this.length++;
+  }
+
+  /** Makes sure that a first batch with schema is sent even if no rows */
+  firstBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    // If there is data, a batch will be sent later
+    if (arrowRecordBatch.numCols !== 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Flush the current batch if conditions are right */
+  flushBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    if (arrowRecordBatch.numCols === 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Get a last batch if any data is left */
+  finishBatch(): ArrowTableBatch | null {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    this.arrowBuilders.forEach((builder) => builder.finish());
+    if (arrowRecordBatch.numCols === 0) {
+      return null;
+    }
+    return {
+      shape: 'arrow-table',
+      batchType: 'data',
+      length: arrowRecordBatch.numRows,
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Return a table with all the accumulated data */
+  finishTable(): ArrowTable {
+    const arrowRecordBatch = this._getArrowRecordBatch();
+    this.arrowBuilders.forEach((builder) => builder.finish());
+    return {
+      shape: 'arrow-table',
+      schema: this.schema,
+      data: new arrow.Table(arrowRecordBatch)
+    };
+  }
+
+  /** Extract a record batch flushing the currently accumulated data in the builders */
+  _getArrowRecordBatch(): arrow.RecordBatch {
+    const {arrowBuilders, arrowSchema} = this;
+    const arrowDatas = arrowBuilders.map((builder) => builder.flush());
+    const length = arrowDatas[0].length;
+    const structField = new arrow.Struct(arrowSchema.fields);
+    const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
+    const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);
+    return arrowRecordBatch;
+  }
+}

package/src/lib/table/batch-builder/base-table-batch-aggregator.ts

@@ -0,0 +1,82 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, TableBatch} from '@loaders.gl/schema';
+import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
+
+const DEFAULT_ROW_COUNT = 100;
+
+export class BaseTableBatchAggregator implements TableBatchAggregator {
+  schema: Schema;
+  options: TableBatchOptions;
+
+  shape?: 'array-row-table' | 'object-row-table';
+  length: number = 0;
+  rows: any[] | null = null;
+  cursor: number = 0;
+  private _headers: string[] = [];
+
+  constructor(schema: Schema, options: TableBatchOptions) {
+    this.options = options;
+    this.schema = schema;
+
+    // schema is an array if there're no headers
+    // object if there are headers
+    if (!Array.isArray(schema)) {
+      this._headers = [];
+      for (const key in schema) {
+        this._headers[schema[key].index] = schema[key].name;
+      }
+    }
+  }
+
+  rowCount(): number {
+    return this.length;
+  }
+
+  addArrayRow(row: any[], cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    this.shape = 'array-row-table';
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor as number;
+    }
+
+    this.shape = 'object-row-table';
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  getBatch(): TableBatch | null {
+    let rows = this.rows;
+    if (!rows) {
+      return null;
+    }
+
+    rows = rows.slice(0, this.length);
+    this.rows = null;
+
+    const batch: TableBatch = {
+      shape: this.shape || 'array-row-table',
+      batchType: 'data',
+      data: rows,
+      length: this.length,
+      schema: this.schema,
+      cursor: this.cursor
+    };
+
+    return batch;
+  }
+}