@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/schema-utils/lib/schema/deduce-table-schema.ts

@@ -0,0 +1,119 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Type deduction
+import type {Schema, Field, ArrayType, Table} from '@loaders.gl/core/schema';
+import {getDataTypeFromArray, getDataTypeFromValue} from './data-type';
+
+/**
+ * SCHEMA SUPPORT - AUTODEDUCTION
+ * @param {*} table
+ * @param {*} schema
+ * @returns
+ */
+export function deduceTableSchema(table: Table): Schema {
+  switch (table.shape) {
+    case 'array-row-table':
+    case 'object-row-table':
+      return deduceSchemaFromRows(table.data);
+
+    case 'geojson-table':
+      return deduceSchemaFromGeoJSON(table.features);
+
+    case 'columnar-table':
+      return deduceSchemaFromColumns(table.data);
+
+    case 'arrow-table':
+    default:
+      throw new Error('Deduce schema');
+  }
+}
+
+export function deduceSchema(
+  data: unknown[][] | {[key: string]: unknown}[] | {[key: string]: unknown[]}
+): Schema {
+  return Array.isArray(data) ? deduceSchemaFromRows(data) : deduceSchemaFromColumns(data);
+}
+
+/** Given an object with columnar arrays, try to deduce a schema */
+function deduceSchemaFromColumns(columnarTable: {[key: string]: ArrayType}): Schema {
+  const fields: Field[] = [];
+  for (const [columnName, column] of Object.entries(columnarTable)) {
+    const field = deduceFieldFromColumn(column, columnName);
+    fields.push(field);
+  }
+  return {fields, metadata: {}};
+}
+
+/** Given an array of rows, try to deduce a schema */
+function deduceSchemaFromRows(rowTable: unknown[][] | {[key: string]: unknown}[]): Schema {
+  if (!rowTable.length) {
+    throw new Error('deduce from empty table');
+  }
+  const fields: Field[] = [];
+  const row0 = rowTable[0];
+  // TODO - fields can be nullable, false detection...
+  // Could look at additional rows if nulls in first row
+  // TODO - if array, column names will be numbers
+  for (const [columnName, value] of Object.entries(row0)) {
+    fields.push(deduceFieldFromValue(value, columnName));
+  }
+
+  return {fields, metadata: {}};
+}
+
+/** Given a GeoJSON, try to deduce a schema */
+function deduceSchemaFromGeoJSON(features: {properties: Record<string, number> | null}[]): Schema {
+  if (!features.length) {
+    throw new Error('deduce from empty table');
+  }
+  const fields: Field[] = [];
+  const row0 = features[0].properties || {};
+  // TODO - fields can be nullable, false detection...
+  // Could look at additional rows if nulls in first row
+  // TODO - if array, column names will be numbers
+  for (const [columnName, value] of Object.entries(row0)) {
+    fields.push(deduceFieldFromValue(value, columnName));
+  }
+
+  return {fields, metadata: {}};
+}
+
+/** Given a column (i.e. array), attempt to deduce an appropriate `Field` */
+function deduceFieldFromColumn(column: ArrayType, name: string): Field {
+  if (ArrayBuffer.isView(column)) {
+    const type = getDataTypeFromArray(column);
+    return {
+      name,
+      type: type.type || 'null',
+      nullable: type.nullable
+      // metadata: {}
+    };
+  }
+
+  if (Array.isArray(column) && column.length > 0) {
+    const value = column[0];
+    const type = getDataTypeFromValue(value);
+    // TODO - support nested schemas?
+    return {
+      name,
+      type,
+      nullable: true
+      // metadata: {},
+    };
+  }
+
+  throw new Error('empty table');
+}
+
+/** Given a value, attempt to deduce an appropriate `Field` */
+function deduceFieldFromValue(value: unknown, name: string): Field {
+  const type = getDataTypeFromValue(value);
+  return {
+    name,
+    type,
+    nullable: true
+    // metadata: {}
+  };
+}

package/src/schema-utils/lib/table/arrow-api/arrow-like-field.ts

@@ -0,0 +1,53 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType} from './arrow-like-type';
+
+/**
+ * ArrowJS `Field` API-compatible class for row-based tables
+ * https://loaders.gl/arrowjs/docs/api-reference/field
+ * A field holds name, nullable, and metadata information about a table "column"
+ * A Schema is essentially a list of fields
+ */
+export class ArrowLikeField {
+  name: string;
+  type: DataType;
+  nullable: boolean;
+  metadata: Map<string, string>;
+
+  constructor(
+    name: string,
+    type: DataType,
+    nullable = false,
+    metadata: Map<string, string> = new Map()
+  ) {
+    this.name = name;
+    this.type = type;
+    this.nullable = nullable;
+    this.metadata = metadata;
+  }
+
+  get typeId(): number {
+    return this.type && this.type.typeId;
+  }
+
+  clone(): ArrowLikeField {
+    return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
+  }
+
+  compareTo(other: this): boolean {
+    return (
+      this.name === other.name &&
+      this.type === other.type &&
+      this.nullable === other.nullable &&
+      this.metadata === other.metadata
+    );
+  }
+
+  toString(): string {
+    return `${JSON.stringify(this.type)}${this.nullable ? ', nullable' : ''}${
+      this.metadata ? `, metadata: ${JSON.stringify(this.metadata)}` : ''
+    }`;
+  }
+}

package/src/schema-utils/lib/table/arrow-api/arrow-like-interface.ts

@@ -0,0 +1,84 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {DataType} from './arrow-like-type';
+
+/**
+ * ArrowJS `Field` API-compatible class for row-based tables
+ * https://loaders.gl/arrowjs/docs/api-reference/field
+ * A field holds name, nullable, and metadata information about a table "column"
+ * A Schema is essentially a list of fields
+ */
+export interface Field {
+  name: string;
+  type: DataType;
+  nullable: boolean;
+  metadata: Map<string, string>;
+
+  // constructor(
+  //   name: string,
+  //   type: DataType,
+  //   nullable?,
+  //   metadata?
+  // );
+
+  typeId(): number;
+
+  clone(): Field;
+
+  compareTo(other: this): boolean;
+
+  toString();
+}
+
+export interface Schema {
+  fields: Field[];
+  metadata: Map<string, string>;
+
+  // constructor(
+  //   fields: Field[]  ,
+  //   metadata: Map<string, string>
+  // )
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: Schema): boolean;
+
+  select(...columnNames: string[]): Schema;
+
+  selectAt(...columnIndices: number[]): Schema;
+
+  assign(schemaOrFields: Schema | Field[]): Schema;
+}
+
+export interface ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  // constructor(table: Table, columnName: string);
+
+  get(rowIndex: number): unknown;
+  toArray(): ArrayLike<unknown>;
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export interface Table {
+  schema: Schema;
+
+  // constructor(table: Table);
+
+  data: any;
+
+  numCols: number;
+
+  length(): number;
+
+  getChild(columnName: string): ArrowLikeVector;
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}

package/src/schema-utils/lib/table/arrow-api/arrow-like-schema.ts

@@ -0,0 +1,101 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {SchemaMetadata, Field} from '@loaders.gl/core/schema';
+import {ArrowLikeField} from './arrow-like-field';
+
+export class ArrowLikeSchema {
+  fields: ArrowLikeField[];
+  metadata: Map<string, string>;
+
+  constructor(
+    fields: ArrowLikeField[] | Field[],
+    metadata: SchemaMetadata | Map<string, string> = new Map<string, string>()
+  ) {
+    // checkNames(fields);
+    // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
+    this.fields = fields.map(
+      field => new ArrowLikeField(field.name, field.type, field.nullable, field.metadata)
+    );
+    this.metadata =
+      metadata instanceof Map ? metadata : new Map<string, string>(Object.entries(metadata));
+  }
+
+  // TODO - arrow only seems to compare fields, not metadata
+  compareTo(other: ArrowLikeSchema): boolean {
+    if (this.metadata !== other.metadata) {
+      return false;
+    }
+    if (this.fields.length !== other.fields.length) {
+      return false;
+    }
+    for (let i = 0; i < this.fields.length; ++i) {
+      if (!this.fields[i].compareTo(other.fields[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  select(...columnNames: string[]): ArrowLikeSchema {
+    // Ensure column names reference valid fields
+    const nameMap = Object.create(null);
+    for (const name of columnNames) {
+      nameMap[name] = true;
+    }
+    const selectedFields = this.fields.filter(field => nameMap[field.name]);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  selectAt(...columnIndices: number[]): ArrowLikeSchema {
+    // Ensure column indices reference valid fields
+    const selectedFields = columnIndices.map(index => this.fields[index]).filter(Boolean);
+    return new ArrowLikeSchema(selectedFields, this.metadata);
+  }
+
+  assign(schemaOrFields: ArrowLikeSchema | ArrowLikeField[]): ArrowLikeSchema {
+    let fields: ArrowLikeField[];
+    let metadata = this.metadata;
+
+    if (schemaOrFields instanceof ArrowLikeSchema) {
+      const otherArrowLikeSchema = schemaOrFields;
+      fields = otherArrowLikeSchema.fields;
+      metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
+    } else {
+      fields = schemaOrFields;
+    }
+
+    // Create a merged list of fields, overwrite fields in place, new fields at end
+    const fieldMap: {[key: string]: ArrowLikeField} = Object.create(null);
+
+    for (const field of this.fields) {
+      fieldMap[field.name] = field;
+    }
+
+    for (const field of fields) {
+      fieldMap[field.name] = field;
+    }
+
+    const mergedFields = Object.values(fieldMap);
+
+    return new ArrowLikeSchema(mergedFields, metadata);
+  }
+}
+
+// Warn if any duplicated field names
+// function checkNames(fields: Field[]): void {
+//   const usedNames: Record<string, boolean> = {};
+//   for (const field of fields) {
+//     if (usedNames[field.name]) {
+//       // eslint-disable-next-line
+//       console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
+//     }
+//     usedNames[field.name] = true;
+//   }
+// }
+
+function mergeMaps<T>(m1: T, m2: T): T {
+  // @ts-ignore
+  return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
+}

package/src/schema-utils/lib/table/arrow-api/arrow-like-table.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Table} from '@loaders.gl/core/schema';
+
+import {ArrowLikeSchema} from './arrow-like-schema';
+import {deduceTableSchema} from '../../schema/deduce-table-schema';
+
+import {
+  getTableCell,
+  getTableLength,
+  getTableNumCols
+  // getTableCell,
+  // getTableRowShape,
+  // getTableColumnIndex,
+  // getTableColumnName,
+  // getTableRowAsObject,
+  // getTableRowAsArray
+} from '../tables/table-accessors';
+
+class ArrowLikeVector {
+  table: Table;
+  columnName: string;
+
+  constructor(table: Table, columnName: string) {
+    this.table = table;
+    this.columnName = columnName;
+  }
+
+  get(rowIndex: number): unknown {
+    return getTableCell(this.table, rowIndex, this.columnName);
+  }
+
+  toArray(): ArrayLike<unknown> {
+    switch (this.table.shape) {
+      case 'arrow-table':
+        const arrowTable = this.table.data as any;
+        return arrowTable.getChild(this.columnName)?.toArray();
+      case 'columnar-table':
+        return this.table.data[this.columnName];
+      default:
+        throw new Error(this.table.shape);
+    }
+  }
+}
+
+/**
+ * Class that provides an API similar to Apache Arrow Table class
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
+ */
+export class ArrowLikeTable {
+  schema: ArrowLikeSchema;
+  table: Table;
+
+  constructor(table: Table) {
+    const schema = table.schema || deduceTableSchema(table);
+    this.schema = new ArrowLikeSchema(schema.fields, schema.metadata);
+    this.table = {...table, schema};
+  }
+
+  // get schema() {
+  //   return this.table.schema;
+  // }
+
+  get data() {
+    return this.table.shape === 'geojson-table' ? this.table.features : this.table.data;
+  }
+
+  get numCols(): number {
+    return getTableNumCols(this.table);
+  }
+
+  get length(): number {
+    return getTableLength(this.table);
+  }
+
+  getChild(columnName: string): ArrowLikeVector {
+    return new ArrowLikeVector(this.table, columnName);
+  }
+
+  // getChildAt(columnIndex: number): ArrowLikeVector {
+  //   return
+  // }
+}