@loaders.gl/schema 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/category/table/table-types.ts

@@ -1,81 +0,0 @@
-import type {Schema} from '../../lib/schema';
-import type {Table as ApacheArrowTable, RecordBatch} from 'apache-arrow/Arrow.dom';
-import type {AnyArray} from '../../types';
-import type {Batch} from '../common';
-
-/** A general table */
-export interface Table {
-  shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
-  schema?: Schema;
-  schemaType?: 'explicit' | 'deduced';
-}
-
-/** A table organized as an array of rows */
-export interface RowTable extends Table {
-  shape: 'row-table' | 'array-row-table' | 'object-row-table';
-  data: any[];
-}
-
-/** A table organized as an array of rows, each row is an array of values */
-export interface ArrayRowTable extends RowTable {
-  shape: 'array-row-table';
-  data: any[][];
-}
-
-/** A table organized as an array of rows, each row is an object mapping columns to values */
-export interface ObjectRowTable extends RowTable {
-  shape: 'object-row-table';
-  data: {[columnName: string]: any}[];
-}
-
-/** A table organized as a map of columns, each column is an array of value */
-export interface ColumnarTable extends Table {
-  shape: 'columnar-table';
-  data: {[columnName: string]: AnyArray};
-}
-
-/** A table organized as an Apache Arrow table */
-export interface ArrowTable extends Table {
-  shape: 'arrow-table';
-  data: ApacheArrowTable;
-}
-
-// Batches
-
-/** Batch for a general table */
-export type TableBatch = Batch & {
-  data: any;
-  length: number;
-  schema?: Schema;
-  schemaType?: 'explicit' | 'deduced';
-};
-
-/** Batch for a table organized as an array of rows */
-export type RowTableBatch = TableBatch & {
-  shape: 'row-table';
-  data: any[];
-};
-
-/** Batch for a table organized as an array of rows, each row is an array of values */
-export type RowArrayTableBatch = RowTableBatch & {
-  shape: 'array-row-table';
-  data: any[][];
-};
-
-/** Batch for a table organized as an array of rows, each row is an object mapping columns to values */
-export type RowObjectTableBatch = RowTableBatch & {
-  shape: 'object-row-table';
-  data: {[columnName: string]: any}[];
-};
-
-/** Batch for a table organized as a map of columns, each column is an array of value */
-export type ColumnarTableBatch = TableBatch & {
-  shape: 'columnar-table';
-  data: {[columnName: string]: AnyArray};
-};
-
-/** Batch for a table organized as an Apache Arrow table */
-export type ArrowTableBatch = TableBatch & {
-  shape: 'arrow-table';
-  data: RecordBatch;
-};

package/src/lib/arrow/arrow-like-type-utils.ts

@@ -1,25 +0,0 @@
-import type {TypedArray} from '../../types';
-import {DataType, Float32, Float64, Int16, Int32, Int8, Uint16, Uint32, Uint8} from '../schema';
-
-export function getArrowTypeFromTypedArray(array: TypedArray): DataType {
-  switch (array.constructor) {
-    case Int8Array:
-      return new Int8();
-    case Uint8Array:
-      return new Uint8();
-    case Int16Array:
-      return new Int16();
-    case Uint16Array:
-      return new Uint16();
-    case Int32Array:
-      return new Int32();
-    case Uint32Array:
-      return new Uint32();
-    case Float32Array:
-      return new Float32();
-    case Float64Array:
-      return new Float64();
-    default:
-      throw new Error('array type not supported');
-  }
-}

package/src/lib/schema/impl/schema.ts

@@ -1,99 +0,0 @@
-import {assert} from '../../utils/assert';
-import Field from './field';
-
-export type SchemaMetadata = Map<string, any>;
-
-/**
- * ArrowJS `Schema` API-compatible class for row-based tables (returned from `DataTable`)
- * https://loaders.gl/arrowjs/docs/api-reference/schema
- */
-export default class Schema {
-  fields: Field[];
-  // TODO - Arrow just allows Map<string, string>
-  metadata: SchemaMetadata;
-
-  constructor(fields: Field[], metadata?: SchemaMetadata) {
-    assert(Array.isArray(fields));
-    checkNames(fields);
-    // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
-    this.fields = fields;
-    this.metadata = metadata || new Map();
-  }
-
-  // TODO - arrow only seems to compare fields, not metadata
-  compareTo(other: Schema): boolean {
-    if (this.metadata !== other.metadata) {
-      return false;
-    }
-    if (this.fields.length !== other.fields.length) {
-      return false;
-    }
-    for (let i = 0; i < this.fields.length; ++i) {
-      if (!this.fields[i].compareTo(other.fields[i])) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  select(...columnNames: string[]): Schema {
-    // Ensure column names reference valid fields
-    const nameMap = Object.create(null);
-    for (const name of columnNames) {
-      nameMap[name] = true;
-    }
-    const selectedFields = this.fields.filter((field) => nameMap[field.name]);
-    return new Schema(selectedFields, this.metadata);
-  }
-
-  selectAt(...columnIndices: number[]): Schema {
-    // Ensure column indices reference valid fields
-    const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
-    return new Schema(selectedFields, this.metadata);
-  }
-
-  assign(schemaOrFields: Schema | Field[]): Schema {
-    let fields: Field[];
-    let metadata: SchemaMetadata = this.metadata;
-
-    if (schemaOrFields instanceof Schema) {
-      const otherSchema = schemaOrFields;
-      fields = otherSchema.fields;
-      metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherSchema.metadata);
-    } else {
-      fields = schemaOrFields;
-    }
-
-    // Create a merged list of fields, overwrite fields in place, new fields at end
-    const fieldMap: {[key: string]: Field} = Object.create(null);
-
-    for (const field of this.fields) {
-      fieldMap[field.name] = field;
-    }
-
-    for (const field of fields) {
-      fieldMap[field.name] = field;
-    }
-
-    const mergedFields = Object.values(fieldMap);
-
-    return new Schema(mergedFields, metadata);
-  }
-}
-
-// Warn if any duplicated field names
-function checkNames(fields: Field[]): void {
-  const usedNames: Record<string, boolean> = {};
-  for (const field of fields) {
-    if (usedNames[field.name]) {
-      // eslint-disable-next-line
-      console.warn('Schema: duplicated field name', field.name, field);
-    }
-    usedNames[field.name] = true;
-  }
-}
-
-function mergeMaps<T>(m1: T, m2: T): T {
-  // @ts-ignore
-  return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
-}

package/src/lib/schema/index.ts

@@ -1,82 +0,0 @@
-/*
-export {
-  Schema,
-  Field,
-  DataType,
-  Null,
-  Bool,
-  Int,
-  Int8,
-  Int16,
-  Int32,
-  Int64,
-  Uint8,
-  Uint16,
-  Uint32,
-  Uint64,
-  Float,
-  Float16,
-  Float32,
-  Float64,
-  Binary,
-  Utf8,
-  Date,
-  DateDay,
-  DateMillisecond,
-  Time,
-  TimeSecond,
-  TimeMillisecond,
-  TimeMicrosecond,
-  TimeNanosecond,
-  Timestamp,
-  TimestampSecond,
-  TimestampMillisecond,
-  TimestampMicrosecond,
-  TimestampNanosecond,
-  Interval,
-  IntervalDayTime,
-  IntervalYearMonth,
-  FixedSizeList
-} from 'apache-arrow';
-*/
-export {default as Schema} from './impl/schema';
-export {default as Field} from './impl/field';
-export {Type} from './impl/type';
-export {
-  DataType,
-  Null,
-  Bool,
-  Int,
-  Int8,
-  Int16,
-  Int32,
-  Int64,
-  Uint8,
-  Uint16,
-  Uint32,
-  Uint64,
-  Float,
-  Float16,
-  Float32,
-  Float64,
-  Binary,
-  Utf8,
-  Date,
-  DateDay,
-  DateMillisecond,
-  Time,
-  TimeSecond,
-  TimeMillisecond,
-  // TimeMicrosecond,
-  // TimeNanosecond,
-  Timestamp,
-  TimestampSecond,
-  TimestampMillisecond,
-  TimestampMicrosecond,
-  TimestampNanosecond,
-  Interval,
-  IntervalDayTime,
-  IntervalYearMonth,
-  FixedSizeList,
-  Struct
-} from './impl/type';

package/src/lib/schema-utils/deduce-column-type.ts

@@ -1,92 +0,0 @@
-// Type deduction
-// import {
-//   Schema
-//   Int,
-//   Int8,
-//   Int16,
-//   Int32,
-//   Uint8,
-//   Uint16,
-//   Uint32,
-//   Float32,
-//   Float64
-//   Bool,
-//   Utf8,
-//   TimestampMillisecond,
-//   Null
-// } from '../schema';
-
-// const TYPED_ARRAY_TO_TYPE = {
-//   Int8Array: new Int8(),
-//   Int16Array: new Int16(),
-//   Int32Array: new Int32(),
-//   Uint8Array: new Uint8(),
-//   Uint8ClampedArray: new Uint8(),
-//   Uint16Array: new Uint16(),
-//   Uint32Array: new Uint32(),
-//   Float32Array: new Float32(),
-//   Float64Array: new Float64()
-// };
-
-// if (typeof BigInt64Array !== 'undefined') {
-//   TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
-//   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
-// }
-
-export function deduceTypeFromColumn(
-  value: unknown
-): StringConstructor | DateConstructor | Float32ArrayConstructor | null {
-  if (value instanceof Date) {
-    return Date;
-  } else if (value instanceof Number) {
-    return Float32Array;
-  } else if (typeof value === 'string') {
-    return String;
-  }
-  return null;
-}
-
-export function deduceTypeFromValue(
-  value: unknown
-): StringConstructor | DateConstructor | Float32ArrayConstructor | null {
-  if (value instanceof Date) {
-    return Date;
-  } else if (value instanceof Number) {
-    return Float32Array;
-  } else if (typeof value === 'string') {
-    return String;
-  }
-  return null;
-}
-
-/*
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function deduceSchema(rows) {
-  const row = rows[0];
-
-  const schema = {};
-  let i = 0;
-  for (const columnName in row) {
-    const value = row[columnName];
-    switch (typeof value) {
-      case 'number':
-      case 'boolean':
-        // TODO - booleans could be handled differently...
-        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
-        break;
-
-      case 'object':
-        schema[columnName] = {name: String(columnName), index: i, type: Array};
-        break;
-
-      case 'string':
-      default:
-        schema[columnName] = {name: String(columnName), index: i, type: Array};
-      // We currently only handle numeric rows
-      // TODO we could offer a function to map strings to numbers?
-    }
-    i++;
-  }
-  return schema;
-}
-*/

package/src/types.ts

@@ -1,19 +0,0 @@
-/** Any typed array */
-export type TypedArray =
-  | Int8Array
-  | Uint8Array
-  | Int16Array
-  | Uint16Array
-  | Int32Array
-  | Uint32Array
-  | Uint8ClampedArray
-  | Float32Array
-  | Float64Array;
-
-export type BigTypedArray = TypedArray | BigInt64Array | BigUint64Array;
-
-/** Any numeric array: typed array or `number[]` */
-export type NumberArray = number[] | TypedArray;
-
-/** Any array: typed array or js array (`any[]`) */
-export type AnyArray = any[] | TypedArray;