@loaders.gl/schema 4.0.0-alpha.9 → 4.0.0-beta.2

package/dist/lib/table/arrow/convert-schema-arrow.js

@@ -1,187 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.deserializeArrowType = exports.serializeArrowType = exports.deserializeArrowField = exports.serializeArrowField = exports.deserializeArrowMetadata = exports.serializeArrowMetadata = exports.deserializeArrowSchema = exports.serializeArrowSchema = void 0;
-const Arrow_dom_1 = require("apache-arrow/Arrow.dom");
-function serializeArrowSchema(arrowSchema) {
-    return {
-        fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
-        metadata: serializeArrowMetadata(arrowSchema.metadata)
-    };
-}
-exports.serializeArrowSchema = serializeArrowSchema;
-function deserializeArrowSchema(schema) {
-    return new Arrow_dom_1.Schema(schema.fields.map((field) => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
-}
-exports.deserializeArrowSchema = deserializeArrowSchema;
-function serializeArrowMetadata(arrowMetadata) {
-    return Object.fromEntries(arrowMetadata);
-}
-exports.serializeArrowMetadata = serializeArrowMetadata;
-function deserializeArrowMetadata(metadata) {
-    return metadata ? new Map(Object.entries(metadata)) : new Map();
-}
-exports.deserializeArrowMetadata = deserializeArrowMetadata;
-function serializeArrowField(field) {
-    return {
-        name: field.name,
-        type: serializeArrowType(field.type),
-        nullable: field.nullable,
-        metadata: serializeArrowMetadata(field.metadata)
-    };
-}
-exports.serializeArrowField = serializeArrowField;
-function deserializeArrowField(field) {
-    return new Arrow_dom_1.Field(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
-}
-exports.deserializeArrowField = deserializeArrowField;
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-// eslint-disable-next-line complexity
-function serializeArrowType(arrowType) {
-    switch (arrowType.constructor) {
-        case Arrow_dom_1.Null:
-            return 'null';
-        case Arrow_dom_1.Binary:
-            return 'binary';
-        case Arrow_dom_1.Bool:
-            return 'bool';
-        // case Int: return 'int';
-        case Arrow_dom_1.Int8:
-            return 'int8';
-        case Arrow_dom_1.Int16:
-            return 'int16';
-        case Arrow_dom_1.Int32:
-            return 'int32';
-        case Arrow_dom_1.Int64:
-            return 'int64';
-        case Arrow_dom_1.Uint8:
-            return 'uint8';
-        case Arrow_dom_1.Uint16:
-            return 'uint16';
-        case Arrow_dom_1.Uint32:
-            return 'uint32';
-        case Arrow_dom_1.Uint64:
-            return 'uint64';
-        // case Float: return 'float';
-        case Arrow_dom_1.Float16:
-            return 'float16';
-        case Arrow_dom_1.Float32:
-            return 'float32';
-        case Arrow_dom_1.Float64:
-            return 'float64';
-        case Arrow_dom_1.Utf8:
-            return 'utf8';
-        // case Date: return 'date';
-        case Arrow_dom_1.DateDay:
-            return 'date-day';
-        case Arrow_dom_1.DateMillisecond:
-            return 'date-millisecond';
-        // case Time: return 'time';
-        case Arrow_dom_1.TimeMillisecond:
-            return 'time-millisecond';
-        case Arrow_dom_1.TimeSecond:
-            return 'time-second';
-        // case Timestamp: return 'timestamp';
-        case Arrow_dom_1.TimestampSecond:
-            return 'timestamp-second';
-        case Arrow_dom_1.TimestampMillisecond:
-            return 'timestamp-millisecond';
-        case Arrow_dom_1.TimestampMicrosecond:
-            return 'timestamp-microsecond';
-        case Arrow_dom_1.TimestampNanosecond:
-            return 'timestamp-nanosecond';
-        // case Interval: return 'interval';
-        case Arrow_dom_1.IntervalDayTime:
-            return 'interval-daytime';
-        case Arrow_dom_1.IntervalYearMonth:
-            return 'interval-yearmonth';
-        case Arrow_dom_1.FixedSizeList:
-            return {
-                type: 'fixed-size-list',
-                listSize: arrowType.listSize,
-                children: [serializeArrowField(arrowType.children[0])]
-            };
-        // case Struct:
-        //   return {type: 'struct', children: (arrowType as Struct).children};
-        default:
-            throw new Error('array type not supported');
-    }
-}
-exports.serializeArrowType = serializeArrowType;
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-// eslint-disable-next-line complexity
-function deserializeArrowType(dataType) {
-    if (typeof dataType === 'object') {
-        switch (dataType.type) {
-            case 'fixed-size-list':
-                const child = deserializeArrowField(dataType.children[0]);
-                return new Arrow_dom_1.FixedSizeList(dataType.listSize, child);
-            case 'struct':
-                const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
-                return new Arrow_dom_1.Struct(children);
-            default:
-                throw new Error('array type not supported');
-        }
-    }
-    switch (dataType) {
-        case 'null':
-            return new Arrow_dom_1.Null();
-        case 'binary':
-            return new Arrow_dom_1.Binary();
-        case 'bool':
-            return new Arrow_dom_1.Bool();
-        // case 'int': return new Int();
-        case 'int8':
-            return new Arrow_dom_1.Int8();
-        case 'int16':
-            return new Arrow_dom_1.Int16();
-        case 'int32':
-            return new Arrow_dom_1.Int32();
-        case 'int64':
-            return new Arrow_dom_1.Int64();
-        case 'uint8':
-            return new Arrow_dom_1.Uint8();
-        case 'uint16':
-            return new Arrow_dom_1.Uint16();
-        case 'uint32':
-            return new Arrow_dom_1.Uint32();
-        case 'uint64':
-            return new Arrow_dom_1.Uint64();
-        // case 'float': return new Float();
-        case 'float16':
-            return new Arrow_dom_1.Float16();
-        case 'float32':
-            return new Arrow_dom_1.Float32();
-        case 'float64':
-            return new Arrow_dom_1.Float64();
-        case 'utf8':
-            return new Arrow_dom_1.Utf8();
-        // case 'date': return new Date();
-        case 'date-day':
-            return new Arrow_dom_1.DateDay();
-        case 'date-millisecond':
-            return new Arrow_dom_1.DateMillisecond();
-        // case 'time': return new Time();
-        case 'time-millisecond':
-            return new Arrow_dom_1.TimeMillisecond();
-        case 'time-second':
-            return new Arrow_dom_1.TimeSecond();
-        // case 'timestamp': return new Timestamp();
-        case 'timestamp-second':
-            return new Arrow_dom_1.TimestampSecond();
-        case 'timestamp-millisecond':
-            return new Arrow_dom_1.TimestampMillisecond();
-        case 'timestamp-microsecond':
-            return new Arrow_dom_1.TimestampMicrosecond();
-        case 'timestamp-nanosecond':
-            return new Arrow_dom_1.TimestampNanosecond();
-        // case 'interval': return new Interval();
-        case 'interval-daytime':
-            return new Arrow_dom_1.IntervalDayTime();
-        case 'interval-yearmonth':
-            return new Arrow_dom_1.IntervalYearMonth();
-        default:
-            throw new Error('array type not supported');
-    }
-}
-exports.deserializeArrowType = deserializeArrowType;

package/dist/lib/table/arrow/convert-table-to-arrow.js

@@ -1,57 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-// import {
-//   Table as ApacheArrowTable,
-//   Schema as ApacheArrowSchema,
-//   RecordBatch,
-//   FixedSizeList,
-//   Field,
-//   Data,
-//   FixedSizeListVector
-// } from 'apache-arrow/Arrow.dom';
-// import {AbstractVector} from 'apache-arrow/vector';
-// import {Table} from '../../types/category-table';
-// import {getArrowType, getArrowVector} from './arrow-type-utils';
-// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
-// import {getTableLength, getTableNumCols} from './table-accessors';
-/**
- * * Convert a loaders.gl Mesh to an Apache Arrow Table
- * @param mesh
- * @param metadata
- * @param batchSize
- * @returns
- *
-export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
-  const vectors: AbstractVector[] = [];
-  const fields: Field[] = [];
-
-  const length = getTableLength(table);
-  const batchSize = options?.batchSize || length;
-
-  const recordBatches: RecordBatch[] = [];
-  for (let i = 0; i < length; i += batchSize) {
-    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
-      const field_ = table.schema.fields[columnIndex];
-      const column = getTableColumnAt(table, columnIndex, i, batchSize);
-      const type = getArrowType(column);
-      const vector = getArrowVector(column);
-      const listType = new FixedSizeList(size, new Field('value', type));
-      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
-      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
-      const listVector = new FixedSizeListVector(data);
-      vectors.push(listVector);
-      fields.push(field);
-    }
-
-  for (const attributeKey in table.columns) {
-    const attribute = mesh.attributes[attributeKey];
-    const {value, size = 1} = attribute;
-  }
-
-  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
-  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
-  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
-
-  return apacheArrowTable;
-}
-*/

package/dist/lib/table/arrow-api/arrow-like-field.js

@@ -1,34 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowLikeField = void 0;
-/**
- * ArrowJS `Field` API-compatible class for row-based tables
- * https://loaders.gl/arrowjs/docs/api-reference/field
- * A field holds name, nullable, and metadata information about a table "column"
- * A Schema is essentially a list of fields
- */
-class ArrowLikeField {
-    constructor(name, type, nullable = false, metadata = new Map()) {
-        this.name = name;
-        this.type = type;
-        this.nullable = nullable;
-        this.metadata = metadata;
-    }
-    get typeId() {
-        return this.type && this.type.typeId;
-    }
-    clone() {
-        return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
-    }
-    compareTo(other) {
-        return (this.name === other.name &&
-            this.type === other.type &&
-            this.nullable === other.nullable &&
-            this.metadata === other.metadata);
-    }
-    toString() {
-        return `${this.type}${this.nullable ? ', nullable' : ''}${this.metadata ? `, metadata: ${this.metadata}` : ''}`;
-    }
-}
-exports.ArrowLikeField = ArrowLikeField;

package/dist/lib/table/arrow-api/arrow-like-schema.js

@@ -1,81 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowLikeSchema = void 0;
-const arrow_like_field_1 = require("./arrow-like-field");
-class ArrowLikeSchema {
-    constructor(fields, metadata = new Map()) {
-        // checkNames(fields);
-        // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
-        this.fields = fields.map((field) => new arrow_like_field_1.ArrowLikeField(field.name, field.type, field.nullable, field.metadata));
-        this.metadata =
-            metadata instanceof Map ? metadata : new Map(Object.entries(metadata));
-    }
-    // TODO - arrow only seems to compare fields, not metadata
-    compareTo(other) {
-        if (this.metadata !== other.metadata) {
-            return false;
-        }
-        if (this.fields.length !== other.fields.length) {
-            return false;
-        }
-        for (let i = 0; i < this.fields.length; ++i) {
-            if (!this.fields[i].compareTo(other.fields[i])) {
-                return false;
-            }
-        }
-        return true;
-    }
-    select(...columnNames) {
-        // Ensure column names reference valid fields
-        const nameMap = Object.create(null);
-        for (const name of columnNames) {
-            nameMap[name] = true;
-        }
-        const selectedFields = this.fields.filter((field) => nameMap[field.name]);
-        return new ArrowLikeSchema(selectedFields, this.metadata);
-    }
-    selectAt(...columnIndices) {
-        // Ensure column indices reference valid fields
-        const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
-        return new ArrowLikeSchema(selectedFields, this.metadata);
-    }
-    assign(schemaOrFields) {
-        let fields;
-        let metadata = this.metadata;
-        if (schemaOrFields instanceof ArrowLikeSchema) {
-            const otherArrowLikeSchema = schemaOrFields;
-            fields = otherArrowLikeSchema.fields;
-            metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
-        }
-        else {
-            fields = schemaOrFields;
-        }
-        // Create a merged list of fields, overwrite fields in place, new fields at end
-        const fieldMap = Object.create(null);
-        for (const field of this.fields) {
-            fieldMap[field.name] = field;
-        }
-        for (const field of fields) {
-            fieldMap[field.name] = field;
-        }
-        const mergedFields = Object.values(fieldMap);
-        return new ArrowLikeSchema(mergedFields, metadata);
-    }
-}
-exports.ArrowLikeSchema = ArrowLikeSchema;
-// Warn if any duplicated field names
-// function checkNames(fields: Field[]): void {
-//   const usedNames: Record<string, boolean> = {};
-//   for (const field of fields) {
-//     if (usedNames[field.name]) {
-//       // eslint-disable-next-line
-//       console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
-//     }
-//     usedNames[field.name] = true;
-//   }
-// }
-function mergeMaps(m1, m2) {
-    // @ts-ignore
-    return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
-}

package/dist/lib/table/arrow-api/arrow-like-table.js

@@ -1,53 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowLikeTable = void 0;
-const arrow_like_schema_1 = require("./arrow-like-schema");
-const table_schema_1 = require("../simple-table/table-schema");
-const table_accessors_1 = require("../simple-table/table-accessors");
-class ArrowLikeVector {
-    constructor(table, columnName) {
-        this.table = table;
-        this.columnName = columnName;
-    }
-    get(rowIndex) {
-        return (0, table_accessors_1.getTableCell)(this.table, rowIndex, this.columnName);
-    }
-    toArray() {
-        switch (this.table.shape) {
-            case 'arrow-table':
-                return this.table.data.getChild(this.columnName)?.toArray();
-            case 'columnar-table':
-                return this.table.data[this.columnName];
-            default:
-                throw new Error(this.table.shape);
-        }
-    }
-}
-/**
- * Class that provides an API similar to Apache Arrow Table class
- * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
- */
-class ArrowLikeTable {
-    constructor(table) {
-        const schema = table.schema || (0, table_schema_1.deduceTableSchema)(table);
-        this.schema = new arrow_like_schema_1.ArrowLikeSchema(schema.fields, schema.metadata);
-        this.table = { ...table, schema };
-    }
-    // get schema() {
-    //   return this.table.schema;
-    // }
-    get data() {
-        return this.table.data;
-    }
-    get numCols() {
-        return (0, table_accessors_1.getTableNumCols)(this.table);
-    }
-    get length() {
-        return (0, table_accessors_1.getTableLength)(this.table);
-    }
-    getChild(columnName) {
-        return new ArrowLikeVector(this.table, columnName);
-    }
-}
-exports.ArrowLikeTable = ArrowLikeTable;