@loaders.gl/schema 3.4.10 → 3.4.12

package/dist/lib/batches/columnar-table-batch-aggregator.js

@@ -1,90 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const DEFAULT_ROW_COUNT = 100;
-class ColumnarTableBatchAggregator {
-    constructor(schema, options) {
-        this.length = 0;
-        this.allocated = 0;
-        this.columns = {};
-        this.schema = schema;
-        this._reallocateColumns();
-    }
-    rowCount() {
-        return this.length;
-    }
-    addArrayRow(row) {
-        // If user keeps pushing rows beyond batch size, reallocate
-        this._reallocateColumns();
-        let i = 0;
-        // TODO what if no csv header, columns not populated?
-        for (const fieldName in this.columns) {
-            this.columns[fieldName][this.length] = row[i++];
-        }
-        this.length++;
-    }
-    addObjectRow(row) {
-        // If user keeps pushing rows beyond batch size, reallocate
-        this._reallocateColumns();
-        for (const fieldName in row) {
-            this.columns[fieldName][this.length] = row[fieldName];
-        }
-        this.length++;
-    }
-    getBatch() {
-        this._pruneColumns();
-        const columns = Array.isArray(this.schema) ? this.columns : {};
-        // schema is an array if there're no headers
-        // object if there are headers
-        // columns should match schema format
-        if (!Array.isArray(this.schema)) {
-            for (const fieldName in this.schema) {
-                const field = this.schema[fieldName];
-                columns[field.name] = this.columns[field.index];
-            }
-        }
-        this.columns = {};
-        const batch = {
-            shape: 'columnar-table',
-            batchType: 'data',
-            data: columns,
-            schema: this.schema,
-            length: this.length
-        };
-        return batch;
-    }
-    // HELPERS
-    _reallocateColumns() {
-        if (this.length < this.allocated) {
-            return;
-        }
-        // @ts-ignore TODO
-        this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
-        this.columns = {};
-        for (const fieldName in this.schema) {
-            const field = this.schema[fieldName];
-            const ArrayType = field.type || Float32Array;
-            const oldColumn = this.columns[field.index];
-            if (oldColumn && ArrayBuffer.isView(oldColumn)) {
-                // Copy the old data to the new array
-                const typedArray = new ArrayType(this.allocated);
-                typedArray.set(oldColumn);
-                this.columns[field.index] = typedArray;
-            }
-            else if (oldColumn) {
-                // Plain array
-                oldColumn.length = this.allocated;
-                this.columns[field.index] = oldColumn;
-            }
-            else {
-                // Create new
-                this.columns[field.index] = new ArrayType(this.allocated);
-            }
-        }
-    }
-    _pruneColumns() {
-        for (const [columnName, column] of Object.entries(this.columns)) {
-            this.columns[columnName] = column.slice(0, this.length);
-        }
-    }
-}
-exports.default = ColumnarTableBatchAggregator;

package/dist/lib/batches/row-table-batch-aggregator.js

@@ -1,79 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
-const row_utils_1 = require("../utils/row-utils");
-const DEFAULT_ROW_COUNT = 100;
-class RowTableBatchAggregator {
-    constructor(schema, options) {
-        this.length = 0;
-        this.objectRows = null;
-        this.arrayRows = null;
-        this.cursor = 0;
-        this._headers = [];
-        this.options = options;
-        this.schema = schema;
-        // schema is an array if there're no headers
-        // object if there are headers
-        if (!Array.isArray(schema)) {
-            this._headers = [];
-            for (const key in schema) {
-                this._headers[schema[key].index] = schema[key].name;
-            }
-        }
-    }
-    rowCount() {
-        return this.length;
-    }
-    addArrayRow(row, cursor) {
-        if (Number.isFinite(cursor)) {
-            this.cursor = cursor;
-        }
-        // eslint-disable-next-line default-case
-        switch (this.options.shape) {
-            case 'object-row-table':
-                const rowObject = (0, row_utils_1.convertToObjectRow)(row, this._headers);
-                this.addObjectRow(rowObject, cursor);
-                break;
-            case 'array-row-table':
-                this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
-                this.arrayRows[this.length] = row;
-                this.length++;
-                break;
-        }
-    }
-    addObjectRow(row, cursor) {
-        if (Number.isFinite(cursor)) {
-            this.cursor = cursor;
-        }
-        // eslint-disable-next-line default-case
-        switch (this.options.shape) {
-            case 'array-row-table':
-                const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
-                this.addArrayRow(rowArray, cursor);
-                break;
-            case 'object-row-table':
-                this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
-                this.objectRows[this.length] = row;
-                this.length++;
-                break;
-        }
-    }
-    getBatch() {
-        let rows = this.arrayRows || this.objectRows;
-        if (!rows) {
-            return null;
-        }
-        rows = rows.slice(0, this.length);
-        this.arrayRows = null;
-        this.objectRows = null;
-        return {
-            shape: this.options.shape,
-            batchType: 'data',
-            data: rows,
-            length: this.length,
-            schema: this.schema,
-            cursor: this.cursor
-        };
-    }
-}
-exports.default = RowTableBatchAggregator;

package/dist/lib/batches/table-batch-aggregator.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lib/batches/table-batch-builder.js

@@ -1,153 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const base_table_batch_aggregator_1 = __importDefault(require("./base-table-batch-aggregator"));
-const row_table_batch_aggregator_1 = __importDefault(require("./row-table-batch-aggregator"));
-const columnar_table_batch_aggregator_1 = __importDefault(require("./columnar-table-batch-aggregator"));
-const DEFAULT_OPTIONS = {
-    shape: 'array-row-table',
-    batchSize: 'auto',
-    batchDebounceMs: 0,
-    limit: 0,
-    _limitMB: 0
-};
-const ERR_MESSAGE = 'TableBatchBuilder';
-/** Incrementally builds batches from a stream of rows */
-class TableBatchBuilder {
-    constructor(schema, options) {
-        this.aggregator = null;
-        this.batchCount = 0;
-        this.bytesUsed = 0;
-        this.isChunkComplete = false;
-        this.lastBatchEmittedMs = Date.now();
-        this.totalLength = 0;
-        this.totalBytes = 0;
-        this.rowBytes = 0;
-        this.schema = schema;
-        this.options = { ...DEFAULT_OPTIONS, ...options };
-    }
-    limitReached() {
-        if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
-            return true;
-        }
-        if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
-            return true;
-        }
-        return false;
-    }
-    /** @deprecated Use addArrayRow or addObjectRow */
-    addRow(row) {
-        if (this.limitReached()) {
-            return;
-        }
-        this.totalLength++;
-        this.rowBytes = this.rowBytes || this._estimateRowMB(row);
-        this.totalBytes += this.rowBytes;
-        if (Array.isArray(row)) {
-            this.addArrayRow(row);
-        }
-        else {
-            this.addObjectRow(row);
-        }
-    }
-    /** Add one row to the batch */
-    addArrayRow(row) {
-        if (!this.aggregator) {
-            const TableBatchType = this._getTableBatchType();
-            this.aggregator = new TableBatchType(this.schema, this.options);
-        }
-        this.aggregator.addArrayRow(row);
-    }
-    /** Add one row to the batch */
-    addObjectRow(row) {
-        if (!this.aggregator) {
-            const TableBatchType = this._getTableBatchType();
-            this.aggregator = new TableBatchType(this.schema, this.options);
-        }
-        this.aggregator.addObjectRow(row);
-    }
-    /** Mark an incoming raw memory chunk has completed */
-    chunkComplete(chunk) {
-        if (chunk instanceof ArrayBuffer) {
-            this.bytesUsed += chunk.byteLength;
-        }
-        if (typeof chunk === 'string') {
-            this.bytesUsed += chunk.length;
-        }
-        this.isChunkComplete = true;
-    }
-    getFullBatch(options) {
-        return this._isFull() ? this._getBatch(options) : null;
-    }
-    getFinalBatch(options) {
-        return this._getBatch(options);
-    }
-    // INTERNAL
-    _estimateRowMB(row) {
-        return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
-    }
-    _isFull() {
-        // No batch, not ready
-        if (!this.aggregator || this.aggregator.rowCount() === 0) {
-            return false;
-        }
-        // if batchSize === 'auto' we wait for chunk to complete
-        // if batchSize === number, ensure we have enough rows
-        if (this.options.batchSize === 'auto') {
-            if (!this.isChunkComplete) {
-                return false;
-            }
-        }
-        else if (this.options.batchSize > this.aggregator.rowCount()) {
-            return false;
-        }
-        // Debounce batches
-        if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
-            return false;
-        }
-        // Emit batch
-        this.isChunkComplete = false;
-        this.lastBatchEmittedMs = Date.now();
-        return true;
-    }
-    /**
-     * bytesUsed can be set via chunkComplete or via getBatch*
-     */
-    _getBatch(options) {
-        if (!this.aggregator) {
-            return null;
-        }
-        // TODO - this can overly increment bytes used?
-        if (options?.bytesUsed) {
-            this.bytesUsed = options.bytesUsed;
-        }
-        const normalizedBatch = this.aggregator.getBatch();
-        normalizedBatch.count = this.batchCount;
-        normalizedBatch.bytesUsed = this.bytesUsed;
-        Object.assign(normalizedBatch, options);
-        this.batchCount++;
-        this.aggregator = null;
-        return normalizedBatch;
-    }
-    _getTableBatchType() {
-        switch (this.options.shape) {
-            case 'row-table':
-                return base_table_batch_aggregator_1.default;
-            case 'array-row-table':
-            case 'object-row-table':
-                return row_table_batch_aggregator_1.default;
-            case 'columnar-table':
-                return columnar_table_batch_aggregator_1.default;
-            case 'arrow-table':
-                if (!TableBatchBuilder.ArrowBatch) {
-                    throw new Error(ERR_MESSAGE);
-                }
-                return TableBatchBuilder.ArrowBatch;
-            default:
-                throw new Error(ERR_MESSAGE);
-        }
-    }
-}
-exports.default = TableBatchBuilder;

package/dist/lib/schema/impl/enum.js

@@ -1,97 +0,0 @@
-"use strict";
-// This code is adapted from ArrowJS https://github.com/apache/arrow
-// under Apache license http://www.apache.org/licenses/LICENSE-2.0
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Type = void 0;
-/**
- * Main data type enumeration.
- *
- * Data types in this library are all *logical*. They can be expressed as
- * either a primitive physical type (bytes or bits of some fixed size), a
- * nested type consisting of other data types, or another data type (e.g. a
- * timestamp encoded as an int64).
- *
- * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
- * IPC payload.
- *
- * The rest of the values are specified here so TypeScript can narrow the type
- * signatures further beyond the base Arrow Types. The Arrow DataTypes include
- * metadata like `bitWidth` that impact the type signatures of the values we
- * accept and return.
- *
- * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
- * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
- * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
- * underlying `Int32Array`.
- *
- * Library consumers benefit by knowing the narrowest type, since we can ensure
- * the types across all public methods are propagated, and never bail to `any`.
- * These values are _never_ used at runtime, and they will _never_ be written
- * to the flatbuffers metadata of serialized Arrow IPC payloads.
- */
-var Type;
-(function (Type) {
-    /** The default placeholder type */
-    Type[Type["NONE"] = 0] = "NONE";
-    /** A NULL type having no physical storage */
-    Type[Type["Null"] = 1] = "Null";
-    /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
-    Type[Type["Int"] = 2] = "Int";
-    /** 2, 4, or 8-byte floating point value */
-    Type[Type["Float"] = 3] = "Float";
-    /** Variable-length bytes (no guarantee of UTF8-ness) */
-    Type[Type["Binary"] = 4] = "Binary";
-    /** UTF8 variable-length string as List<Char> */
-    Type[Type["Utf8"] = 5] = "Utf8";
-    /** Boolean as 1 bit, LSB bit-packed ordering */
-    Type[Type["Bool"] = 6] = "Bool";
-    /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
-    Type[Type["Decimal"] = 7] = "Decimal";
-    /** int32_t days or int64_t milliseconds since the UNIX epoch */
-    Type[Type["Date"] = 8] = "Date";
-    /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
-    Type[Type["Time"] = 9] = "Time";
-    /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
-    Type[Type["Timestamp"] = 10] = "Timestamp";
-    /** YEAR_MONTH or DAY_TIME interval in SQL style */
-    Type[Type["Interval"] = 11] = "Interval";
-    /** A list of some logical data type */
-    Type[Type["List"] = 12] = "List";
-    /** Struct of logical types */
-    Type[Type["Struct"] = 13] = "Struct";
-    /** Union of logical types */
-    Type[Type["Union"] = 14] = "Union";
-    /** Fixed-size binary. Each value occupies the same number of bytes */
-    Type[Type["FixedSizeBinary"] = 15] = "FixedSizeBinary";
-    /** Fixed-size list. Each value occupies the same number of bytes */
-    Type[Type["FixedSizeList"] = 16] = "FixedSizeList";
-    /** Map of named logical types */
-    Type[Type["Map"] = 17] = "Map";
-    /** Dictionary aka Category type */
-    Type[Type["Dictionary"] = -1] = "Dictionary";
-    Type[Type["Int8"] = -2] = "Int8";
-    Type[Type["Int16"] = -3] = "Int16";
-    Type[Type["Int32"] = -4] = "Int32";
-    Type[Type["Int64"] = -5] = "Int64";
-    Type[Type["Uint8"] = -6] = "Uint8";
-    Type[Type["Uint16"] = -7] = "Uint16";
-    Type[Type["Uint32"] = -8] = "Uint32";
-    Type[Type["Uint64"] = -9] = "Uint64";
-    Type[Type["Float16"] = -10] = "Float16";
-    Type[Type["Float32"] = -11] = "Float32";
-    Type[Type["Float64"] = -12] = "Float64";
-    Type[Type["DateDay"] = -13] = "DateDay";
-    Type[Type["DateMillisecond"] = -14] = "DateMillisecond";
-    Type[Type["TimestampSecond"] = -15] = "TimestampSecond";
-    Type[Type["TimestampMillisecond"] = -16] = "TimestampMillisecond";
-    Type[Type["TimestampMicrosecond"] = -17] = "TimestampMicrosecond";
-    Type[Type["TimestampNanosecond"] = -18] = "TimestampNanosecond";
-    Type[Type["TimeSecond"] = -19] = "TimeSecond";
-    Type[Type["TimeMillisecond"] = -20] = "TimeMillisecond";
-    Type[Type["TimeMicrosecond"] = -21] = "TimeMicrosecond";
-    Type[Type["TimeNanosecond"] = -22] = "TimeNanosecond";
-    Type[Type["DenseUnion"] = -23] = "DenseUnion";
-    Type[Type["SparseUnion"] = -24] = "SparseUnion";
-    Type[Type["IntervalDayTime"] = -25] = "IntervalDayTime";
-    Type[Type["IntervalYearMonth"] = -26] = "IntervalYearMonth";
-})(Type = exports.Type || (exports.Type = {}));

package/dist/lib/schema/impl/field.js

@@ -1,32 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-/**
- * ArrowJS `Field` API-compatible class for row-based tables
- * https://loaders.gl/arrowjs/docs/api-reference/field
- * A field holds name, nullable, and metadata information about a table "column"
- * A Schema is essentially a list of fields
- */
-class Field {
-    constructor(name, type, nullable = false, metadata = new Map()) {
-        this.name = name;
-        this.type = type;
-        this.nullable = nullable;
-        this.metadata = metadata;
-    }
-    get typeId() {
-        return this.type && this.type.typeId;
-    }
-    clone() {
-        return new Field(this.name, this.type, this.nullable, this.metadata);
-    }
-    compareTo(other) {
-        return (this.name === other.name &&
-            this.type === other.type &&
-            this.nullable === other.nullable &&
-            this.metadata === other.metadata);
-    }
-    toString() {
-        return `${this.type}${this.nullable ? ', nullable' : ''}${this.metadata ? `, metadata: ${this.metadata}` : ''}`;
-    }
-}
-exports.default = Field;

package/dist/lib/schema/impl/schema.js

@@ -1,83 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = require("../../utils/assert");
-/**
- * ArrowJS `Schema` API-compatible class for row-based tables (returned from `DataTable`)
- * https://loaders.gl/arrowjs/docs/api-reference/schema
- */
-class Schema {
-    constructor(fields, metadata) {
-        (0, assert_1.assert)(Array.isArray(fields));
-        checkNames(fields);
-        // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
-        this.fields = fields;
-        this.metadata = metadata || new Map();
-    }
-    // TODO - arrow only seems to compare fields, not metadata
-    compareTo(other) {
-        if (this.metadata !== other.metadata) {
-            return false;
-        }
-        if (this.fields.length !== other.fields.length) {
-            return false;
-        }
-        for (let i = 0; i < this.fields.length; ++i) {
-            if (!this.fields[i].compareTo(other.fields[i])) {
-                return false;
-            }
-        }
-        return true;
-    }
-    select(...columnNames) {
-        // Ensure column names reference valid fields
-        const nameMap = Object.create(null);
-        for (const name of columnNames) {
-            nameMap[name] = true;
-        }
-        const selectedFields = this.fields.filter((field) => nameMap[field.name]);
-        return new Schema(selectedFields, this.metadata);
-    }
-    selectAt(...columnIndices) {
-        // Ensure column indices reference valid fields
-        const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
-        return new Schema(selectedFields, this.metadata);
-    }
-    assign(schemaOrFields) {
-        let fields;
-        let metadata = this.metadata;
-        if (schemaOrFields instanceof Schema) {
-            const otherSchema = schemaOrFields;
-            fields = otherSchema.fields;
-            metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherSchema.metadata);
-        }
-        else {
-            fields = schemaOrFields;
-        }
-        // Create a merged list of fields, overwrite fields in place, new fields at end
-        const fieldMap = Object.create(null);
-        for (const field of this.fields) {
-            fieldMap[field.name] = field;
-        }
-        for (const field of fields) {
-            fieldMap[field.name] = field;
-        }
-        const mergedFields = Object.values(fieldMap);
-        return new Schema(mergedFields, metadata);
-    }
-}
-exports.default = Schema;
-// Warn if any duplicated field names
-function checkNames(fields) {
-    const usedNames = {};
-    for (const field of fields) {
-        if (usedNames[field.name]) {
-            // eslint-disable-next-line
-            console.warn('Schema: duplicated field name', field.name, field);
-        }
-        usedNames[field.name] = true;
-    }
-}
-function mergeMaps(m1, m2) {
-    // @ts-ignore
-    return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
-}