@loaders.gl/schema 4.0.0-alpha.22 → 4.0.0-alpha.24

package/dist/lib/table/batches/columnar-table-batch-aggregator.js

@@ -1,92 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ColumnarTableBatchAggregator = void 0;
-const DEFAULT_ROW_COUNT = 100;
-class ColumnarTableBatchAggregator {
-    constructor(schema, options) {
-        this.length = 0;
-        this.allocated = 0;
-        this.columns = {};
-        this.schema = schema;
-        this._reallocateColumns();
-    }
-    rowCount() {
-        return this.length;
-    }
-    addArrayRow(row) {
-        // If user keeps pushing rows beyond batch size, reallocate
-        this._reallocateColumns();
-        let i = 0;
-        // TODO what if no csv header, columns not populated?
-        for (const fieldName in this.columns) {
-            this.columns[fieldName][this.length] = row[i++];
-        }
-        this.length++;
-    }
-    addObjectRow(row) {
-        // If user keeps pushing rows beyond batch size, reallocate
-        this._reallocateColumns();
-        for (const fieldName in row) {
-            this.columns[fieldName][this.length] = row[fieldName];
-        }
-        this.length++;
-    }
-    getBatch() {
-        this._pruneColumns();
-        const columns = Array.isArray(this.schema) ? this.columns : {};
-        // schema is an array if there're no headers
-        // object if there are headers
-        // columns should match schema format
-        if (!Array.isArray(this.schema)) {
-            for (const fieldName in this.schema) {
-                const field = this.schema[fieldName];
-                columns[field.name] = this.columns[field.index];
-            }
-        }
-        this.columns = {};
-        const batch = {
-            shape: 'columnar-table',
-            batchType: 'data',
-            data: columns,
-            schema: this.schema,
-            length: this.length
-        };
-        return batch;
-    }
-    // HELPERS
-    _reallocateColumns() {
-        if (this.length < this.allocated) {
-            return;
-        }
-        // @ts-ignore TODO
-        this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
-        this.columns = {};
-        for (const fieldName in this.schema) {
-            const field = this.schema[fieldName];
-            const ArrayType = field.type || Float32Array;
-            const oldColumn = this.columns[field.index];
-            if (oldColumn && ArrayBuffer.isView(oldColumn)) {
-                // Copy the old data to the new array
-                const typedArray = new ArrayType(this.allocated);
-                typedArray.set(oldColumn);
-                this.columns[field.index] = typedArray;
-            }
-            else if (oldColumn) {
-                // Plain array
-                oldColumn.length = this.allocated;
-                this.columns[field.index] = oldColumn;
-            }
-            else {
-                // Create new
-                this.columns[field.index] = new ArrayType(this.allocated);
-            }
-        }
-    }
-    _pruneColumns() {
-        for (const [columnName, column] of Object.entries(this.columns)) {
-            this.columns[columnName] = column.slice(0, this.length);
-        }
-    }
-}
-exports.ColumnarTableBatchAggregator = ColumnarTableBatchAggregator;

package/dist/lib/table/batches/row-table-batch-aggregator.js

@@ -1,81 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RowTableBatchAggregator = void 0;
-// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
-const row_utils_1 = require("../simple-table/row-utils");
-const DEFAULT_ROW_COUNT = 100;
-class RowTableBatchAggregator {
-    constructor(schema, options) {
-        this.length = 0;
-        this.objectRows = null;
-        this.arrayRows = null;
-        this.cursor = 0;
-        this._headers = [];
-        this.options = options;
-        this.schema = schema;
-        // schema is an array if there're no headers
-        // object if there are headers
-        if (!Array.isArray(schema)) {
-            this._headers = [];
-            for (const key in schema) {
-                this._headers[schema[key].index] = schema[key].name;
-            }
-        }
-    }
-    rowCount() {
-        return this.length;
-    }
-    addArrayRow(row, cursor) {
-        if (Number.isFinite(cursor)) {
-            this.cursor = cursor;
-        }
-        // eslint-disable-next-line default-case
-        switch (this.options.shape) {
-            case 'object-row-table':
-                const rowObject = (0, row_utils_1.convertToObjectRow)(row, this._headers);
-                this.addObjectRow(rowObject, cursor);
-                break;
-            case 'array-row-table':
-                this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
-                this.arrayRows[this.length] = row;
-                this.length++;
-                break;
-        }
-    }
-    addObjectRow(row, cursor) {
-        if (Number.isFinite(cursor)) {
-            this.cursor = cursor;
-        }
-        // eslint-disable-next-line default-case
-        switch (this.options.shape) {
-            case 'array-row-table':
-                const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
-                this.addArrayRow(rowArray, cursor);
-                break;
-            case 'object-row-table':
-                this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
-                this.objectRows[this.length] = row;
-                this.length++;
-                break;
-        }
-    }
-    getBatch() {
-        let rows = this.arrayRows || this.objectRows;
-        if (!rows) {
-            return null;
-        }
-        rows = rows.slice(0, this.length);
-        this.arrayRows = null;
-        this.objectRows = null;
-        return {
-            shape: this.options.shape,
-            batchType: 'data',
-            data: rows,
-            length: this.length,
-            schema: this.schema,
-            cursor: this.cursor
-        };
-    }
-}
-exports.RowTableBatchAggregator = RowTableBatchAggregator;

package/dist/lib/table/batches/table-batch-aggregator.js

@@ -1,3 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lib/table/batches/table-batch-builder.js

@@ -1,152 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TableBatchBuilder = void 0;
-const base_table_batch_aggregator_1 = require("./base-table-batch-aggregator");
-const row_table_batch_aggregator_1 = require("./row-table-batch-aggregator");
-const columnar_table_batch_aggregator_1 = require("./columnar-table-batch-aggregator");
-const DEFAULT_OPTIONS = {
-    shape: 'array-row-table',
-    batchSize: 'auto',
-    batchDebounceMs: 0,
-    limit: 0,
-    _limitMB: 0
-};
-const ERR_MESSAGE = 'TableBatchBuilder';
-/** Incrementally builds batches from a stream of rows */
-class TableBatchBuilder {
-    constructor(schema, options) {
-        this.aggregator = null;
-        this.batchCount = 0;
-        this.bytesUsed = 0;
-        this.isChunkComplete = false;
-        this.lastBatchEmittedMs = Date.now();
-        this.totalLength = 0;
-        this.totalBytes = 0;
-        this.rowBytes = 0;
-        this.schema = schema;
-        this.options = { ...DEFAULT_OPTIONS, ...options };
-    }
-    limitReached() {
-        if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
-            return true;
-        }
-        if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
-            return true;
-        }
-        return false;
-    }
-    /** @deprecated Use addArrayRow or addObjectRow */
-    addRow(row) {
-        if (this.limitReached()) {
-            return;
-        }
-        this.totalLength++;
-        this.rowBytes = this.rowBytes || this._estimateRowMB(row);
-        this.totalBytes += this.rowBytes;
-        if (Array.isArray(row)) {
-            this.addArrayRow(row);
-        }
-        else {
-            this.addObjectRow(row);
-        }
-    }
-    /** Add one row to the batch */
-    addArrayRow(row) {
-        if (!this.aggregator) {
-            const TableBatchType = this._getTableBatchType();
-            this.aggregator = new TableBatchType(this.schema, this.options);
-        }
-        this.aggregator.addArrayRow(row);
-    }
-    /** Add one row to the batch */
-    addObjectRow(row) {
-        if (!this.aggregator) {
-            const TableBatchType = this._getTableBatchType();
-            this.aggregator = new TableBatchType(this.schema, this.options);
-        }
-        this.aggregator.addObjectRow(row);
-    }
-    /** Mark an incoming raw memory chunk has completed */
-    chunkComplete(chunk) {
-        if (chunk instanceof ArrayBuffer) {
-            this.bytesUsed += chunk.byteLength;
-        }
-        if (typeof chunk === 'string') {
-            this.bytesUsed += chunk.length;
-        }
-        this.isChunkComplete = true;
-    }
-    getFullBatch(options) {
-        return this._isFull() ? this._getBatch(options) : null;
-    }
-    getFinalBatch(options) {
-        return this._getBatch(options);
-    }
-    // INTERNAL
-    _estimateRowMB(row) {
-        return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
-    }
-    _isFull() {
-        // No batch, not ready
-        if (!this.aggregator || this.aggregator.rowCount() === 0) {
-            return false;
-        }
-        // if batchSize === 'auto' we wait for chunk to complete
-        // if batchSize === number, ensure we have enough rows
-        if (this.options.batchSize === 'auto') {
-            if (!this.isChunkComplete) {
-                return false;
-            }
-        }
-        else if (this.options.batchSize > this.aggregator.rowCount()) {
-            return false;
-        }
-        // Debounce batches
-        if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
-            return false;
-        }
-        // Emit batch
-        this.isChunkComplete = false;
-        this.lastBatchEmittedMs = Date.now();
-        return true;
-    }
-    /**
-     * bytesUsed can be set via chunkComplete or via getBatch*
-     */
-    _getBatch(options) {
-        if (!this.aggregator) {
-            return null;
-        }
-        // TODO - this can overly increment bytes used?
-        if (options?.bytesUsed) {
-            this.bytesUsed = options.bytesUsed;
-        }
-        const normalizedBatch = this.aggregator.getBatch();
-        normalizedBatch.count = this.batchCount;
-        normalizedBatch.bytesUsed = this.bytesUsed;
-        Object.assign(normalizedBatch, options);
-        this.batchCount++;
-        this.aggregator = null;
-        return normalizedBatch;
-    }
-    _getTableBatchType() {
-        switch (this.options.shape) {
-            case 'row-table':
-                return base_table_batch_aggregator_1.BaseTableBatchAggregator;
-            case 'array-row-table':
-            case 'object-row-table':
-                return row_table_batch_aggregator_1.RowTableBatchAggregator;
-            case 'columnar-table':
-                return columnar_table_batch_aggregator_1.ColumnarTableBatchAggregator;
-            case 'arrow-table':
-                if (!TableBatchBuilder.ArrowBatch) {
-                    throw new Error(ERR_MESSAGE);
-                }
-                return TableBatchBuilder.ArrowBatch;
-            default:
-                throw new Error(ERR_MESSAGE);
-        }
-    }
-}
-exports.TableBatchBuilder = TableBatchBuilder;

package/dist/lib/table/simple-table/convert-table.js

@@ -1,65 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeObjectRowTable = exports.makeArrayRowTable = exports.makeColumnarTable = void 0;
-// loaders.gl, MIT license
-const table_accessors_1 = require("./table-accessors");
-const table_schema_1 = require("./table-schema");
-const table_column_1 = require("./table-column");
-/** Convert any simple table into columnar format */
-function makeColumnarTable(table) {
-    // TODO - should schema really be optional?
-    const schema = table.schema || (0, table_schema_1.deduceTableSchema)(table);
-    const fields = table.schema?.fields || [];
-    if (table.shape === 'columnar-table') {
-        return { ...table, schema };
-    }
-    const length = (0, table_accessors_1.getTableLength)(table);
-    const columns = {};
-    for (const field of fields) {
-        const column = (0, table_column_1.makeColumnFromField)(field, length);
-        columns[field.name] = column;
-        for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-            column[rowIndex] = (0, table_accessors_1.getTableCell)(table, rowIndex, field.name);
-        }
-    }
-    return {
-        shape: 'columnar-table',
-        schema,
-        data: columns
-    };
-}
-exports.makeColumnarTable = makeColumnarTable;
-/** Convert any table into array row format */
-function makeArrayRowTable(table) {
-    if (table.shape === 'array-row-table') {
-        return table;
-    }
-    const length = (0, table_accessors_1.getTableLength)(table);
-    const data = new Array(length);
-    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-        data[rowIndex] = (0, table_accessors_1.getTableRowAsArray)(table, rowIndex);
-    }
-    return {
-        shape: 'array-row-table',
-        schema: table.schema,
-        data
-    };
-}
-exports.makeArrayRowTable = makeArrayRowTable;
-/** Convert any table into object row format */
-function makeObjectRowTable(table) {
-    if (table.shape === 'object-row-table') {
-        return table;
-    }
-    const length = (0, table_accessors_1.getTableLength)(table);
-    const data = new Array(length);
-    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-        data[rowIndex] = (0, table_accessors_1.getTableRowAsObject)(table, rowIndex);
-    }
-    return {
-        shape: 'object-row-table',
-        schema: table.schema,
-        data
-    };
-}
-exports.makeObjectRowTable = makeObjectRowTable;

package/dist/lib/table/simple-table/data-type.js

@@ -1,93 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getArrayTypeFromDataType = exports.getDataTypeFromTypedArray = exports.getDataTypeFromArray = exports.getDataTypeFromValue = void 0;
-/** Deduce column types from values */
-function getDataTypeFromValue(value, defaultNumberType = 'float32') {
-    if (value instanceof Date) {
-        return 'date-millisecond';
-    }
-    if (value instanceof Number) {
-        return defaultNumberType;
-    }
-    if (typeof value === 'string') {
-        return 'utf8';
-    }
-    if (value === null || value === 'undefined') {
-        return 'null';
-    }
-    return 'null';
-}
-exports.getDataTypeFromValue = getDataTypeFromValue;
-/**
- * Deduces a simple data type "descriptor from a typed array instance
- */
-function getDataTypeFromArray(array) {
-    let type = getDataTypeFromTypedArray(array);
-    if (type !== 'null') {
-        return { type, nullable: false };
-    }
-    if (array.length > 0) {
-        type = getDataTypeFromValue(array[0]);
-        return { type, nullable: true };
-    }
-    return { type: 'null', nullable: true };
-}
-exports.getDataTypeFromArray = getDataTypeFromArray;
-/**
- * Deduces a simple data type "descriptor from a typed array instance
- */
-function getDataTypeFromTypedArray(array) {
-    switch (array.constructor) {
-        case Int8Array:
-            return 'int8';
-        case Uint8Array:
-        case Uint8ClampedArray:
-            return 'uint8';
-        case Int16Array:
-            return 'int16';
-        case Uint16Array:
-            return 'uint16';
-        case Int32Array:
-            return 'int32';
-        case Uint32Array:
-            return 'uint32';
-        case Float32Array:
-            return 'float32';
-        case Float64Array:
-            return 'float64';
-        default:
-            return 'null';
-    }
-}
-exports.getDataTypeFromTypedArray = getDataTypeFromTypedArray;
-function getArrayTypeFromDataType(type, nullable) {
-    if (!nullable) {
-        switch (type) {
-            case 'int8':
-                return Int8Array;
-            case 'uint8':
-                return Uint8Array;
-            case 'int16':
-                return Int16Array;
-            case 'uint16':
-                return Uint16Array;
-            case 'int32':
-                return Int32Array;
-            case 'uint32':
-                return Uint32Array;
-            case 'float32':
-                return Float32Array;
-            case 'float64':
-                return Float64Array;
-            default:
-                break;
-        }
-    }
-    // if (typeof BigInt64Array !== 'undefined') {
-    //   TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
-    //   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
-    // }
-    return Array;
-}
-exports.getArrayTypeFromDataType = getArrayTypeFromDataType;

package/dist/lib/table/simple-table/make-table.js

@@ -1,95 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeTableFromData = void 0;
-const table_schema_1 = require("./table-schema");
-function makeTableFromData(data) {
-    let table;
-    switch (getTableShapeFromData(data)) {
-        case 'array-row-table':
-            table = { shape: 'array-row-table', data: data };
-            break;
-        case 'object-row-table':
-            table = { shape: 'object-row-table', data: data };
-            break;
-        case 'columnar-table':
-            table = { shape: 'columnar-table', data: data };
-            break;
-        default:
-            throw new Error('table');
-    }
-    const schema = (0, table_schema_1.deduceTableSchema)(table);
-    return { ...table, schema };
-}
-exports.makeTableFromData = makeTableFromData;
-/** Helper function to get shape of data */
-function getTableShapeFromData(data) {
-    if (Array.isArray(data)) {
-        if (data.length === 0) {
-            throw new Error('cannot deduce type of empty table');
-        }
-        // Deduce the table shape from the first row
-        const firstRow = data[0];
-        if (Array.isArray(firstRow)) {
-            return 'array-row-table';
-        }
-        if (firstRow && typeof firstRow === 'object') {
-            return 'object-row-table';
-        }
-    }
-    if (data && typeof data === 'object') {
-        return 'columnar-table';
-    }
-    throw new Error('invalid table');
-}
-/** Convert any table into object row format *
-export function makeColumnarTable(table: Table): ColumnarTable {
-  if (table.shape === 'columnar-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<{[key: string]: unknown}>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsObject(table, rowIndex);
-  }
-  return {
-    shape: 'columnar-table',
-    schema: table.schema,
-    data
-  };
-}
-
-
-/** Convert any table into array row format *
-export function makeArrayRowTable(table: TableLike): ArrayRowTable {
-  if (table.shape === 'array-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<unknown[]>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsArray(table, rowIndex);
-  }
-  return {
-    shape: 'array-row-table',
-    schema: table.schema,
-    data
-  };
-}
-
-/** Convert any table into object row format *
-export function makeObjectRowTable(table: Table): ObjectRowTable {
-  if (table.shape === 'object-row-table') {
-    return table;
-  }
-  const length = getTableLength(table);
-  const data = new Array<{[key: string]: unknown}>(length);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    data[rowIndex] = getTableRowAsObject(table, rowIndex);
-  }
-  return {
-    shape: 'object-row-table',
-    schema: table.schema,
-    data
-  };
-}
-*/

package/dist/lib/table/simple-table/row-utils.js

@@ -1,33 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertToArrayRow = exports.convertToObjectRow = void 0;
-/** Convert an object row to an array row */
-function convertToObjectRow(arrayRow, headers) {
-    if (!arrayRow) {
-        throw new Error('null row');
-    }
-    if (!headers) {
-        throw new Error('no headers');
-    }
-    const objectRow = {};
-    for (let i = 0; i < headers.length; i++) {
-        objectRow[headers[i]] = arrayRow[i];
-    }
-    return objectRow;
-}
-exports.convertToObjectRow = convertToObjectRow;
-/** Convert an object row to an array row */
-function convertToArrayRow(objectRow, headers) {
-    if (!objectRow) {
-        throw new Error('null row');
-    }
-    if (!headers) {
-        throw new Error('no headers');
-    }
-    const arrayRow = new Array(headers.length);
-    for (let i = 0; i < headers.length; i++) {
-        arrayRow[i] = objectRow[headers[i]];
-    }
-    return arrayRow;
-}
-exports.convertToArrayRow = convertToArrayRow;