@loaders.gl/schema-utils 4.4.0-alpha.1

package/dist/lib/table/arrow-api/enum.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enum.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/enum.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,oBAAY,IAAI;IACd,mCAAmC;IACnC,IAAI,IAAI;IACR,6CAA6C;IAC7C,IAAI,IAAI;IACR,oEAAoE;IACpE,GAAG,IAAI;IACP,2CAA2C;IAC3C,KAAK,IAAI;IACT,wDAAwD;IACxD,MAAM,IAAI;IACV,gDAAgD;IAChD,IAAI,IAAI;IACR,gDAAgD;IAChD,IAAI,IAAI;IACR,sFAAsF;IACtF,OAAO,IAAI;IACX,gEAAgE;IAChE,IAAI,IAAI;IACR,iJAAiJ;IACjJ,IAAI,IAAI;IACR,qFAAqF;IACrF,SAAS,KAAK;IACd,mDAAmD;IACnD,QAAQ,KAAK;IACb,uCAAuC;IACvC,IAAI,KAAK;IACT,8BAA8B;IAC9B,MAAM,KAAK;IACX,6BAA6B;IAC7B,KAAK,KAAK;IACV,sEAAsE;IACtE,eAAe,KAAK;IACpB,oEAAoE;IACpE,aAAa,KAAK;IAClB,iCAAiC;IACjC,GAAG,KAAK;IAER,mCAAmC;IACnC,UAAU,KAAK;IACf,IAAI,KAAK;IACT,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,eAAe,MAAM;IACrB,eAAe,MAAM;IACrB,oBAAoB,MAAM;IAC1B,oBAAoB,MAAM;IAC1B,mBAAmB,MAAM;IACzB,UAAU,MAAM;IAChB,eAAe,MAAM;IACrB,eAAe,MAAM;IACrB,cAAc,MAAM;IACpB,UAAU,MAAM;IAChB,WAAW,MAAM;IACjB,eAAe,MAAM;IACrB,iBAAiB,MAAM;CACxB"}

package/dist/lib/table/arrow-api/enum.js

@@ -0,0 +1,97 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+// This code is adapted from ArrowJS https://github.com/apache/arrow
+// under Apache license http://www.apache.org/licenses/LICENSE-2.0
+/**
+ * Main data type enumeration.
+ *
+ * Data types in this library are all *logical*. They can be expressed as
+ * either a primitive physical type (bytes or bits of some fixed size), a
+ * nested type consisting of other data types, or another data type (e.g. a
+ * timestamp encoded as an int64).
+ *
+ * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
+ * IPC payload.
+ *
+ * The rest of the values are specified here so TypeScript can narrow the type
+ * signatures further beyond the base Arrow Types. The Arrow DataTypes include
+ * metadata like `bitWidth` that impact the type signatures of the values we
+ * accept and return.
+ *
+ * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
+ * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
+ * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
+ * underlying `Int32Array`.
+ *
+ * Library consumers benefit by knowing the narrowest type, since we can ensure
+ * the types across all public methods are propagated, and never bail to `any`.
+ * These values are _never_ used at runtime, and they will _never_ be written
+ * to the flatbuffers metadata of serialized Arrow IPC payloads.
+ */
+export var Type;
+(function (Type) {
+    /** The default placeholder type */
+    Type[Type["NONE"] = 0] = "NONE";
+    /** A NULL type having no physical storage */
+    Type[Type["Null"] = 1] = "Null";
+    /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
+    Type[Type["Int"] = 2] = "Int";
+    /** 2, 4, or 8-byte floating point value */
+    Type[Type["Float"] = 3] = "Float";
+    /** Variable-length bytes (no guarantee of UTF8-ness) */
+    Type[Type["Binary"] = 4] = "Binary";
+    /** UTF8 variable-length string as List<Char> */
+    Type[Type["Utf8"] = 5] = "Utf8";
+    /** Boolean as 1 bit, LSB bit-packed ordering */
+    Type[Type["Bool"] = 6] = "Bool";
+    /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
+    Type[Type["Decimal"] = 7] = "Decimal";
+    /** int32_t days or int64_t milliseconds since the UNIX epoch */
+    Type[Type["Date"] = 8] = "Date";
+    /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
+    Type[Type["Time"] = 9] = "Time";
+    /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
+    Type[Type["Timestamp"] = 10] = "Timestamp";
+    /** YEAR_MONTH or DAY_TIME interval in SQL style */
+    Type[Type["Interval"] = 11] = "Interval";
+    /** A list of some logical data type */
+    Type[Type["List"] = 12] = "List";
+    /** Struct of logical types */
+    Type[Type["Struct"] = 13] = "Struct";
+    /** Union of logical types */
+    Type[Type["Union"] = 14] = "Union";
+    /** Fixed-size binary. Each value occupies the same number of bytes */
+    Type[Type["FixedSizeBinary"] = 15] = "FixedSizeBinary";
+    /** Fixed-size list. Each value occupies the same number of bytes */
+    Type[Type["FixedSizeList"] = 16] = "FixedSizeList";
+    /** Map of named logical types */
+    Type[Type["Map"] = 17] = "Map";
+    /** Dictionary aka Category type */
+    Type[Type["Dictionary"] = -1] = "Dictionary";
+    Type[Type["Int8"] = -2] = "Int8";
+    Type[Type["Int16"] = -3] = "Int16";
+    Type[Type["Int32"] = -4] = "Int32";
+    Type[Type["Int64"] = -5] = "Int64";
+    Type[Type["Uint8"] = -6] = "Uint8";
+    Type[Type["Uint16"] = -7] = "Uint16";
+    Type[Type["Uint32"] = -8] = "Uint32";
+    Type[Type["Uint64"] = -9] = "Uint64";
+    Type[Type["Float16"] = -10] = "Float16";
+    Type[Type["Float32"] = -11] = "Float32";
+    Type[Type["Float64"] = -12] = "Float64";
+    Type[Type["DateDay"] = -13] = "DateDay";
+    Type[Type["DateMillisecond"] = -14] = "DateMillisecond";
+    Type[Type["TimestampSecond"] = -15] = "TimestampSecond";
+    Type[Type["TimestampMillisecond"] = -16] = "TimestampMillisecond";
+    Type[Type["TimestampMicrosecond"] = -17] = "TimestampMicrosecond";
+    Type[Type["TimestampNanosecond"] = -18] = "TimestampNanosecond";
+    Type[Type["TimeSecond"] = -19] = "TimeSecond";
+    Type[Type["TimeMillisecond"] = -20] = "TimeMillisecond";
+    Type[Type["TimeMicrosecond"] = -21] = "TimeMicrosecond";
+    Type[Type["TimeNanosecond"] = -22] = "TimeNanosecond";
+    Type[Type["DenseUnion"] = -23] = "DenseUnion";
+    Type[Type["SparseUnion"] = -24] = "SparseUnion";
+    Type[Type["IntervalDayTime"] = -25] = "IntervalDayTime";
+    Type[Type["IntervalYearMonth"] = -26] = "IntervalYearMonth";
+})(Type || (Type = {}));

package/dist/lib/table/arrow-api/get-type-info.d.ts

@@ -0,0 +1,13 @@
+import { Type } from "./arrow-like-type.js";
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export declare function getTypeInfo(arrowTypeLike: any): {
+    typeId: Type;
+    ArrayType: ArrayLike<unknown>;
+    typeName: string;
+    typeEnumName?: string;
+    precision?: number;
+};
+//# sourceMappingURL=get-type-info.d.ts.map

package/dist/lib/table/arrow-api/get-type-info.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-type-info.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/get-type-info.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,IAAI,EAAC,6BAA0B;AAEvC;;;GAGG;AACH,wBAAgB,WAAW,CAAC,aAAa,EAAE,GAAG,GAAG;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAQA"}

package/dist/lib/table/arrow-api/get-type-info.js

@@ -0,0 +1,27 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { Type } from "./arrow-like-type.js";
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export function getTypeInfo(arrowTypeLike) {
+    return {
+        typeId: arrowTypeLike.typeId,
+        ArrayType: arrowTypeLike.ArrayType,
+        typeName: arrowTypeLike.toString(),
+        typeEnumName: getTypeKey(arrowTypeLike.typeId),
+        precision: arrowTypeLike.precision
+    };
+}
+let ReverseType = null;
+function getTypeKey(typeKey) {
+    if (!ReverseType) {
+        ReverseType = {};
+        for (const key in Type) {
+            ReverseType[Type[key]] = key;
+        }
+    }
+    return ReverseType[typeKey];
+}

package/dist/lib/table/arrow-api/index.d.ts

@@ -0,0 +1,5 @@
+export { ArrowLikeField as Field } from "./arrow-like-field.js";
+export { ArrowLikeSchema as Schema } from "./arrow-like-schema.js";
+export { ArrowLikeTable as Table } from "./arrow-like-table.js";
+export * from "./arrow-like-type.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/table/arrow-api/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,OAAO,EAAC,eAAe,IAAI,MAAM,EAAC,+BAA4B;AAC9D,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,qCAAkC"}

package/dist/lib/table/arrow-api/index.js

@@ -0,0 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+export { ArrowLikeField as Field } from "./arrow-like-field.js";
+export { ArrowLikeSchema as Schema } from "./arrow-like-schema.js";
+export { ArrowLikeTable as Table } from "./arrow-like-table.js";
+export * from "./arrow-like-type.js";

package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts

@@ -0,0 +1,10 @@
+import type { ArrowTableBatch } from '@loaders.gl/schema';
+import { ColumnarTableBatchAggregator } from "./columnar-table-batch-aggregator.js";
+import * as arrow from 'apache-arrow';
+/** @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency */
+export declare class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+    arrowSchema: arrow.Schema | null;
+    constructor(schema: any, options: any);
+    getBatch(): ArrowTableBatch | null;
+}
+//# sourceMappingURL=arrow-table-batch-aggregator.d.ts.map

package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/arrow-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAC,4BAA4B,EAAC,6CAA0C;AAC/E,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,gGAAgG;AAChG,qBAAa,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;gBAErB,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}

package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.js

@@ -0,0 +1,67 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { ColumnarTableBatchAggregator } from "./columnar-table-batch-aggregator.js";
+import * as arrow from 'apache-arrow';
+/** @note Should be part of @loaders.gl/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+    arrowSchema;
+    constructor(schema, options) {
+        super(schema, options);
+        this.arrowSchema = null;
+    }
+    getBatch() {
+        const batch = super.getBatch();
+        if (batch) {
+            // Get the arrow schema
+            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+            // Get arrow format vectors
+            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+            // Create the record batch
+            const recordBatch = new arrow.RecordBatch(this.arrowSchema, arrow.makeData({
+                type: new arrow.Struct(this.arrowSchema.fields),
+                children: arrowVectors.map(({ data }) => data[0])
+            }));
+            return {
+                shape: 'arrow-table',
+                batchType: 'data',
+                data: new arrow.Table([recordBatch]),
+                length: batch.length
+            };
+        }
+        return null;
+    }
+}
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema) {
+    const arrowFields = [];
+    for (const key in schema) {
+        const field = schema[key];
+        if (field.type === Float32Array) {
+            // TODO - just store the original field as metadata?
+            const metadata = new Map(); // field;
+            // arrow: new arrow.Field(name, nullable, metadata)
+            const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+            arrowFields.push(arrowField);
+        }
+    }
+    if (arrowFields.length === 0) {
+        throw new Error('No arrow convertible fields');
+    }
+    return new arrow.Schema(arrowFields);
+}
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data) {
+    const arrowVectors = [];
+    for (const field of arrowSchema.fields) {
+        const vector = data[field.name];
+        if (vector instanceof Float32Array) {
+            const arrowVector = arrow.makeVector(vector);
+            arrowVectors.push(arrowVector);
+        }
+    }
+    if (arrowSchema.fields.length !== arrowVectors.length) {
+        throw new Error('Some columns not arrow convertible');
+    }
+    return arrowVectors;
+}

package/dist/lib/table/batch-builder/arrow-table-builder.d.ts

@@ -0,0 +1,25 @@
+import type { Schema, ArrowTable, ArrowTableBatch } from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+/** Builds an arrow table or batches */
+export declare class ArrowTableBuilder {
+    schema: Schema;
+    arrowSchema: arrow.Schema;
+    arrowBuilders: arrow.Builder[];
+    length: number;
+    constructor(schema: Schema);
+    addObjectRow(row: {
+        [key: string]: any;
+    }): void;
+    addArrayRow(row: any[]): void;
+    /** Makes sure that a first batch with schema is sent even if no rows */
+    firstBatch(): ArrowTableBatch | null;
+    /** Flush the current batch if conditions are right */
+    flushBatch(): ArrowTableBatch | null;
+    /** Get a last batch if any data is left */
+    finishBatch(): ArrowTableBatch | null;
+    /** Return a table with all the accumulated data */
+    finishTable(): ArrowTable;
+    /** Extract a record batch flushing the currently accumulated data in the builders */
+    _getArrowRecordBatch(): arrow.RecordBatch;
+}
+//# sourceMappingURL=arrow-table-builder.d.ts.map

package/dist/lib/table/batch-builder/arrow-table-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-builder.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/arrow-table-builder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAGtC,uCAAuC;AACvC,qBAAa,iBAAiB;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC;IAC1B,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;gBAEH,MAAM,EAAE,MAAM;IAS1B,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC;IAYtC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAOtB,wEAAwE;IACxE,UAAU,IAAI,eAAe,GAAG,IAAI;IAepC,sDAAsD;IACtD,UAAU,IAAI,eAAe,GAAG,IAAI;IAcpC,2CAA2C;IAC3C,WAAW,IAAI,eAAe,GAAG,IAAI;IAerC,mDAAmD;IACnD,WAAW,IAAI,UAAU;IAUzB,qFAAqF;IACrF,oBAAoB,IAAI,KAAK,CAAC,WAAW;CAS1C"}

package/dist/lib/table/batch-builder/arrow-table-builder.js

@@ -0,0 +1,99 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import * as arrow from 'apache-arrow';
+import { convertSchemaToArrow } from '@loaders.gl/schema-utils';
+/** Builds an arrow table or batches */
+export class ArrowTableBuilder {
+    schema;
+    arrowSchema;
+    arrowBuilders;
+    length;
+    constructor(schema) {
+        this.schema = schema;
+        this.arrowSchema = convertSchemaToArrow(schema);
+        this.arrowBuilders = this.arrowSchema.fields.map((field) => arrow.makeBuilder({ type: field.type, nullValues: [null] }));
+        this.length = 0;
+    }
+    addObjectRow(row) {
+        for (let i = 0; i < this.arrowBuilders.length; i++) {
+            const columnName = this.schema.fields[i].name;
+            const value = row[columnName];
+            // if (this.schema.fields[i].type.toString() === 'bool') {
+            //   debugger;
+            // }
+            this.arrowBuilders[i].append(value);
+        }
+        this.length++;
+    }
+    addArrayRow(row) {
+        for (let i = 0; i < this.arrowBuilders.length; i++) {
+            this.arrowBuilders[i].append(row[i]);
+        }
+        this.length++;
+    }
+    /** Makes sure that a first batch with schema is sent even if no rows */
+    firstBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        // If there is data, a batch will be sent later
+        if (arrowRecordBatch.numCols !== 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Flush the current batch if conditions are right */
+    flushBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        if (arrowRecordBatch.numCols === 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Get a last batch if any data is left */
+    finishBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        this.arrowBuilders.forEach((builder) => builder.finish());
+        if (arrowRecordBatch.numCols === 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Return a table with all the accumulated data */
+    finishTable() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        this.arrowBuilders.forEach((builder) => builder.finish());
+        return {
+            shape: 'arrow-table',
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Extract a record batch flushing the currently accumulated data in the builders */
+    _getArrowRecordBatch() {
+        const { arrowBuilders, arrowSchema } = this;
+        const arrowDatas = arrowBuilders.map((builder) => builder.flush());
+        const length = arrowDatas[0].length;
+        const structField = new arrow.Struct(arrowSchema.fields);
+        const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
+        const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);
+        return arrowRecordBatch;
+    }
+}

package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts

@@ -0,0 +1,19 @@
+import type { Schema, TableBatch } from '@loaders.gl/schema';
+import { TableBatchAggregator, TableBatchOptions } from "./table-batch-aggregator.js";
+export declare class BaseTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    options: TableBatchOptions;
+    shape?: 'array-row-table' | 'object-row-table';
+    length: number;
+    rows: any[] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=base-table-batch-aggregator.d.ts.map

package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/base-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,oCAAiC;AAIjF,qBAAa,wBAAyB,YAAW,oBAAoB;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAE3B,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;IAC/C,MAAM,EAAE,MAAM,CAAK;IACnB,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAQ;IAC1B,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IActD,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAY9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAYrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAoB9B"}

package/dist/lib/table/batch-builder/base-table-batch-aggregator.js

@@ -0,0 +1,63 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+const DEFAULT_ROW_COUNT = 100;
+export class BaseTableBatchAggregator {
+    schema;
+    options;
+    shape;
+    length = 0;
+    rows = null;
+    cursor = 0;
+    _headers = [];
+    constructor(schema, options) {
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (!Array.isArray(schema)) {
+            this._headers = [];
+            for (const key in schema) {
+                this._headers[schema[key].index] = schema[key].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.shape = 'array-row-table';
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.shape = 'object-row-table';
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    getBatch() {
+        let rows = this.rows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.rows = null;
+        const batch = {
+            shape: this.shape || 'array-row-table',
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            schema: this.schema,
+            cursor: this.cursor
+        };
+        return batch;
+    }
+}

package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts

@@ -0,0 +1,20 @@
+import type { Schema, ColumnarTableBatch, ArrowTableBatch, TypedArray } from '@loaders.gl/schema';
+import { TableBatchAggregator } from "./table-batch-aggregator.js";
+type ColumnarTableBatchOptions = {};
+export declare class ColumnarTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    length: number;
+    allocated: number;
+    columns: Record<string, TypedArray | Array<any>>;
+    constructor(schema: Schema, options: ColumnarTableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[]): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    getBatch(): ColumnarTableBatch | ArrowTableBatch | null;
+    _reallocateColumns(): void;
+    _pruneColumns(): void;
+}
+export {};
+//# sourceMappingURL=columnar-table-batch-aggregator.d.ts.map

package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"columnar-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/columnar-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAGhG,OAAO,EAAC,oBAAoB,EAAC,oCAAiC;AAC9D,KAAK,yBAAyB,GAAG,EAAE,CAAC;AAIpC,qBAAa,4BAA6B,YAAW,oBAAoB;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAK;IACnB,SAAS,EAAE,MAAM,CAAK;IACtB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM;gBAE1C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,yBAAyB;IAK9D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAWtB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IASpD,QAAQ,IAAI,kBAAkB,GAAG,eAAe,GAAG,IAAI;IAgBvD,kBAAkB;IA8BlB,aAAa;CAKd"}

package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.js

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { isTypedArray } from '@math.gl/types';
+import { getArrayTypeFromDataType } from "../../schema/data-type.js";
+const DEFAULT_ROW_COUNT = 100;
+export class ColumnarTableBatchAggregator {
+    schema;
+    length = 0;
+    allocated = 0;
+    columns = {};
+    constructor(schema, options) {
+        this.schema = schema;
+        this._reallocateColumns();
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        let i = 0;
+        // TODO what if no csv header, columns not populated?
+        for (const fieldName in this.columns) {
+            this.columns[fieldName][this.length] = row[i++];
+        }
+        this.length++;
+    }
+    addObjectRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        for (const fieldName in row) {
+            this.columns[fieldName][this.length] = row[fieldName];
+        }
+        this.length++;
+    }
+    getBatch() {
+        this._pruneColumns();
+        const batch = {
+            shape: 'columnar-table',
+            batchType: 'data',
+            data: this.columns,
+            schema: this.schema,
+            length: this.length
+        };
+        return batch;
+    }
+    // HELPERS
+    _reallocateColumns() {
+        if (this.length < this.allocated) {
+            return;
+        }
+        // @ts-ignore TODO
+        this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
+        this.columns = {};
+        for (const field of this.schema.fields) {
+            const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+            const oldColumn = this.columns[field.name];
+            if (!oldColumn) {
+                // Create new
+                this.columns[field.name] = new ArrayType(this.allocated);
+            }
+            else if (Array.isArray(oldColumn)) {
+                // Plain array, just increase its size
+                oldColumn.length = this.allocated;
+            }
+            else if (isTypedArray(oldColumn)) {
+                const typedArray = new ArrayType(this.allocated);
+                // Copy the old data to the new array
+                typedArray.set(oldColumn);
+                this.columns[field.name] = typedArray;
+            }
+            else {
+                throw new Error('Invalid column type');
+            }
+        }
+    }
+    _pruneColumns() {
+        for (const [columnName, column] of Object.entries(this.columns)) {
+            this.columns[columnName] = column.slice(0, this.length);
+        }
+    }
+}

package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts

@@ -0,0 +1,21 @@
+import type { Schema, TableBatch } from '@loaders.gl/schema';
+import { TableBatchAggregator, TableBatchOptions } from "./table-batch-aggregator.js";
+export declare class RowTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema | null;
+    options: TableBatchOptions;
+    length: number;
+    objectRows: {
+        [columnName: string]: unknown;
+    }[] | null;
+    arrayRows: unknown[][] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema | null, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=row-table-batch-aggregator.d.ts.map

package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/row-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,oCAAiC;AAUjF,qBAAa,uBAAwB,YAAW,oBAAoB;IAClE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,iBAAiB,CAAC;IAE3B,MAAM,EAAE,MAAM,CAAK;IACnB,UAAU,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;KAAC,EAAE,GAAG,IAAI,CAAQ;IAC5D,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,IAAI,CAAQ;IACrC,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAyB;gBAE7B,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,iBAAiB;IAc7D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAsB9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAsBrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAoB9B"}