@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/dist/schema-utils/lib/table/arrow-api/get-type-info.js

@@ -0,0 +1,28 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { Type } from "./arrow-like-type.js";
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export function getTypeInfo(arrowTypeLike) {
+    return {
+        typeId: arrowTypeLike.typeId,
+        ArrayType: arrowTypeLike.ArrayType,
+        typeName: arrowTypeLike.toString(),
+        typeEnumName: getTypeKey(arrowTypeLike.typeId),
+        precision: arrowTypeLike.precision
+    };
+}
+let ReverseType = null;
+function getTypeKey(typeKey) {
+    if (!ReverseType) {
+        ReverseType = {};
+        for (const key in Type) {
+            ReverseType[Type[key]] = key;
+        }
+    }
+    return ReverseType[typeKey];
+}
+//# sourceMappingURL=get-type-info.js.map

package/dist/schema-utils/lib/table/arrow-api/get-type-info.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-type-info.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/arrow-api/get-type-info.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,IAAI,EAAC,6BAA0B;AAEvC;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,aAAkB;IAO5C,OAAO;QACL,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;QAClC,YAAY,EAAE,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC;QAC9C,SAAS,EAAE,aAAa,CAAC,SAAS;KACnC,CAAC;AACJ,CAAC;AAED,IAAI,WAAW,GAAmC,IAAI,CAAC;AAEvD,SAAS,UAAU,CAAC,OAAO;IACzB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC"}

package/dist/schema-utils/lib/table/arrow-api/index.d.ts

@@ -0,0 +1,6 @@
+export { ArrowLikeField as Field } from "./arrow-like-field.js";
+export { ArrowLikeSchema as Schema } from "./arrow-like-schema.js";
+export { ArrowLikeTable as Table } from "./arrow-like-table.js";
+export { Type, DataType, Null, Bool, Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, Float, Float16, Float32, Float64, Binary, Utf8, Date, DateDay, DateMillisecond, Time, TimeSecond, TimeMillisecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, Interval, IntervalDayTime, IntervalYearMonth, FixedSizeList, Struct } from "./arrow-like-type.js";
+export type { TypedIntArray, TypedFloatArray, TypedArray, AnyArrayType } from "./arrow-like-type.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/schema-utils/lib/table/arrow-api/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/arrow-api/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,OAAO,EAAC,eAAe,IAAI,MAAM,EAAC,+BAA4B;AAC9D,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,IAAI,EACJ,IAAI,EACJ,GAAG,EACH,IAAI,EACJ,KAAK,EACL,KAAK,EACL,KAAK,EACL,KAAK,EACL,MAAM,EACN,MAAM,EACN,MAAM,EACN,KAAK,EACL,OAAO,EACP,OAAO,EACP,OAAO,EACP,MAAM,EACN,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,eAAe,EACf,IAAI,EACJ,UAAU,EACV,eAAe,EACf,SAAS,EACT,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,QAAQ,EACR,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,MAAM,EACP,6BAA0B;AAC3B,YAAY,EACV,aAAa,EACb,eAAe,EACf,UAAU,EACV,YAAY,EACb,6BAA0B"}

package/dist/schema-utils/lib/table/arrow-api/index.js

@@ -0,0 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+export { ArrowLikeField as Field } from "./arrow-like-field.js";
+export { ArrowLikeSchema as Schema } from "./arrow-like-schema.js";
+export { ArrowLikeTable as Table } from "./arrow-like-table.js";
+export { Type, DataType, Null, Bool, Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, Float, Float16, Float32, Float64, Binary, Utf8, Date, DateDay, DateMillisecond, Time, TimeSecond, TimeMillisecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, Interval, IntervalDayTime, IntervalYearMonth, FixedSizeList, Struct } from "./arrow-like-type.js";
+//# sourceMappingURL=index.js.map

package/dist/schema-utils/lib/table/arrow-api/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/arrow-api/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,OAAO,EAAC,eAAe,IAAI,MAAM,EAAC,+BAA4B;AAC9D,OAAO,EAAC,cAAc,IAAI,KAAK,EAAC,8BAA2B;AAC3D,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,IAAI,EACJ,IAAI,EACJ,GAAG,EACH,IAAI,EACJ,KAAK,EACL,KAAK,EACL,KAAK,EACL,KAAK,EACL,MAAM,EACN,MAAM,EACN,MAAM,EACN,KAAK,EACL,OAAO,EACP,OAAO,EACP,OAAO,EACP,MAAM,EACN,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,eAAe,EACf,IAAI,EACJ,UAAU,EACV,eAAe,EACf,SAAS,EACT,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,QAAQ,EACR,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,MAAM,EACP,6BAA0B"}

package/dist/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts

@@ -0,0 +1,10 @@
+import type { ArrowTableBatch } from '@loaders.gl/core/schema';
+import { ColumnarTableBatchAggregator } from "./columnar-table-batch-aggregator.js";
+import * as arrow from 'apache-arrow';
+/** @note Should be part of @loaders.gl/core/schema - kept separate to avoid hard arrow dependency */
+export declare class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+    arrowSchema: arrow.Schema | null;
+    constructor(schema: any, options: any);
+    getBatch(): ArrowTableBatch | null;
+}
+//# sourceMappingURL=arrow-table-batch-aggregator.d.ts.map

package/dist/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAC,4BAA4B,EAAC,6CAA0C;AAC/E,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,qGAAqG;AACrG,qBAAa,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;gBAErB,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}

package/dist/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.js

@@ -0,0 +1,71 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { ColumnarTableBatchAggregator } from "./columnar-table-batch-aggregator.js";
+import * as arrow from 'apache-arrow';
+/** @note Should be part of @loaders.gl/core/schema - kept separate to avoid hard arrow dependency */
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+    arrowSchema;
+    constructor(schema, options) {
+        super(schema, options);
+        this.arrowSchema = null;
+    }
+    getBatch() {
+        const batch = super.getBatch();
+        if (batch) {
+            // Get the arrow schema
+            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+            // Get arrow format vectors
+            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+            // Create the record batch
+            const recordBatch = new arrow.RecordBatch(this.arrowSchema, arrow.makeData({
+                type: new arrow.Struct(this.arrowSchema.fields),
+                children: arrowVectors.map(({ data }) => data[0])
+            }));
+            return {
+                shape: 'arrow-table',
+                batchType: 'data',
+                data: new arrow.Table([recordBatch]),
+                length: batch.length
+            };
+        }
+        return null;
+    }
+}
+// Convert from a simple loaders.gl schema to an Arrow schema
+function getArrowSchema(schema) {
+    const arrowFields = [];
+    const fields = Array.isArray(schema.fields) ? schema.fields : Object.values(schema);
+    for (const field of fields) {
+        if (field.type === 'float32' || field.type === Float32Array) {
+            // TODO - just store the original field as metadata?
+            const metadata = new Map(); // field;
+            // arrow: new arrow.Field(name, nullable, metadata)
+            const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
+            arrowFields.push(arrowField);
+        }
+    }
+    if (arrowFields.length === 0) {
+        throw new Error('No arrow convertible fields');
+    }
+    if (arrowFields.length !== fields.length) {
+        throw new Error('Some schema fields are not arrow convertible');
+    }
+    return new arrow.Schema(arrowFields);
+}
+// Convert from simple loaders.gl arrays to arrow vectors
+function getArrowVectors(arrowSchema, data) {
+    const arrowVectors = [];
+    for (const field of arrowSchema.fields) {
+        const vector = data[field.name];
+        if (vector instanceof Float32Array) {
+            const arrowVector = arrow.makeVector(vector);
+            arrowVectors.push(arrowVector);
+        }
+    }
+    if (arrowSchema.fields.length !== arrowVectors.length) {
+        throw new Error('Some columns not arrow convertible');
+    }
+    return arrowVectors;
+}
+//# sourceMappingURL=arrow-table-batch-aggregator.js.map

package/dist/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-batch-aggregator.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/arrow-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,4BAA4B,EAAC,6CAA0C;AAC/E,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,qGAAqG;AACrG,MAAM,OAAO,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,CAAsB;IAEjC,YAAY,MAAM,EAAE,OAAO;QACzB,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,QAAQ;QACN,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,EAAE,CAAC;YACV,uBAAuB;YACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAEpE,2BAA2B;YAC3B,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAEnE,0BAA0B;YAC1B,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,WAAW,CACvC,IAAI,CAAC,WAAW,EAChB,KAAK,CAAC,QAAQ,CAAC;gBACb,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBAC/C,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,EAAC,IAAI,EAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aAChD,CAAC,CACH,CAAC;YAEF,OAAO;gBACL,KAAK,EAAE,aAAa;gBACpB,SAAS,EAAE,MAAM;gBACjB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;gBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,6DAA6D;AAC7D,SAAS,cAAc,CAAC,MAAM;IAC5B,MAAM,WAAW,GAAkB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpF,KAAK,MAAM,KAAK,IAAI,MAAe,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC5D,oDAAoD;YACpD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,SAAS;YACrC,mDAAmD;YACnD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC9F,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;AACvC,CAAC;AAED,yDAAyD;AACzD,SAAS,eAAe,CAAC,WAAW,EAAE,IAAI;IACxC,MAAM,YAAY,GAAU,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,YAAY,YAAY,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,MAAa,CAAC,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/schema-utils/lib/table/batch-builder/arrow-table-builder.d.ts

@@ -0,0 +1,28 @@
+import type { Schema, ArrowTable, ArrowTableBatch } from '@loaders.gl/core/schema';
+import * as arrow from 'apache-arrow';
+import { type ArrowSchemaConversionOptions } from "../../schema/convert-arrow-schema.js";
+/** Options for constructing Arrow tables from loaders.gl rows. */
+export type ArrowTableBuilderOptions = ArrowSchemaConversionOptions;
+/** Builds an arrow table or batches */
+export declare class ArrowTableBuilder {
+    schema: Schema;
+    arrowSchema: arrow.Schema;
+    arrowBuilders: arrow.Builder[];
+    length: number;
+    constructor(schema: Schema, options?: ArrowTableBuilderOptions);
+    addObjectRow(row: {
+        [key: string]: any;
+    }): void;
+    addArrayRow(row: any[]): void;
+    /** Makes sure that a first batch with schema is sent even if no rows */
+    firstBatch(): ArrowTableBatch | null;
+    /** Flush the current batch if conditions are right */
+    flushBatch(): ArrowTableBatch | null;
+    /** Get a last batch if any data is left */
+    finishBatch(): ArrowTableBatch | null;
+    /** Return a table with all the accumulated data */
+    finishTable(): ArrowTable;
+    /** Extract a record batch flushing the currently accumulated data in the builders */
+    _getArrowRecordBatch(): arrow.RecordBatch;
+}
+//# sourceMappingURL=arrow-table-builder.d.ts.map

package/dist/schema-utils/lib/table/batch-builder/arrow-table-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-builder.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/arrow-table-builder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAE,eAAe,EAAC,MAAM,yBAAyB,CAAC;AACjF,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,EAGL,KAAK,4BAA4B,EAClC,6CAA0C;AAE3C,kEAAkE;AAClE,MAAM,MAAM,wBAAwB,GAAG,4BAA4B,CAAC;AAEpE,uCAAuC;AACvC,qBAAa,iBAAiB;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC;IAC1B,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;gBAEH,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAY9D,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC;IAYtC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAOtB,wEAAwE;IACxE,UAAU,IAAI,eAAe,GAAG,IAAI;IAepC,sDAAsD;IACtD,UAAU,IAAI,eAAe,GAAG,IAAI;IAcpC,2CAA2C;IAC3C,WAAW,IAAI,eAAe,GAAG,IAAI;IAerC,mDAAmD;IACnD,WAAW,IAAI,UAAU;IAUzB,qFAAqF;IACrF,oBAAoB,IAAI,KAAK,CAAC,WAAW;CAU1C"}

package/dist/schema-utils/lib/table/batch-builder/arrow-table-builder.js

@@ -0,0 +1,104 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import * as arrow from 'apache-arrow';
+import { convertArrowToSchema, convertSchemaToArrow } from "../../schema/convert-arrow-schema.js";
+/** Builds an arrow table or batches */
+export class ArrowTableBuilder {
+    schema;
+    arrowSchema;
+    arrowBuilders;
+    length;
+    constructor(schema, options) {
+        this.arrowSchema = convertSchemaToArrow(schema, options);
+        this.schema =
+            options?.viewTypes && options.viewTypes !== 'never'
+                ? convertArrowToSchema(this.arrowSchema)
+                : schema;
+        this.arrowBuilders = this.arrowSchema.fields.map(field => arrow.makeBuilder({ type: field.type, nullValues: [null] }));
+        this.length = 0;
+    }
+    addObjectRow(row) {
+        for (let i = 0; i < this.arrowBuilders.length; i++) {
+            const columnName = this.schema.fields[i].name;
+            const value = row[columnName];
+            // if (this.schema.fields[i].type.toString() === 'bool') {
+            //   debugger;
+            // }
+            this.arrowBuilders[i].append(value);
+        }
+        this.length++;
+    }
+    addArrayRow(row) {
+        for (let i = 0; i < this.arrowBuilders.length; i++) {
+            this.arrowBuilders[i].append(row[i]);
+        }
+        this.length++;
+    }
+    /** Makes sure that a first batch with schema is sent even if no rows */
+    firstBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        // If there is data, a batch will be sent later
+        if (arrowRecordBatch.numCols !== 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Flush the current batch if conditions are right */
+    flushBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        if (arrowRecordBatch.numCols === 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Get a last batch if any data is left */
+    finishBatch() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        this.arrowBuilders.forEach(builder => builder.finish());
+        if (arrowRecordBatch.numCols === 0 && arrowRecordBatch.numRows === 0) {
+            return null;
+        }
+        return {
+            shape: 'arrow-table',
+            batchType: 'data',
+            length: arrowRecordBatch.numRows,
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Return a table with all the accumulated data */
+    finishTable() {
+        const arrowRecordBatch = this._getArrowRecordBatch();
+        this.arrowBuilders.forEach(builder => builder.finish());
+        return {
+            shape: 'arrow-table',
+            schema: this.schema,
+            data: new arrow.Table(arrowRecordBatch)
+        };
+    }
+    /** Extract a record batch flushing the currently accumulated data in the builders */
+    _getArrowRecordBatch() {
+        const { arrowBuilders, arrowSchema } = this;
+        const arrowDatas = arrowBuilders.map(builder => builder.flush());
+        const length = arrowDatas[0]?.length || this.length;
+        this.length = 0;
+        const structField = new arrow.Struct(arrowSchema.fields);
+        const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
+        const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);
+        return arrowRecordBatch;
+    }
+}
+//# sourceMappingURL=arrow-table-builder.js.map

package/dist/schema-utils/lib/table/batch-builder/arrow-table-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-builder.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/arrow-table-builder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EAErB,6CAA0C;AAK3C,uCAAuC;AACvC,MAAM,OAAO,iBAAiB;IAC5B,MAAM,CAAS;IACf,WAAW,CAAe;IAC1B,aAAa,CAAkB;IAC/B,MAAM,CAAS;IAEf,YAAY,MAAc,EAAE,OAAkC;QAC5D,IAAI,CAAC,WAAW,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM;YACT,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO;gBACjD,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,WAAW,CAAC;gBACxC,CAAC,CAAC,MAAM,CAAC;QACb,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACvD,KAAK,CAAC,WAAW,CAAC,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,EAAC,CAAC,CAC1D,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,YAAY,CAAC,GAAyB;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;YAC9B,0DAA0D;YAC1D,cAAc;YACd,IAAI;YACJ,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,WAAW,CAAC,GAAU;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,wEAAwE;IACxE,UAAU;QACR,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACrD,+CAA+C;QAC/C,IAAI,gBAAgB,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,gBAAgB,CAAC,OAAO;YAChC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,sDAAsD;IACtD,UAAU;QACR,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACrD,IAAI,gBAAgB,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,gBAAgB,CAAC,OAAO;YAChC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,WAAW;QACT,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACrD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxD,IAAI,gBAAgB,CAAC,OAAO,KAAK,CAAC,IAAI,gBAAgB,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,gBAAgB,CAAC,OAAO;YAChC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,mDAAmD;IACnD,WAAW;QACT,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACrD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxD,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,qFAAqF;IACrF,oBAAoB;QAClB,MAAM,EAAC,aAAa,EAAE,WAAW,EAAC,GAAG,IAAI,CAAC;QAC1C,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QACzF,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QAC7E,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF"}

package/dist/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.d.ts

@@ -0,0 +1,19 @@
+import type { Schema, TableBatch } from '@loaders.gl/core/schema';
+import { TableBatchAggregator, TableBatchOptions } from "./table-batch-aggregator.js";
+export declare class BaseTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    options: TableBatchOptions;
+    shape?: 'array-row-table' | 'object-row-table';
+    length: number;
+    rows: any[] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=base-table-batch-aggregator.d.ts.map

package/dist/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,oCAAiC;AAIjF,qBAAa,wBAAyB,YAAW,oBAAoB;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAE3B,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;IAC/C,MAAM,EAAE,MAAM,CAAK;IACnB,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAQ;IAC1B,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IActD,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAY9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAYrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAoB9B"}

package/dist/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.js

@@ -0,0 +1,64 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+const DEFAULT_ROW_COUNT = 100;
+export class BaseTableBatchAggregator {
+    schema;
+    options;
+    shape;
+    length = 0;
+    rows = null;
+    cursor = 0;
+    _headers = [];
+    constructor(schema, options) {
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (!Array.isArray(schema)) {
+            this._headers = [];
+            for (const key in schema) {
+                this._headers[schema[key].index] = schema[key].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.shape = 'array-row-table';
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.shape = 'object-row-table';
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    getBatch() {
+        let rows = this.rows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.rows = null;
+        const batch = {
+            shape: this.shape || 'array-row-table',
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            schema: this.schema,
+            cursor: this.cursor
+        };
+        return batch;
+    }
+}
+//# sourceMappingURL=base-table-batch-aggregator.js.map

package/dist/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-table-batch-aggregator.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/base-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,wBAAwB;IACnC,MAAM,CAAS;IACf,OAAO,CAAoB;IAE3B,KAAK,CAA0C;IAC/C,MAAM,GAAW,CAAC,CAAC;IACnB,IAAI,GAAiB,IAAI,CAAC;IAC1B,MAAM,GAAW,CAAC,CAAC;IACX,QAAQ,GAAa,EAAE,CAAC;IAEhC,YAAY,MAAc,EAAE,OAA0B;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,4CAA4C;QAC5C,8BAA8B;QAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,GAAU,EAAE,MAAe;QACrC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,GAAG,MAAgB,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,iBAAiB,CAAC;QAE/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC7B,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,YAAY,CAAC,GAAgC,EAAE,MAAe;QAC5D,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,GAAG,MAAgB,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,kBAAkB,CAAC;QAEhC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC7B,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACrB,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,MAAM,KAAK,GAAe;YACxB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,iBAAiB;YACtC,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts

@@ -0,0 +1,20 @@
+import type { Schema, ColumnarTableBatch, ArrowTableBatch, TypedArray } from '@loaders.gl/core/schema';
+import { TableBatchAggregator } from "./table-batch-aggregator.js";
+type ColumnarTableBatchOptions = {};
+export declare class ColumnarTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    length: number;
+    allocated: number;
+    columns: Record<string, TypedArray | Array<any>>;
+    constructor(schema: Schema, options: ColumnarTableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[]): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    getBatch(): ColumnarTableBatch | ArrowTableBatch | null;
+    _reallocateColumns(): void;
+    _pruneColumns(): void;
+}
+export {};
+//# sourceMappingURL=columnar-table-batch-aggregator.d.ts.map

package/dist/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"columnar-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,MAAM,EACN,kBAAkB,EAClB,eAAe,EACf,UAAU,EACX,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAC,oBAAoB,EAAC,oCAAiC;AAC9D,KAAK,yBAAyB,GAAG,EAAE,CAAC;AAIpC,qBAAa,4BAA6B,YAAW,oBAAoB;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAK;IACnB,SAAS,EAAE,MAAM,CAAK;IACtB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM;gBAE1C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,yBAAyB;IAK9D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAWtB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IASpD,QAAQ,IAAI,kBAAkB,GAAG,eAAe,GAAG,IAAI;IAgBvD,kBAAkB;IA8BlB,aAAa;CAKd"}

package/dist/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.js

@@ -0,0 +1,84 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { isTypedArray } from '@math.gl/core/types';
+import { getArrayTypeFromDataType } from "../../schema/data-type.js";
+const DEFAULT_ROW_COUNT = 100;
+export class ColumnarTableBatchAggregator {
+    schema;
+    length = 0;
+    allocated = 0;
+    columns = {};
+    constructor(schema, options) {
+        this.schema = schema;
+        this._reallocateColumns();
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        let i = 0;
+        // TODO what if no csv header, columns not populated?
+        for (const fieldName in this.columns) {
+            this.columns[fieldName][this.length] = row[i++];
+        }
+        this.length++;
+    }
+    addObjectRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        for (const fieldName in row) {
+            this.columns[fieldName][this.length] = row[fieldName];
+        }
+        this.length++;
+    }
+    getBatch() {
+        this._pruneColumns();
+        const batch = {
+            shape: 'columnar-table',
+            batchType: 'data',
+            data: this.columns,
+            schema: this.schema,
+            length: this.length
+        };
+        return batch;
+    }
+    // HELPERS
+    _reallocateColumns() {
+        if (this.length < this.allocated) {
+            return;
+        }
+        // @ts-ignore TODO
+        this.allocated = this.allocated > 0 ? this.allocated * 2 : DEFAULT_ROW_COUNT;
+        this.columns = {};
+        for (const field of this.schema.fields) {
+            const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
+            const oldColumn = this.columns[field.name];
+            if (!oldColumn) {
+                // Create new
+                this.columns[field.name] = new ArrayType(this.allocated);
+            }
+            else if (Array.isArray(oldColumn)) {
+                // Plain array, just increase its size
+                oldColumn.length = this.allocated;
+            }
+            else if (isTypedArray(oldColumn)) {
+                const typedArray = new ArrayType(this.allocated);
+                // Copy the old data to the new array
+                typedArray.set(oldColumn);
+                this.columns[field.name] = typedArray;
+            }
+            else {
+                throw new Error('Invalid column type');
+            }
+        }
+    }
+    _pruneColumns() {
+        for (const [columnName, column] of Object.entries(this.columns)) {
+            this.columns[columnName] = column.slice(0, this.length);
+        }
+    }
+}
+//# sourceMappingURL=columnar-table-batch-aggregator.js.map

package/dist/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"columnar-table-batch-aggregator.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/columnar-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAQpC,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAC,wBAAwB,EAAC,kCAA+B;AAIhE,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,4BAA4B;IACvC,MAAM,CAAS;IACf,MAAM,GAAW,CAAC,CAAC;IACnB,SAAS,GAAW,CAAC,CAAC;IACtB,OAAO,GAA4C,EAAE,CAAC;IAEtD,YAAY,MAAc,EAAE,OAAkC;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,GAAU;QACpB,2DAA2D;QAC3D,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,qDAAqD;QACrD,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,YAAY,CAAC,GAAgC;QAC3C,2DAA2D;QAC3D,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,KAAK,MAAM,SAAS,IAAI,GAAG,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,MAAM,KAAK,GAAuB;YAChC,KAAK,EAAE,gBAAgB;YACvB,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,IAAI,CAAC,OAAO;YAClB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,UAAU;IAEV,kBAAkB;QAChB,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,aAAa;gBACb,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3D,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpC,sCAAsC;gBACtC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;YACpC,CAAC;iBAAM,IAAI,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;gBACnC,MAAM,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,SAAS,CAAe,CAAC;gBAC/D,qCAAqC;gBACrC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,aAAa;QACX,KAAK,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;CACF"}

package/dist/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.d.ts

@@ -0,0 +1,21 @@
+import type { Schema, TableBatch } from '@loaders.gl/core/schema';
+import { TableBatchAggregator, TableBatchOptions } from "./table-batch-aggregator.js";
+export declare class RowTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema | null;
+    options: TableBatchOptions;
+    length: number;
+    objectRows: {
+        [columnName: string]: unknown;
+    }[] | null;
+    arrayRows: unknown[][] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema | null, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=row-table-batch-aggregator.d.ts.map

package/dist/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,oCAAiC;AAUjF,qBAAa,uBAAwB,YAAW,oBAAoB;IAClE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,iBAAiB,CAAC;IAE3B,MAAM,EAAE,MAAM,CAAK;IACnB,UAAU,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;KAAC,EAAE,GAAG,IAAI,CAAQ;IAC5D,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,IAAI,CAAQ;IACrC,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAyB;gBAE7B,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,iBAAiB;IAc7D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAsB9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAsBrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAoB9B"}

package/dist/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.js

@@ -0,0 +1,86 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { convertToArrayRow, convertToObjectRow, inferHeadersFromArrayRow, inferHeadersFromObjectRow } from "../tables/row-utils.js";
+const DEFAULT_ROW_COUNT = 100;
+export class RowTableBatchAggregator {
+    schema;
+    options;
+    length = 0;
+    objectRows = null;
+    arrayRows = null;
+    cursor = 0;
+    _headers = null;
+    constructor(schema, options) {
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (schema) {
+            this._headers = [];
+            for (let i = 0; i < schema.fields.length; i++) {
+                this._headers[i] = schema.fields[i].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // TODO - infer schema at a higher level, instead of hacking headers here?
+        this._headers ||= inferHeadersFromArrayRow(row);
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'object-row-table':
+                const rowObject = convertToObjectRow(row, this._headers);
+                this.addObjectRow(rowObject, cursor);
+                break;
+            case 'array-row-table':
+                this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
+                this.arrayRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // TODO - infer schema at a higher level, instead of hacking headers here?
+        this._headers ||= inferHeadersFromObjectRow(row);
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'array-row-table':
+                const rowArray = convertToArrayRow(row, this._headers);
+                this.addArrayRow(rowArray, cursor);
+                break;
+            case 'object-row-table':
+                this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
+                this.objectRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    getBatch() {
+        let rows = this.arrayRows || this.objectRows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.arrayRows = null;
+        this.objectRows = null;
+        return {
+            shape: this.options.shape,
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            // @ts-expect-error we should infer a schema
+            schema: this.schema,
+            cursor: this.cursor
+        };
+    }
+}
+//# sourceMappingURL=row-table-batch-aggregator.js.map

package/dist/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-table-batch-aggregator.js","sourceRoot":"","sources":["../../../../../src/schema-utils/lib/table/batch-builder/row-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EACxB,yBAAyB,EAC1B,+BAA4B;AAE7B,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,uBAAuB;IAClC,MAAM,CAAgB;IACtB,OAAO,CAAoB;IAE3B,MAAM,GAAW,CAAC,CAAC;IACnB,UAAU,GAA6C,IAAI,CAAC;IAC5D,SAAS,GAAuB,IAAI,CAAC;IACrC,MAAM,GAAW,CAAC,CAAC;IACX,QAAQ,GAAoB,IAAI,CAAC;IAEzC,YAAY,MAAqB,EAAE,OAA0B;QAC3D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,4CAA4C;QAC5C,8BAA8B;QAC9B,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;YACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,GAAU,EAAE,MAAe;QACrC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,GAAG,MAAgB,CAAC;QACjC,CAAC;QAED,0EAA0E;QAC1E,IAAI,CAAC,QAAQ,KAAK,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAEhD,wCAAwC;QACxC,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,KAAK,kBAAkB;gBACrB,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzD,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM;YACR,KAAK,iBAAiB;gBACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;gBAClC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,MAAM;QACV,CAAC;IACH,CAAC;IAED,YAAY,CAAC,GAAgC,EAAE,MAAe;QAC5D,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,GAAG,MAAgB,CAAC;QACjC,CAAC;QAED,0EAA0E;QAC1E,IAAI,CAAC,QAAQ,KAAK,yBAAyB,CAAC,GAAG,CAAC,CAAC;QAEjD,wCAAwC;QACxC,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,KAAK,iBAAiB;gBACpB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,kBAAkB;gBACrB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAClE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;gBACnC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,MAAM;QACV,CAAC;IACH,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC;QAC7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,4CAA4C;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;IACJ,CAAC;CACF"}