@loaders.gl/schema-utils 4.4.0-alpha.1

package/dist/lib/table/batch-builder/row-table-batch-aggregator.js

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { convertToArrayRow, convertToObjectRow, inferHeadersFromArrayRow, inferHeadersFromObjectRow } from "../tables/row-utils.js";
+const DEFAULT_ROW_COUNT = 100;
+export class RowTableBatchAggregator {
+    schema;
+    options;
+    length = 0;
+    objectRows = null;
+    arrayRows = null;
+    cursor = 0;
+    _headers = null;
+    constructor(schema, options) {
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (schema) {
+            this._headers = [];
+            for (let i = 0; i < schema.fields.length; i++) {
+                this._headers[i] = schema.fields[i].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // TODO - infer schema at a higher level, instead of hacking headers here?
+        this._headers ||= inferHeadersFromArrayRow(row);
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'object-row-table':
+                const rowObject = convertToObjectRow(row, this._headers);
+                this.addObjectRow(rowObject, cursor);
+                break;
+            case 'array-row-table':
+                this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
+                this.arrayRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // TODO - infer schema at a higher level, instead of hacking headers here?
+        this._headers ||= inferHeadersFromObjectRow(row);
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'array-row-table':
+                const rowArray = convertToArrayRow(row, this._headers);
+                this.addArrayRow(rowArray, cursor);
+                break;
+            case 'object-row-table':
+                this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
+                this.objectRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    getBatch() {
+        let rows = this.arrayRows || this.objectRows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.arrayRows = null;
+        this.objectRows = null;
+        return {
+            shape: this.options.shape,
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            // @ts-expect-error we should infer a schema
+            schema: this.schema,
+            cursor: this.cursor
+        };
+    }
+}

package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts

@@ -0,0 +1,24 @@
+import type { Schema, TableBatch } from '@loaders.gl/schema';
+export interface TableBatchOptions {
+    batchSize: number | string;
+    [key: string]: any;
+}
+export interface TableBatchConstructor {
+    new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
+}
+/**
+ * TableBatchBuilder delegates batch building to this interface
+ */
+export interface TableBatchAggregator {
+    /** Number of rows */
+    rowCount(): number;
+    /** Add one row */
+    addArrayRow(row: any[]): void;
+    /** Add one row */
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    /** return a batch object */
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=table-batch-aggregator.d.ts.map

package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/table-batch-aggregator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAE3D,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,CAAC;CACxE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,qBAAqB;IACrB,QAAQ,IAAI,MAAM,CAAC;IACnB,kBAAkB;IAClB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC9B,kBAAkB;IAClB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI,CAAC;IACrD,4BAA4B;IAC5B,QAAQ,IAAI,UAAU,GAAG,IAAI,CAAC;CAC/B"}

package/dist/lib/table/batch-builder/table-batch-aggregator.js

@@ -0,0 +1,4 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+export {};

package/dist/lib/table/batch-builder/table-batch-builder.d.ts

@@ -0,0 +1,52 @@
+import type { Schema, TableBatch } from '@loaders.gl/schema';
+import type { TableBatchConstructor } from "./table-batch-aggregator.js";
+type TableBatchBuilderOptions = {
+    shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
+    batchSize?: number | 'auto';
+    batchDebounceMs?: number;
+    limit?: number;
+    _limitMB?: number;
+};
+type GetBatchOptions = {
+    bytesUsed?: number;
+    [key: string]: any;
+};
+/** Incrementally builds batches from a stream of rows */
+export declare class TableBatchBuilder {
+    schema: Schema;
+    options: Required<TableBatchBuilderOptions>;
+    private aggregator;
+    private batchCount;
+    private bytesUsed;
+    private isChunkComplete;
+    private lastBatchEmittedMs;
+    private totalLength;
+    private totalBytes;
+    private rowBytes;
+    static ArrowBatch?: TableBatchConstructor;
+    constructor(schema: Schema, options?: TableBatchBuilderOptions);
+    limitReached(): boolean;
+    /** @deprecated Use addArrayRow or addObjectRow */
+    addRow(row: any[] | {
+        [columnName: string]: any;
+    }): void;
+    /** Add one row to the batch */
+    protected addArrayRow(row: any[]): void;
+    /** Add one row to the batch */
+    protected addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    /** Mark an incoming raw memory chunk has completed */
+    chunkComplete(chunk: ArrayBuffer | string): void;
+    getFullBatch(options?: GetBatchOptions): TableBatch | null;
+    getFinalBatch(options?: GetBatchOptions): TableBatch | null;
+    _estimateRowMB(row: any[] | object): number;
+    private _isFull;
+    /**
+     * bytesUsed can be set via chunkComplete or via getBatch*
+     */
+    private _getBatch;
+    private _getTableBatchType;
+}
+export {};
+//# sourceMappingURL=table-batch-builder.d.ts.map

package/dist/lib/table/batch-builder/table-batch-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-batch-builder.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batch-builder/table-batch-builder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAuB,qBAAqB,EAAC,oCAAiC;AAO1F,KAAK,wBAAwB,GAAG;IAC9B,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,aAAa,CAAC;IAClF,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAUF,yDAAyD;AACzD,qBAAa,iBAAiB;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,QAAQ,CAAC,wBAAwB,CAAC,CAAC;IAE5C,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,kBAAkB,CAAsB;IAChD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAa;IAE7B,MAAM,CAAC,UAAU,CAAC,EAAE,qBAAqB,CAAC;gBAE9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAK9D,YAAY,IAAI,OAAO;IAUvB,kDAAkD;IAClD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IActD,+BAA+B;IAC/B,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAQhC,+BAA+B;IAC/B,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IAQ9D,sDAAsD;IACtD,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAUhD,YAAY,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAI1D,aAAa,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAM3D,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,MAAM;IAI3C,OAAO,CAAC,OAAO;IA2Bf;;OAEG;IACH,OAAO,CAAC,SAAS;IAmBjB,OAAO,CAAC,kBAAkB;CAa3B"}

package/dist/lib/table/batch-builder/table-batch-builder.js

@@ -0,0 +1,148 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { BaseTableBatchAggregator } from "./base-table-batch-aggregator.js";
+import { RowTableBatchAggregator } from "./row-table-batch-aggregator.js";
+import { ColumnarTableBatchAggregator } from "./columnar-table-batch-aggregator.js";
+import { ArrowTableBatchAggregator } from "./arrow-table-batch-aggregator.js";
+const DEFAULT_OPTIONS = {
+    shape: undefined,
+    batchSize: 'auto',
+    batchDebounceMs: 0,
+    limit: 0,
+    _limitMB: 0
+};
+/** Incrementally builds batches from a stream of rows */
+export class TableBatchBuilder {
+    schema;
+    options;
+    aggregator = null;
+    batchCount = 0;
+    bytesUsed = 0;
+    isChunkComplete = false;
+    lastBatchEmittedMs = Date.now();
+    totalLength = 0;
+    totalBytes = 0;
+    rowBytes = 0;
+    static ArrowBatch;
+    constructor(schema, options) {
+        this.schema = schema;
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+    }
+    limitReached() {
+        if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
+            return true;
+        }
+        if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
+            return true;
+        }
+        return false;
+    }
+    /** @deprecated Use addArrayRow or addObjectRow */
+    addRow(row) {
+        if (this.limitReached()) {
+            return;
+        }
+        this.totalLength++;
+        this.rowBytes = this.rowBytes || this._estimateRowMB(row);
+        this.totalBytes += this.rowBytes;
+        if (Array.isArray(row)) {
+            this.addArrayRow(row);
+        }
+        else {
+            this.addObjectRow(row);
+        }
+    }
+    /** Add one row to the batch */
+    addArrayRow(row) {
+        if (!this.aggregator) {
+            const TableBatchType = this._getTableBatchType();
+            this.aggregator = new TableBatchType(this.schema, this.options);
+        }
+        this.aggregator.addArrayRow(row);
+    }
+    /** Add one row to the batch */
+    addObjectRow(row) {
+        if (!this.aggregator) {
+            const TableBatchType = this._getTableBatchType();
+            this.aggregator = new TableBatchType(this.schema, this.options);
+        }
+        this.aggregator.addObjectRow(row);
+    }
+    /** Mark an incoming raw memory chunk has completed */
+    chunkComplete(chunk) {
+        if (chunk instanceof ArrayBuffer) {
+            this.bytesUsed += chunk.byteLength;
+        }
+        if (typeof chunk === 'string') {
+            this.bytesUsed += chunk.length;
+        }
+        this.isChunkComplete = true;
+    }
+    getFullBatch(options) {
+        return this._isFull() ? this._getBatch(options) : null;
+    }
+    getFinalBatch(options) {
+        return this._getBatch(options);
+    }
+    // INTERNAL
+    _estimateRowMB(row) {
+        return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
+    }
+    _isFull() {
+        // No batch, not ready
+        if (!this.aggregator || this.aggregator.rowCount() === 0) {
+            return false;
+        }
+        // if batchSize === 'auto' we wait for chunk to complete
+        // if batchSize === number, ensure we have enough rows
+        if (this.options.batchSize === 'auto') {
+            if (!this.isChunkComplete) {
+                return false;
+            }
+        }
+        else if (this.options.batchSize > this.aggregator.rowCount()) {
+            return false;
+        }
+        // Debounce batches
+        if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
+            return false;
+        }
+        // Emit batch
+        this.isChunkComplete = false;
+        this.lastBatchEmittedMs = Date.now();
+        return true;
+    }
+    /**
+     * bytesUsed can be set via chunkComplete or via getBatch*
+     */
+    _getBatch(options) {
+        if (!this.aggregator) {
+            return null;
+        }
+        // TODO - this can overly increment bytes used?
+        if (options?.bytesUsed) {
+            this.bytesUsed = options.bytesUsed;
+        }
+        const normalizedBatch = this.aggregator.getBatch();
+        normalizedBatch.count = this.batchCount;
+        normalizedBatch.bytesUsed = this.bytesUsed;
+        Object.assign(normalizedBatch, options);
+        this.batchCount++;
+        this.aggregator = null;
+        return normalizedBatch;
+    }
+    _getTableBatchType() {
+        switch (this.options.shape) {
+            case 'array-row-table':
+            case 'object-row-table':
+                return RowTableBatchAggregator;
+            case 'columnar-table':
+                return ColumnarTableBatchAggregator;
+            case 'arrow-table':
+                return ArrowTableBatchAggregator;
+            default:
+                return BaseTableBatchAggregator;
+        }
+    }
+}

package/dist/lib/table/batches/convert-batches.d.ts

@@ -0,0 +1,10 @@
+import type { TableBatch, ArrayRowTableBatch, ObjectRowTableBatch, ColumnarTableBatch, ArrowTableBatch } from '@loaders.gl/schema';
+export declare function convertBatch(batches: TableBatch, shape: 'object-row-table'): ObjectRowTableBatch;
+export declare function convertBatch(batches: TableBatch, shape: 'array-row-table'): ArrayRowTableBatch;
+export declare function convertBatch(batches: TableBatch, shape: 'columnar-table'): ColumnarTableBatch;
+export declare function convertBatch(batches: TableBatch, shape: 'arrow-table'): ArrowTableBatch;
+export declare function convertBatches(batches: Iterable<TableBatch> | AsyncIterable<TableBatch>, shape: 'object-row-table'): AsyncIterableIterator<ObjectRowTableBatch>;
+export declare function convertBatches(batches: Iterable<TableBatch> | AsyncIterable<TableBatch>, shape: 'array-row-table'): AsyncIterableIterator<ArrayRowTableBatch>;
+export declare function convertBatches(batches: Iterable<TableBatch> | AsyncIterable<TableBatch>, shape: 'columnar-table'): AsyncIterableIterator<ColumnarTableBatch>;
+export declare function convertBatches(batches: Iterable<TableBatch> | AsyncIterable<TableBatch>, shape: 'arrow-table'): AsyncIterableIterator<ArrowTableBatch>;
+//# sourceMappingURL=convert-batches.d.ts.map

package/dist/lib/table/batches/convert-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-batches.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batches/convert-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,UAAU,EACV,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAG5B,wBAAgB,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,GAAG,mBAAmB,CAAC;AAClG,wBAAgB,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;AAChG,wBAAgB,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,GAAG,kBAAkB,CAAC;AAC/F,wBAAgB,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;AAqBzF,wBAAgB,cAAc,CAC5B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACzD,KAAK,EAAE,kBAAkB,GACxB,qBAAqB,CAAC,mBAAmB,CAAC,CAAC;AAC9C,wBAAgB,cAAc,CAC5B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACzD,KAAK,EAAE,iBAAiB,GACvB,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;AAC7C,wBAAgB,cAAc,CAC5B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACzD,KAAK,EAAE,gBAAgB,GACtB,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;AAC7C,wBAAgB,cAAc,CAC5B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACzD,KAAK,EAAE,aAAa,GACnB,qBAAqB,CAAC,eAAe,CAAC,CAAC"}

package/dist/lib/table/batches/convert-batches.js

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { convertTable } from "../tables/convert-table.js";
+/** Convert a table batch to a different shape */
+export function convertBatch(batch, shape) {
+    switch (shape) {
+        case 'object-row-table':
+            return { ...batch, ...convertTable(batch, 'object-row-table') };
+        case 'array-row-table':
+            return { ...batch, ...convertTable(batch, 'array-row-table') };
+        case 'columnar-table':
+            return { ...batch, ...convertTable(batch, 'columnar-table') };
+        case 'arrow-table':
+            return { ...batch, ...convertTable(batch, 'arrow-table') };
+        default:
+            throw new Error(shape);
+    }
+}
+/**
+ * Convert batches to a different shape
+ * @param table
+ * @param shape
+ * @returns
+ */
+export async function* convertBatches(batches, shape) {
+    for await (const batch of batches) {
+        switch (shape) {
+            case 'object-row-table':
+                yield convertBatch(batch, 'object-row-table');
+                break;
+            case 'array-row-table':
+                yield convertBatch(batch, 'array-row-table');
+                break;
+            case 'columnar-table':
+                yield convertBatch(batch, 'columnar-table');
+                break;
+            case 'arrow-table':
+                yield convertBatch(batch, 'arrow-table');
+                break;
+            default:
+                throw new Error(shape);
+        }
+    }
+}

package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts

@@ -0,0 +1,17 @@
+import * as arrow from 'apache-arrow';
+import type { Table, ArrowTableBatch } from '@loaders.gl/schema';
+/**
+ * Returns an iterator that yields a single table as a sequence of ArrowTable batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export declare function makeArrowTableBatchIterator(table: Table, options?: {
+    batchSize?: number;
+}): IterableIterator<ArrowTableBatch>;
+/**
+ * Returns an iterator that yields a single table as a sequence of arrow.RecordBatch batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export declare function makeArrowRecordBatchIterator(table: Table, options?: {
+    batchSize?: number;
+}): IterableIterator<arrow.RecordBatch>;
+//# sourceMappingURL=make-arrow-batch-iterator.d.ts.map

package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-arrow-batch-iterator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batches/make-arrow-batch-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,KAAK,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAK/D;;;GAGG;AACH,wBAAiB,2BAA2B,CAC1C,KAAK,EAAE,KAAK,EACZ,OAAO,CAAC,EAAE;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAC,GAC7B,gBAAgB,CAAC,eAAe,CAAC,CAYnC;AAED;;;GAGG;AACH,wBAAiB,4BAA4B,CAC3C,KAAK,EAAE,KAAK,EACZ,OAAO,CAAC,EAAE;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAC,GAC7B,gBAAgB,CAAC,KAAK,CAAC,WAAW,CAAC,CAoCrC"}

package/dist/lib/table/batches/make-arrow-batch-iterator.js

@@ -0,0 +1,57 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import * as arrow from 'apache-arrow';
+import { convertSchemaToArrow } from "../../schema/convert-arrow-schema.js";
+import { getTableLength, getTableNumCols, getTableCellAt } from "../tables/table-accessors.js";
+/**
+ * Returns an iterator that yields a single table as a sequence of ArrowTable batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowTableBatchIterator(table, options) {
+    for (const batch of makeArrowRecordBatchIterator(table, options)) {
+        const arrowTable = new arrow.Table([batch]);
+        yield {
+            ...batch,
+            shape: 'arrow-table',
+            schema: table.schema,
+            batchType: 'data',
+            length: arrowTable.numRows,
+            data: arrowTable
+        };
+    }
+}
+/**
+ * Returns an iterator that yields a single table as a sequence of arrow.RecordBatch batches.
+ * @note All batches will have the same shape and schema as the original table.
+ */
+export function* makeArrowRecordBatchIterator(table, options) {
+    const arrowSchema = convertSchemaToArrow(table.schema);
+    const length = getTableLength(table);
+    const numColumns = getTableNumCols(table);
+    const batchSize = options?.batchSize || length;
+    const builders = arrowSchema?.fields.map((arrowField) => arrow.makeBuilder(arrowField));
+    const structField = new arrow.Struct(arrowSchema.fields);
+    let batchLength = 0;
+    for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+        for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+            const value = getTableCellAt(table, rowIndex, columnIndex);
+            const builder = builders[columnIndex];
+            builder.append(value);
+            batchLength++;
+            if (batchLength >= batchSize) {
+                const datas = builders.map((builder) => builder.flush());
+                const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+                yield new arrow.RecordBatch(arrowSchema, structData);
+                batchLength = 0;
+            }
+        }
+    }
+    if (batchLength > 0) {
+        const datas = builders.map((builder) => builder.flush());
+        const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+        yield new arrow.RecordBatch(arrowSchema, structData);
+        batchLength = 0;
+    }
+    builders.map((builder) => builder.finish());
+}

package/dist/lib/table/batches/make-table-batch-iterator.d.ts

@@ -0,0 +1,15 @@
+import type { TableBatch, Table } from '@loaders.gl/schema';
+/**
+ * Returns an iterator that yields the contents of a table as a sequence of batches.
+ * @todo Currently only a single batch is yielded.
+ * @note All batches will have the same shape and schema as the original table.
+ * @returns
+ */
+export declare function makeTableBatchIterator(table: Table): IterableIterator<TableBatch>;
+/**
+ * Returns a table packaged as a single table batch
+ * @note The batch will have the same shape and schema as the original table.
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+export declare function makeBatchFromTable(table: Table): TableBatch;
+//# sourceMappingURL=make-table-batch-iterator.d.ts.map

package/dist/lib/table/batches/make-table-batch-iterator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-table-batch-iterator.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batches/make-table-batch-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAE,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAG1D;;;;;GAKG;AACH,wBAAiB,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAElF;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAE3D"}

package/dist/lib/table/batches/make-table-batch-iterator.js

@@ -0,0 +1,21 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { getTableLength } from "../tables/table-accessors.js";
+/**
+ * Returns an iterator that yields the contents of a table as a sequence of batches.
+ * @todo Currently only a single batch is yielded.
+ * @note All batches will have the same shape and schema as the original table.
+ * @returns
+ */
+export function* makeTableBatchIterator(table) {
+    yield makeBatchFromTable(table);
+}
+/**
+ * Returns a table packaged as a single table batch
+ * @note The batch will have the same shape and schema as the original table.
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+export function makeBatchFromTable(table) {
+    return { ...table, length: getTableLength(table), batchType: 'data' };
+}

package/dist/lib/table/batches/make-table-from-batches.d.ts

@@ -0,0 +1,9 @@
+import type { TableBatch, Table } from '@loaders.gl/schema';
+/**
+ * Assembles all batches from an async iterator into a single table.
+ * @note All batches must have the same shape and schema
+ * @param batchIterator
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+export declare function makeTableFromBatches(batchIterator: AsyncIterable<TableBatch> | Iterable<TableBatch>): Promise<Table | null>;
+//# sourceMappingURL=make-table-from-batches.d.ts.map

package/dist/lib/table/batches/make-table-from-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-table-from-batches.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/batches/make-table-from-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,UAAU,EACV,KAAK,EAIN,MAAM,oBAAoB,CAAC;AAG5B;;;;;GAKG;AAEH,wBAAsB,oBAAoB,CACxC,aAAa,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,GAC9D,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CA4DvB"}

package/dist/lib/table/batches/make-table-from-batches.js

@@ -0,0 +1,62 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { getTableLength } from "../tables/table-accessors.js";
+/**
+ * Assembles all batches from an async iterator into a single table.
+ * @note All batches must have the same shape and schema
+ * @param batchIterator
+ * @returns `null` if no batches are yielded by the async iterator
+ */
+// eslint-disable-next-line complexity
+export async function makeTableFromBatches(batchIterator) {
+    let arrayRows;
+    let objectRows;
+    let features;
+    let shape = null;
+    let schema;
+    for await (const batch of batchIterator) {
+        shape = shape || batch.shape;
+        schema = schema || batch.schema;
+        switch (batch.shape) {
+            case 'array-row-table':
+                arrayRows = arrayRows || [];
+                for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+                    const row = batch.data[rowIndex];
+                    arrayRows.push(row);
+                }
+                break;
+            case 'object-row-table':
+                objectRows = objectRows || [];
+                for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+                    const row = batch.data[rowIndex];
+                    objectRows.push(row);
+                }
+                break;
+            case 'geojson-table':
+                features = features || [];
+                for (let rowIndex = 0; rowIndex < getTableLength(batch); rowIndex++) {
+                    const row = batch.features[rowIndex];
+                    features.push(row);
+                }
+                break;
+            case 'columnar-table':
+            case 'arrow-table':
+            default:
+                throw new Error('shape');
+        }
+    }
+    if (!shape) {
+        return null;
+    }
+    switch (shape) {
+        case 'array-row-table':
+            return { shape: 'array-row-table', data: arrayRows, schema };
+        case 'object-row-table':
+            return { shape: 'object-row-table', data: objectRows, schema };
+        case 'geojson-table':
+            return { shape: 'geojson-table', type: 'FeatureCollection', features: features, schema };
+        default:
+            return null;
+    }
+}

package/dist/lib/table/tables/convert-arrow-table.d.ts

@@ -0,0 +1,24 @@
+import * as arrow from 'apache-arrow';
+import type { Table, ArrayRowTable, ColumnarTable, ObjectRowTable, GeoJSONTable, ArrowTable } from '@loaders.gl/schema';
+/**
+ * * Convert a loaders.gl Table to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ */
+export declare function convertTableToArrow(table: Table, options?: {
+    batchSize?: number;
+}): arrow.Table;
+/**
+ * Convert an Apache Arrow table to a loaders.gl Table
+ * @note Currently does not convert schema
+ */
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'arrow-table'): ArrowTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'object-row-table'): ObjectRowTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'array-row-table'): ArrayRowTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'geojson-table'): GeoJSONTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export declare function convertArrowToTable(arrow: arrow.Table, shape: Table['shape']): Table;
+//# sourceMappingURL=convert-arrow-table.d.ts.map

package/dist/lib/table/tables/convert-arrow-table.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-arrow-table.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/tables/convert-arrow-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EACV,KAAK,EACL,aAAa,EACb,aAAa,EACb,cAAc,EACd,YAAY,EACZ,UAAU,EAEX,MAAM,oBAAoB,CAAC;AAM5B;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAC,GAAG,KAAK,CAAC,KAAK,CAe7F;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,aAAa,GAAG,UAAU,CAAC;AAC1F,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,gBAAgB,GAAG,aAAa,CAAC;AAChG,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,kBAAkB,GAAG,cAAc,CAAC;AACnG,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,iBAAiB,GAAG,aAAa,CAAC;AACjG,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,GAAG,YAAY,CAAC;AAC9F,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,gBAAgB,GAAG,aAAa,CAAC;AAChG,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC"}