@loaders.gl/schema 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/index.js

@@ -0,0 +1,75 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTypeInfo = exports.deduceTypeFromValue = exports.deduceTypeFromColumn = exports.Struct = exports.FixedSizeList = exports.IntervalYearMonth = exports.IntervalDayTime = exports.Interval = exports.TimestampNanosecond = exports.TimestampMicrosecond = exports.TimestampMillisecond = exports.TimestampSecond = exports.Timestamp = exports.TimeSecond = exports.TimeMillisecond = exports.Time = exports.DateMillisecond = exports.DateDay = exports.Date = exports.Utf8 = exports.Float64 = exports.Float32 = exports.Float16 = exports.Float = exports.Uint64 = exports.Uint32 = exports.Uint16 = exports.Uint8 = exports.Int64 = exports.Int32 = exports.Int16 = exports.Int8 = exports.Int = exports.Bool = exports.Binary = exports.Null = exports.DataType = exports.Field = exports.Schema = exports.makeMeshAttributeMetadata = exports.deduceMeshField = exports.deduceMeshSchema = exports.convertMesh = exports.getMeshBoundingBox = exports.getMeshSize = exports.convertToArrayRow = exports.convertToObjectRow = exports.ColumnarTableBatchAggregator = exports.RowTableBatchAggregator = exports.TableBatchBuilder = void 0;
+exports.AsyncQueue = exports.getArrowTypeFromTypedArray = void 0;
+// TABLE CATEGORY UTILS
+var table_batch_builder_1 = require("./lib/batches/table-batch-builder");
+Object.defineProperty(exports, "TableBatchBuilder", { enumerable: true, get: function () { return __importDefault(table_batch_builder_1).default; } });
+var row_table_batch_aggregator_1 = require("./lib/batches/row-table-batch-aggregator");
+Object.defineProperty(exports, "RowTableBatchAggregator", { enumerable: true, get: function () { return __importDefault(row_table_batch_aggregator_1).default; } });
+var columnar_table_batch_aggregator_1 = require("./lib/batches/columnar-table-batch-aggregator");
+Object.defineProperty(exports, "ColumnarTableBatchAggregator", { enumerable: true, get: function () { return __importDefault(columnar_table_batch_aggregator_1).default; } });
+var row_utils_1 = require("./lib/utils/row-utils");
+Object.defineProperty(exports, "convertToObjectRow", { enumerable: true, get: function () { return row_utils_1.convertToObjectRow; } });
+Object.defineProperty(exports, "convertToArrayRow", { enumerable: true, get: function () { return row_utils_1.convertToArrayRow; } });
+var mesh_utils_1 = require("./category/mesh/mesh-utils");
+Object.defineProperty(exports, "getMeshSize", { enumerable: true, get: function () { return mesh_utils_1.getMeshSize; } });
+Object.defineProperty(exports, "getMeshBoundingBox", { enumerable: true, get: function () { return mesh_utils_1.getMeshBoundingBox; } });
+var convert_mesh_1 = require("./category/mesh/convert-mesh");
+Object.defineProperty(exports, "convertMesh", { enumerable: true, get: function () { return convert_mesh_1.convertMesh; } });
+var deduce_mesh_schema_1 = require("./category/mesh/deduce-mesh-schema");
+Object.defineProperty(exports, "deduceMeshSchema", { enumerable: true, get: function () { return deduce_mesh_schema_1.deduceMeshSchema; } });
+Object.defineProperty(exports, "deduceMeshField", { enumerable: true, get: function () { return deduce_mesh_schema_1.deduceMeshField; } });
+Object.defineProperty(exports, "makeMeshAttributeMetadata", { enumerable: true, get: function () { return deduce_mesh_schema_1.makeMeshAttributeMetadata; } });
+// SCHEMA
+var schema_1 = require("./lib/schema/schema");
+Object.defineProperty(exports, "Schema", { enumerable: true, get: function () { return schema_1.Schema; } });
+Object.defineProperty(exports, "Field", { enumerable: true, get: function () { return schema_1.Field; } });
+Object.defineProperty(exports, "DataType", { enumerable: true, get: function () { return schema_1.DataType; } });
+Object.defineProperty(exports, "Null", { enumerable: true, get: function () { return schema_1.Null; } });
+Object.defineProperty(exports, "Binary", { enumerable: true, get: function () { return schema_1.Binary; } });
+Object.defineProperty(exports, "Bool", { enumerable: true, get: function () { return schema_1.Bool; } });
+Object.defineProperty(exports, "Int", { enumerable: true, get: function () { return schema_1.Int; } });
+Object.defineProperty(exports, "Int8", { enumerable: true, get: function () { return schema_1.Int8; } });
+Object.defineProperty(exports, "Int16", { enumerable: true, get: function () { return schema_1.Int16; } });
+Object.defineProperty(exports, "Int32", { enumerable: true, get: function () { return schema_1.Int32; } });
+Object.defineProperty(exports, "Int64", { enumerable: true, get: function () { return schema_1.Int64; } });
+Object.defineProperty(exports, "Uint8", { enumerable: true, get: function () { return schema_1.Uint8; } });
+Object.defineProperty(exports, "Uint16", { enumerable: true, get: function () { return schema_1.Uint16; } });
+Object.defineProperty(exports, "Uint32", { enumerable: true, get: function () { return schema_1.Uint32; } });
+Object.defineProperty(exports, "Uint64", { enumerable: true, get: function () { return schema_1.Uint64; } });
+Object.defineProperty(exports, "Float", { enumerable: true, get: function () { return schema_1.Float; } });
+Object.defineProperty(exports, "Float16", { enumerable: true, get: function () { return schema_1.Float16; } });
+Object.defineProperty(exports, "Float32", { enumerable: true, get: function () { return schema_1.Float32; } });
+Object.defineProperty(exports, "Float64", { enumerable: true, get: function () { return schema_1.Float64; } });
+Object.defineProperty(exports, "Utf8", { enumerable: true, get: function () { return schema_1.Utf8; } });
+Object.defineProperty(exports, "Date", { enumerable: true, get: function () { return schema_1.Date; } });
+Object.defineProperty(exports, "DateDay", { enumerable: true, get: function () { return schema_1.DateDay; } });
+Object.defineProperty(exports, "DateMillisecond", { enumerable: true, get: function () { return schema_1.DateMillisecond; } });
+Object.defineProperty(exports, "Time", { enumerable: true, get: function () { return schema_1.Time; } });
+Object.defineProperty(exports, "TimeMillisecond", { enumerable: true, get: function () { return schema_1.TimeMillisecond; } });
+Object.defineProperty(exports, "TimeSecond", { enumerable: true, get: function () { return schema_1.TimeSecond; } });
+Object.defineProperty(exports, "Timestamp", { enumerable: true, get: function () { return schema_1.Timestamp; } });
+Object.defineProperty(exports, "TimestampSecond", { enumerable: true, get: function () { return schema_1.TimestampSecond; } });
+Object.defineProperty(exports, "TimestampMillisecond", { enumerable: true, get: function () { return schema_1.TimestampMillisecond; } });
+Object.defineProperty(exports, "TimestampMicrosecond", { enumerable: true, get: function () { return schema_1.TimestampMicrosecond; } });
+Object.defineProperty(exports, "TimestampNanosecond", { enumerable: true, get: function () { return schema_1.TimestampNanosecond; } });
+Object.defineProperty(exports, "Interval", { enumerable: true, get: function () { return schema_1.Interval; } });
+Object.defineProperty(exports, "IntervalDayTime", { enumerable: true, get: function () { return schema_1.IntervalDayTime; } });
+Object.defineProperty(exports, "IntervalYearMonth", { enumerable: true, get: function () { return schema_1.IntervalYearMonth; } });
+Object.defineProperty(exports, "FixedSizeList", { enumerable: true, get: function () { return schema_1.FixedSizeList; } });
+Object.defineProperty(exports, "Struct", { enumerable: true, get: function () { return schema_1.Struct; } });
+// EXPERIMENTAL APIs
+// SCHEMA UTILS
+var deduce_column_type_1 = require("./lib/schema-utils/deduce-column-type");
+Object.defineProperty(exports, "deduceTypeFromColumn", { enumerable: true, get: function () { return deduce_column_type_1.deduceTypeFromColumn; } });
+Object.defineProperty(exports, "deduceTypeFromValue", { enumerable: true, get: function () { return deduce_column_type_1.deduceTypeFromValue; } });
+var get_type_info_1 = require("./lib/arrow/get-type-info");
+Object.defineProperty(exports, "getTypeInfo", { enumerable: true, get: function () { return get_type_info_1.getTypeInfo; } });
+var arrow_like_type_utils_1 = require("./lib/arrow/arrow-like-type-utils");
+Object.defineProperty(exports, "getArrowTypeFromTypedArray", { enumerable: true, get: function () { return arrow_like_type_utils_1.getArrowTypeFromTypedArray; } });
+var async_queue_1 = require("./lib/utils/async-queue");
+Object.defineProperty(exports, "AsyncQueue", { enumerable: true, get: function () { return __importDefault(async_queue_1).default; } });

package/dist/lib/arrow/arrow-like-type-utils.d.ts

@@ -0,0 +1,4 @@
+import type { TypedArray } from '../../types';
+import { DataType } from '../schema/schema';
+export declare function getArrowTypeFromTypedArray(array: TypedArray): DataType;
+//# sourceMappingURL=arrow-like-type-utils.d.ts.map

package/dist/lib/arrow/arrow-like-type-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-like-type-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/arrow-like-type-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,QAAQ,EAST,MAAM,kBAAkB,CAAC;AAE1B,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,UAAU,GAAG,QAAQ,CAqBtE"}

package/dist/lib/arrow/arrow-like-type-utils.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getArrowTypeFromTypedArray = void 0;
+const schema_1 = require("../schema/schema");
+function getArrowTypeFromTypedArray(array) {
+    switch (array.constructor) {
+        case Int8Array:
+            return new schema_1.Int8();
+        case Uint8Array:
+            return new schema_1.Uint8();
+        case Int16Array:
+            return new schema_1.Int16();
+        case Uint16Array:
+            return new schema_1.Uint16();
+        case Int32Array:
+            return new schema_1.Int32();
+        case Uint32Array:
+            return new schema_1.Uint32();
+        case Float32Array:
+            return new schema_1.Float32();
+        case Float64Array:
+            return new schema_1.Float64();
+        default:
+            throw new Error('array type not supported');
+    }
+}
+exports.getArrowTypeFromTypedArray = getArrowTypeFromTypedArray;

package/dist/lib/arrow/arrow-type-utils.d.ts

@@ -0,0 +1 @@
+//# sourceMappingURL=arrow-type-utils.d.ts.map

package/dist/lib/arrow/arrow-type-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-type-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/arrow-type-utils.ts"],"names":[],"mappings":""}

package/dist/lib/arrow/arrow-type-utils.js

@@ -0,0 +1,70 @@
+"use strict";
+/*
+import type {TypedArray} from '../../types';
+import {
+  DataType,
+  Float32,
+  Float64,
+  Int16,
+  Int32,
+  Int8,
+  Uint16,
+  Uint32,
+  Uint8,
+  Int8Vector,
+  Uint8Vector,
+  Int16Vector,
+  Uint16Vector,
+  Int32Vector,
+  Uint32Vector,
+  Float32Vector,
+  Float64Vector
+} from 'apache-arrow/Arrow.dom';
+import {AbstractVector} from 'apache-arrow/vector';
+
+export function getArrowType(array: TypedArray): DataType {
+  switch (array.constructor) {
+    case Int8Array:
+      return new Int8();
+    case Uint8Array:
+      return new Uint8();
+    case Int16Array:
+      return new Int16();
+    case Uint16Array:
+      return new Uint16();
+    case Int32Array:
+      return new Int32();
+    case Uint32Array:
+      return new Uint32();
+    case Float32Array:
+      return new Float32();
+    case Float64Array:
+      return new Float64();
+    default:
+      throw new Error('array type not supported');
+  }
+}
+
+export function getArrowVector(array: TypedArray): AbstractVector {
+  switch (array.constructor) {
+    case Int8Array:
+      return Int8Vector.from(array);
+    case Uint8Array:
+      return Uint8Vector.from(array);
+    case Int16Array:
+      return Int16Vector.from(array);
+    case Uint16Array:
+      return Uint16Vector.from(array);
+    case Int32Array:
+      return Int32Vector.from(array);
+    case Uint32Array:
+      return Uint32Vector.from(array);
+    case Float32Array:
+      return Float32Vector.from(array);
+    case Float64Array:
+      return Float64Vector.from(array);
+    default:
+      throw new Error('array type not supported');
+  }
+}
+*/

package/dist/lib/arrow/get-type-info.d.ts

@@ -0,0 +1,14 @@
+import { Type } from '../schema/schema';
+import { AnyArray } from '../../types';
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+export declare function getTypeInfo(arrowTypeLike: any): {
+    typeId: Type;
+    ArrayType: AnyArray;
+    typeName: string;
+    typeEnumName?: string;
+    precision?: number;
+};
+//# sourceMappingURL=get-type-info.d.ts.map

package/dist/lib/arrow/get-type-info.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-type-info.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/get-type-info.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,IAAI,EAAC,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAErC;;;GAGG;AACH,wBAAgB,WAAW,CAAC,aAAa,EAAE,GAAG,GAAG;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,SAAS,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAQA"}

package/dist/lib/arrow/get-type-info.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTypeInfo = void 0;
+const schema_1 = require("../schema/schema");
+/**
+ * Gets type information from an Arrow type object or "mock" Arrow type object
+ * @param arrowTypeLike Arrow Type or type object of similar shape
+ */
+function getTypeInfo(arrowTypeLike) {
+    return {
+        typeId: arrowTypeLike.typeId,
+        ArrayType: arrowTypeLike.ArrayType,
+        typeName: arrowTypeLike.toString(),
+        typeEnumName: getTypeKey(arrowTypeLike.typeId),
+        precision: arrowTypeLike.precision
+    };
+}
+exports.getTypeInfo = getTypeInfo;
+let ReverseType = null;
+function getTypeKey(typeKey) {
+    if (!ReverseType) {
+        ReverseType = {};
+        for (const key in schema_1.Type) {
+            ReverseType[schema_1.Type[key]] = key;
+        }
+    }
+    return ReverseType[typeKey];
+}

package/dist/lib/batches/base-table-batch-aggregator.d.ts

@@ -0,0 +1,19 @@
+import type { Schema } from '../schema/schema';
+import type { TableBatch } from '../../category/table/table-types';
+import { TableBatchAggregator, TableBatchOptions } from './table-batch-aggregator';
+export default class RowTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    options: TableBatchOptions;
+    length: number;
+    rows: any[] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=base-table-batch-aggregator.d.ts.map

package/dist/lib/batches/base-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/base-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIjF,MAAM,CAAC,OAAO,OAAO,uBAAwB,YAAW,oBAAoB;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAE3B,MAAM,EAAE,MAAM,CAAK;IACnB,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAQ;IAC1B,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IActD,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAU9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAUrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAoB9B"}

package/dist/lib/batches/base-table-batch-aggregator.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const DEFAULT_ROW_COUNT = 100;
+class RowTableBatchAggregator {
+    constructor(schema, options) {
+        this.length = 0;
+        this.rows = null;
+        this.cursor = 0;
+        this._headers = [];
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (!Array.isArray(schema)) {
+            this._headers = [];
+            for (const key in schema) {
+                this._headers[schema[key].index] = schema[key].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+        this.rows[this.length] = row;
+        this.length++;
+    }
+    getBatch() {
+        let rows = this.rows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.rows = null;
+        const batch = {
+            shape: this.options.shape,
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            schema: this.schema,
+            cursor: this.cursor
+        };
+        return batch;
+    }
+}
+exports.default = RowTableBatchAggregator;

package/dist/lib/batches/columnar-table-batch-aggregator.d.ts

@@ -0,0 +1,23 @@
+import type { Schema } from '../schema/schema';
+import type { ColumnarTableBatch, ArrowTableBatch } from '../../category/table/table-types';
+import { TableBatchAggregator } from './table-batch-aggregator';
+declare type ColumnarTableBatchOptions = {};
+export default class ColumnarTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    length: number;
+    allocated: number;
+    columns: {
+        [columnName: string]: any[];
+    };
+    constructor(schema: Schema, options: ColumnarTableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[]): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    getBatch(): ColumnarTableBatch | ArrowTableBatch | null;
+    _reallocateColumns(): void;
+    _pruneColumns(): void;
+}
+export {};
+//# sourceMappingURL=columnar-table-batch-aggregator.d.ts.map

package/dist/lib/batches/columnar-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"columnar-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,kBAAkB,EAAE,eAAe,EAAC,MAAM,kCAAkC,CAAC;AAC1F,OAAO,EAAC,oBAAoB,EAAC,MAAM,0BAA0B,CAAC;AAE9D,aAAK,yBAAyB,GAAG,EAAE,CAAC;AAIpC,MAAM,CAAC,OAAO,OAAO,4BAA6B,YAAW,oBAAoB;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAK;IACnB,SAAS,EAAE,MAAM,CAAK;IACtB,OAAO,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;KAAC,CAAM;gBAEhC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,yBAAyB;IAK9D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAWtB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IASpD,QAAQ,IAAI,kBAAkB,GAAG,eAAe,GAAG,IAAI;IA6BvD,kBAAkB;IA8BlB,aAAa;CAKd"}

package/dist/lib/batches/columnar-table-batch-aggregator.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const DEFAULT_ROW_COUNT = 100;
+class ColumnarTableBatchAggregator {
+    constructor(schema, options) {
+        this.length = 0;
+        this.allocated = 0;
+        this.columns = {};
+        this.schema = schema;
+        this._reallocateColumns();
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        let i = 0;
+        // TODO what if no csv header, columns not populated?
+        for (const fieldName in this.columns) {
+            this.columns[fieldName][this.length] = row[i++];
+        }
+        this.length++;
+    }
+    addObjectRow(row) {
+        // If user keeps pushing rows beyond batch size, reallocate
+        this._reallocateColumns();
+        for (const fieldName in row) {
+            this.columns[fieldName][this.length] = row[fieldName];
+        }
+        this.length++;
+    }
+    getBatch() {
+        this._pruneColumns();
+        const columns = Array.isArray(this.schema) ? this.columns : {};
+        // schema is an array if there're no headers
+        // object if there are headers
+        // columns should match schema format
+        if (!Array.isArray(this.schema)) {
+            for (const fieldName in this.schema) {
+                const field = this.schema[fieldName];
+                columns[field.name] = this.columns[field.index];
+            }
+        }
+        this.columns = {};
+        const batch = {
+            shape: 'columnar-table',
+            batchType: 'data',
+            data: columns,
+            schema: this.schema,
+            length: this.length
+        };
+        return batch;
+    }
+    // HELPERS
+    _reallocateColumns() {
+        if (this.length < this.allocated) {
+            return;
+        }
+        // @ts-ignore TODO
+        this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
+        this.columns = {};
+        for (const fieldName in this.schema) {
+            const field = this.schema[fieldName];
+            const ArrayType = field.type || Float32Array;
+            const oldColumn = this.columns[field.index];
+            if (oldColumn && ArrayBuffer.isView(oldColumn)) {
+                // Copy the old data to the new array
+                const typedArray = new ArrayType(this.allocated);
+                typedArray.set(oldColumn);
+                this.columns[field.index] = typedArray;
+            }
+            else if (oldColumn) {
+                // Plain array
+                oldColumn.length = this.allocated;
+                this.columns[field.index] = oldColumn;
+            }
+            else {
+                // Create new
+                this.columns[field.index] = new ArrayType(this.allocated);
+            }
+        }
+    }
+    _pruneColumns() {
+        for (const [columnName, column] of Object.entries(this.columns)) {
+            this.columns[columnName] = column.slice(0, this.length);
+        }
+    }
+}
+exports.default = ColumnarTableBatchAggregator;

package/dist/lib/batches/row-table-batch-aggregator.d.ts

@@ -0,0 +1,22 @@
+import type { Schema } from '../schema/schema';
+import type { TableBatch } from '../../category/table/table-types';
+import { TableBatchAggregator, TableBatchOptions } from './table-batch-aggregator';
+export default class RowTableBatchAggregator implements TableBatchAggregator {
+    schema: Schema;
+    options: TableBatchOptions;
+    length: number;
+    objectRows: {
+        [columnName: string]: any;
+    } | null;
+    arrayRows: any[] | null;
+    cursor: number;
+    private _headers;
+    constructor(schema: Schema, options: TableBatchOptions);
+    rowCount(): number;
+    addArrayRow(row: any[], cursor?: number): void;
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }, cursor?: number): void;
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=row-table-batch-aggregator.d.ts.map

package/dist/lib/batches/row-table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/row-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AAGjE,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIjF,MAAM,CAAC,OAAO,OAAO,uBAAwB,YAAW,oBAAoB;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAE3B,MAAM,EAAE,MAAM,CAAK;IACnB,UAAU,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI,CAAQ;IACtD,SAAS,EAAE,GAAG,EAAE,GAAG,IAAI,CAAQ;IAC/B,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IActD,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAmB9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAmBrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAmB9B"}

package/dist/lib/batches/row-table-batch-aggregator.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
+const row_utils_1 = require("../utils/row-utils");
+const DEFAULT_ROW_COUNT = 100;
+class RowTableBatchAggregator {
+    constructor(schema, options) {
+        this.length = 0;
+        this.objectRows = null;
+        this.arrayRows = null;
+        this.cursor = 0;
+        this._headers = [];
+        this.options = options;
+        this.schema = schema;
+        // schema is an array if there're no headers
+        // object if there are headers
+        if (!Array.isArray(schema)) {
+            this._headers = [];
+            for (const key in schema) {
+                this._headers[schema[key].index] = schema[key].name;
+            }
+        }
+    }
+    rowCount() {
+        return this.length;
+    }
+    addArrayRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'object-row-table':
+                const rowObject = (0, row_utils_1.convertToObjectRow)(row, this._headers);
+                this.addObjectRow(rowObject, cursor);
+                break;
+            case 'array-row-table':
+                this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
+                this.arrayRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    addObjectRow(row, cursor) {
+        if (Number.isFinite(cursor)) {
+            this.cursor = cursor;
+        }
+        // eslint-disable-next-line default-case
+        switch (this.options.shape) {
+            case 'array-row-table':
+                const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
+                this.addArrayRow(rowArray, cursor);
+                break;
+            case 'object-row-table':
+                this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
+                this.objectRows[this.length] = row;
+                this.length++;
+                break;
+        }
+    }
+    getBatch() {
+        let rows = this.arrayRows || this.objectRows;
+        if (!rows) {
+            return null;
+        }
+        rows = rows.slice(0, this.length);
+        this.arrayRows = null;
+        this.objectRows = null;
+        return {
+            shape: this.options.shape,
+            batchType: 'data',
+            data: rows,
+            length: this.length,
+            schema: this.schema,
+            cursor: this.cursor
+        };
+    }
+}
+exports.default = RowTableBatchAggregator;

package/dist/lib/batches/table-batch-aggregator.d.ts

@@ -0,0 +1,25 @@
+import type { Schema } from '../schema/schema';
+import type { TableBatch } from '../../category/table/table-types';
+export interface TableBatchOptions {
+    batchSize: number | string;
+    [key: string]: any;
+}
+export interface TableBatchConstructor {
+    new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
+}
+/**
+ * TableBatchBuilder delegates batch building to this interface
+ */
+export interface TableBatchAggregator {
+    /** Number of rows */
+    rowCount(): number;
+    /** Add one row */
+    addArrayRow(row: any[]): void;
+    /** Add one row */
+    addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    /** return a batch object */
+    getBatch(): TableBatch | null;
+}
+//# sourceMappingURL=table-batch-aggregator.d.ts.map

package/dist/lib/batches/table-batch-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AAEjE,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,CAAC;CACxE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,qBAAqB;IACrB,QAAQ,IAAI,MAAM,CAAC;IACnB,kBAAkB;IAClB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC9B,kBAAkB;IAClB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI,CAAC;IACrD,4BAA4B;IAC5B,QAAQ,IAAI,UAAU,GAAG,IAAI,CAAC;CAC/B"}

package/dist/lib/batches/table-batch-aggregator.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lib/batches/table-batch-builder.d.ts

@@ -0,0 +1,53 @@
+import type { Schema } from '../schema/schema';
+import type { TableBatch } from '../../category/table/table-types';
+import type { TableBatchConstructor } from './table-batch-aggregator';
+declare type TableBatchBuilderOptions = {
+    shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
+    batchSize?: number | 'auto';
+    batchDebounceMs?: number;
+    limit: number;
+    _limitMB: number;
+};
+declare type GetBatchOptions = {
+    bytesUsed?: number;
+    [key: string]: any;
+};
+/** Incrementally builds batches from a stream of rows */
+export default class TableBatchBuilder {
+    schema: Schema;
+    options: Required<TableBatchBuilderOptions>;
+    private aggregator;
+    private batchCount;
+    private bytesUsed;
+    private isChunkComplete;
+    private lastBatchEmittedMs;
+    private totalLength;
+    private totalBytes;
+    private rowBytes;
+    static ArrowBatch?: TableBatchConstructor;
+    constructor(schema: Schema, options?: TableBatchBuilderOptions);
+    limitReached(): boolean;
+    /** @deprecated Use addArrayRow or addObjectRow */
+    addRow(row: any[] | {
+        [columnName: string]: any;
+    }): void;
+    /** Add one row to the batch */
+    protected addArrayRow(row: any[]): void;
+    /** Add one row to the batch */
+    protected addObjectRow(row: {
+        [columnName: string]: any;
+    }): void;
+    /** Mark an incoming raw memory chunk has completed */
+    chunkComplete(chunk: ArrayBuffer | string): void;
+    getFullBatch(options?: GetBatchOptions): TableBatch | null;
+    getFinalBatch(options?: GetBatchOptions): TableBatch | null;
+    _estimateRowMB(row: any[] | object): number;
+    private _isFull;
+    /**
+     * bytesUsed can be set via chunkComplete or via getBatch*
+     */
+    private _getBatch;
+    private _getTableBatchType;
+}
+export {};
+//# sourceMappingURL=table-batch-builder.d.ts.map

package/dist/lib/batches/table-batch-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-batch-builder.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/table-batch-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EAAuB,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AAM1F,aAAK,wBAAwB,GAAG;IAC9B,KAAK,EAAE,WAAW,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,aAAa,CAAC;IAC/F,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,aAAK,eAAe,GAAG;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAYF,yDAAyD;AACzD,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,QAAQ,CAAC,wBAAwB,CAAC,CAAC;IAE5C,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,kBAAkB,CAAsB;IAChD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAa;IAE7B,MAAM,CAAC,UAAU,CAAC,EAAE,qBAAqB,CAAC;gBAE9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAK9D,YAAY,IAAI,OAAO;IAUvB,kDAAkD;IAClD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IActD,+BAA+B;IAC/B,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAQhC,+BAA+B;IAC/B,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IAQ9D,sDAAsD;IACtD,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAUhD,YAAY,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAI1D,aAAa,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAM3D,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,MAAM;IAI3C,OAAO,CAAC,OAAO;IA2Bf;;OAEG;IACH,OAAO,CAAC,SAAS;IAmBjB,OAAO,CAAC,kBAAkB;CAkB3B"}