@loaders.gl/schema 3.1.0-alpha.3 → 3.1.0-beta.2

package/dist/es5/index.js

@@ -0,0 +1,344 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "TableBatchBuilder", {
+  enumerable: true,
+  get: function () {
+    return _tableBatchBuilder.default;
+  }
+});
+Object.defineProperty(exports, "RowTableBatchAggregator", {
+  enumerable: true,
+  get: function () {
+    return _rowTableBatchAggregator.default;
+  }
+});
+Object.defineProperty(exports, "ColumnarTableBatchAggregator", {
+  enumerable: true,
+  get: function () {
+    return _columnarTableBatchAggregator.default;
+  }
+});
+Object.defineProperty(exports, "convertToObjectRow", {
+  enumerable: true,
+  get: function () {
+    return _rowUtils.convertToObjectRow;
+  }
+});
+Object.defineProperty(exports, "convertToArrayRow", {
+  enumerable: true,
+  get: function () {
+    return _rowUtils.convertToArrayRow;
+  }
+});
+Object.defineProperty(exports, "getMeshSize", {
+  enumerable: true,
+  get: function () {
+    return _meshUtils.getMeshSize;
+  }
+});
+Object.defineProperty(exports, "getMeshBoundingBox", {
+  enumerable: true,
+  get: function () {
+    return _meshUtils.getMeshBoundingBox;
+  }
+});
+Object.defineProperty(exports, "convertMesh", {
+  enumerable: true,
+  get: function () {
+    return _convertMesh.convertMesh;
+  }
+});
+Object.defineProperty(exports, "deduceMeshSchema", {
+  enumerable: true,
+  get: function () {
+    return _deduceMeshSchema.deduceMeshSchema;
+  }
+});
+Object.defineProperty(exports, "deduceMeshField", {
+  enumerable: true,
+  get: function () {
+    return _deduceMeshSchema.deduceMeshField;
+  }
+});
+Object.defineProperty(exports, "makeMeshAttributeMetadata", {
+  enumerable: true,
+  get: function () {
+    return _deduceMeshSchema.makeMeshAttributeMetadata;
+  }
+});
+Object.defineProperty(exports, "Schema", {
+  enumerable: true,
+  get: function () {
+    return _schema.Schema;
+  }
+});
+Object.defineProperty(exports, "Field", {
+  enumerable: true,
+  get: function () {
+    return _schema.Field;
+  }
+});
+Object.defineProperty(exports, "DataType", {
+  enumerable: true,
+  get: function () {
+    return _schema.DataType;
+  }
+});
+Object.defineProperty(exports, "Null", {
+  enumerable: true,
+  get: function () {
+    return _schema.Null;
+  }
+});
+Object.defineProperty(exports, "Binary", {
+  enumerable: true,
+  get: function () {
+    return _schema.Binary;
+  }
+});
+Object.defineProperty(exports, "Bool", {
+  enumerable: true,
+  get: function () {
+    return _schema.Bool;
+  }
+});
+Object.defineProperty(exports, "Int", {
+  enumerable: true,
+  get: function () {
+    return _schema.Int;
+  }
+});
+Object.defineProperty(exports, "Int8", {
+  enumerable: true,
+  get: function () {
+    return _schema.Int8;
+  }
+});
+Object.defineProperty(exports, "Int16", {
+  enumerable: true,
+  get: function () {
+    return _schema.Int16;
+  }
+});
+Object.defineProperty(exports, "Int32", {
+  enumerable: true,
+  get: function () {
+    return _schema.Int32;
+  }
+});
+Object.defineProperty(exports, "Int64", {
+  enumerable: true,
+  get: function () {
+    return _schema.Int64;
+  }
+});
+Object.defineProperty(exports, "Uint8", {
+  enumerable: true,
+  get: function () {
+    return _schema.Uint8;
+  }
+});
+Object.defineProperty(exports, "Uint16", {
+  enumerable: true,
+  get: function () {
+    return _schema.Uint16;
+  }
+});
+Object.defineProperty(exports, "Uint32", {
+  enumerable: true,
+  get: function () {
+    return _schema.Uint32;
+  }
+});
+Object.defineProperty(exports, "Uint64", {
+  enumerable: true,
+  get: function () {
+    return _schema.Uint64;
+  }
+});
+Object.defineProperty(exports, "Float", {
+  enumerable: true,
+  get: function () {
+    return _schema.Float;
+  }
+});
+Object.defineProperty(exports, "Float16", {
+  enumerable: true,
+  get: function () {
+    return _schema.Float16;
+  }
+});
+Object.defineProperty(exports, "Float32", {
+  enumerable: true,
+  get: function () {
+    return _schema.Float32;
+  }
+});
+Object.defineProperty(exports, "Float64", {
+  enumerable: true,
+  get: function () {
+    return _schema.Float64;
+  }
+});
+Object.defineProperty(exports, "Utf8", {
+  enumerable: true,
+  get: function () {
+    return _schema.Utf8;
+  }
+});
+Object.defineProperty(exports, "Date", {
+  enumerable: true,
+  get: function () {
+    return _schema.Date;
+  }
+});
+Object.defineProperty(exports, "DateDay", {
+  enumerable: true,
+  get: function () {
+    return _schema.DateDay;
+  }
+});
+Object.defineProperty(exports, "DateMillisecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.DateMillisecond;
+  }
+});
+Object.defineProperty(exports, "Time", {
+  enumerable: true,
+  get: function () {
+    return _schema.Time;
+  }
+});
+Object.defineProperty(exports, "TimeMillisecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimeMillisecond;
+  }
+});
+Object.defineProperty(exports, "TimeSecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimeSecond;
+  }
+});
+Object.defineProperty(exports, "Timestamp", {
+  enumerable: true,
+  get: function () {
+    return _schema.Timestamp;
+  }
+});
+Object.defineProperty(exports, "TimestampSecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimestampSecond;
+  }
+});
+Object.defineProperty(exports, "TimestampMillisecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimestampMillisecond;
+  }
+});
+Object.defineProperty(exports, "TimestampMicrosecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimestampMicrosecond;
+  }
+});
+Object.defineProperty(exports, "TimestampNanosecond", {
+  enumerable: true,
+  get: function () {
+    return _schema.TimestampNanosecond;
+  }
+});
+Object.defineProperty(exports, "Interval", {
+  enumerable: true,
+  get: function () {
+    return _schema.Interval;
+  }
+});
+Object.defineProperty(exports, "IntervalDayTime", {
+  enumerable: true,
+  get: function () {
+    return _schema.IntervalDayTime;
+  }
+});
+Object.defineProperty(exports, "IntervalYearMonth", {
+  enumerable: true,
+  get: function () {
+    return _schema.IntervalYearMonth;
+  }
+});
+Object.defineProperty(exports, "FixedSizeList", {
+  enumerable: true,
+  get: function () {
+    return _schema.FixedSizeList;
+  }
+});
+Object.defineProperty(exports, "Struct", {
+  enumerable: true,
+  get: function () {
+    return _schema.Struct;
+  }
+});
+Object.defineProperty(exports, "deduceTypeFromColumn", {
+  enumerable: true,
+  get: function () {
+    return _deduceColumnType.deduceTypeFromColumn;
+  }
+});
+Object.defineProperty(exports, "deduceTypeFromValue", {
+  enumerable: true,
+  get: function () {
+    return _deduceColumnType.deduceTypeFromValue;
+  }
+});
+Object.defineProperty(exports, "getTypeInfo", {
+  enumerable: true,
+  get: function () {
+    return _getTypeInfo.getTypeInfo;
+  }
+});
+Object.defineProperty(exports, "getArrowTypeFromTypedArray", {
+  enumerable: true,
+  get: function () {
+    return _arrowLikeTypeUtils.getArrowTypeFromTypedArray;
+  }
+});
+Object.defineProperty(exports, "AsyncQueue", {
+  enumerable: true,
+  get: function () {
+    return _asyncQueue.default;
+  }
+});
+
+var _tableBatchBuilder = _interopRequireDefault(require("./lib/batches/table-batch-builder"));
+
+var _rowTableBatchAggregator = _interopRequireDefault(require("./lib/batches/row-table-batch-aggregator"));
+
+var _columnarTableBatchAggregator = _interopRequireDefault(require("./lib/batches/columnar-table-batch-aggregator"));
+
+var _rowUtils = require("./lib/utils/row-utils");
+
+var _meshUtils = require("./category/mesh/mesh-utils");
+
+var _convertMesh = require("./category/mesh/convert-mesh");
+
+var _deduceMeshSchema = require("./category/mesh/deduce-mesh-schema");
+
+var _schema = require("./lib/schema/schema");
+
+var _deduceColumnType = require("./lib/schema-utils/deduce-column-type");
+
+var _getTypeInfo = require("./lib/arrow/get-type-info");
+
+var _arrowLikeTypeUtils = require("./lib/arrow/arrow-like-type-utils");
+
+var _asyncQueue = _interopRequireDefault(require("./lib/utils/async-queue"));
+//# sourceMappingURL=index.js.map

package/dist/es5/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;;AAEA;;AACA;;AAEA;;AAYA;;AACA;;AACA;;AAmCA;;AA0CA;;AACA;;AACA;;AAEA","sourcesContent":["// COMMON CATEGORY\nexport type {TypedArray, NumberArray, AnyArray} from './types';\n\nexport type {Batch} from './category/common';\n\n// TABLE CATEGORY TYPES\n\nexport type {\n  Table,\n  ArrayRowTable,\n  ObjectRowTable,\n  ColumnarTable,\n  ArrowTable\n} from './category/table/table-types';\nexport type {\n  TableBatch,\n  RowArrayTableBatch,\n  RowObjectTableBatch,\n  ColumnarTableBatch,\n  ArrowTableBatch\n} from './category/table/table-types';\n\n// TABLE CATEGORY UTILS\nexport {default as TableBatchBuilder} from './lib/batches/table-batch-builder';\nexport type {TableBatchAggregator} from './lib/batches/table-batch-aggregator';\nexport {default as RowTableBatchAggregator} from './lib/batches/row-table-batch-aggregator';\nexport {default as ColumnarTableBatchAggregator} from './lib/batches/columnar-table-batch-aggregator';\n\nexport {convertToObjectRow, convertToArrayRow} from './lib/utils/row-utils';\n\n// MESH CATEGORY\nexport type {\n  MeshTable,\n  MeshArrowTable,\n  Mesh,\n  MeshGeometry,\n  MeshAttribute,\n  MeshAttributes\n} from './category/mesh/mesh-types';\n\nexport {getMeshSize, getMeshBoundingBox} from './category/mesh/mesh-utils';\nexport {convertMesh} from './category/mesh/convert-mesh';\nexport {\n  deduceMeshSchema,\n  deduceMeshField,\n  makeMeshAttributeMetadata\n} from './category/mesh/deduce-mesh-schema';\n\n// TYPES\n// GIS CATEGORY - GEOJSON\nexport type {GeoJSON, Feature, Geometry, Position, GeoJsonProperties} from './category/gis';\nexport type {\n  Point,\n  MultiPoint,\n  LineString,\n  MultiLineString,\n  Polygon,\n  MultiPolygon\n} from './category/gis';\n\n// GIS CATEGORY - BINARY\nexport type {\n  BinaryGeometryType,\n  BinaryGeometry,\n  BinaryPointGeometry,\n  BinaryLineGeometry,\n  BinaryPolygonGeometry,\n  BinaryAttribute\n} from './category/gis';\nexport type {\n  BinaryFeatures,\n  BinaryPointFeatures,\n  BinaryLineFeatures,\n  BinaryPolygonFeatures\n} from './category/gis';\n\n// SCHEMA\nexport {\n  Schema,\n  Field,\n  DataType,\n  Null,\n  Binary,\n  Bool,\n  Int,\n  Int8,\n  Int16,\n  Int32,\n  Int64,\n  Uint8,\n  Uint16,\n  Uint32,\n  Uint64,\n  Float,\n  Float16,\n  Float32,\n  Float64,\n  Utf8,\n  Date,\n  DateDay,\n  DateMillisecond,\n  Time,\n  TimeMillisecond,\n  TimeSecond,\n  Timestamp,\n  TimestampSecond,\n  TimestampMillisecond,\n  TimestampMicrosecond,\n  TimestampNanosecond,\n  Interval,\n  IntervalDayTime,\n  IntervalYearMonth,\n  FixedSizeList,\n  Struct\n} from './lib/schema/schema';\n\n// EXPERIMENTAL APIs\n\n// SCHEMA UTILS\nexport {deduceTypeFromColumn, deduceTypeFromValue} from './lib/schema-utils/deduce-column-type';\nexport {getTypeInfo} from './lib/arrow/get-type-info';\nexport {getArrowTypeFromTypedArray} from './lib/arrow/arrow-like-type-utils';\n\nexport {default as AsyncQueue} from './lib/utils/async-queue';\n"],"file":"index.js"}

package/dist/es5/lib/arrow/arrow-like-type-utils.js

@@ -0,0 +1,40 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getArrowTypeFromTypedArray = getArrowTypeFromTypedArray;
+
+var _schema = require("../schema/schema");
+
+function getArrowTypeFromTypedArray(array) {
+  switch (array.constructor) {
+    case Int8Array:
+      return new _schema.Int8();
+
+    case Uint8Array:
+      return new _schema.Uint8();
+
+    case Int16Array:
+      return new _schema.Int16();
+
+    case Uint16Array:
+      return new _schema.Uint16();
+
+    case Int32Array:
+      return new _schema.Int32();
+
+    case Uint32Array:
+      return new _schema.Uint32();
+
+    case Float32Array:
+      return new _schema.Float32();
+
+    case Float64Array:
+      return new _schema.Float64();
+
+    default:
+      throw new Error('array type not supported');
+  }
+}
+//# sourceMappingURL=arrow-like-type-utils.js.map

package/dist/es5/lib/arrow/arrow-like-type-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/arrow/arrow-like-type-utils.ts"],"names":["getArrowTypeFromTypedArray","array","constructor","Int8Array","Int8","Uint8Array","Uint8","Int16Array","Int16","Uint16Array","Uint16","Int32Array","Int32","Uint32Array","Uint32","Float32Array","Float32","Float64Array","Float64","Error"],"mappings":";;;;;;;AACA;;AAYO,SAASA,0BAAT,CAAoCC,KAApC,EAAiE;AACtE,UAAQA,KAAK,CAACC,WAAd;AACE,SAAKC,SAAL;AACE,aAAO,IAAIC,YAAJ,EAAP;;AACF,SAAKC,UAAL;AACE,aAAO,IAAIC,aAAJ,EAAP;;AACF,SAAKC,UAAL;AACE,aAAO,IAAIC,aAAJ,EAAP;;AACF,SAAKC,WAAL;AACE,aAAO,IAAIC,cAAJ,EAAP;;AACF,SAAKC,UAAL;AACE,aAAO,IAAIC,aAAJ,EAAP;;AACF,SAAKC,WAAL;AACE,aAAO,IAAIC,cAAJ,EAAP;;AACF,SAAKC,YAAL;AACE,aAAO,IAAIC,eAAJ,EAAP;;AACF,SAAKC,YAAL;AACE,aAAO,IAAIC,eAAJ,EAAP;;AACF;AACE,YAAM,IAAIC,KAAJ,CAAU,0BAAV,CAAN;AAlBJ;AAoBD","sourcesContent":["import type {TypedArray} from '../../types';\nimport {\n  DataType,\n  Float32,\n  Float64,\n  Int16,\n  Int32,\n  Int8,\n  Uint16,\n  Uint32,\n  Uint8\n} from '../schema/schema';\n\nexport function getArrowTypeFromTypedArray(array: TypedArray): DataType {\n  switch (array.constructor) {\n    case Int8Array:\n      return new Int8();\n    case Uint8Array:\n      return new Uint8();\n    case Int16Array:\n      return new Int16();\n    case Uint16Array:\n      return new Uint16();\n    case Int32Array:\n      return new Int32();\n    case Uint32Array:\n      return new Uint32();\n    case Float32Array:\n      return new Float32();\n    case Float64Array:\n      return new Float64();\n    default:\n      throw new Error('array type not supported');\n  }\n}\n"],"file":"arrow-like-type-utils.js"}

package/dist/es5/lib/arrow/arrow-type-utils.js

@@ -0,0 +1,2 @@
+"use strict";
+//# sourceMappingURL=arrow-type-utils.js.map

package/dist/es5/lib/arrow/arrow-type-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","sourcesContent":[],"file":"arrow-type-utils.js"}

package/dist/es5/lib/arrow/get-type-info.js

@@ -0,0 +1,33 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getTypeInfo = getTypeInfo;
+
+var _schema = require("../schema/schema");
+
+function getTypeInfo(arrowTypeLike) {
+  return {
+    typeId: arrowTypeLike.typeId,
+    ArrayType: arrowTypeLike.ArrayType,
+    typeName: arrowTypeLike.toString(),
+    typeEnumName: getTypeKey(arrowTypeLike.typeId),
+    precision: arrowTypeLike.precision
+  };
+}
+
+let ReverseType = null;
+
+function getTypeKey(typeKey) {
+  if (!ReverseType) {
+    ReverseType = {};
+
+    for (const key in _schema.Type) {
+      ReverseType[_schema.Type[key]] = key;
+    }
+  }
+
+  return ReverseType[typeKey];
+}
+//# sourceMappingURL=get-type-info.js.map

package/dist/es5/lib/arrow/get-type-info.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/arrow/get-type-info.ts"],"names":["getTypeInfo","arrowTypeLike","typeId","ArrayType","typeName","toString","typeEnumName","getTypeKey","precision","ReverseType","typeKey","key","Type"],"mappings":";;;;;;;AAAA;;AAOO,SAASA,WAAT,CAAqBC,aAArB,EAML;AACA,SAAO;AACLC,IAAAA,MAAM,EAAED,aAAa,CAACC,MADjB;AAELC,IAAAA,SAAS,EAAEF,aAAa,CAACE,SAFpB;AAGLC,IAAAA,QAAQ,EAAEH,aAAa,CAACI,QAAd,EAHL;AAILC,IAAAA,YAAY,EAAEC,UAAU,CAACN,aAAa,CAACC,MAAf,CAJnB;AAKLM,IAAAA,SAAS,EAAEP,aAAa,CAACO;AALpB,GAAP;AAOD;;AAED,IAAIC,WAA2C,GAAG,IAAlD;;AAEA,SAASF,UAAT,CAAoBG,OAApB,EAA6B;AAC3B,MAAI,CAACD,WAAL,EAAkB;AAChBA,IAAAA,WAAW,GAAG,EAAd;;AACA,SAAK,MAAME,GAAX,IAAkBC,YAAlB,EAAwB;AACtBH,MAAAA,WAAW,CAACG,aAAKD,GAAL,CAAD,CAAX,GAAyBA,GAAzB;AACD;AACF;;AAED,SAAOF,WAAW,CAACC,OAAD,CAAlB;AACD","sourcesContent":["import {Type} from '../schema/schema';\nimport {AnyArray} from '../../types';\n\n/**\n * Gets type information from an Arrow type object or \"mock\" Arrow type object\n * @param arrowTypeLike Arrow Type or type object of similar shape\n */\nexport function getTypeInfo(arrowTypeLike: any): {\n  typeId: Type;\n  ArrayType: AnyArray;\n  typeName: string;\n  typeEnumName?: string;\n  precision?: number;\n} {\n  return {\n    typeId: arrowTypeLike.typeId,\n    ArrayType: arrowTypeLike.ArrayType,\n    typeName: arrowTypeLike.toString(),\n    typeEnumName: getTypeKey(arrowTypeLike.typeId),\n    precision: arrowTypeLike.precision\n  };\n}\n\nlet ReverseType: {[key: string]: string} | null = null;\n\nfunction getTypeKey(typeKey) {\n  if (!ReverseType) {\n    ReverseType = {};\n    for (const key in Type) {\n      ReverseType[Type[key]] = key;\n    }\n  }\n\n  return ReverseType[typeKey];\n}\n"],"file":"get-type-info.js"}

package/dist/es5/lib/batches/base-table-batch-aggregator.js

@@ -0,0 +1,81 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+const DEFAULT_ROW_COUNT = 100;
+
+class RowTableBatchAggregator {
+  constructor(schema, options) {
+    (0, _defineProperty2.default)(this, "schema", void 0);
+    (0, _defineProperty2.default)(this, "options", void 0);
+    (0, _defineProperty2.default)(this, "length", 0);
+    (0, _defineProperty2.default)(this, "rows", null);
+    (0, _defineProperty2.default)(this, "cursor", 0);
+    (0, _defineProperty2.default)(this, "_headers", []);
+    this.options = options;
+    this.schema = schema;
+
+    if (!Array.isArray(schema)) {
+      this._headers = [];
+
+      for (const key in schema) {
+        this._headers[schema[key].index] = schema[key].name;
+      }
+    }
+  }
+
+  rowCount() {
+    return this.length;
+  }
+
+  addArrayRow(row, cursor) {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor;
+    }
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  addObjectRow(row, cursor) {
+    if (Number.isFinite(cursor)) {
+      this.cursor = cursor;
+    }
+
+    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
+    this.rows[this.length] = row;
+    this.length++;
+  }
+
+  getBatch() {
+    let rows = this.rows;
+
+    if (!rows) {
+      return null;
+    }
+
+    rows = rows.slice(0, this.length);
+    this.rows = null;
+    const batch = {
+      shape: this.options.shape,
+      batchType: 'data',
+      data: rows,
+      length: this.length,
+      schema: this.schema,
+      cursor: this.cursor
+    };
+    return batch;
+  }
+
+}
+
+exports.default = RowTableBatchAggregator;
+//# sourceMappingURL=base-table-batch-aggregator.js.map

package/dist/es5/lib/batches/base-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/batches/base-table-batch-aggregator.ts"],"names":["DEFAULT_ROW_COUNT","RowTableBatchAggregator","constructor","schema","options","Array","isArray","_headers","key","index","name","rowCount","length","addArrayRow","row","cursor","Number","isFinite","rows","addObjectRow","getBatch","slice","batch","shape","batchType","data"],"mappings":";;;;;;;;;;;AAIA,MAAMA,iBAAiB,GAAG,GAA1B;;AAEe,MAAMC,uBAAN,CAA8D;AAS3EC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAA6C;AAAA;AAAA;AAAA,kDALvC,CAKuC;AAAA,gDAJnC,IAImC;AAAA,kDAHvC,CAGuC;AAAA,oDAF3B,EAE2B;AACtD,SAAKA,OAAL,GAAeA,OAAf;AACA,SAAKD,MAAL,GAAcA,MAAd;;AAIA,QAAI,CAACE,KAAK,CAACC,OAAN,CAAcH,MAAd,CAAL,EAA4B;AAC1B,WAAKI,QAAL,GAAgB,EAAhB;;AACA,WAAK,MAAMC,GAAX,IAAkBL,MAAlB,EAA0B;AACxB,aAAKI,QAAL,CAAcJ,MAAM,CAACK,GAAD,CAAN,CAAYC,KAA1B,IAAmCN,MAAM,CAACK,GAAD,CAAN,CAAYE,IAA/C;AACD;AACF;AACF;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAaC,MAAb,EAAoC;AAC7C,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAED,SAAKG,IAAL,GAAY,KAAKA,IAAL,IAAa,IAAIb,KAAJ,CAAUL,iBAAV,CAAzB;AACA,SAAKkB,IAAL,CAAU,KAAKN,MAAf,IAAyBE,GAAzB;AACA,SAAKF,MAAL;AACD;;AAEDO,EAAAA,YAAY,CAACL,GAAD,EAAmCC,MAAnC,EAA0D;AACpE,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAED,SAAKG,IAAL,GAAY,KAAKA,IAAL,IAAa,IAAIb,KAAJ,CAAUL,iBAAV,CAAzB;AACA,SAAKkB,IAAL,CAAU,KAAKN,MAAf,IAAyBE,GAAzB;AACA,SAAKF,MAAL;AACD;;AAEDQ,EAAAA,QAAQ,GAAsB;AAC5B,QAAIF,IAAI,GAAG,KAAKA,IAAhB;;AACA,QAAI,CAACA,IAAL,EAAW;AACT,aAAO,IAAP;AACD;;AAEDA,IAAAA,IAAI,GAAGA,IAAI,CAACG,KAAL,CAAW,CAAX,EAAc,KAAKT,MAAnB,CAAP;AACA,SAAKM,IAAL,GAAY,IAAZ;AAEA,UAAMI,KAAiB,GAAG;AACxBC,MAAAA,KAAK,EAAE,KAAKnB,OAAL,CAAamB,KADI;AAExBC,MAAAA,SAAS,EAAE,MAFa;AAGxBC,MAAAA,IAAI,EAAEP,IAHkB;AAIxBN,MAAAA,MAAM,EAAE,KAAKA,MAJW;AAKxBT,MAAAA,MAAM,EAAE,KAAKA,MALW;AAMxBY,MAAAA,MAAM,EAAE,KAAKA;AANW,KAA1B;AASA,WAAOO,KAAP;AACD;;AAlE0E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class RowTableBatchAggregator implements TableBatchAggregator {\n  schema: Schema;\n  options: TableBatchOptions;\n\n  length: number = 0;\n  rows: any[] | null = null;\n  cursor: number = 0;\n  private _headers: string[] = [];\n\n  constructor(schema: Schema, options: TableBatchOptions) {\n    this.options = options;\n    this.schema = schema;\n\n    // schema is an array if there're no headers\n    // object if there are headers\n    if (!Array.isArray(schema)) {\n      this._headers = [];\n      for (const key in schema) {\n        this._headers[schema[key].index] = schema[key].name;\n      }\n    }\n  }\n\n  rowCount(): number {\n    return this.length;\n  }\n\n  addArrayRow(row: any[], cursor?: number): void {\n    if (Number.isFinite(cursor)) {\n      this.cursor = cursor as number;\n    }\n\n    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);\n    this.rows[this.length] = row;\n    this.length++;\n  }\n\n  addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {\n    if (Number.isFinite(cursor)) {\n      this.cursor = cursor as number;\n    }\n\n    this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);\n    this.rows[this.length] = row;\n    this.length++;\n  }\n\n  getBatch(): TableBatch | null {\n    let rows = this.rows;\n    if (!rows) {\n      return null;\n    }\n\n    rows = rows.slice(0, this.length);\n    this.rows = null;\n\n    const batch: TableBatch = {\n      shape: this.options.shape,\n      batchType: 'data',\n      data: rows,\n      length: this.length,\n      schema: this.schema,\n      cursor: this.cursor\n    };\n\n    return batch;\n  }\n}\n"],"file":"base-table-batch-aggregator.js"}

package/dist/es5/lib/batches/columnar-table-batch-aggregator.js

@@ -0,0 +1,109 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+const DEFAULT_ROW_COUNT = 100;
+
+class ColumnarTableBatchAggregator {
+  constructor(schema, options) {
+    (0, _defineProperty2.default)(this, "schema", void 0);
+    (0, _defineProperty2.default)(this, "length", 0);
+    (0, _defineProperty2.default)(this, "allocated", 0);
+    (0, _defineProperty2.default)(this, "columns", {});
+    this.schema = schema;
+
+    this._reallocateColumns();
+  }
+
+  rowCount() {
+    return this.length;
+  }
+
+  addArrayRow(row) {
+    this._reallocateColumns();
+
+    let i = 0;
+
+    for (const fieldName in this.columns) {
+      this.columns[fieldName][this.length] = row[i++];
+    }
+
+    this.length++;
+  }
+
+  addObjectRow(row) {
+    this._reallocateColumns();
+
+    for (const fieldName in row) {
+      this.columns[fieldName][this.length] = row[fieldName];
+    }
+
+    this.length++;
+  }
+
+  getBatch() {
+    this._pruneColumns();
+
+    const columns = Array.isArray(this.schema) ? this.columns : {};
+
+    if (!Array.isArray(this.schema)) {
+      for (const fieldName in this.schema) {
+        const field = this.schema[fieldName];
+        columns[field.name] = this.columns[field.index];
+      }
+    }
+
+    this.columns = {};
+    const batch = {
+      shape: 'columnar-table',
+      batchType: 'data',
+      data: columns,
+      schema: this.schema,
+      length: this.length
+    };
+    return batch;
+  }
+
+  _reallocateColumns() {
+    if (this.length < this.allocated) {
+      return;
+    }
+
+    this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT;
+    this.columns = {};
+
+    for (const fieldName in this.schema) {
+      const field = this.schema[fieldName];
+      const ArrayType = field.type || Float32Array;
+      const oldColumn = this.columns[field.index];
+
+      if (oldColumn && ArrayBuffer.isView(oldColumn)) {
+        const typedArray = new ArrayType(this.allocated);
+        typedArray.set(oldColumn);
+        this.columns[field.index] = typedArray;
+      } else if (oldColumn) {
+        oldColumn.length = this.allocated;
+        this.columns[field.index] = oldColumn;
+      } else {
+        this.columns[field.index] = new ArrayType(this.allocated);
+      }
+    }
+  }
+
+  _pruneColumns() {
+    for (const [columnName, column] of Object.entries(this.columns)) {
+      this.columns[columnName] = column.slice(0, this.length);
+    }
+  }
+
+}
+
+exports.default = ColumnarTableBatchAggregator;
+//# sourceMappingURL=columnar-table-batch-aggregator.js.map

package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"names":["DEFAULT_ROW_COUNT","ColumnarTableBatchAggregator","constructor","schema","options","_reallocateColumns","rowCount","length","addArrayRow","row","i","fieldName","columns","addObjectRow","getBatch","_pruneColumns","Array","isArray","field","name","index","batch","shape","batchType","data","allocated","ArrayType","type","Float32Array","oldColumn","ArrayBuffer","isView","typedArray","set","columnName","column","Object","entries","slice"],"mappings":";;;;;;;;;;;AAMA,MAAMA,iBAAiB,GAAG,GAA1B;;AAEe,MAAMC,4BAAN,CAAmE;AAMhFC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAAqD;AAAA;AAAA,kDAJ/C,CAI+C;AAAA,qDAH5C,CAG4C;AAAA,mDAFvB,EAEuB;AAC9D,SAAKD,MAAL,GAAcA,MAAd;;AACA,SAAKE,kBAAL;AACD;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAa;AAEtB,SAAKJ,kBAAL;;AACA,QAAIK,CAAC,GAAG,CAAR;;AAEA,SAAK,MAAMC,SAAX,IAAwB,KAAKC,OAA7B,EAAsC;AACpC,WAAKA,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACC,CAAC,EAAF,CAA1C;AACD;;AACD,SAAKH,MAAL;AACD;;AAEDM,EAAAA,YAAY,CAACJ,GAAD,EAAyC;AAEnD,SAAKJ,kBAAL;;AACA,SAAK,MAAMM,SAAX,IAAwBF,GAAxB,EAA6B;AAC3B,WAAKG,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACE,SAAD,CAA1C;AACD;;AACD,SAAKJ,MAAL;AACD;;AAEDO,EAAAA,QAAQ,GAAgD;AACtD,SAAKC,aAAL;;AACA,UAAMH,OAAO,GAAGI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,IAA6B,KAAKS,OAAlC,GAA4C,EAA5D;;AAKA,QAAI,CAACI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,CAAL,EAAiC;AAC/B,WAAK,MAAMQ,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,cAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACAC,QAAAA,OAAO,CAACM,KAAK,CAACC,IAAP,CAAP,GAAsB,KAAKP,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAtB;AACD;AACF;;AAED,SAAKR,OAAL,GAAe,EAAf;AAEA,UAAMS,KAAyB,GAAG;AAChCC,MAAAA,KAAK,EAAE,gBADyB;AAEhCC,MAAAA,SAAS,EAAE,MAFqB;AAGhCC,MAAAA,IAAI,EAAEZ,OAH0B;AAIhCT,MAAAA,MAAM,EAAE,KAAKA,MAJmB;AAKhCI,MAAAA,MAAM,EAAE,KAAKA;AALmB,KAAlC;AAQA,WAAOc,KAAP;AACD;;AAIDhB,EAAAA,kBAAkB,GAAG;AACnB,QAAI,KAAKE,MAAL,GAAc,KAAKkB,SAAvB,EAAkC;AAChC;AACD;;AAGD,SAAKA,SAAL,GAAiB,KAAKA,SAAL,GAAiB,CAAjB,GAAsB,KAAKA,SAAL,IAAkB,CAAxC,GAA6CzB,iBAA9D;AACA,SAAKY,OAAL,GAAe,EAAf;;AAEA,SAAK,MAAMD,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,YAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACA,YAAMe,SAAS,GAAGR,KAAK,CAACS,IAAN,IAAcC,YAAhC;AACA,YAAMC,SAAS,GAAG,KAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAlB;;AAEA,UAAIS,SAAS,IAAIC,WAAW,CAACC,MAAZ,CAAmBF,SAAnB,CAAjB,EAAgD;AAE9C,cAAMG,UAAU,GAAG,IAAIN,SAAJ,CAAc,KAAKD,SAAnB,CAAnB;AACAO,QAAAA,UAAU,CAACC,GAAX,CAAeJ,SAAf;AACA,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BY,UAA5B;AACD,OALD,MAKO,IAAIH,SAAJ,EAAe;AAEpBA,QAAAA,SAAS,CAACtB,MAAV,GAAmB,KAAKkB,SAAxB;AACA,aAAKb,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BS,SAA5B;AACD,OAJM,MAIA;AAEL,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4B,IAAIM,SAAJ,CAAc,KAAKD,SAAnB,CAA5B;AACD;AACF;AACF;;AAEDV,EAAAA,aAAa,GAAG;AACd,SAAK,MAAM,CAACmB,UAAD,EAAaC,MAAb,CAAX,IAAmCC,MAAM,CAACC,OAAP,CAAe,KAAKzB,OAApB,CAAnC,EAAiE;AAC/D,WAAKA,OAAL,CAAasB,UAAb,IAA2BC,MAAM,CAACG,KAAP,CAAa,CAAb,EAAgB,KAAK/B,MAArB,CAA3B;AACD;AACF;;AAlG+E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator} from './table-batch-aggregator';\n\ntype ColumnarTableBatchOptions = {};\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class ColumnarTableBatchAggregator implements TableBatchAggregator {\n  schema: Schema;\n  length: number = 0;\n  allocated: number = 0;\n  columns: {[columnName: string]: any[]} = {};\n\n  constructor(schema: Schema, options: ColumnarTableBatchOptions) {\n    this.schema = schema;\n    this._reallocateColumns();\n  }\n\n  rowCount(): number {\n    return this.length;\n  }\n\n  addArrayRow(row: any[]) {\n    // If user keeps pushing rows beyond batch size, reallocate\n    this._reallocateColumns();\n    let i = 0;\n    // TODO what if no csv header, columns not populated?\n    for (const fieldName in this.columns) {\n      this.columns[fieldName][this.length] = row[i++];\n    }\n    this.length++;\n  }\n\n  addObjectRow(row: {[columnName: string]: any}): void {\n    // If user keeps pushing rows beyond batch size, reallocate\n    this._reallocateColumns();\n    for (const fieldName in row) {\n      this.columns[fieldName][this.length] = row[fieldName];\n    }\n    this.length++;\n  }\n\n  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {\n    this._pruneColumns();\n    const columns = Array.isArray(this.schema) ? this.columns : {};\n\n    // schema is an array if there're no headers\n    // object if there are headers\n    // columns should match schema format\n    if (!Array.isArray(this.schema)) {\n      for (const fieldName in this.schema) {\n        const field = this.schema[fieldName];\n        columns[field.name] = this.columns[field.index];\n      }\n    }\n\n    this.columns = {};\n\n    const batch: ColumnarTableBatch = {\n      shape: 'columnar-table',\n      batchType: 'data',\n      data: columns,\n      schema: this.schema,\n      length: this.length\n    };\n\n    return batch;\n  }\n\n  // HELPERS\n\n  _reallocateColumns() {\n    if (this.length < this.allocated) {\n      return;\n    }\n\n    // @ts-ignore TODO\n    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;\n    this.columns = {};\n\n    for (const fieldName in this.schema) {\n      const field = this.schema[fieldName];\n      const ArrayType = field.type || Float32Array;\n      const oldColumn = this.columns[field.index];\n\n      if (oldColumn && ArrayBuffer.isView(oldColumn)) {\n        // Copy the old data to the new array\n        const typedArray = new ArrayType(this.allocated);\n        typedArray.set(oldColumn);\n        this.columns[field.index] = typedArray;\n      } else if (oldColumn) {\n        // Plain array\n        oldColumn.length = this.allocated;\n        this.columns[field.index] = oldColumn;\n      } else {\n        // Create new\n        this.columns[field.index] = new ArrayType(this.allocated);\n      }\n    }\n  }\n\n  _pruneColumns() {\n    for (const [columnName, column] of Object.entries(this.columns)) {\n      this.columns[columnName] = column.slice(0, this.length);\n    }\n  }\n}\n"],"file":"columnar-table-batch-aggregator.js"}