@loaders.gl/arrow 4.0.0-alpha.22 → 4.0.0-alpha.24

package/dist/es5/arrow-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ArrowLoader = void 0;
-var VERSION = typeof "4.0.0-alpha.22" !== 'undefined' ? "4.0.0-alpha.22" : 'latest';
+var VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
 var ArrowLoader = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/es5/arrow-writer.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ArrowWriter = void 0;
 var _encodeArrow = require("./lib/encode-arrow");
-var VERSION = typeof "4.0.0-alpha.22" !== 'undefined' ? "4.0.0-alpha.22" : 'latest';
+var VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
 var ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/es5/index.js

@@ -23,20 +23,19 @@ Object.defineProperty(exports, "VECTOR_TYPES", {
     return _types.VECTOR_TYPES;
   }
 });
-exports._typecheckArrowLoader = void 0;
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+var _schema = require("@loaders.gl/schema");
 var _arrowLoader = require("./arrow-loader");
 var _parseArrowSync = _interopRequireDefault(require("./lib/parse-arrow-sync"));
 var _parseArrowInBatches = require("./lib/parse-arrow-in-batches");
-var _schema = require("@loaders.gl/schema");
-var _arrowTableBatch = _interopRequireDefault(require("./lib/arrow-table-batch"));
+var _arrowTableBatch = require("./lib/arrow-table-batch");
 var _types = require("./types");
 var _arrowWriter = require("./arrow-writer");
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-_schema.TableBatchBuilder.ArrowBatch = _arrowTableBatch.default;
+_schema.TableBatchBuilder.ArrowBatch = _arrowTableBatch.ArrowTableBatchAggregator;
 var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {}, {
   parse: function () {
     var _parse = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(arraybuffer, options) {
@@ -59,6 +58,4 @@ var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {},
   parseInBatches: _parseArrowInBatches.parseArrowInBatches
 });
 exports.ArrowLoader = ArrowLoader;
-var _typecheckArrowLoader = ArrowLoader;
-exports._typecheckArrowLoader = _typecheckArrowLoader;
 //# sourceMappingURL=index.js.map

package/dist/es5/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["_arrowLoader","require","_parseArrowSync","_interopRequireDefault","_parseArrowInBatches","_schema","_arrowTableBatch","_types","_arrowWriter","ownKeys","object","enumerableOnly","keys","Object","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","i","arguments","length","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","TableBatchBuilder","ArrowBatch","ArrowTableBatchAggregator","ArrowLoader","ArrowWorkerLoader","parse","_parse","_asyncToGenerator2","_regenerator","mark","_callee","arraybuffer","options","wrap","_callee$","_context","prev","next","abrupt","parseSync","stop","_x","_x2","parseInBatches","parseArrowInBatches","exports","_typecheckArrowLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,YAAA,GAAAC,OAAA;AACA,IAAAC,eAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,oBAAA,GAAAH,OAAA;AAEA,IAAAI,OAAA,GAAAJ,OAAA;AACA,IAAAK,gBAAA,GAAAH,sBAAA,CAAAF,OAAA;AAMA,IAAAM,MAAA,GAAAN,OAAA;AAIA,IAAAO,YAAA,GAAAP,OAAA;AAA2C,SAAAQ,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAC,MAAA,CAAAD,IAAA,CAAAF,MAAA,OAAAG,MAAA,CAAAC,qBAAA,QAAAC,OAAA,GAAAF,MAAA,CAAAC,qBAAA,CAAAJ,MAAA,GAAAC,cAAA,KAAAI,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAJ,MAAA,CAAAK,wBAAA,CAAAR,MAAA,EAAAO,GAAA,EAAAE,UAAA,OAAAP,IAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,IAAA,EAAAG,OAAA,YAAAH,IAAA;AAAA,SAAAU,cAAAC,MAAA,aAAAC,CAAA,MAAAA,CAAA,GAAAC,SAAA,CAAAC,MAAA,EAAAF,CAAA,UAAAG,MAAA,WAAAF,SAAA,CAAAD,CAAA,IAAAC,SAAA,CAAAD,CAAA,QAAAA,CAAA,OAAAf,OAAA,CAAAI,MAAA,CAAAc,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAR,MAAA,EAAAM,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAAhB,MAAA,CAAAmB,yBAAA,GAAAnB,MAAA,CAAAoB,gBAAA,CAAAV,MAAA,EAAAV,MAAA,CAAAmB,yBAAA,CAAAL,MAAA,KAAAlB,OAAA,CAAAI,MAAA,CAAAc,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAAhB,MAAA,CAAAqB,cAAA,CAAAX,MAAA,EAAAM,GAAA,EAAAhB,MAAA,CAAAK,wBAAA,CAAAS,MAAA,EAAAE,GAAA,iBAAAN,MAAA;AAP3CY,yBAAiB,CAACC,UAAU,GAAGC,wBAAyB;AAejD,IAAMC,WAA6B,GAAAhB,aAAA,CAAAA,aAAA,KACrCiB,wBAAiB;EACpBC,KAAK;IAAA,IAAAC,MAAA,OAAAC,kBAAA,CAAAX,OAAA,EAAAY,YAAA,CAAAZ,OAAA,CAAAa,IAAA,CAAE,SAAAC,QAAOC,WAAwB,EAAEC,OAA4B;MAAA,OAAAJ,YAAA,CAAAZ,OAAA,CAAAiB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;UAAA;YAAA,OAAAF,QAAA,CAAAG,MAAA,WAClE,IAAAC,uBAAS,EAACR,WAAW,EAAEC,OAAO,CAAC;UAAA;UAAA;YAAA,OAAAG,QAAA,CAAAK,IAAA;QAAA;MAAA,GAAAV,OAAA;IAAA;IAAA,SAAAL,MAAAgB,EAAA,EAAAC,GAAA;MAAA,OAAAhB,MAAA,CAAApB,KAAA,OAAAI,SAAA;IAAA;IAAA,OAAAe,KAAA;EAAA;EACjCc,SAAS,EAATA,uBAAS;EACTI,cAAc,EAAEC;AAAmB,EACpC;AAACC,OAAA,CAAAtB,WAAA,GAAAA,WAAA;AAEK,IAAMuB,qBAAuC,GAAGvB,WAAW;AAACsB,OAAA,CAAAC,qBAAA,GAAAA,qBAAA"}
+{"version":3,"file":"index.js","names":["_schema","require","_arrowLoader","_parseArrowSync","_interopRequireDefault","_parseArrowInBatches","_arrowTableBatch","_types","_arrowWriter","ownKeys","object","enumerableOnly","keys","Object","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","i","arguments","length","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","TableBatchBuilder","ArrowBatch","ArrowTableBatchAggregator","ArrowLoader","ArrowWorkerLoader","parse","_parse","_asyncToGenerator2","_regenerator","mark","_callee","arraybuffer","options","wrap","_callee$","_context","prev","next","abrupt","parseSync","stop","_x","_x2","parseInBatches","parseArrowInBatches","exports"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowTable, ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {ArrowTableBatchAggregator} from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser<\n  ArrowTable | ColumnarTable | ObjectRowTable,\n  ArrowTableBatch,\n  ArrowLoaderOptions\n> = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AACA,IAAAE,eAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,oBAAA,GAAAJ,OAAA;AAEA,IAAAK,gBAAA,GAAAL,OAAA;AAMA,IAAAM,MAAA,GAAAN,OAAA;AAIA,IAAAO,YAAA,GAAAP,OAAA;AAA2C,SAAAQ,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAC,MAAA,CAAAD,IAAA,CAAAF,MAAA,OAAAG,MAAA,CAAAC,qBAAA,QAAAC,OAAA,GAAAF,MAAA,CAAAC,qBAAA,CAAAJ,MAAA,GAAAC,cAAA,KAAAI,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAJ,MAAA,CAAAK,wBAAA,CAAAR,MAAA,EAAAO,GAAA,EAAAE,UAAA,OAAAP,IAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,IAAA,EAAAG,OAAA,YAAAH,IAAA;AAAA,SAAAU,cAAAC,MAAA,aAAAC,CAAA,MAAAA,CAAA,GAAAC,SAAA,CAAAC,MAAA,EAAAF,CAAA,UAAAG,MAAA,WAAAF,SAAA,CAAAD,CAAA,IAAAC,SAAA,CAAAD,CAAA,QAAAA,CAAA,OAAAf,OAAA,CAAAI,MAAA,CAAAc,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAR,MAAA,EAAAM,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAAhB,MAAA,CAAAmB,yBAAA,GAAAnB,MAAA,CAAAoB,gBAAA,CAAAV,MAAA,EAAAV,MAAA,CAAAmB,yBAAA,CAAAL,MAAA,KAAAlB,OAAA,CAAAI,MAAA,CAAAc,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAAhB,MAAA,CAAAqB,cAAA,CAAAX,MAAA,EAAAM,GAAA,EAAAhB,MAAA,CAAAK,wBAAA,CAAAS,MAAA,EAAAE,GAAA,iBAAAN,MAAA;AAP3CY,yBAAiB,CAACC,UAAU,GAAGC,0CAAyB;AAejD,IAAMC,WAIZ,GAAAhB,aAAA,CAAAA,aAAA,KACIiB,wBAAiB;EACpBC,KAAK;IAAA,IAAAC,MAAA,OAAAC,kBAAA,CAAAX,OAAA,EAAAY,YAAA,CAAAZ,OAAA,CAAAa,IAAA,CAAE,SAAAC,QAAOC,WAAwB,EAAEC,OAA4B;MAAA,OAAAJ,YAAA,CAAAZ,OAAA,CAAAiB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;UAAA;YAAA,OAAAF,QAAA,CAAAG,MAAA,WAClE,IAAAC,uBAAS,EAACR,WAAW,EAAEC,OAAO,CAAC;UAAA;UAAA;YAAA,OAAAG,QAAA,CAAAK,IAAA;QAAA;MAAA,GAAAV,OAAA;IAAA;IAAA,SAAAL,MAAAgB,EAAA,EAAAC,GAAA;MAAA,OAAAhB,MAAA,CAAApB,KAAA,OAAAI,SAAA;IAAA;IAAA,OAAAe,KAAA;EAAA;EACjCc,SAAS,EAATA,uBAAS;EACTI,cAAc,EAAEC;AAAmB,EACpC;AAACC,OAAA,CAAAtB,WAAA,GAAAA,WAAA"}

package/dist/es5/lib/arrow-table-batch.js

@@ -4,7 +4,7 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.default = void 0;
+exports.ArrowTableBatchAggregator = void 0;
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
@@ -57,7 +57,7 @@ var ArrowTableBatchAggregator = function (_ColumnarTableBatchAg) {
   }]);
   return ArrowTableBatchAggregator;
 }(_schema.ColumnarTableBatchAggregator);
-exports.default = ArrowTableBatchAggregator;
+exports.ArrowTableBatchAggregator = ArrowTableBatchAggregator;
 function getArrowSchema(schema) {
   var arrowFields = [];
   for (var key in schema) {

package/dist/es5/lib/arrow-table-batch.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.js","names":["_apacheArrow","require","_schema","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","_createSuper","Derived","hasNativeReflectConstruct","_isNativeReflectConstruct","_createSuperInternal","Super","_getPrototypeOf2","default","result","NewTarget","Reflect","construct","arguments","apply","_possibleConstructorReturn2","sham","Proxy","Boolean","valueOf","ArrowTableBatchAggregator","_ColumnarTableBatchAg","_inherits2","_super","schema","options","_this","_classCallCheck2","_defineProperty2","_assertThisInitialized2","arrowSchema","_createClass2","key","getBatch","batch","_get2","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","makeData","type","Struct","fields","children","map","_ref","shape","batchType","ColumnarTableBatchAggregator","exports","arrowFields","field","Float32Array","metadata","Map","arrowField","Field","Float32","nullable","push","Error","Schema","_iterator","_step","vector","arrowVector","makeVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";;;;;;;;;;;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAUA,IAAAC,OAAA,GAAAD,OAAA;AAAgE,SAAAE,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,aAAAC,OAAA,QAAAC,yBAAA,GAAAC,yBAAA,oBAAAC,qBAAA,QAAAC,KAAA,OAAAC,gBAAA,CAAAC,OAAA,EAAAN,OAAA,GAAAO,MAAA,MAAAN,yBAAA,QAAAO,SAAA,OAAAH,gBAAA,CAAAC,OAAA,QAAAd,WAAA,EAAAe,MAAA,GAAAE,OAAA,CAAAC,SAAA,CAAAN,KAAA,EAAAO,SAAA,EAAAH,SAAA,YAAAD,MAAA,GAAAH,KAAA,CAAAQ,KAAA,OAAAD,SAAA,gBAAAE,2BAAA,CAAAP,OAAA,QAAAC,MAAA;AAAA,SAAAL,0BAAA,eAAAO,OAAA,qBAAAA,OAAA,CAAAC,SAAA,oBAAAD,OAAA,CAAAC,SAAA,CAAAI,IAAA,2BAAAC,KAAA,oCAAAC,OAAA,CAAA3B,SAAA,CAAA4B,OAAA,CAAApC,IAAA,CAAA4B,OAAA,CAAAC,SAAA,CAAAM,OAAA,8CAAA1C,CAAA;AAAA,IAE3C4C,yBAAyB,aAAAC,qBAAA;EAAA,IAAAC,UAAA,CAAAd,OAAA,EAAAY,yBAAA,EAAAC,qBAAA;EAAA,IAAAE,MAAA,GAAAtB,YAAA,CAAAmB,yBAAA;EAG5C,SAAAA,0BAAYI,MAAM,EAAEC,OAAO,EAAE;IAAA,IAAAC,KAAA;IAAA,IAAAC,gBAAA,CAAAnB,OAAA,QAAAY,yBAAA;IAC3BM,KAAA,GAAAH,MAAA,CAAAxC,IAAA,OAAMyC,MAAM,EAAEC,OAAO;IAAE,IAAAG,gBAAA,CAAApB,OAAA,MAAAqB,uBAAA,CAAArB,OAAA,EAAAkB,KAAA;IACvBA,KAAA,CAAKI,WAAW,GAAG,IAAI;IAAC,OAAAJ,KAAA;EAC1B;EAAC,IAAAK,aAAA,CAAAvB,OAAA,EAAAY,yBAAA;IAAAY,GAAA;IAAAzD,KAAA,EAED,SAAA0D,SAAA,EAAmC;MACjC,IAAMC,KAAK,OAAAC,KAAA,CAAA3B,OAAA,MAAAD,gBAAA,CAAAC,OAAA,EAAAY,yBAAA,CAAA7B,SAAA,qBAAAR,IAAA,MAAmB;MAC9B,IAAImD,KAAK,EAAE;QAET,IAAI,CAACJ,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIM,cAAc,CAACF,KAAK,CAACV,MAAM,CAAC;QAGnE,IAAMa,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACR,WAAW,EAAEI,KAAK,CAACK,IAAI,CAAC;QAGlE,IAAMC,WAAW,GAAG,IAAIC,wBAAW,CACjC,IAAI,CAACX,WAAW,EAChB,IAAAY,qBAAQ,EAAC;UACPC,IAAI,EAAE,IAAIC,mBAAM,CAAC,IAAI,CAACd,WAAW,CAACe,MAAM,CAAC;UACzCC,QAAQ,EAAET,YAAY,CAACU,GAAG,CAAC,UAAAC,IAAA;YAAA,IAAET,IAAI,GAAAS,IAAA,CAAJT,IAAI;YAAA,OAAMA,IAAI,CAAC,CAAC,CAAC;UAAA;QAChD,CAAC,CACH,CAAC;QAED,OAAO;UACLU,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBX,IAAI,EAAEC,WAAW;UACjBvE,MAAM,EAAEiE,KAAK,CAACjE;QAChB,CAAC;MACH;MAEA,OAAO,IAAI;IACb;EAAC;EAAA,OAAAmD,yBAAA;AAAA,EAnCoD+B,oCAA4B;AAAAC,OAAA,CAAA5C,OAAA,GAAAY,yBAAA;AAuCnF,SAASgB,cAAcA,CAACZ,MAAM,EAAU;EACtC,IAAM6B,WAAoB,GAAG,EAAE;EAC/B,KAAK,IAAMrB,GAAG,IAAIR,MAAM,EAAE;IACxB,IAAM8B,KAAK,GAAG9B,MAAM,CAACQ,GAAG,CAAC;IACzB,IAAIsB,KAAK,CAACX,IAAI,KAAKY,YAAY,EAAE;MAE/B,IAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,IAAMC,UAAU,GAAG,IAAIC,kBAAK,CAACL,KAAK,CAAC3D,IAAI,EAAE,IAAIiE,oBAAO,CAAC,CAAC,EAAEN,KAAK,CAACO,QAAQ,EAAEL,QAAQ,CAAC;MACjFH,WAAW,CAACS,IAAI,CAACJ,UAAU,CAAC;IAC9B;EACF;EACA,IAAIL,WAAW,CAACpF,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAI8F,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIC,mBAAM,CAACX,WAAW,CAAC;AAChC;AAGA,SAASf,eAAeA,CAACR,WAAW,EAAES,IAAI,EAAY;EACpD,IAAMF,YAAmB,GAAG,EAAE;EAAC,IAAA4B,SAAA,GAAAzG,0BAAA,CACXsE,WAAW,CAACe,MAAM;IAAAqB,KAAA;EAAA;IAAtC,KAAAD,SAAA,CAAA7F,CAAA,MAAA8F,KAAA,GAAAD,SAAA,CAAA5F,CAAA,IAAAC,IAAA,GAAwC;MAAA,IAA7BgF,KAAK,GAAAY,KAAA,CAAA3F,KAAA;MACd,IAAM4F,MAAM,GAAG5B,IAAI,CAACe,KAAK,CAAC3D,IAAI,CAAC;MAC/B,IAAIwE,MAAM,YAAYZ,YAAY,EAAE;QAClC,IAAMa,WAAW,GAAG,IAAAC,uBAAU,EAACF,MAAM,CAAC;QACtC9B,YAAY,CAACyB,IAAI,CAACM,WAAW,CAAC;MAChC;IACF;EAAC,SAAAtF,GAAA;IAAAmF,SAAA,CAAAzF,CAAA,CAAAM,GAAA;EAAA;IAAAmF,SAAA,CAAAvF,CAAA;EAAA;EACD,IAAIoD,WAAW,CAACe,MAAM,CAAC5E,MAAM,KAAKoE,YAAY,CAACpE,MAAM,EAAE;IACrD,MAAM,IAAI8F,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAO1B,YAAY;AACrB"}
+{"version":3,"file":"arrow-table-batch.js","names":["_apacheArrow","require","_schema","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","_createSuper","Derived","hasNativeReflectConstruct","_isNativeReflectConstruct","_createSuperInternal","Super","_getPrototypeOf2","default","result","NewTarget","Reflect","construct","arguments","apply","_possibleConstructorReturn2","sham","Proxy","Boolean","valueOf","ArrowTableBatchAggregator","_ColumnarTableBatchAg","_inherits2","_super","schema","options","_this","_classCallCheck2","_defineProperty2","_assertThisInitialized2","arrowSchema","_createClass2","key","getBatch","batch","_get2","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","makeData","type","Struct","fields","children","map","_ref","shape","batchType","ColumnarTableBatchAggregator","exports","arrowFields","field","Float32Array","metadata","Map","arrowField","Field","Float32","nullable","push","Error","Schema","_iterator","_step","vector","arrowVector","makeVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";;;;;;;;;;;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAUA,IAAAC,OAAA,GAAAD,OAAA;AAAgE,SAAAE,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,aAAAC,OAAA,QAAAC,yBAAA,GAAAC,yBAAA,oBAAAC,qBAAA,QAAAC,KAAA,OAAAC,gBAAA,CAAAC,OAAA,EAAAN,OAAA,GAAAO,MAAA,MAAAN,yBAAA,QAAAO,SAAA,OAAAH,gBAAA,CAAAC,OAAA,QAAAd,WAAA,EAAAe,MAAA,GAAAE,OAAA,CAAAC,SAAA,CAAAN,KAAA,EAAAO,SAAA,EAAAH,SAAA,YAAAD,MAAA,GAAAH,KAAA,CAAAQ,KAAA,OAAAD,SAAA,gBAAAE,2BAAA,CAAAP,OAAA,QAAAC,MAAA;AAAA,SAAAL,0BAAA,eAAAO,OAAA,qBAAAA,OAAA,CAAAC,SAAA,oBAAAD,OAAA,CAAAC,SAAA,CAAAI,IAAA,2BAAAC,KAAA,oCAAAC,OAAA,CAAA3B,SAAA,CAAA4B,OAAA,CAAApC,IAAA,CAAA4B,OAAA,CAAAC,SAAA,CAAAM,OAAA,8CAAA1C,CAAA;AAAA,IAEnD4C,yBAAyB,aAAAC,qBAAA;EAAA,IAAAC,UAAA,CAAAd,OAAA,EAAAY,yBAAA,EAAAC,qBAAA;EAAA,IAAAE,MAAA,GAAAtB,YAAA,CAAAmB,yBAAA;EAGpC,SAAAA,0BAAYI,MAAM,EAAEC,OAAO,EAAE;IAAA,IAAAC,KAAA;IAAA,IAAAC,gBAAA,CAAAnB,OAAA,QAAAY,yBAAA;IAC3BM,KAAA,GAAAH,MAAA,CAAAxC,IAAA,OAAMyC,MAAM,EAAEC,OAAO;IAAE,IAAAG,gBAAA,CAAApB,OAAA,MAAAqB,uBAAA,CAAArB,OAAA,EAAAkB,KAAA;IACvBA,KAAA,CAAKI,WAAW,GAAG,IAAI;IAAC,OAAAJ,KAAA;EAC1B;EAAC,IAAAK,aAAA,CAAAvB,OAAA,EAAAY,yBAAA;IAAAY,GAAA;IAAAzD,KAAA,EAED,SAAA0D,SAAA,EAAmC;MACjC,IAAMC,KAAK,OAAAC,KAAA,CAAA3B,OAAA,MAAAD,gBAAA,CAAAC,OAAA,EAAAY,yBAAA,CAAA7B,SAAA,qBAAAR,IAAA,MAAmB;MAC9B,IAAImD,KAAK,EAAE;QAET,IAAI,CAACJ,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIM,cAAc,CAACF,KAAK,CAACV,MAAM,CAAC;QAGnE,IAAMa,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACR,WAAW,EAAEI,KAAK,CAACK,IAAI,CAAC;QAGlE,IAAMC,WAAW,GAAG,IAAIC,wBAAW,CACjC,IAAI,CAACX,WAAW,EAChB,IAAAY,qBAAQ,EAAC;UACPC,IAAI,EAAE,IAAIC,mBAAM,CAAC,IAAI,CAACd,WAAW,CAACe,MAAM,CAAC;UACzCC,QAAQ,EAAET,YAAY,CAACU,GAAG,CAAC,UAAAC,IAAA;YAAA,IAAET,IAAI,GAAAS,IAAA,CAAJT,IAAI;YAAA,OAAMA,IAAI,CAAC,CAAC,CAAC;UAAA;QAChD,CAAC,CACH,CAAC;QAED,OAAO;UACLU,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBX,IAAI,EAAEC,WAAW;UACjBvE,MAAM,EAAEiE,KAAK,CAACjE;QAChB,CAAC;MACH;MAEA,OAAO,IAAI;IACb;EAAC;EAAA,OAAAmD,yBAAA;AAAA,EAnC4C+B,oCAA4B;AAAAC,OAAA,CAAAhC,yBAAA,GAAAA,yBAAA;AAuC3E,SAASgB,cAAcA,CAACZ,MAAM,EAAU;EACtC,IAAM6B,WAAoB,GAAG,EAAE;EAC/B,KAAK,IAAMrB,GAAG,IAAIR,MAAM,EAAE;IACxB,IAAM8B,KAAK,GAAG9B,MAAM,CAACQ,GAAG,CAAC;IACzB,IAAIsB,KAAK,CAACX,IAAI,KAAKY,YAAY,EAAE;MAE/B,IAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,IAAMC,UAAU,GAAG,IAAIC,kBAAK,CAACL,KAAK,CAAC3D,IAAI,EAAE,IAAIiE,oBAAO,CAAC,CAAC,EAAEN,KAAK,CAACO,QAAQ,EAAEL,QAAQ,CAAC;MACjFH,WAAW,CAACS,IAAI,CAACJ,UAAU,CAAC;IAC9B;EACF;EACA,IAAIL,WAAW,CAACpF,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAI8F,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIC,mBAAM,CAACX,WAAW,CAAC;AAChC;AAGA,SAASf,eAAeA,CAACR,WAAW,EAAES,IAAI,EAAY;EACpD,IAAMF,YAAmB,GAAG,EAAE;EAAC,IAAA4B,SAAA,GAAAzG,0BAAA,CACXsE,WAAW,CAACe,MAAM;IAAAqB,KAAA;EAAA;IAAtC,KAAAD,SAAA,CAAA7F,CAAA,MAAA8F,KAAA,GAAAD,SAAA,CAAA5F,CAAA,IAAAC,IAAA,GAAwC;MAAA,IAA7BgF,KAAK,GAAAY,KAAA,CAAA3F,KAAA;MACd,IAAM4F,MAAM,GAAG5B,IAAI,CAACe,KAAK,CAAC3D,IAAI,CAAC;MAC/B,IAAIwE,MAAM,YAAYZ,YAAY,EAAE;QAClC,IAAMa,WAAW,GAAG,IAAAC,uBAAU,EAACF,MAAM,CAAC;QACtC9B,YAAY,CAACyB,IAAI,CAACM,WAAW,CAAC;MAChC;IACF;EAAC,SAAAtF,GAAA;IAAAmF,SAAA,CAAAzF,CAAA,CAAAM,GAAA;EAAA;IAAAmF,SAAA,CAAAvF,CAAA;EAAA;EACD,IAAIoD,WAAW,CAACe,MAAM,CAAC5E,MAAM,KAAKoE,YAAY,CAACpE,MAAM,EAAE;IACrD,MAAM,IAAI8F,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAO1B,YAAY;AACrB"}

package/dist/es5/lib/convert-table.js

@@ -0,0 +1,57 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.convertApacheArrowToArrowTable = convertApacheArrowToArrowTable;
+exports.convertArrowToColumnarTable = convertArrowToColumnarTable;
+exports.convertColumnarToRowFormatTable = convertColumnarToRowFormatTable;
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+function convertApacheArrowToArrowTable(arrowTable) {
+  return {
+    shape: 'arrow-table',
+    data: arrowTable
+  };
+}
+function convertArrowToColumnarTable(table) {
+  var arrowTable = table.data;
+  var columnarTable = {};
+  var _iterator = _createForOfIteratorHelper(arrowTable.schema.fields),
+    _step;
+  try {
+    for (_iterator.s(); !(_step = _iterator.n()).done;) {
+      var field = _step.value;
+      var arrowColumn = arrowTable.getChild(field.name);
+      var values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
+      columnarTable[field.name] = values;
+    }
+  } catch (err) {
+    _iterator.e(err);
+  } finally {
+    _iterator.f();
+  }
+  return {
+    shape: 'columnar-table',
+    data: columnarTable
+  };
+}
+function convertColumnarToRowFormatTable(columnarTable) {
+  var tableKeys = Object.keys(columnarTable);
+  var tableRowsCount = columnarTable[tableKeys[0]].length;
+  var rowFormatTable = [];
+  for (var index = 0; index < tableRowsCount; index++) {
+    var tableItem = {};
+    for (var keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      var fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+  return {
+    shape: 'object-row-table',
+    data: rowFormatTable
+  };
+}
+//# sourceMappingURL=convert-table.js.map

package/dist/es5/lib/convert-table.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","_iterator","_createForOfIteratorHelper","schema","fields","_step","s","n","done","field","value","arrowColumn","getChild","name","values","toArray","err","e","f","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport type {ColumnarTable, ObjectRowTable, ArrowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n  return {\n    shape: 'arrow-table',\n    data: arrowTable\n  };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n\n  const arrowTable = table.data;\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  return {\n    shape: 'columnar-table',\n    data: columnarTable\n  };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n\n  return {\n    shape: 'object-row-table',\n    data: rowFormatTable\n  };\n}\n"],"mappings":";;;;;;;;;;;AAUO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,IAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,IAAMG,aAAa,GAAG,CAAC,CAAC;EAAC,IAAAC,SAAA,GAAAC,0BAAA,CAELP,UAAU,CAACQ,MAAM,CAACC,MAAM;IAAAC,KAAA;EAAA;IAA5C,KAAAJ,SAAA,CAAAK,CAAA,MAAAD,KAAA,GAAAJ,SAAA,CAAAM,CAAA,IAAAC,IAAA,GAA8C;MAAA,IAAnCC,KAAK,GAAAJ,KAAA,CAAAK,KAAA;MAEd,IAAMC,WAAW,GAAGhB,UAAU,CAACiB,QAAQ,CAACH,KAAK,CAACI,IAAI,CAAC;MACnD,IAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;MACrCf,aAAa,CAACS,KAAK,CAACI,IAAI,CAAC,GAAGC,MAAM;IACpC;EAAC,SAAAE,GAAA;IAAAf,SAAA,CAAAgB,CAAA,CAAAD,GAAA;EAAA;IAAAf,SAAA,CAAAiB,CAAA;EAAA;EAED,OAAO;IACLtB,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMO,SAASmB,+BAA+BA,CAACnB,aAA4B,EAAkB;EAC5F,IAAMoB,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACtB,aAAa,CAAC;EAC5C,IAAMuB,cAAc,GAAGvB,aAAa,CAACoB,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,IAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,IAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,IAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAG7B,aAAa,CAAC6B,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACL/B,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAE4B;EACR,CAAC;AACH"}

package/dist/es5/lib/parse-arrow-in-batches.js

@@ -17,7 +17,7 @@ function parseArrowInBatches(asyncIterator) {
   }
   function _makeArrowAsyncIterator() {
     _makeArrowAsyncIterator = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee() {
-      var readers, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, reader, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch;
+      var readers, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, reader, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, recordBatch, arrowTabledBatch;
       return _regenerator.default.wrap(function _callee$(_context) {
         while (1) switch (_context.prev = _context.next) {
           case 0:
@@ -31,7 +31,7 @@ function parseArrowInBatches(asyncIterator) {
             return (0, _awaitAsyncGenerator2.default)(_iterator.next());
           case 7:
             if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
-              _context.next = 42;
+              _context.next = 43;
               break;
             }
             reader = _step.value;
@@ -44,97 +44,92 @@ function parseArrowInBatches(asyncIterator) {
             return (0, _awaitAsyncGenerator2.default)(_iterator2.next());
           case 15:
             if (!(_iteratorAbruptCompletion2 = !(_step2 = _context.sent).done)) {
-              _context.next = 22;
+              _context.next = 23;
               break;
             }
-            batch = _step2.value;
-            _context.next = 19;
-            return processBatch(batch);
-          case 19:
+            recordBatch = _step2.value;
+            arrowTabledBatch = {
+              shape: 'arrow-table',
+              batchType: 'data',
+              data: recordBatch,
+              length: recordBatch.data.length
+            };
+            _context.next = 20;
+            return arrowTabledBatch;
+          case 20:
             _iteratorAbruptCompletion2 = false;
             _context.next = 13;
             break;
-          case 22:
-            _context.next = 28;
+          case 23:
+            _context.next = 29;
             break;
-          case 24:
-            _context.prev = 24;
+          case 25:
+            _context.prev = 25;
             _context.t0 = _context["catch"](11);
             _didIteratorError2 = true;
             _iteratorError2 = _context.t0;
-          case 28:
-            _context.prev = 28;
+          case 29:
             _context.prev = 29;
+            _context.prev = 30;
             if (!(_iteratorAbruptCompletion2 && _iterator2.return != null)) {
-              _context.next = 33;
+              _context.next = 34;
               break;
             }
-            _context.next = 33;
+            _context.next = 34;
             return (0, _awaitAsyncGenerator2.default)(_iterator2.return());
-          case 33:
-            _context.prev = 33;
+          case 34:
+            _context.prev = 34;
             if (!_didIteratorError2) {
-              _context.next = 36;
+              _context.next = 37;
               break;
             }
             throw _iteratorError2;
-          case 36:
-            return _context.finish(33);
           case 37:
-            return _context.finish(28);
+            return _context.finish(34);
           case 38:
-            return _context.abrupt("break", 42);
+            return _context.finish(29);
           case 39:
+            return _context.abrupt("break", 43);
+          case 40:
             _iteratorAbruptCompletion = false;
             _context.next = 5;
             break;
-          case 42:
-            _context.next = 48;
+          case 43:
+            _context.next = 49;
             break;
-          case 44:
-            _context.prev = 44;
+          case 45:
+            _context.prev = 45;
             _context.t1 = _context["catch"](3);
             _didIteratorError = true;
             _iteratorError = _context.t1;
-          case 48:
-            _context.prev = 48;
+          case 49:
             _context.prev = 49;
+            _context.prev = 50;
             if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
-              _context.next = 53;
+              _context.next = 54;
               break;
             }
-            _context.next = 53;
+            _context.next = 54;
             return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-          case 53:
-            _context.prev = 53;
+          case 54:
+            _context.prev = 54;
             if (!_didIteratorError) {
-              _context.next = 56;
+              _context.next = 57;
               break;
             }
             throw _iteratorError;
-          case 56:
-            return _context.finish(53);
           case 57:
-            return _context.finish(48);
+            return _context.finish(54);
           case 58:
+            return _context.finish(49);
+          case 59:
           case "end":
             return _context.stop();
         }
-      }, _callee, null, [[3, 44, 48, 58], [11, 24, 28, 38], [29,, 33, 37], [49,, 53, 57]]);
+      }, _callee, null, [[3, 45, 49, 59], [11, 25, 29, 39], [30,, 34, 38], [50,, 54, 58]]);
     }));
     return _makeArrowAsyncIterator.apply(this, arguments);
   }
   return makeArrowAsyncIterator();
 }
-function processBatch(batch) {
-  var values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach(function (_ref, index) {
-    var name = _ref.name;
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
-}
 //# sourceMappingURL=parse-arrow-in-batches.js.map

package/dist/es5/lib/parse-arrow-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["_apacheArrow","require","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseArrowInBatches","makeArrowAsyncIterator","_makeArrowAsyncIterator","_wrapAsyncGenerator2","default","_regenerator","mark","_callee","readers","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","reader","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","batch","wrap","_callee$","_context","prev","RecordBatchReader","readAll","_awaitAsyncGenerator2","sent","processBatch","t0","finish","abrupt","t1","stop","values","metadata","schema","length","fields","forEach","_ref","index","name","getChildAt","toArray"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator() {\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const batch of reader) {\n        yield processBatch(batch);\n      }\n      break; // only processing one stream of batches\n    }\n  }\n  return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n  const values = {\n    metadata: batch.schema.metadata,\n    length: batch.length\n  };\n  batch.schema.fields.forEach(({name}, index) => {\n    values[name] = batch.getChildAt(index).toArray();\n  });\n  return values;\n}\n"],"mappings":";;;;;;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAA+C,SAAAC,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAKxC,SAASqB,mBAAmBA,CACjC1B,aAAiE,EAC7C;EAAA,SAmBJ2B,sBAAsBA,CAAA;IAAA,OAAAC,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAAA,SAAAS,wBAAA;IAAAA,uBAAA,OAAAC,oBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAtC,SAAAC,QAAA;MAAA,IAAAC,OAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,KAAA;MAAA,OAAAf,YAAA,CAAAD,OAAA,CAAAiB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAjC,IAAA;UAAA;YACQkB,OAAO,GAAGiB,8BAAiB,CAACC,OAAO,CAACpD,aAAa,CAAC;YAAAmC,yBAAA;YAAAC,iBAAA;YAAAa,QAAA,CAAAC,IAAA;YAAAZ,SAAA,GAAA7C,cAAA,CAC7ByC,OAAO;UAAA;YAAAe,QAAA,CAAAjC,IAAA;YAAA,WAAAqC,qBAAA,CAAAvB,OAAA,EAAAQ,SAAA,CAAAtB,IAAA;UAAA;YAAA,MAAAmB,yBAAA,KAAAI,KAAA,GAAAU,QAAA,CAAAK,IAAA,EAAA3C,IAAA;cAAAsC,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAjBwB,MAAM,GAAAD,KAAA,CAAA1B,KAAA;YAAA4B,0BAAA;YAAAC,kBAAA;YAAAO,QAAA,CAAAC,IAAA;YAAAN,UAAA,GAAAnD,cAAA,CACK+C,MAAM;UAAA;YAAAS,QAAA,CAAAjC,IAAA;YAAA,WAAAqC,qBAAA,CAAAvB,OAAA,EAAAc,UAAA,CAAA5B,IAAA;UAAA;YAAA,MAAAyB,0BAAA,KAAAI,MAAA,GAAAI,QAAA,CAAAK,IAAA,EAAA3C,IAAA;cAAAsC,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAf8B,KAAK,GAAAD,MAAA,CAAAhC,KAAA;YAAAoC,QAAA,CAAAjC,IAAA;YACpB,OAAMuC,YAAY,CAACT,KAAK,CAAC;UAAA;YAAAL,0BAAA;YAAAQ,QAAA,CAAAjC,IAAA;YAAA;UAAA;YAAAiC,QAAA,CAAAjC,IAAA;YAAA;UAAA;YAAAiC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAO,EAAA,GAAAP,QAAA;YAAAP,kBAAA;YAAAC,eAAA,GAAAM,QAAA,CAAAO,EAAA;UAAA;YAAAP,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAT,0BAAA,IAAAG,UAAA,CAAAxB,MAAA;cAAA6B,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAAiC,QAAA,CAAAjC,IAAA;YAAA,WAAAqC,qBAAA,CAAAvB,OAAA,EAAAc,UAAA,CAAAxB,MAAA;UAAA;YAAA6B,QAAA,CAAAC,IAAA;YAAA,KAAAR,kBAAA;cAAAO,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAA,MAAA2B,eAAA;UAAA;YAAA,OAAAM,QAAA,CAAAQ,MAAA;UAAA;YAAA,OAAAR,QAAA,CAAAQ,MAAA;UAAA;YAAA,OAAAR,QAAA,CAAAS,MAAA;UAAA;YAAAvB,yBAAA;YAAAc,QAAA,CAAAjC,IAAA;YAAA;UAAA;YAAAiC,QAAA,CAAAjC,IAAA;YAAA;UAAA;YAAAiC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAU,EAAA,GAAAV,QAAA;YAAAb,iBAAA;YAAAC,cAAA,GAAAY,QAAA,CAAAU,EAAA;UAAA;YAAAV,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAf,yBAAA,IAAAG,SAAA,CAAAlB,MAAA;cAAA6B,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAAiC,QAAA,CAAAjC,IAAA;YAAA,WAAAqC,qBAAA,CAAAvB,OAAA,EAAAQ,SAAA,CAAAlB,MAAA;UAAA;YAAA6B,QAAA,CAAAC,IAAA;YAAA,KAAAd,iBAAA;cAAAa,QAAA,CAAAjC,IAAA;cAAA;YAAA;YAAA,MAAAqB,cAAA;UAAA;YAAA,OAAAY,QAAA,CAAAQ,MAAA;UAAA;YAAA,OAAAR,QAAA,CAAAQ,MAAA;UAAA;UAAA;YAAA,OAAAR,QAAA,CAAAW,IAAA;QAAA;MAAA,GAAA3B,OAAA;IAAA,CAI9B;IAAA,OAAAL,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EACD,OAAOQ,sBAAsB,CAAC,CAAC;AACjC;AAEA,SAAS4B,YAAYA,CAACT,KAAK,EAAE;EAC3B,IAAMe,MAAM,GAAG;IACbC,QAAQ,EAAEhB,KAAK,CAACiB,MAAM,CAACD,QAAQ;IAC/BE,MAAM,EAAElB,KAAK,CAACkB;EAChB,CAAC;EACDlB,KAAK,CAACiB,MAAM,CAACE,MAAM,CAACC,OAAO,CAAC,UAAAC,IAAA,EAASC,KAAK,EAAK;IAAA,IAAjBC,IAAI,GAAAF,IAAA,CAAJE,IAAI;IAChCR,MAAM,CAACQ,IAAI,CAAC,GAAGvB,KAAK,CAACwB,UAAU,CAACF,KAAK,CAAC,CAACG,OAAO,CAAC,CAAC;EAClD,CAAC,CAAC;EACF,OAAOV,MAAM;AACf"}
+{"version":3,"file":"parse-arrow-in-batches.js","names":["_apacheArrow","require","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseArrowInBatches","makeArrowAsyncIterator","_makeArrowAsyncIterator","_wrapAsyncGenerator2","default","_regenerator","mark","_callee","readers","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","reader","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","recordBatch","arrowTabledBatch","wrap","_callee$","_context","prev","RecordBatchReader","readAll","_awaitAsyncGenerator2","sent","shape","batchType","data","length","t0","finish","abrupt","t1","stop"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: recordBatch,\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,YAAA,GAAAC,OAAA;AAA+C,SAAAC,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAKxC,SAASqB,mBAAmBA,CACjC1B,aAAiE,EACjC;EAAA,SAmBhB2B,sBAAsBA,CAAA;IAAA,OAAAC,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAAA,SAAAS,wBAAA;IAAAA,uBAAA,OAAAC,oBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAtC,SAAAC,QAAA;MAAA,IAAAC,OAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,WAAA,EAAAC,gBAAA;MAAA,OAAAhB,YAAA,CAAAD,OAAA,CAAAkB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAlC,IAAA;UAAA;YAEQkB,OAAO,GAAGkB,8BAAiB,CAACC,OAAO,CAACrD,aAAa,CAAC;YAAAmC,yBAAA;YAAAC,iBAAA;YAAAc,QAAA,CAAAC,IAAA;YAAAb,SAAA,GAAA7C,cAAA,CAC7ByC,OAAO;UAAA;YAAAgB,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAtB,IAAA;UAAA;YAAA,MAAAmB,yBAAA,KAAAI,KAAA,GAAAW,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAjBwB,MAAM,GAAAD,KAAA,CAAA1B,KAAA;YAAA4B,0BAAA;YAAAC,kBAAA;YAAAQ,QAAA,CAAAC,IAAA;YAAAP,UAAA,GAAAnD,cAAA,CACW+C,MAAM;UAAA;YAAAU,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAA5B,IAAA;UAAA;YAAA,MAAAyB,0BAAA,KAAAI,MAAA,GAAAK,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAArB8B,WAAW,GAAAD,MAAA,CAAAhC,KAAA;YACpBkC,gBAAiC,GAAG;cACxCS,KAAK,EAAE,aAAa;cACpBC,SAAS,EAAE,MAAM;cACjBC,IAAI,EAAEZ,WAAW;cACjBa,MAAM,EAAEb,WAAW,CAACY,IAAI,CAACC;YAC3B,CAAC;YAAAT,QAAA,CAAAlC,IAAA;YAED,OAAM+B,gBAAgB;UAAA;YAAAN,0BAAA;YAAAS,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAU,EAAA,GAAAV,QAAA;YAAAR,kBAAA;YAAAC,eAAA,GAAAO,QAAA,CAAAU,EAAA;UAAA;YAAAV,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAV,0BAAA,IAAAG,UAAA,CAAAxB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAAxB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAT,kBAAA;cAAAQ,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAA2B,eAAA;UAAA;YAAA,OAAAO,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAY,MAAA;UAAA;YAAA3B,yBAAA;YAAAe,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAa,EAAA,GAAAb,QAAA;YAAAd,iBAAA;YAAAC,cAAA,GAAAa,QAAA,CAAAa,EAAA;UAAA;YAAAb,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAhB,yBAAA,IAAAG,SAAA,CAAAlB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAlB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAf,iBAAA;cAAAc,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAAqB,cAAA;UAAA;YAAA,OAAAa,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAW,MAAA;UAAA;UAAA;YAAA,OAAAX,QAAA,CAAAc,IAAA;QAAA;MAAA,GAAA/B,OAAA;IAAA,CAI3B;IAAA,OAAAL,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAED,OAAOQ,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/es5/lib/parse-arrow-sync.js

@@ -4,50 +4,24 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseArrowSync;
+var _schema = require("@loaders.gl/schema");
 var _apacheArrow = require("apache-arrow");
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+var _convertTable = require("./convert-table");
 function parseArrowSync(arrayBuffer, options) {
   var _options$arrow;
-  var arrowTable = (0, _apacheArrow.tableFromIPC)([new Uint8Array(arrayBuffer)]);
-  var columnarTable = {};
-  var _iterator = _createForOfIteratorHelper(arrowTable.schema.fields),
-    _step;
-  try {
-    for (_iterator.s(); !(_step = _iterator.n()).done;) {
-      var field = _step.value;
-      var arrowColumn = arrowTable.getChild(field.name);
-      var values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
-      columnarTable[field.name] = values;
-    }
-  } catch (err) {
-    _iterator.e(err);
-  } finally {
-    _iterator.f();
-  }
-  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
+  var apacheArrowTable = (0, _apacheArrow.tableFromIPC)([new Uint8Array(arrayBuffer)]);
+  var arrowTable = (0, _convertTable.convertApacheArrowToArrowTable)(apacheArrowTable);
+  var shape = (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) || 'arrow-table';
+  switch (shape) {
     case 'arrow-table':
       return arrowTable;
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
     case 'columnar-table':
+      return (0, _convertTable.convertArrowToColumnarTable)(arrowTable);
+    case 'object-row-table':
+      var columnarTable = (0, _convertTable.convertArrowToColumnarTable)(arrowTable);
+      return (0, _schema.convertTable)(columnarTable, 'object-row-table');
     default:
-      return columnarTable;
-  }
-}
-function convertColumnarToRowFormatTable(columnarTable) {
-  var tableKeys = Object.keys(columnarTable);
-  var tableRowsCount = columnarTable[tableKeys[0]].length;
-  var rowFormatTable = [];
-  for (var index = 0; index < tableRowsCount; index++) {
-    var tableItem = {};
-    for (var keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      var fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
+      throw new Error(shape);
   }
-  return rowFormatTable;
 }
 //# sourceMappingURL=parse-arrow-sync.js.map

package/dist/es5/lib/parse-arrow-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-sync.js","names":["_apacheArrow","require","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","parseArrowSync","arrayBuffer","options","_options$arrow","arrowTable","tableFromIPC","Uint8Array","columnarTable","_iterator","schema","fields","_step","field","arrowColumn","getChild","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","keys","tableRowsCount","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {tableFromIPC} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);\n\n  // Extract columns\n\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  switch (options?.arrow?.shape) {\n    case 'arrow-table':\n      return arrowTable;\n    case 'object-row-table':\n      return convertColumnarToRowFormatTable(columnarTable);\n    case 'columnar-table':\n    default:\n      return columnarTable;\n  }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n  return rowFormatTable;\n}\n"],"mappings":";;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAA0C,SAAAC,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAG3B,SAASC,cAAcA,CAACC,WAAW,EAAEC,OAA4B,EAAE;EAAA,IAAAC,cAAA;EAChF,IAAMC,UAAU,GAAG,IAAAC,yBAAY,EAAC,CAAC,IAAIC,UAAU,CAACL,WAAW,CAAC,CAAC,CAAC;EAM9D,IAAMM,aAAa,GAAG,CAAC,CAAC;EAAC,IAAAC,SAAA,GAAAjD,0BAAA,CAEL6C,UAAU,CAACK,MAAM,CAACC,MAAM;IAAAC,KAAA;EAAA;IAA5C,KAAAH,SAAA,CAAArC,CAAA,MAAAwC,KAAA,GAAAH,SAAA,CAAApC,CAAA,IAAAC,IAAA,GAA8C;MAAA,IAAnCuC,KAAK,GAAAD,KAAA,CAAArC,KAAA;MAEd,IAAMuC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACF,KAAK,CAAClB,IAAI,CAAC;MACnD,IAAMqB,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,OAAO,CAAC,CAAC;MACrCT,aAAa,CAACK,KAAK,CAAClB,IAAI,CAAC,GAAGqB,MAAM;IACpC;EAAC,SAAAlC,GAAA;IAAA2B,SAAA,CAAAjC,CAAA,CAAAM,GAAA;EAAA;IAAA2B,SAAA,CAAA/B,CAAA;EAAA;EAED,QAAQyB,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEe,KAAK,cAAAd,cAAA,uBAAdA,cAAA,CAAgBe,KAAK;IAC3B,KAAK,aAAa;MAChB,OAAOd,UAAU;IACnB,KAAK,kBAAkB;MACrB,OAAOe,+BAA+B,CAACZ,aAAa,CAAC;IACvD,KAAK,gBAAgB;IACrB;MACE,OAAOA,aAAa;EACxB;AACF;AAEA,SAASY,+BAA+BA,CAACZ,aAAa,EAAE;EACtD,IAAMa,SAAS,GAAG/B,MAAM,CAACgC,IAAI,CAACd,aAAa,CAAC;EAC5C,IAAMe,cAAc,GAAGf,aAAa,CAACa,SAAS,CAAC,CAAC,CAAC,CAAC,CAACpD,MAAM;EACzD,IAAMuD,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGF,cAAc,EAAEE,KAAK,EAAE,EAAE;IACnD,IAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGN,SAAS,CAACpD,MAAM,EAAE0D,QAAQ,EAAE,EAAE;MAC9D,IAAMC,SAAS,GAAGP,SAAS,CAACM,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGpB,aAAa,CAACoB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EACA,OAAOF,cAAc;AACvB"}
+{"version":3,"file":"parse-arrow-sync.js","names":["_schema","require","_apacheArrow","_convertTable","parseArrowSync","arrayBuffer","options","_options$arrow","apacheArrowTable","tableFromIPC","Uint8Array","arrowTable","convertApacheArrowToArrowTable","shape","arrow","convertArrowToColumnarTable","columnarTable","convertTable","Error"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport {convertTable} from '@loaders.gl/schema';\nimport {tableFromIPC} from 'apache-arrow';\nimport type {ArrowLoaderOptions} from '../arrow-loader';\nimport {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(\n  arrayBuffer,\n  options?: ArrowLoaderOptions\n): ArrowTable | ColumnarTable | ObjectRowTable {\n  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);\n  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);\n\n  const shape = options?.arrow?.shape || 'arrow-table';\n  switch (shape) {\n    case 'arrow-table':\n      return arrowTable;\n\n    case 'columnar-table':\n      return convertArrowToColumnarTable(arrowTable);\n\n    case 'object-row-table':\n      const columnarTable = convertArrowToColumnarTable(arrowTable);\n      return convertTable(columnarTable, 'object-row-table');\n\n    default:\n      // TODO\n      throw new Error(shape);\n  }\n}\n"],"mappings":";;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAEA,IAAAE,aAAA,GAAAF,OAAA;AAGe,SAASG,cAAcA,CACpCC,WAAW,EACXC,OAA4B,EACiB;EAAA,IAAAC,cAAA;EAC7C,IAAMC,gBAAgB,GAAG,IAAAC,yBAAY,EAAC,CAAC,IAAIC,UAAU,CAACL,WAAW,CAAC,CAAC,CAAC;EACpE,IAAMM,UAAU,GAAG,IAAAC,4CAA8B,EAACJ,gBAAgB,CAAC;EAEnE,IAAMK,KAAK,GAAG,CAAAP,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEQ,KAAK,cAAAP,cAAA,uBAAdA,cAAA,CAAgBM,KAAK,KAAI,aAAa;EACpD,QAAQA,KAAK;IACX,KAAK,aAAa;MAChB,OAAOF,UAAU;IAEnB,KAAK,gBAAgB;MACnB,OAAO,IAAAI,yCAA2B,EAACJ,UAAU,CAAC;IAEhD,KAAK,kBAAkB;MACrB,IAAMK,aAAa,GAAG,IAAAD,yCAA2B,EAACJ,UAAU,CAAC;MAC7D,OAAO,IAAAM,oBAAY,EAACD,aAAa,EAAE,kBAAkB,CAAC;IAExD;MAEE,MAAM,IAAIE,KAAK,CAACL,KAAK,CAAC;EAC1B;AACF"}

package/dist/esm/arrow-loader.js

@@ -1,4 +1,4 @@
-const VERSION = typeof "4.0.0-alpha.22" !== 'undefined' ? "4.0.0-alpha.22" : 'latest';
+const VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
 export const ArrowLoader = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/esm/arrow-writer.js

@@ -1,5 +1,5 @@
 import { encodeArrowSync } from './lib/encode-arrow';
-const VERSION = typeof "4.0.0-alpha.22" !== 'undefined' ? "4.0.0-alpha.22" : 'latest';
+const VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
 export const ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/esm/index.js

@@ -1,8 +1,8 @@
+import { TableBatchBuilder } from '@loaders.gl/schema';
 import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
 import parseSync from './lib/parse-arrow-sync';
 import { parseArrowInBatches } from './lib/parse-arrow-in-batches';
-import { TableBatchBuilder } from '@loaders.gl/schema';
-import ArrowTableBatchAggregator from './lib/arrow-table-batch';
+import { ArrowTableBatchAggregator } from './lib/arrow-table-batch';
 TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
 export { VECTOR_TYPES } from './types';
 export { ArrowWriter } from './arrow-writer';
@@ -13,5 +13,4 @@ export const ArrowLoader = {
   parseSync,
   parseInBatches: parseArrowInBatches
 };
-export const _typecheckArrowLoader = ArrowLoader;
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["ArrowLoader","ArrowWorkerLoader","parseSync","parseArrowInBatches","TableBatchBuilder","ArrowTableBatchAggregator","ArrowBatch","VECTOR_TYPES","ArrowWriter","parse","arraybuffer","options","parseInBatches","_typecheckArrowLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"mappings":"AAEA,SAAQA,WAAW,IAAIC,iBAAiB,QAAO,gBAAgB;AAC/D,OAAOC,SAAS,MAAM,wBAAwB;AAC9C,SAAQC,mBAAmB,QAAO,8BAA8B;AAEhE,SAAQC,iBAAiB,QAAO,oBAAoB;AACpD,OAAOC,yBAAyB,MAAM,yBAAyB;AAG/DD,iBAAiB,CAACE,UAAU,GAAGD,yBAAyB;AAGxD,SAAQE,YAAY,QAAO,SAAS;AAIpC,SAAQC,WAAW,QAAO,gBAAgB;AAK1C,SAAQP,iBAAiB;AAGzB,OAAO,MAAMD,WAA6B,GAAG;EAC3C,GAAGC,iBAAiB;EACpBQ,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEC,OAA4B,KAClET,SAAS,CAACQ,WAAW,EAAEC,OAAO,CAAC;EACjCT,SAAS;EACTU,cAAc,EAAET;AAClB,CAAC;AAED,OAAO,MAAMU,qBAAuC,GAAGb,WAAW"}
+{"version":3,"file":"index.js","names":["TableBatchBuilder","ArrowLoader","ArrowWorkerLoader","parseSync","parseArrowInBatches","ArrowTableBatchAggregator","ArrowBatch","VECTOR_TYPES","ArrowWriter","parse","arraybuffer","options","parseInBatches"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowTable, ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {ArrowTableBatchAggregator} from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser<\n  ArrowTable | ColumnarTable | ObjectRowTable,\n  ArrowTableBatch,\n  ArrowLoaderOptions\n> = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n"],"mappings":"AAGA,SAAQA,iBAAiB,QAAO,oBAAoB;AACpD,SAAQC,WAAW,IAAIC,iBAAiB,QAAO,gBAAgB;AAC/D,OAAOC,SAAS,MAAM,wBAAwB;AAC9C,SAAQC,mBAAmB,QAAO,8BAA8B;AAEhE,SAAQC,yBAAyB,QAAO,yBAAyB;AAGjEL,iBAAiB,CAACM,UAAU,GAAGD,yBAAyB;AAGxD,SAAQE,YAAY,QAAO,SAAS;AAIpC,SAAQC,WAAW,QAAO,gBAAgB;AAK1C,SAAQN,iBAAiB;AAGzB,OAAO,MAAMD,WAIZ,GAAG;EACF,GAAGC,iBAAiB;EACpBO,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEC,OAA4B,KAClER,SAAS,CAACO,WAAW,EAAEC,OAAO,CAAC;EACjCR,SAAS;EACTS,cAAc,EAAER;AAClB,CAAC"}

package/dist/esm/lib/arrow-table-batch.js

@@ -1,7 +1,7 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 import { Schema, Field, RecordBatch, Struct, makeVector, makeData, Float32 } from 'apache-arrow';
 import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
-export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
   constructor(schema, options) {
     super(schema, options);
     _defineProperty(this, "arrowSchema", void 0);

package/dist/esm/lib/arrow-table-batch.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.js","names":["Schema","Field","RecordBatch","Struct","makeVector","makeData","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","_defineProperty","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","type","fields","children","map","_ref","shape","batchType","length","arrowFields","key","field","Float32Array","metadata","Map","arrowField","name","nullable","push","Error","vector","arrowVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";AACA,SACEA,MAAM,EACNC,KAAK,EACLC,WAAW,EACXC,MAAM,EACNC,UAAU,EACVC,QAAQ,EAERC,OAAO,QACF,cAAc;AACrB,SAAQC,4BAA4B,QAAO,oBAAoB;AAE/D,eAAe,MAAMC,yBAAyB,SAASD,4BAA4B,CAAC;EAGlFE,WAAWA,CAACC,MAAM,EAAEC,OAAO,EAAE;IAC3B,KAAK,CAACD,MAAM,EAAEC,OAAO,CAAC;IAACC,eAAA;IACvB,IAAI,CAACC,WAAW,GAAG,IAAI;EACzB;EAEAC,QAAQA,CAAA,EAA2B;IACjC,MAAMC,KAAK,GAAG,KAAK,CAACD,QAAQ,CAAC,CAAC;IAC9B,IAAIC,KAAK,EAAE;MAET,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIG,cAAc,CAACD,KAAK,CAACL,MAAM,CAAC;MAGnE,MAAMO,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACL,WAAW,EAAEE,KAAK,CAACI,IAAI,CAAC;MAGlE,MAAMC,WAAW,GAAG,IAAIlB,WAAW,CACjC,IAAI,CAACW,WAAW,EAChBR,QAAQ,CAAC;QACPgB,IAAI,EAAE,IAAIlB,MAAM,CAAC,IAAI,CAACU,WAAW,CAACS,MAAM,CAAC;QACzCC,QAAQ,EAAEN,YAAY,CAACO,GAAG,CAACC,IAAA;UAAA,IAAC;YAACN;UAAI,CAAC,GAAAM,IAAA;UAAA,OAAKN,IAAI,CAAC,CAAC,CAAC;QAAA;MAChD,CAAC,CACH,CAAC;MAED,OAAO;QACLO,KAAK,EAAE,aAAa;QACpBC,SAAS,EAAE,MAAM;QACjBR,IAAI,EAAEC,WAAW;QACjBQ,MAAM,EAAEb,KAAK,CAACa;MAChB,CAAC;IACH;IAEA,OAAO,IAAI;EACb;AACF;AAGA,SAASZ,cAAcA,CAACN,MAAM,EAAU;EACtC,MAAMmB,WAAoB,GAAG,EAAE;EAC/B,KAAK,MAAMC,GAAG,IAAIpB,MAAM,EAAE;IACxB,MAAMqB,KAAK,GAAGrB,MAAM,CAACoB,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACV,IAAI,KAAKW,YAAY,EAAE;MAE/B,MAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,MAAMC,UAAU,GAAG,IAAIlC,KAAK,CAAC8B,KAAK,CAACK,IAAI,EAAE,IAAI9B,OAAO,CAAC,CAAC,EAAEyB,KAAK,CAACM,QAAQ,EAAEJ,QAAQ,CAAC;MACjFJ,WAAW,CAACS,IAAI,CAACH,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACD,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIW,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIvC,MAAM,CAAC6B,WAAW,CAAC;AAChC;AAGA,SAASX,eAAeA,CAACL,WAAW,EAAEM,IAAI,EAAY;EACpD,MAAMF,YAAmB,GAAG,EAAE;EAC9B,KAAK,MAAMc,KAAK,IAAIlB,WAAW,CAACS,MAAM,EAAE;IACtC,MAAMkB,MAAM,GAAGrB,IAAI,CAACY,KAAK,CAACK,IAAI,CAAC;IAC/B,IAAII,MAAM,YAAYR,YAAY,EAAE;MAClC,MAAMS,WAAW,GAAGrC,UAAU,CAACoC,MAAM,CAAC;MACtCvB,YAAY,CAACqB,IAAI,CAACG,WAAW,CAAC;IAChC;EACF;EACA,IAAI5B,WAAW,CAACS,MAAM,CAACM,MAAM,KAAKX,YAAY,CAACW,MAAM,EAAE;IACrD,MAAM,IAAIW,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOtB,YAAY;AACrB"}
+{"version":3,"file":"arrow-table-batch.js","names":["Schema","Field","RecordBatch","Struct","makeVector","makeData","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","_defineProperty","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","type","fields","children","map","_ref","shape","batchType","length","arrowFields","key","field","Float32Array","metadata","Map","arrowField","name","nullable","push","Error","vector","arrowVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";AACA,SACEA,MAAM,EACNC,KAAK,EACLC,WAAW,EACXC,MAAM,EACNC,UAAU,EACVC,QAAQ,EAERC,OAAO,QACF,cAAc;AACrB,SAAQC,4BAA4B,QAAO,oBAAoB;AAE/D,OAAO,MAAMC,yBAAyB,SAASD,4BAA4B,CAAC;EAG1EE,WAAWA,CAACC,MAAM,EAAEC,OAAO,EAAE;IAC3B,KAAK,CAACD,MAAM,EAAEC,OAAO,CAAC;IAACC,eAAA;IACvB,IAAI,CAACC,WAAW,GAAG,IAAI;EACzB;EAEAC,QAAQA,CAAA,EAA2B;IACjC,MAAMC,KAAK,GAAG,KAAK,CAACD,QAAQ,CAAC,CAAC;IAC9B,IAAIC,KAAK,EAAE;MAET,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIG,cAAc,CAACD,KAAK,CAACL,MAAM,CAAC;MAGnE,MAAMO,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACL,WAAW,EAAEE,KAAK,CAACI,IAAI,CAAC;MAGlE,MAAMC,WAAW,GAAG,IAAIlB,WAAW,CACjC,IAAI,CAACW,WAAW,EAChBR,QAAQ,CAAC;QACPgB,IAAI,EAAE,IAAIlB,MAAM,CAAC,IAAI,CAACU,WAAW,CAACS,MAAM,CAAC;QACzCC,QAAQ,EAAEN,YAAY,CAACO,GAAG,CAACC,IAAA;UAAA,IAAC;YAACN;UAAI,CAAC,GAAAM,IAAA;UAAA,OAAKN,IAAI,CAAC,CAAC,CAAC;QAAA;MAChD,CAAC,CACH,CAAC;MAED,OAAO;QACLO,KAAK,EAAE,aAAa;QACpBC,SAAS,EAAE,MAAM;QACjBR,IAAI,EAAEC,WAAW;QACjBQ,MAAM,EAAEb,KAAK,CAACa;MAChB,CAAC;IACH;IAEA,OAAO,IAAI;EACb;AACF;AAGA,SAASZ,cAAcA,CAACN,MAAM,EAAU;EACtC,MAAMmB,WAAoB,GAAG,EAAE;EAC/B,KAAK,MAAMC,GAAG,IAAIpB,MAAM,EAAE;IACxB,MAAMqB,KAAK,GAAGrB,MAAM,CAACoB,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACV,IAAI,KAAKW,YAAY,EAAE;MAE/B,MAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,MAAMC,UAAU,GAAG,IAAIlC,KAAK,CAAC8B,KAAK,CAACK,IAAI,EAAE,IAAI9B,OAAO,CAAC,CAAC,EAAEyB,KAAK,CAACM,QAAQ,EAAEJ,QAAQ,CAAC;MACjFJ,WAAW,CAACS,IAAI,CAACH,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACD,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIW,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIvC,MAAM,CAAC6B,WAAW,CAAC;AAChC;AAGA,SAASX,eAAeA,CAACL,WAAW,EAAEM,IAAI,EAAY;EACpD,MAAMF,YAAmB,GAAG,EAAE;EAC9B,KAAK,MAAMc,KAAK,IAAIlB,WAAW,CAACS,MAAM,EAAE;IACtC,MAAMkB,MAAM,GAAGrB,IAAI,CAACY,KAAK,CAACK,IAAI,CAAC;IAC/B,IAAII,MAAM,YAAYR,YAAY,EAAE;MAClC,MAAMS,WAAW,GAAGrC,UAAU,CAACoC,MAAM,CAAC;MACtCvB,YAAY,CAACqB,IAAI,CAACG,WAAW,CAAC;IAChC;EACF;EACA,IAAI5B,WAAW,CAACS,MAAM,CAACM,MAAM,KAAKX,YAAY,CAACW,MAAM,EAAE;IACrD,MAAM,IAAIW,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOtB,YAAY;AACrB"}