@loaders.gl/arrow 3.1.3 → 4.0.0-alpha.5

package/dist/es5/lib/parse-arrow-sync.js

@@ -1,53 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = parseArrowSync;
-
-var _apacheArrow = require("apache-arrow");
-
-function parseArrowSync(arrayBuffer, options) {
-  var _options$arrow;
-
-  var arrowTable = _apacheArrow.Table.from([new Uint8Array(arrayBuffer)]);
-
-  var columnarTable = {};
-  arrowTable.schema.fields.forEach(function (field) {
-    var arrowColumn = arrowTable.getColumn(field.name);
-    var values = arrowColumn.toArray();
-    columnarTable[field.name] = values;
-  });
-
-  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
-    case 'arrow-table':
-      return arrowTable;
-
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
-
-    case 'columnar-table':
-    default:
-      return columnarTable;
-  }
-}
-
-function convertColumnarToRowFormatTable(columnarTable) {
-  var tableKeys = Object.keys(columnarTable);
-  var tableRowsCount = columnarTable[tableKeys[0]].length;
-  var rowFormatTable = [];
-
-  for (var index = 0; index < tableRowsCount; index++) {
-    var tableItem = {};
-
-    for (var keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      var fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-
-    rowFormatTable.push(tableItem);
-  }
-
-  return rowFormatTable;
-}
-//# sourceMappingURL=parse-arrow-sync.js.map

package/dist/es5/lib/parse-arrow-sync.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/parse-arrow-sync.ts"],"names":["parseArrowSync","arrayBuffer","options","arrowTable","Table","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":";;;;;;;AACA;;AAGe,SAASA,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,MAAMC,UAAU,GAAGC,mBAAMC,IAAN,CAAW,CAAC,IAAIC,UAAJ,CAAeL,WAAf,CAAD,CAAX,CAAnB;;AAMA,MAAMM,aAAa,GAAG,EAAtB;AAEAJ,EAAAA,UAAU,CAACK,MAAX,CAAkBC,MAAlB,CAAyBC,OAAzB,CAAiC,UAACC,KAAD,EAAW;AAE1C,QAAMC,WAAW,GAAGT,UAAU,CAACU,SAAX,CAAqBF,KAAK,CAACG,IAA3B,CAApB;AACA,QAAMC,MAAM,GAAGH,WAAW,CAACI,OAAZ,EAAf;AACAT,IAAAA,aAAa,CAACI,KAAK,CAACG,IAAP,CAAb,GAA4BC,MAA5B;AACD,GALD;;AAOA,UAAQb,OAAR,aAAQA,OAAR,yCAAQA,OAAO,CAAEe,KAAjB,mDAAQ,eAAgBC,KAAxB;AACE,SAAK,aAAL;AACE,aAAOf,UAAP;;AACF,SAAK,kBAAL;AACE,aAAOgB,+BAA+B,CAACZ,aAAD,CAAtC;;AACF,SAAK,gBAAL;AACA;AACE,aAAOA,aAAP;AAPJ;AASD;;AAED,SAASY,+BAAT,CAAyCZ,aAAzC,EAAwD;AACtD,MAAMa,SAAS,GAAGC,MAAM,CAACC,IAAP,CAAYf,aAAZ,CAAlB;AACA,MAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAD,CAAV,CAAb,CAA4BI,MAAnD;AACA,MAAMC,cAAoB,GAAG,EAA7B;;AAEA,OAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGH,cAA5B,EAA4CG,KAAK,EAAjD,EAAqD;AACnD,QAAMC,SAAS,GAAG,EAAlB;;AACA,SAAK,IAAIC,QAAQ,GAAG,CAApB,EAAuBA,QAAQ,GAAGR,SAAS,CAACI,MAA5C,EAAoDI,QAAQ,EAA5D,EAAgE;AAC9D,UAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAD,CAA3B;AACAD,MAAAA,SAAS,CAACE,SAAD,CAAT,GAAuBtB,aAAa,CAACsB,SAAD,CAAb,CAAyBH,KAAzB,CAAvB;AACD;;AACDD,IAAAA,cAAc,CAACK,IAAf,CAAoBH,SAApB;AACD;;AACD,SAAOF,cAAP;AACD","sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n  const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n  // Extract columns\n\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n  const columnarTable = {};\n\n  arrowTable.schema.fields.forEach((field) => {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getColumn(field.name);\n    const values = arrowColumn.toArray();\n    columnarTable[field.name] = values;\n  });\n\n  switch (options?.arrow?.shape) {\n    case 'arrow-table':\n      return arrowTable;\n    case 'object-row-table':\n      return convertColumnarToRowFormatTable(columnarTable);\n    case 'columnar-table':\n    default:\n      return columnarTable;\n  }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n  return rowFormatTable;\n}\n"],"file":"parse-arrow-sync.js"}

package/dist/es5/types.js

@@ -1,14 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.VECTOR_TYPES = void 0;
-var VECTOR_TYPES;
-exports.VECTOR_TYPES = VECTOR_TYPES;
-
-(function (VECTOR_TYPES) {
-  VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
-  VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
-})(VECTOR_TYPES || (exports.VECTOR_TYPES = VECTOR_TYPES = {}));
-//# sourceMappingURL=types.js.map

package/dist/es5/types.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/types.ts"],"names":["VECTOR_TYPES"],"mappings":";;;;;;IAgBYA,Y;;;WAAAA,Y;AAAAA,EAAAA,Y,CAAAA,Y;AAAAA,EAAAA,Y,CAAAA,Y;GAAAA,Y,4BAAAA,Y","sourcesContent":["type TypedIntArray =\n  | Int8Array\n  | Uint8Array\n  | Int16Array\n  | Uint16Array\n  | Int32Array\n  | Uint32Array\n  | Int32Array\n  | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n  FLOAT,\n  DATE\n}\n"],"file":"types.js"}

package/dist/es5/workers/arrow-worker.js

@@ -1,8 +0,0 @@
-"use strict";
-
-var _loaderUtils = require("@loaders.gl/loader-utils");
-
-var _index = require("../index");
-
-(0, _loaderUtils.createLoaderWorker)(_index.ArrowLoader);
-//# sourceMappingURL=arrow-worker.js.map

package/dist/es5/workers/arrow-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/workers/arrow-worker.ts"],"names":["ArrowLoader"],"mappings":";;AAAA;;AACA;;AAEA,qCAAmBA,kBAAnB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"file":"arrow-worker.js"}

package/dist/esm/arrow-loader.js

@@ -1,21 +0,0 @@
-const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
-const DEFAULT_ARROW_LOADER_OPTIONS = {
-  arrow: {
-    shape: 'columnar-table'
-  }
-};
-export const ArrowLoader = {
-  name: 'Apache Arrow',
-  id: 'arrow',
-  module: 'arrow',
-  version: VERSION,
-  worker: true,
-  category: 'table',
-  extensions: ['arrow', 'feather'],
-  mimeTypes: ['application/octet-stream'],
-  binary: true,
-  tests: ['ARROW'],
-  options: DEFAULT_ARROW_LOADER_OPTIONS
-};
-export const _typecheckArrowLoader = ArrowLoader;
-//# sourceMappingURL=arrow-loader.js.map

package/dist/esm/arrow-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/arrow-loader.ts"],"names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAQA,MAAMC,4BAA4B,GAAG;AACnCC,EAAAA,KAAK,EAAE;AACLC,IAAAA,KAAK,EAAE;AADF;AAD4B,CAArC;AAOA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAER,OAJgB;AAKzBS,EAAAA,MAAM,EAAE,IALiB;AAMzBC,EAAAA,QAAQ,EAAE,OANe;AAOzBC,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CAPa;AAQzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARc;AASzBC,EAAAA,MAAM,EAAE,IATiB;AAUzBC,EAAAA,KAAK,EAAE,CAAC,OAAD,CAVkB;AAWzBC,EAAAA,OAAO,EAAEd;AAXgB,CAApB;AAcP,OAAO,MAAMe,qBAA6B,GAAGZ,WAAtC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n  arrow?: {\n    shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n  };\n};\n\nconst DEFAULT_ARROW_LOADER_OPTIONS = {\n  arrow: {\n    shape: 'columnar-table'\n  }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['arrow', 'feather'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['ARROW'],\n  options: DEFAULT_ARROW_LOADER_OPTIONS\n};\n\nexport const _typecheckArrowLoader: Loader = ArrowLoader;\n"],"file":"arrow-loader.js"}

package/dist/esm/arrow-writer.js

@@ -1,18 +0,0 @@
-import { encodeArrowSync } from './lib/encode-arrow';
-const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
-export const ArrowWriter = {
-  name: 'Apache Arrow',
-  id: 'arrow',
-  module: 'arrow',
-  version: VERSION,
-  extensions: ['arrow', 'feather'],
-  mimeTypes: ['application/octet-stream'],
-  encodeSync,
-  binary: true,
-  options: {}
-};
-
-function encodeSync(data, options) {
-  return encodeArrowSync(data);
-}
-//# sourceMappingURL=arrow-writer.js.map

package/dist/esm/arrow-writer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/arrow-writer.ts"],"names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,oBAA9B;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAKA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAEL,OAJgB;AAKzBM,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CALa;AAMzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANc;AAOzBC,EAAAA,UAPyB;AAQzBC,EAAAA,MAAM,EAAE,IARiB;AASzBC,EAAAA,OAAO,EAAE;AATgB,CAApB;;AAYP,SAASF,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAAwD;AACtD,SAAOX,eAAe,CAACY,IAAD,CAAtB;AACD","sourcesContent":["// import type {Writer} from '@loaders.gl/loader-utils';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions;\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  extensions: ['arrow', 'feather'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: {}\n};\n\nfunction encodeSync(data, options?: ArrowWriterOptions) {\n  return encodeArrowSync(data);\n}\n"],"file":"arrow-writer.js"}

package/dist/esm/bundle.js

@@ -1,5 +0,0 @@
-const moduleExports = require('./index');
-
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);
-//# sourceMappingURL=bundle.js.map

package/dist/esm/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/esm/index.js

@@ -1,16 +0,0 @@
-import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
-import parseSync from './lib/parse-arrow-sync';
-import { parseArrowInBatches } from './lib/parse-arrow-in-batches';
-import { TableBatchBuilder } from '@loaders.gl/schema';
-import ArrowTableBatchAggregator from './lib/arrow-table-batch';
-TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
-export { VECTOR_TYPES } from './types';
-export { ArrowWriter } from './arrow-writer';
-export { ArrowWorkerLoader };
-export const ArrowLoader = { ...ArrowWorkerLoader,
-  parse: async (arraybuffer, options) => parseSync(arraybuffer, options),
-  parseSync,
-  parseInBatches: parseArrowInBatches
-};
-export const _typecheckArrowLoader = ArrowLoader;
-//# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ArrowLoader","ArrowWorkerLoader","parseSync","parseArrowInBatches","TableBatchBuilder","ArrowTableBatchAggregator","ArrowBatch","VECTOR_TYPES","ArrowWriter","parse","arraybuffer","options","parseInBatches","_typecheckArrowLoader"],"mappings":"AAEA,SAAQA,WAAW,IAAIC,iBAAvB,QAA+C,gBAA/C;AACA,OAAOC,SAAP,MAAsB,wBAAtB;AACA,SAAQC,mBAAR,QAAkC,8BAAlC;AAEA,SAAQC,iBAAR,QAAgC,oBAAhC;AACA,OAAOC,yBAAP,MAAsC,yBAAtC;AAGAD,iBAAiB,CAACE,UAAlB,GAA+BD,yBAA/B;AAGA,SAAQE,YAAR,QAA2B,SAA3B;AAIA,SAAQC,WAAR,QAA0B,gBAA1B;AAKA,SAAQP,iBAAR;AAGA,OAAO,MAAMD,WAA6B,GAAG,EAC3C,GAAGC,iBADwC;AAE3CQ,EAAAA,KAAK,EAAE,OAAOC,WAAP,EAAiCC,OAAjC,KACLT,SAAS,CAACQ,WAAD,EAAcC,OAAd,CAHgC;AAI3CT,EAAAA,SAJ2C;AAK3CU,EAAAA,cAAc,EAAET;AAL2B,CAAtC;AAQP,OAAO,MAAMU,qBAAuC,GAAGb,WAAhD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"file":"index.js"}

package/dist/esm/lib/arrow-table-batch.js

@@ -1,71 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Schema, Field, RecordBatch, Float32Vector, Float32 } from 'apache-arrow';
-import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
-export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
-  constructor(schema, options) {
-    super(schema, options);
-
-    _defineProperty(this, "arrowSchema", void 0);
-
-    this.arrowSchema = null;
-  }
-
-  getBatch() {
-    const batch = super.getBatch();
-
-    if (batch) {
-      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-      const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);
-      return {
-        shape: 'arrow-table',
-        batchType: 'data',
-        data: recordBatch,
-        length: batch.length
-      };
-    }
-
-    return null;
-  }
-
-}
-
-function getArrowSchema(schema) {
-  const arrowFields = [];
-
-  for (const key in schema) {
-    const field = schema[key];
-
-    if (field.type === Float32Array) {
-      const metadata = field;
-      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
-      arrowFields.push(arrowField);
-    }
-  }
-
-  if (arrowFields.length === 0) {
-    throw new Error('No arrow convertible fields');
-  }
-
-  return new Schema(arrowFields);
-}
-
-function getArrowVectors(arrowSchema, data) {
-  const arrowVectors = [];
-
-  for (const field of arrowSchema.fields) {
-    const vector = data[field.name];
-
-    if (vector instanceof Float32Array) {
-      const arrowVector = Float32Vector.from(vector);
-      arrowVectors.push(arrowVector);
-    }
-  }
-
-  if (arrowSchema.fields.length !== arrowVectors.length) {
-    throw new Error('Some columns not arrow convertible');
-  }
-
-  return arrowVectors;
-}
-//# sourceMappingURL=arrow-table-batch.js.map

package/dist/esm/lib/arrow-table-batch.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/arrow-table-batch.ts"],"names":["Schema","Field","RecordBatch","Float32Vector","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","length","shape","batchType","arrowFields","key","field","type","Float32Array","metadata","arrowField","name","nullable","push","Error","fields","vector","arrowVector","from"],"mappings":";AACA,SAAQA,MAAR,EAAgBC,KAAhB,EAAuBC,WAAvB,EAAoCC,aAApC,EAAmDC,OAAnD,QAAiE,cAAjE;AACA,SAAQC,4BAAR,QAA2C,oBAA3C;AAEA,eAAe,MAAMC,yBAAN,SAAwCD,4BAAxC,CAAqE;AAGlFE,EAAAA,WAAW,CAACC,MAAD,EAASC,OAAT,EAAkB;AAC3B,UAAMD,MAAN,EAAcC,OAAd;;AAD2B;;AAE3B,SAAKC,WAAL,GAAmB,IAAnB;AACD;;AAEDC,EAAAA,QAAQ,GAA2B;AACjC,UAAMC,KAAK,GAAG,MAAMD,QAAN,EAAd;;AACA,QAAIC,KAAJ,EAAW;AAET,WAAKF,WAAL,GAAmB,KAAKA,WAAL,IAAoBG,cAAc,CAACD,KAAK,CAACJ,MAAP,CAArD;AAEA,YAAMM,YAAY,GAAGC,eAAe,CAAC,KAAKL,WAAN,EAAmBE,KAAK,CAACI,IAAzB,CAApC;AAGA,YAAMC,WAAW,GAAG,IAAIf,WAAJ,CAAgB,KAAKQ,WAArB,EAAkCE,KAAK,CAACM,MAAxC,EAAgDJ,YAAhD,CAApB;AACA,aAAO;AACLK,QAAAA,KAAK,EAAE,aADF;AAELC,QAAAA,SAAS,EAAE,MAFN;AAGLJ,QAAAA,IAAI,EAAEC,WAHD;AAILC,QAAAA,MAAM,EAAEN,KAAK,CAACM;AAJT,OAAP;AAMD;;AAED,WAAO,IAAP;AACD;;AA3BiF;;AA+BpF,SAASL,cAAT,CAAwBL,MAAxB,EAAgC;AAC9B,QAAMa,WAAoB,GAAG,EAA7B;;AACA,OAAK,MAAMC,GAAX,IAAkBd,MAAlB,EAA0B;AACxB,UAAMe,KAAK,GAAGf,MAAM,CAACc,GAAD,CAApB;;AACA,QAAIC,KAAK,CAACC,IAAN,KAAeC,YAAnB,EAAiC;AAC/B,YAAMC,QAAQ,GAAGH,KAAjB;AAEA,YAAMI,UAAU,GAAG,IAAI1B,KAAJ,CAAUsB,KAAK,CAACK,IAAhB,EAAsB,IAAIxB,OAAJ,EAAtB,EAAqCmB,KAAK,CAACM,QAA3C,EAAqDH,QAArD,CAAnB;AACAL,MAAAA,WAAW,CAACS,IAAZ,CAAiBH,UAAjB;AACD;AACF;;AACD,MAAIN,WAAW,CAACH,MAAZ,KAAuB,CAA3B,EAA8B;AAC5B,UAAM,IAAIa,KAAJ,CAAU,6BAAV,CAAN;AACD;;AAED,SAAO,IAAI/B,MAAJ,CAAWqB,WAAX,CAAP;AACD;;AAGD,SAASN,eAAT,CAAyBL,WAAzB,EAAsCM,IAAtC,EAA4C;AAC1C,QAAMF,YAAmB,GAAG,EAA5B;;AACA,OAAK,MAAMS,KAAX,IAAoBb,WAAW,CAACsB,MAAhC,EAAwC;AACtC,UAAMC,MAAM,GAAGjB,IAAI,CAACO,KAAK,CAACK,IAAP,CAAnB;;AACA,QAAIK,MAAM,YAAYR,YAAtB,EAAoC;AAClC,YAAMS,WAAW,GAAG/B,aAAa,CAACgC,IAAd,CAAmBF,MAAnB,CAApB;AACAnB,MAAAA,YAAY,CAACgB,IAAb,CAAkBI,WAAlB;AACD;AACF;;AACD,MAAIxB,WAAW,CAACsB,MAAZ,CAAmBd,MAAnB,KAA8BJ,YAAY,CAACI,MAA/C,EAAuD;AACrD,UAAM,IAAIa,KAAJ,CAAU,oCAAV,CAAN;AACD;;AACD,SAAOjB,YAAP;AACD","sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {Schema, Field, RecordBatch, Float32Vector, Float32} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n      // Create the record batch\n      // new RecordBatch(schema, numRows, vectors, ...);\n      const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema) {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      const metadata = field; // just store the original field as metadata\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data) {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = Float32Vector.from(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"file":"arrow-table-batch.js"}

package/dist/esm/lib/encode-arrow.js

@@ -1,28 +0,0 @@
-import { Table, FloatVector, DateVector } from 'apache-arrow';
-import { VECTOR_TYPES } from '../types';
-export function encodeArrowSync(data) {
-  const vectors = [];
-  const arrayNames = [];
-
-  for (const arrayData of data) {
-    arrayNames.push(arrayData.name);
-    const arrayVector = createVector(arrayData.array, arrayData.type);
-    vectors.push(arrayVector);
-  }
-
-  const table = Table.new(vectors, arrayNames);
-  const arrowBuffer = table.serialize();
-  return arrowBuffer;
-}
-
-function createVector(array, type) {
-  switch (type) {
-    case VECTOR_TYPES.DATE:
-      return DateVector.from(array);
-
-    case VECTOR_TYPES.FLOAT:
-    default:
-      return FloatVector.from(array);
-  }
-}
-//# sourceMappingURL=encode-arrow.js.map

package/dist/esm/lib/encode-arrow.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/encode-arrow.ts"],"names":["Table","FloatVector","DateVector","VECTOR_TYPES","encodeArrowSync","data","vectors","arrayNames","arrayData","push","name","arrayVector","createVector","array","type","table","new","arrowBuffer","serialize","DATE","from","FLOAT"],"mappings":"AAAA,SAAQA,KAAR,EAAeC,WAAf,EAA4BC,UAA5B,QAA6C,cAA7C;AACA,SAAsBC,YAAtB,QAAyC,UAAzC;AAeA,OAAO,SAASC,eAAT,CAAyBC,IAAzB,EAA2D;AAChE,QAAMC,OAAc,GAAG,EAAvB;AACA,QAAMC,UAAoB,GAAG,EAA7B;;AACA,OAAK,MAAMC,SAAX,IAAwBH,IAAxB,EAA8B;AAC5BE,IAAAA,UAAU,CAACE,IAAX,CAAgBD,SAAS,CAACE,IAA1B;AACA,UAAMC,WAAW,GAAGC,YAAY,CAACJ,SAAS,CAACK,KAAX,EAAkBL,SAAS,CAACM,IAA5B,CAAhC;AACAR,IAAAA,OAAO,CAACG,IAAR,CAAaE,WAAb;AACD;;AACD,QAAMI,KAAK,GAAGf,KAAK,CAACgB,GAAN,CAAUV,OAAV,EAAmBC,UAAnB,CAAd;AACA,QAAMU,WAAW,GAAGF,KAAK,CAACG,SAAN,EAApB;AACA,SAAOD,WAAP;AACD;;AAQD,SAASL,YAAT,CAAsBC,KAAtB,EAA6BC,IAA7B,EAAmC;AACjC,UAAQA,IAAR;AACE,SAAKX,YAAY,CAACgB,IAAlB;AACE,aAAOjB,UAAU,CAACkB,IAAX,CAAgBP,KAAhB,CAAP;;AACF,SAAKV,YAAY,CAACkB,KAAlB;AACA;AACE,aAAOpB,WAAW,CAACmB,IAAZ,CAAiBP,KAAjB,CAAP;AALJ;AAOD","sourcesContent":["import {Table, FloatVector, DateVector} from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\n\ntype ColumnarTable = {\n  name: string;\n  array: AnyArrayType;\n  type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n  const vectors: any[] = [];\n  const arrayNames: string[] = [];\n  for (const arrayData of data) {\n    arrayNames.push(arrayData.name);\n    const arrayVector = createVector(arrayData.array, arrayData.type);\n    vectors.push(arrayVector);\n  }\n  const table = Table.new(vectors, arrayNames);\n  const arrowBuffer = table.serialize();\n  return arrowBuffer;\n}\n\n/**\n * Create Arrow Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type) {\n  switch (type) {\n    case VECTOR_TYPES.DATE:\n      return DateVector.from(array);\n    case VECTOR_TYPES.FLOAT:\n    default:\n      return FloatVector.from(array);\n  }\n}\n"],"file":"encode-arrow.js"}

package/dist/esm/lib/parse-arrow-in-batches.js

@@ -1,30 +0,0 @@
-import { RecordBatchReader } from 'apache-arrow';
-export function parseArrowInBatches(asyncIterator) {
-  async function* makeArrowAsyncIterator() {
-    const readers = RecordBatchReader.readAll(asyncIterator);
-
-    for await (const reader of readers) {
-      for await (const batch of reader) {
-        yield processBatch(batch);
-      }
-
-      break;
-    }
-  }
-
-  return makeArrowAsyncIterator();
-}
-
-function processBatch(batch) {
-  const values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach(({
-    name
-  }, index) => {
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
-}
-//# sourceMappingURL=parse-arrow-in-batches.js.map

package/dist/esm/lib/parse-arrow-in-batches.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","name","index","getChildAt","toArray"],"mappings":"AACA,SAAQA,iBAAR,QAAgC,cAAhC;AAKA,OAAO,SAASC,mBAAT,CACLC,aADK,EAEe;AAmBpB,kBAAgBC,sBAAhB,GAAyC;AACvC,UAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAlB,CAA0BH,aAA1B,CAAhB;;AACA,eAAW,MAAMI,MAAjB,IAA2BF,OAA3B,EAAoC;AAClC,iBAAW,MAAMG,KAAjB,IAA0BD,MAA1B,EAAkC;AAChC,cAAME,YAAY,CAACD,KAAD,CAAlB;AACD;;AACD;AACD;AACF;;AACD,SAAOJ,sBAAsB,EAA7B;AACD;;AAED,SAASK,YAAT,CAAsBD,KAAtB,EAA6B;AAC3B,QAAME,MAAM,GAAG;AACbC,IAAAA,QAAQ,EAAEH,KAAK,CAACI,MAAN,CAAaD,QADV;AAEbE,IAAAA,MAAM,EAAEL,KAAK,CAACK;AAFD,GAAf;AAIAL,EAAAA,KAAK,CAACI,MAAN,CAAaE,MAAb,CAAoBC,OAApB,CAA4B,CAAC;AAACC,IAAAA;AAAD,GAAD,EAASC,KAAT,KAAmB;AAC7CP,IAAAA,MAAM,CAACM,IAAD,CAAN,GAAeR,KAAK,CAACU,UAAN,CAAiBD,KAAjB,EAAwBE,OAAxB,EAAf;AACD,GAFD;AAGA,SAAOT,MAAP;AACD","sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator() {\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const batch of reader) {\n        yield processBatch(batch);\n      }\n      break; // only processing one stream of batches\n    }\n  }\n  return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n  const values = {\n    metadata: batch.schema.metadata,\n    length: batch.length\n  };\n  batch.schema.fields.forEach(({name}, index) => {\n    values[name] = batch.getChildAt(index).toArray();\n  });\n  return values;\n}\n"],"file":"parse-arrow-in-batches.js"}

package/dist/esm/lib/parse-arrow-sync.js

@@ -1,44 +0,0 @@
-import { Table } from 'apache-arrow';
-export default function parseArrowSync(arrayBuffer, options) {
-  var _options$arrow;
-
-  const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);
-  const columnarTable = {};
-  arrowTable.schema.fields.forEach(field => {
-    const arrowColumn = arrowTable.getColumn(field.name);
-    const values = arrowColumn.toArray();
-    columnarTable[field.name] = values;
-  });
-
-  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
-    case 'arrow-table':
-      return arrowTable;
-
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
-
-    case 'columnar-table':
-    default:
-      return columnarTable;
-  }
-}
-
-function convertColumnarToRowFormatTable(columnarTable) {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-  const rowFormatTable = [];
-
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-
-    rowFormatTable.push(tableItem);
-  }
-
-  return rowFormatTable;
-}
-//# sourceMappingURL=parse-arrow-sync.js.map

package/dist/esm/lib/parse-arrow-sync.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/parse-arrow-sync.ts"],"names":["Table","parseArrowSync","arrayBuffer","options","arrowTable","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":"AACA,SAAQA,KAAR,QAAoB,cAApB;AAGA,eAAe,SAASC,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,QAAMC,UAAU,GAAGJ,KAAK,CAACK,IAAN,CAAW,CAAC,IAAIC,UAAJ,CAAeJ,WAAf,CAAD,CAAX,CAAnB;AAMA,QAAMK,aAAa,GAAG,EAAtB;AAEAH,EAAAA,UAAU,CAACI,MAAX,CAAkBC,MAAlB,CAAyBC,OAAzB,CAAkCC,KAAD,IAAW;AAE1C,UAAMC,WAAW,GAAGR,UAAU,CAACS,SAAX,CAAqBF,KAAK,CAACG,IAA3B,CAApB;AACA,UAAMC,MAAM,GAAGH,WAAW,CAACI,OAAZ,EAAf;AACAT,IAAAA,aAAa,CAACI,KAAK,CAACG,IAAP,CAAb,GAA4BC,MAA5B;AACD,GALD;;AAOA,UAAQZ,OAAR,aAAQA,OAAR,yCAAQA,OAAO,CAAEc,KAAjB,mDAAQ,eAAgBC,KAAxB;AACE,SAAK,aAAL;AACE,aAAOd,UAAP;;AACF,SAAK,kBAAL;AACE,aAAOe,+BAA+B,CAACZ,aAAD,CAAtC;;AACF,SAAK,gBAAL;AACA;AACE,aAAOA,aAAP;AAPJ;AASD;;AAED,SAASY,+BAAT,CAAyCZ,aAAzC,EAAwD;AACtD,QAAMa,SAAS,GAAGC,MAAM,CAACC,IAAP,CAAYf,aAAZ,CAAlB;AACA,QAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAD,CAAV,CAAb,CAA4BI,MAAnD;AACA,QAAMC,cAAoB,GAAG,EAA7B;;AAEA,OAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGH,cAA5B,EAA4CG,KAAK,EAAjD,EAAqD;AACnD,UAAMC,SAAS,GAAG,EAAlB;;AACA,SAAK,IAAIC,QAAQ,GAAG,CAApB,EAAuBA,QAAQ,GAAGR,SAAS,CAACI,MAA5C,EAAoDI,QAAQ,EAA5D,EAAgE;AAC9D,YAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAD,CAA3B;AACAD,MAAAA,SAAS,CAACE,SAAD,CAAT,GAAuBtB,aAAa,CAACsB,SAAD,CAAb,CAAyBH,KAAzB,CAAvB;AACD;;AACDD,IAAAA,cAAc,CAACK,IAAf,CAAoBH,SAApB;AACD;;AACD,SAAOF,cAAP;AACD","sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n  const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n  // Extract columns\n\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n  const columnarTable = {};\n\n  arrowTable.schema.fields.forEach((field) => {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getColumn(field.name);\n    const values = arrowColumn.toArray();\n    columnarTable[field.name] = values;\n  });\n\n  switch (options?.arrow?.shape) {\n    case 'arrow-table':\n      return arrowTable;\n    case 'object-row-table':\n      return convertColumnarToRowFormatTable(columnarTable);\n    case 'columnar-table':\n    default:\n      return columnarTable;\n  }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n  return rowFormatTable;\n}\n"],"file":"parse-arrow-sync.js"}

package/dist/esm/types.js

@@ -1,7 +0,0 @@
-export let VECTOR_TYPES;
-
-(function (VECTOR_TYPES) {
-  VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
-  VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
-})(VECTOR_TYPES || (VECTOR_TYPES = {}));
-//# sourceMappingURL=types.js.map

package/dist/esm/types.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/types.ts"],"names":["VECTOR_TYPES"],"mappings":"AAgBA,WAAYA,YAAZ;;WAAYA,Y;AAAAA,EAAAA,Y,CAAAA,Y;AAAAA,EAAAA,Y,CAAAA,Y;GAAAA,Y,KAAAA,Y","sourcesContent":["type TypedIntArray =\n  | Int8Array\n  | Uint8Array\n  | Int16Array\n  | Uint16Array\n  | Int32Array\n  | Uint32Array\n  | Int32Array\n  | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n  FLOAT,\n  DATE\n}\n"],"file":"types.js"}

package/dist/esm/workers/arrow-worker.js

@@ -1,4 +0,0 @@
-import { createLoaderWorker } from '@loaders.gl/loader-utils';
-import { ArrowLoader } from '../index';
-createLoaderWorker(ArrowLoader);
-//# sourceMappingURL=arrow-worker.js.map

package/dist/esm/workers/arrow-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/workers/arrow-worker.ts"],"names":["createLoaderWorker","ArrowLoader"],"mappings":"AAAA,SAAQA,kBAAR,QAAiC,0BAAjC;AACA,SAAQC,WAAR,QAA0B,UAA1B;AAEAD,kBAAkB,CAACC,WAAD,CAAlB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"file":"arrow-worker.js"}