@loaders.gl/arrow 3.1.3 → 4.0.0-alpha.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "3.1.3",
+  "version": "4.0.0-alpha.5",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "publishConfig": {
@@ -19,8 +19,8 @@
     "PLY"
   ],
   "types": "dist/index.d.ts",
-  "main": "dist/es5/index.js",
-  "module": "dist/esm/index.js",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -33,15 +33,12 @@
   "scripts": {
     "pre-build": "npm run build-worker && npm run build-bundle",
     "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
-    "build-worker": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js"
+    "build-worker": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "3.1.3",
-    "@loaders.gl/schema": "3.1.3",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
+    "@loaders.gl/schema": "4.0.0-alpha.5",
     "apache-arrow": "^4.0.0"
   },
-  "peerDependencies": {
-    "@loaders.gl/core": "3.1.0"
-  },
-  "gitHead": "4a690c369779346d73c9a27395d1c08d77d279a4"
+  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
 }

package/dist/es5/arrow-loader.js

@@ -1,29 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
-var VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
-var DEFAULT_ARROW_LOADER_OPTIONS = {
-  arrow: {
-    shape: 'columnar-table'
-  }
-};
-var ArrowLoader = {
-  name: 'Apache Arrow',
-  id: 'arrow',
-  module: 'arrow',
-  version: VERSION,
-  worker: true,
-  category: 'table',
-  extensions: ['arrow', 'feather'],
-  mimeTypes: ['application/octet-stream'],
-  binary: true,
-  tests: ['ARROW'],
-  options: DEFAULT_ARROW_LOADER_OPTIONS
-};
-exports.ArrowLoader = ArrowLoader;
-var _typecheckArrowLoader = ArrowLoader;
-exports._typecheckArrowLoader = _typecheckArrowLoader;
-//# sourceMappingURL=arrow-loader.js.map

package/dist/es5/arrow-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/arrow-loader.ts"],"names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAQA,IAAMC,4BAA4B,GAAG;AACnCC,EAAAA,KAAK,EAAE;AACLC,IAAAA,KAAK,EAAE;AADF;AAD4B,CAArC;AAOO,IAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAER,OAJgB;AAKzBS,EAAAA,MAAM,EAAE,IALiB;AAMzBC,EAAAA,QAAQ,EAAE,OANe;AAOzBC,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CAPa;AAQzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARc;AASzBC,EAAAA,MAAM,EAAE,IATiB;AAUzBC,EAAAA,KAAK,EAAE,CAAC,OAAD,CAVkB;AAWzBC,EAAAA,OAAO,EAAEd;AAXgB,CAApB;;AAcA,IAAMe,qBAA6B,GAAGZ,WAAtC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n  arrow?: {\n    shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n  };\n};\n\nconst DEFAULT_ARROW_LOADER_OPTIONS = {\n  arrow: {\n    shape: 'columnar-table'\n  }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['arrow', 'feather'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['ARROW'],\n  options: DEFAULT_ARROW_LOADER_OPTIONS\n};\n\nexport const _typecheckArrowLoader: Loader = ArrowLoader;\n"],"file":"arrow-loader.js"}

package/dist/es5/arrow-writer.js

@@ -1,27 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.ArrowWriter = void 0;
-
-var _encodeArrow = require("./lib/encode-arrow");
-
-var VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
-var ArrowWriter = {
-  name: 'Apache Arrow',
-  id: 'arrow',
-  module: 'arrow',
-  version: VERSION,
-  extensions: ['arrow', 'feather'],
-  mimeTypes: ['application/octet-stream'],
-  encodeSync: encodeSync,
-  binary: true,
-  options: {}
-};
-exports.ArrowWriter = ArrowWriter;
-
-function encodeSync(data, options) {
-  return (0, _encodeArrow.encodeArrowSync)(data);
-}
-//# sourceMappingURL=arrow-writer.js.map

package/dist/es5/arrow-writer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/arrow-writer.ts"],"names":["VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data"],"mappings":";;;;;;;AAEA;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAKO,IAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAEL,OAJgB;AAKzBM,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CALa;AAMzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANc;AAOzBC,EAAAA,UAAU,EAAVA,UAPyB;AAQzBC,EAAAA,MAAM,EAAE,IARiB;AASzBC,EAAAA,OAAO,EAAE;AATgB,CAApB;;;AAYP,SAASF,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAAwD;AACtD,SAAO,kCAAgBC,IAAhB,CAAP;AACD","sourcesContent":["// import type {Writer} from '@loaders.gl/loader-utils';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions;\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  extensions: ['arrow', 'feather'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: {}\n};\n\nfunction encodeSync(data, options?: ArrowWriterOptions) {\n  return encodeArrowSync(data);\n}\n"],"file":"arrow-writer.js"}

package/dist/es5/bundle.js

@@ -1,7 +0,0 @@
-"use strict";
-
-var moduleExports = require('./index');
-
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);
-//# sourceMappingURL=bundle.js.map

package/dist/es5/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":";;AACA,IAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/es5/index.js

@@ -1,84 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-Object.defineProperty(exports, "ArrowWorkerLoader", {
-  enumerable: true,
-  get: function get() {
-    return _arrowLoader.ArrowLoader;
-  }
-});
-Object.defineProperty(exports, "VECTOR_TYPES", {
-  enumerable: true,
-  get: function get() {
-    return _types.VECTOR_TYPES;
-  }
-});
-Object.defineProperty(exports, "ArrowWriter", {
-  enumerable: true,
-  get: function get() {
-    return _arrowWriter.ArrowWriter;
-  }
-});
-exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
-
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _arrowLoader = require("./arrow-loader");
-
-var _parseArrowSync = _interopRequireDefault(require("./lib/parse-arrow-sync"));
-
-var _parseArrowInBatches = require("./lib/parse-arrow-in-batches");
-
-var _schema = require("@loaders.gl/schema");
-
-var _arrowTableBatch = _interopRequireDefault(require("./lib/arrow-table-batch"));
-
-var _types = require("./types");
-
-var _arrowWriter = require("./arrow-writer");
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-_schema.TableBatchBuilder.ArrowBatch = _arrowTableBatch.default;
-
-var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {}, {
-  parse: function () {
-    var _parse = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(arraybuffer, options) {
-      return _regenerator.default.wrap(function _callee$(_context) {
-        while (1) {
-          switch (_context.prev = _context.next) {
-            case 0:
-              return _context.abrupt("return", (0, _parseArrowSync.default)(arraybuffer, options));
-
-            case 1:
-            case "end":
-              return _context.stop();
-          }
-        }
-      }, _callee);
-    }));
-
-    function parse(_x, _x2) {
-      return _parse.apply(this, arguments);
-    }
-
-    return parse;
-  }(),
-  parseSync: _parseArrowSync.default,
-  parseInBatches: _parseArrowInBatches.parseArrowInBatches
-});
-
-exports.ArrowLoader = ArrowLoader;
-var _typecheckArrowLoader = ArrowLoader;
-exports._typecheckArrowLoader = _typecheckArrowLoader;
-//# sourceMappingURL=index.js.map

package/dist/es5/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["TableBatchBuilder","ArrowBatch","ArrowTableBatchAggregator","ArrowLoader","ArrowWorkerLoader","parse","arraybuffer","options","parseSync","parseInBatches","parseArrowInBatches","_typecheckArrowLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AACA;;AAEA;;AACA;;AAMA;;AAIA;;;;;;AAPAA,0BAAkBC,UAAlB,GAA+BC,wBAA/B;;AAeO,IAAMC,WAA6B,mCACrCC,wBADqC;AAExCC,EAAAA,KAAK;AAAA,2EAAE,iBAAOC,WAAP,EAAiCC,OAAjC;AAAA;AAAA;AAAA;AAAA;AAAA,+CACL,6BAAUD,WAAV,EAAuBC,OAAvB,CADK;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,KAFmC;AAIxCC,EAAAA,SAAS,EAATA,uBAJwC;AAKxCC,EAAAA,cAAc,EAAEC;AALwB,EAAnC;;;AAQA,IAAMC,qBAAuC,GAAGR,WAAhD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"file":"index.js"}

package/dist/es5/lib/arrow-table-batch.js

@@ -1,129 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
-var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
-
-var _get2 = _interopRequireDefault(require("@babel/runtime/helpers/get"));
-
-var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
-
-var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
-
-var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _apacheArrow = require("apache-arrow");
-
-var _schema = require("@loaders.gl/schema");
-
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
-function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
-
-function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
-
-var ArrowTableBatchAggregator = function (_ColumnarTableBatchAg) {
-  (0, _inherits2.default)(ArrowTableBatchAggregator, _ColumnarTableBatchAg);
-
-  var _super = _createSuper(ArrowTableBatchAggregator);
-
-  function ArrowTableBatchAggregator(schema, options) {
-    var _this;
-
-    (0, _classCallCheck2.default)(this, ArrowTableBatchAggregator);
-    _this = _super.call(this, schema, options);
-    (0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "arrowSchema", void 0);
-    _this.arrowSchema = null;
-    return _this;
-  }
-
-  (0, _createClass2.default)(ArrowTableBatchAggregator, [{
-    key: "getBatch",
-    value: function getBatch() {
-      var batch = (0, _get2.default)((0, _getPrototypeOf2.default)(ArrowTableBatchAggregator.prototype), "getBatch", this).call(this);
-
-      if (batch) {
-        this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-        var arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-        var recordBatch = new _apacheArrow.RecordBatch(this.arrowSchema, batch.length, arrowVectors);
-        return {
-          shape: 'arrow-table',
-          batchType: 'data',
-          data: recordBatch,
-          length: batch.length
-        };
-      }
-
-      return null;
-    }
-  }]);
-  return ArrowTableBatchAggregator;
-}(_schema.ColumnarTableBatchAggregator);
-
-exports.default = ArrowTableBatchAggregator;
-
-function getArrowSchema(schema) {
-  var arrowFields = [];
-
-  for (var key in schema) {
-    var field = schema[key];
-
-    if (field.type === Float32Array) {
-      var metadata = field;
-      var arrowField = new _apacheArrow.Field(field.name, new _apacheArrow.Float32(), field.nullable, metadata);
-      arrowFields.push(arrowField);
-    }
-  }
-
-  if (arrowFields.length === 0) {
-    throw new Error('No arrow convertible fields');
-  }
-
-  return new _apacheArrow.Schema(arrowFields);
-}
-
-function getArrowVectors(arrowSchema, data) {
-  var arrowVectors = [];
-
-  var _iterator = _createForOfIteratorHelper(arrowSchema.fields),
-      _step;
-
-  try {
-    for (_iterator.s(); !(_step = _iterator.n()).done;) {
-      var field = _step.value;
-      var vector = data[field.name];
-
-      if (vector instanceof Float32Array) {
-        var arrowVector = _apacheArrow.Float32Vector.from(vector);
-
-        arrowVectors.push(arrowVector);
-      }
-    }
-  } catch (err) {
-    _iterator.e(err);
-  } finally {
-    _iterator.f();
-  }
-
-  if (arrowSchema.fields.length !== arrowVectors.length) {
-    throw new Error('Some columns not arrow convertible');
-  }
-
-  return arrowVectors;
-}
-//# sourceMappingURL=arrow-table-batch.js.map

package/dist/es5/lib/arrow-table-batch.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/arrow-table-batch.ts"],"names":["ArrowTableBatchAggregator","schema","options","arrowSchema","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","length","shape","batchType","ColumnarTableBatchAggregator","arrowFields","key","field","type","Float32Array","metadata","arrowField","Field","name","Float32","nullable","push","Error","Schema","fields","vector","arrowVector","Float32Vector","from"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA;;AACA;;;;;;;;;;;;IAEqBA,yB;;;;;AAGnB,qCAAYC,MAAZ,EAAoBC,OAApB,EAA6B;AAAA;;AAAA;AAC3B,8BAAMD,MAAN,EAAcC,OAAd;AAD2B;AAE3B,UAAKC,WAAL,GAAmB,IAAnB;AAF2B;AAG5B;;;;WAED,oBAAmC;AACjC,UAAMC,KAAK,sHAAX;;AACA,UAAIA,KAAJ,EAAW;AAET,aAAKD,WAAL,GAAmB,KAAKA,WAAL,IAAoBE,cAAc,CAACD,KAAK,CAACH,MAAP,CAArD;AAEA,YAAMK,YAAY,GAAGC,eAAe,CAAC,KAAKJ,WAAN,EAAmBC,KAAK,CAACI,IAAzB,CAApC;AAGA,YAAMC,WAAW,GAAG,IAAIC,wBAAJ,CAAgB,KAAKP,WAArB,EAAkCC,KAAK,CAACO,MAAxC,EAAgDL,YAAhD,CAApB;AACA,eAAO;AACLM,UAAAA,KAAK,EAAE,aADF;AAELC,UAAAA,SAAS,EAAE,MAFN;AAGLL,UAAAA,IAAI,EAAEC,WAHD;AAILE,UAAAA,MAAM,EAAEP,KAAK,CAACO;AAJT,SAAP;AAMD;;AAED,aAAO,IAAP;AACD;;;EA3BoDG,oC;;;;AA+BvD,SAAST,cAAT,CAAwBJ,MAAxB,EAAgC;AAC9B,MAAMc,WAAoB,GAAG,EAA7B;;AACA,OAAK,IAAMC,GAAX,IAAkBf,MAAlB,EAA0B;AACxB,QAAMgB,KAAK,GAAGhB,MAAM,CAACe,GAAD,CAApB;;AACA,QAAIC,KAAK,CAACC,IAAN,KAAeC,YAAnB,EAAiC;AAC/B,UAAMC,QAAQ,GAAGH,KAAjB;AAEA,UAAMI,UAAU,GAAG,IAAIC,kBAAJ,CAAUL,KAAK,CAACM,IAAhB,EAAsB,IAAIC,oBAAJ,EAAtB,EAAqCP,KAAK,CAACQ,QAA3C,EAAqDL,QAArD,CAAnB;AACAL,MAAAA,WAAW,CAACW,IAAZ,CAAiBL,UAAjB;AACD;AACF;;AACD,MAAIN,WAAW,CAACJ,MAAZ,KAAuB,CAA3B,EAA8B;AAC5B,UAAM,IAAIgB,KAAJ,CAAU,6BAAV,CAAN;AACD;;AAED,SAAO,IAAIC,mBAAJ,CAAWb,WAAX,CAAP;AACD;;AAGD,SAASR,eAAT,CAAyBJ,WAAzB,EAAsCK,IAAtC,EAA4C;AAC1C,MAAMF,YAAmB,GAAG,EAA5B;;AAD0C,6CAEtBH,WAAW,CAAC0B,MAFU;AAAA;;AAAA;AAE1C,wDAAwC;AAAA,UAA7BZ,KAA6B;AACtC,UAAMa,MAAM,GAAGtB,IAAI,CAACS,KAAK,CAACM,IAAP,CAAnB;;AACA,UAAIO,MAAM,YAAYX,YAAtB,EAAoC;AAClC,YAAMY,WAAW,GAAGC,2BAAcC,IAAd,CAAmBH,MAAnB,CAApB;;AACAxB,QAAAA,YAAY,CAACoB,IAAb,CAAkBK,WAAlB;AACD;AACF;AARyC;AAAA;AAAA;AAAA;AAAA;;AAS1C,MAAI5B,WAAW,CAAC0B,MAAZ,CAAmBlB,MAAnB,KAA8BL,YAAY,CAACK,MAA/C,EAAuD;AACrD,UAAM,IAAIgB,KAAJ,CAAU,oCAAV,CAAN;AACD;;AACD,SAAOrB,YAAP;AACD","sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {Schema, Field, RecordBatch, Float32Vector, Float32} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n      // Create the record batch\n      // new RecordBatch(schema, numRows, vectors, ...);\n      const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema) {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      const metadata = field; // just store the original field as metadata\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data) {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = Float32Vector.from(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"file":"arrow-table-batch.js"}

package/dist/es5/lib/encode-arrow.js

@@ -1,54 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.encodeArrowSync = encodeArrowSync;
-
-var _apacheArrow = require("apache-arrow");
-
-var _types = require("../types");
-
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
-function encodeArrowSync(data) {
-  var vectors = [];
-  var arrayNames = [];
-
-  var _iterator = _createForOfIteratorHelper(data),
-      _step;
-
-  try {
-    for (_iterator.s(); !(_step = _iterator.n()).done;) {
-      var arrayData = _step.value;
-      arrayNames.push(arrayData.name);
-      var arrayVector = createVector(arrayData.array, arrayData.type);
-      vectors.push(arrayVector);
-    }
-  } catch (err) {
-    _iterator.e(err);
-  } finally {
-    _iterator.f();
-  }
-
-  var table = _apacheArrow.Table.new(vectors, arrayNames);
-
-  var arrowBuffer = table.serialize();
-  return arrowBuffer;
-}
-
-function createVector(array, type) {
-  switch (type) {
-    case _types.VECTOR_TYPES.DATE:
-      return _apacheArrow.DateVector.from(array);
-
-    case _types.VECTOR_TYPES.FLOAT:
-    default:
-      return _apacheArrow.FloatVector.from(array);
-  }
-}
-//# sourceMappingURL=encode-arrow.js.map

package/dist/es5/lib/encode-arrow.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/encode-arrow.ts"],"names":["encodeArrowSync","data","vectors","arrayNames","arrayData","push","name","arrayVector","createVector","array","type","table","Table","new","arrowBuffer","serialize","VECTOR_TYPES","DATE","DateVector","from","FLOAT","FloatVector"],"mappings":";;;;;;;AAAA;;AACA;;;;;;;;AAeO,SAASA,eAAT,CAAyBC,IAAzB,EAA2D;AAChE,MAAMC,OAAc,GAAG,EAAvB;AACA,MAAMC,UAAoB,GAAG,EAA7B;;AAFgE,6CAGxCF,IAHwC;AAAA;;AAAA;AAGhE,wDAA8B;AAAA,UAAnBG,SAAmB;AAC5BD,MAAAA,UAAU,CAACE,IAAX,CAAgBD,SAAS,CAACE,IAA1B;AACA,UAAMC,WAAW,GAAGC,YAAY,CAACJ,SAAS,CAACK,KAAX,EAAkBL,SAAS,CAACM,IAA5B,CAAhC;AACAR,MAAAA,OAAO,CAACG,IAAR,CAAaE,WAAb;AACD;AAP+D;AAAA;AAAA;AAAA;AAAA;;AAQhE,MAAMI,KAAK,GAAGC,mBAAMC,GAAN,CAAUX,OAAV,EAAmBC,UAAnB,CAAd;;AACA,MAAMW,WAAW,GAAGH,KAAK,CAACI,SAAN,EAApB;AACA,SAAOD,WAAP;AACD;;AAQD,SAASN,YAAT,CAAsBC,KAAtB,EAA6BC,IAA7B,EAAmC;AACjC,UAAQA,IAAR;AACE,SAAKM,oBAAaC,IAAlB;AACE,aAAOC,wBAAWC,IAAX,CAAgBV,KAAhB,CAAP;;AACF,SAAKO,oBAAaI,KAAlB;AACA;AACE,aAAOC,yBAAYF,IAAZ,CAAiBV,KAAjB,CAAP;AALJ;AAOD","sourcesContent":["import {Table, FloatVector, DateVector} from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\n\ntype ColumnarTable = {\n  name: string;\n  array: AnyArrayType;\n  type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n  const vectors: any[] = [];\n  const arrayNames: string[] = [];\n  for (const arrayData of data) {\n    arrayNames.push(arrayData.name);\n    const arrayVector = createVector(arrayData.array, arrayData.type);\n    vectors.push(arrayVector);\n  }\n  const table = Table.new(vectors, arrayNames);\n  const arrowBuffer = table.serialize();\n  return arrowBuffer;\n}\n\n/**\n * Create Arrow Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type) {\n  switch (type) {\n    case VECTOR_TYPES.DATE:\n      return DateVector.from(array);\n    case VECTOR_TYPES.FLOAT:\n    default:\n      return FloatVector.from(array);\n  }\n}\n"],"file":"encode-arrow.js"}

package/dist/es5/lib/parse-arrow-in-batches.js

@@ -1,198 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.parseArrowInBatches = parseArrowInBatches;
-
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
-
-var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
-
-var _apacheArrow = require("apache-arrow");
-
-function parseArrowInBatches(asyncIterator) {
-  function makeArrowAsyncIterator() {
-    return _makeArrowAsyncIterator.apply(this, arguments);
-  }
-
-  function _makeArrowAsyncIterator() {
-    _makeArrowAsyncIterator = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee() {
-      var readers, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, reader, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, _value2, batch;
-
-      return _regenerator.default.wrap(function _callee$(_context) {
-        while (1) {
-          switch (_context.prev = _context.next) {
-            case 0:
-              readers = _apacheArrow.RecordBatchReader.readAll(asyncIterator);
-              _iteratorNormalCompletion = true;
-              _didIteratorError = false;
-              _context.prev = 3;
-              _iterator = (0, _asyncIterator2.default)(readers);
-
-            case 5:
-              _context.next = 7;
-              return (0, _awaitAsyncGenerator2.default)(_iterator.next());
-
-            case 7:
-              _step = _context.sent;
-              _iteratorNormalCompletion = _step.done;
-              _context.next = 11;
-              return (0, _awaitAsyncGenerator2.default)(_step.value);
-
-            case 11:
-              _value = _context.sent;
-
-              if (_iteratorNormalCompletion) {
-                _context.next = 52;
-                break;
-              }
-
-              reader = _value;
-              _iteratorNormalCompletion2 = true;
-              _didIteratorError2 = false;
-              _context.prev = 16;
-              _iterator2 = (0, _asyncIterator2.default)(reader);
-
-            case 18:
-              _context.next = 20;
-              return (0, _awaitAsyncGenerator2.default)(_iterator2.next());
-
-            case 20:
-              _step2 = _context.sent;
-              _iteratorNormalCompletion2 = _step2.done;
-              _context.next = 24;
-              return (0, _awaitAsyncGenerator2.default)(_step2.value);
-
-            case 24:
-              _value2 = _context.sent;
-
-              if (_iteratorNormalCompletion2) {
-                _context.next = 32;
-                break;
-              }
-
-              batch = _value2;
-              _context.next = 29;
-              return processBatch(batch);
-
-            case 29:
-              _iteratorNormalCompletion2 = true;
-              _context.next = 18;
-              break;
-
-            case 32:
-              _context.next = 38;
-              break;
-
-            case 34:
-              _context.prev = 34;
-              _context.t0 = _context["catch"](16);
-              _didIteratorError2 = true;
-              _iteratorError2 = _context.t0;
-
-            case 38:
-              _context.prev = 38;
-              _context.prev = 39;
-
-              if (!(!_iteratorNormalCompletion2 && _iterator2.return != null)) {
-                _context.next = 43;
-                break;
-              }
-
-              _context.next = 43;
-              return (0, _awaitAsyncGenerator2.default)(_iterator2.return());
-
-            case 43:
-              _context.prev = 43;
-
-              if (!_didIteratorError2) {
-                _context.next = 46;
-                break;
-              }
-
-              throw _iteratorError2;
-
-            case 46:
-              return _context.finish(43);
-
-            case 47:
-              return _context.finish(38);
-
-            case 48:
-              return _context.abrupt("break", 52);
-
-            case 49:
-              _iteratorNormalCompletion = true;
-              _context.next = 5;
-              break;
-
-            case 52:
-              _context.next = 58;
-              break;
-
-            case 54:
-              _context.prev = 54;
-              _context.t1 = _context["catch"](3);
-              _didIteratorError = true;
-              _iteratorError = _context.t1;
-
-            case 58:
-              _context.prev = 58;
-              _context.prev = 59;
-
-              if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
-                _context.next = 63;
-                break;
-              }
-
-              _context.next = 63;
-              return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-
-            case 63:
-              _context.prev = 63;
-
-              if (!_didIteratorError) {
-                _context.next = 66;
-                break;
-              }
-
-              throw _iteratorError;
-
-            case 66:
-              return _context.finish(63);
-
-            case 67:
-              return _context.finish(58);
-
-            case 68:
-            case "end":
-              return _context.stop();
-          }
-        }
-      }, _callee, null, [[3, 54, 58, 68], [16, 34, 38, 48], [39,, 43, 47], [59,, 63, 67]]);
-    }));
-    return _makeArrowAsyncIterator.apply(this, arguments);
-  }
-
-  return makeArrowAsyncIterator();
-}
-
-function processBatch(batch) {
-  var values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach(function (_ref, index) {
-    var name = _ref.name;
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
-}
-//# sourceMappingURL=parse-arrow-in-batches.js.map

package/dist/es5/lib/parse-arrow-in-batches.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"names":["parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","RecordBatchReader","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","index","name","getChildAt","toArray"],"mappings":";;;;;;;;;;;;;;;;;AACA;;AAKO,SAASA,mBAAT,CACLC,aADK,EAEe;AAAA,WAmBJC,sBAnBI;AAAA;AAAA;;AAAA;AAAA,0FAmBpB;AAAA;;AAAA;AAAA;AAAA;AAAA;AACQC,cAAAA,OADR,GACkBC,+BAAkBC,OAAlB,CAA0BJ,aAA1B,CADlB;AAAA;AAAA;AAAA;AAAA,uDAE6BE,OAF7B;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAEmBG,cAAAA,MAFnB;AAAA;AAAA;AAAA;AAAA,wDAG8BA,MAH9B;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAGqBC,cAAAA,KAHrB;AAAA;AAIM,qBAAMC,YAAY,CAACD,KAAD,CAAlB;;AAJN;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAnBoB;AAAA;AAAA;;AA4BpB,SAAOL,sBAAsB,EAA7B;AACD;;AAED,SAASM,YAAT,CAAsBD,KAAtB,EAA6B;AAC3B,MAAME,MAAM,GAAG;AACbC,IAAAA,QAAQ,EAAEH,KAAK,CAACI,MAAN,CAAaD,QADV;AAEbE,IAAAA,MAAM,EAAEL,KAAK,CAACK;AAFD,GAAf;AAIAL,EAAAA,KAAK,CAACI,MAAN,CAAaE,MAAb,CAAoBC,OAApB,CAA4B,gBAASC,KAAT,EAAmB;AAAA,QAAjBC,IAAiB,QAAjBA,IAAiB;AAC7CP,IAAAA,MAAM,CAACO,IAAD,CAAN,GAAeT,KAAK,CAACU,UAAN,CAAiBF,KAAjB,EAAwBG,OAAxB,EAAf;AACD,GAFD;AAGA,SAAOT,MAAP;AACD","sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator() {\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const batch of reader) {\n        yield processBatch(batch);\n      }\n      break; // only processing one stream of batches\n    }\n  }\n  return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n  const values = {\n    metadata: batch.schema.metadata,\n    length: batch.length\n  };\n  batch.schema.fields.forEach(({name}, index) => {\n    values[name] = batch.getChildAt(index).toArray();\n  });\n  return values;\n}\n"],"file":"parse-arrow-in-batches.js"}