@loaders.gl/parquet 3.3.0-alpha.4 → 3.3.0-alpha.6

package/dist/es5/bundle.js

@@ -1,7 +1,6 @@
 "use strict";
 
 var moduleExports = require('./index');
-
 globalThis.loaders = globalThis.loaders || {};
 module.exports = Object.assign(globalThis.loaders, moduleExports);
 //# sourceMappingURL=bundle.js.map

package/dist/es5/bundle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":";;AACA,IAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}
+{"version":3,"file":"bundle.js","names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"sources":["../../src/bundle.ts"],"sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"mappings":";;AACA,IAAMA,aAAa,GAAGC,OAAO,CAAC,SAAS,CAAC;AACxCC,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;AAC7CC,MAAM,CAACC,OAAO,GAAGC,MAAM,CAACC,MAAM,CAACL,UAAU,CAACC,OAAO,EAAEH,aAAa,CAAC"}

package/dist/es5/constants.js

@@ -3,12 +3,14 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.PARQUET_RDLVL_ENCODING = exports.PARQUET_RDLVL_TYPE = exports.PARQUET_VERSION = exports.PARQUET_MAGIC_ENCRYPTED = exports.PARQUET_MAGIC = void 0;
+exports.PARQUET_VERSION = exports.PARQUET_RDLVL_TYPE = exports.PARQUET_RDLVL_ENCODING = exports.PARQUET_MAGIC_ENCRYPTED = exports.PARQUET_MAGIC = void 0;
 var PARQUET_MAGIC = 'PAR1';
 exports.PARQUET_MAGIC = PARQUET_MAGIC;
 var PARQUET_MAGIC_ENCRYPTED = 'PARE';
+
 exports.PARQUET_MAGIC_ENCRYPTED = PARQUET_MAGIC_ENCRYPTED;
 var PARQUET_VERSION = 1;
+
 exports.PARQUET_VERSION = PARQUET_VERSION;
 var PARQUET_RDLVL_TYPE = 'INT32';
 exports.PARQUET_RDLVL_TYPE = PARQUET_RDLVL_TYPE;

package/dist/es5/constants.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/constants.ts"],"names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"mappings":";;;;;;AAIO,IAAMA,aAAa,GAAG,MAAtB;;AACA,IAAMC,uBAAuB,GAAG,MAAhC;;AAKA,IAAMC,eAAe,GAAG,CAAxB;;AAKA,IAAMC,kBAAkB,GAAG,OAA3B;;AACA,IAAMC,sBAAsB,GAAG,KAA/B","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"file":"constants.js"}
+{"version":3,"file":"constants.js","names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"sources":["../../src/constants.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"mappings":";;;;;;AAIO,IAAMA,aAAa,GAAG,MAAM;AAAC;AAC7B,IAAMC,uBAAuB,GAAG,MAAM;;AAAC;AAKvC,IAAMC,eAAe,GAAG,CAAC;;AAAC;AAK1B,IAAMC,kBAAkB,GAAG,OAAO;AAAC;AACnC,IAAMC,sBAAsB,GAAG,KAAK;AAAC"}

package/dist/es5/index.js

@@ -1,101 +1,85 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-Object.defineProperty(exports, "ParquetWasmWorkerLoader", {
+Object.defineProperty(exports, "ParquetEnvelopeReader", {
   enumerable: true,
   get: function get() {
-    return _parquetWasmLoader.ParquetWasmLoader;
+    return _parquetEnvelopeReader.ParquetEnvelopeReader;
   }
 });
-Object.defineProperty(exports, "ParquetWorkerLoader", {
+exports.ParquetLoader = void 0;
+Object.defineProperty(exports, "ParquetReader", {
   enumerable: true,
   get: function get() {
-    return _parquetLoader.ParquetLoader;
+    return _parquetReader.ParquetReader;
   }
 });
-Object.defineProperty(exports, "_ParquetWriter", {
+Object.defineProperty(exports, "ParquetSchema", {
   enumerable: true,
   get: function get() {
-    return _parquetWriter.ParquetWriter;
+    return _schema.ParquetSchema;
   }
 });
-Object.defineProperty(exports, "ParquetWasmWriter", {
+exports.ParquetWasmLoader = void 0;
+Object.defineProperty(exports, "ParquetWasmWorkerLoader", {
   enumerable: true,
   get: function get() {
-    return _parquetWasmWriter.ParquetWasmWriter;
+    return _parquetWasmLoader.ParquetWasmLoader;
   }
 });
-Object.defineProperty(exports, "preloadCompressions", {
+Object.defineProperty(exports, "ParquetWasmWriter", {
   enumerable: true,
   get: function get() {
-    return _compression.preloadCompressions;
+    return _parquetWasmWriter.ParquetWasmWriter;
   }
 });
-Object.defineProperty(exports, "ParquetSchema", {
+Object.defineProperty(exports, "ParquetWorkerLoader", {
   enumerable: true,
   get: function get() {
-    return _schema.ParquetSchema;
+    return _parquetLoader.ParquetLoader;
   }
 });
-Object.defineProperty(exports, "ParquetReader", {
+Object.defineProperty(exports, "_ParquetWriter", {
   enumerable: true,
   get: function get() {
-    return _parquetReader.ParquetReader;
+    return _parquetWriter.ParquetWriter;
   }
 });
-Object.defineProperty(exports, "ParquetEnvelopeReader", {
+exports._typecheckParquetLoader = void 0;
+Object.defineProperty(exports, "convertParquetToArrowSchema", {
   enumerable: true,
   get: function get() {
-    return _parquetEnvelopeReader.ParquetEnvelopeReader;
+    return _convertSchema.convertParquetToArrowSchema;
   }
 });
-Object.defineProperty(exports, "convertParquetToArrowSchema", {
+Object.defineProperty(exports, "preloadCompressions", {
   enumerable: true,
   get: function get() {
-    return _convertSchema.convertParquetToArrowSchema;
+    return _compression.preloadCompressions;
   }
 });
-exports._typecheckParquetLoader = exports.ParquetWasmLoader = exports.ParquetLoader = void 0;
-
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
 var _parquetWasmLoader = require("./parquet-wasm-loader");
-
 var _parquetLoader = require("./parquet-loader");
-
 var _parseParquet = require("./lib/parse-parquet");
-
 var _parseParquetWasm = require("./lib/wasm/parse-parquet-wasm");
-
 var _parquetWriter = require("./parquet-writer");
-
 var _parquetWasmWriter = require("./parquet-wasm-writer");
-
 var _compression = require("./parquetjs/compression");
-
 var _schema = require("./parquetjs/schema/schema");
-
 var _parquetReader = require("./parquetjs/parser/parquet-reader");
-
 var _parquetEnvelopeReader = require("./parquetjs/parser/parquet-envelope-reader");
-
 var _convertSchema = require("./lib/convert-schema");
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
 var ParquetLoader = _objectSpread(_objectSpread({}, _parquetLoader.ParquetLoader), {}, {
   parse: _parseParquet.parseParquet,
   parseFileInBatches: _parseParquet.parseParquetFileInBatches
 });
-
 exports.ParquetLoader = ParquetLoader;
-
 var ParquetWasmLoader = _objectSpread(_objectSpread({}, _parquetWasmLoader.ParquetWasmLoader), {}, {
   parse: _parseParquetWasm.parseParquet
 });

package/dist/es5/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","ParquetWasmLoader","ParquetWasmWorkerLoader","parseParquetWasm","_typecheckParquetLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;;AACA;;AACA;;AACA;;AAkBA;;AACA;;AAIA;;AAEA;;AACA;;AACA;;AAEA;;;;;;AAxBO,IAAMA,aAAa,mCACrBC,4BADqB;AAExBC,EAAAA,KAAK,EAAEC,0BAFiB;AAGxBC,EAAAA,kBAAkB,EAAEC;AAHI,EAAnB;;;;AAMA,IAAMC,iBAAiB,mCACzBC,oCADyB;AAE5BL,EAAAA,KAAK,EAAEM;AAFqB,EAAvB;;;AAqBA,IAAMC,uBAAyC,GAAGT,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
+{"version":3,"file":"index.js","names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","ParquetWasmLoader","ParquetWasmWorkerLoader","parseParquetWasm","_typecheckParquetLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;AACA;AACA;AACA;AAkBA;AACA;AAIA;AAEA;AACA;AACA;AAEA;AAAiE;AAAA;AAxB1D,IAAMA,aAAa,mCACrBC,4BAAmB;EACtBC,KAAK,EAAEC,0BAAY;EACnBC,kBAAkB,EAAEC;AAAyB,EAC9C;AAAC;AAEK,IAAMC,iBAAiB,mCACzBC,oCAAuB;EAC1BL,KAAK,EAAEM;AAAgB,EACxB;;AAAC;AAkBK,IAAMC,uBAAyC,GAAGT,aAAa;AAAC"}

package/dist/es5/lib/convert-schema.js

@@ -3,11 +3,9 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.convertParquetToArrowSchema = convertParquetToArrowSchema;
 exports.PARQUET_TYPE_MAPPING = void 0;
-
+exports.convertParquetToArrowSchema = convertParquetToArrowSchema;
 var _schema = require("@loaders.gl/schema");
-
 var PARQUET_TYPE_MAPPING = {
   BOOLEAN: _schema.Bool,
   INT32: _schema.Int32,
@@ -40,31 +38,25 @@ var PARQUET_TYPE_MAPPING = {
   DECIMAL_FIXED_LEN_BYTE_ARRAY: _schema.Float64
 };
 exports.PARQUET_TYPE_MAPPING = PARQUET_TYPE_MAPPING;
-
 function convertParquetToArrowSchema(parquetSchema) {
   var fields = getFields(parquetSchema.schema);
+
   return new _schema.Schema(fields);
 }
-
 function getFieldMetadata(field) {
   var metadata = new Map();
-
   for (var key in field) {
     if (key !== 'name') {
       var value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
       metadata.set(key, value);
     }
   }
-
   return metadata;
 }
-
 function getFields(schema) {
   var fields = [];
-
   for (var name in schema) {
     var field = schema[name];
-
     if (field.fields) {
       var childField = getFields(field.fields);
       var nestedField = new _schema.Field(name, new _schema.Struct(childField), field.optional);
@@ -76,7 +68,6 @@ function getFields(schema) {
       fields.push(arrowField);
     }
   }
-
   return fields;
 }
 //# sourceMappingURL=convert-schema.js.map

package/dist/es5/lib/convert-schema.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/convert-schema.ts"],"names":["PARQUET_TYPE_MAPPING","BOOLEAN","Bool","INT32","Int32","INT64","Float64","INT96","FLOAT","Float32","DOUBLE","BYTE_ARRAY","Binary","FIXED_LEN_BYTE_ARRAY","UTF8","Utf8","DATE","TIME_MILLIS","Int64","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","Uint16","UINT_32","Uint32","UINT_64","Uint64","INT_8","Int8","INT_16","Int16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","Schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","Field","Struct","optional","push","FieldType","type","arrowField"],"mappings":";;;;;;;;AAGA;;AAmBO,IAAMA,oBAA8D,GAAG;AAC5EC,EAAAA,OAAO,EAAEC,YADmE;AAE5EC,EAAAA,KAAK,EAAEC,aAFqE;AAG5EC,EAAAA,KAAK,EAAEC,eAHqE;AAI5EC,EAAAA,KAAK,EAAED,eAJqE;AAK5EE,EAAAA,KAAK,EAAEC,eALqE;AAM5EC,EAAAA,MAAM,EAAEJ,eANoE;AAO5EK,EAAAA,UAAU,EAAEC,cAPgE;AAQ5EC,EAAAA,oBAAoB,EAAED,cARsD;AAS5EE,EAAAA,IAAI,EAAEC,YATsE;AAU5EC,EAAAA,IAAI,EAAEZ,aAVsE;AAW5Ea,EAAAA,WAAW,EAAEC,aAX+D;AAY5EC,EAAAA,WAAW,EAAED,aAZ+D;AAa5EE,EAAAA,gBAAgB,EAAEF,aAb0D;AAc5EG,EAAAA,gBAAgB,EAAEH,aAd0D;AAe5EI,EAAAA,MAAM,EAAElB,aAfoE;AAgB5EmB,EAAAA,OAAO,EAAEC,cAhBmE;AAiB5EC,EAAAA,OAAO,EAAEC,cAjBmE;AAkB5EC,EAAAA,OAAO,EAAEC,cAlBmE;AAmB5EC,EAAAA,KAAK,EAAEC,YAnBqE;AAoB5EC,EAAAA,MAAM,EAAEC,aApBoE;AAqB5EC,EAAAA,MAAM,EAAE7B,aArBoE;AAsB5E8B,EAAAA,MAAM,EAAEhB,aAtBoE;AAuB5EiB,EAAAA,IAAI,EAAEvB,cAvBsE;AAwB5EwB,EAAAA,IAAI,EAAExB,cAxBsE;AA0B5EyB,EAAAA,QAAQ,EAAEzB,cA1BkE;AA2B5E0B,EAAAA,aAAa,EAAE7B,eA3B6D;AA4B5E8B,EAAAA,aAAa,EAAEjC,eA5B6D;AA6B5EkC,EAAAA,kBAAkB,EAAElC,eA7BwD;AA8B5EmC,EAAAA,4BAA4B,EAAEnC;AA9B8C,CAAvE;;;AAiCA,SAASoC,2BAAT,CAAqCC,aAArC,EAA2E;AAChF,MAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAf,CAAxB;AAGA,SAAO,IAAIC,cAAJ,CAAWH,MAAX,CAAP;AACD;;AAED,SAASI,gBAAT,CAA0BC,KAA1B,EAAoE;AAClE,MAAMC,QAAQ,GAAG,IAAIC,GAAJ,EAAjB;;AAEA,OAAK,IAAMC,GAAX,IAAkBH,KAAlB,EAAyB;AACvB,QAAIG,GAAG,KAAK,MAAZ,EAAoB;AAClB,UAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAD,CAAZ,KAAsB,QAAtB,GAAiCjB,IAAI,CAACmB,SAAL,CAAeL,KAAK,CAACG,GAAD,CAApB,CAAjC,GAA8DH,KAAK,CAACG,GAAD,CAAjF;AACAF,MAAAA,QAAQ,CAACK,GAAT,CAAaH,GAAb,EAAkBC,KAAlB;AACD;AACF;;AAED,SAAOH,QAAP;AACD;;AAED,SAASL,SAAT,CAAmBC,MAAnB,EAAqD;AACnD,MAAMF,MAAe,GAAG,EAAxB;;AAEA,OAAK,IAAMY,IAAX,IAAmBV,MAAnB,EAA2B;AACzB,QAAMG,KAAK,GAAGH,MAAM,CAACU,IAAD,CAApB;;AAEA,QAAIP,KAAK,CAACL,MAAV,EAAkB;AAChB,UAAMa,UAAU,GAAGZ,SAAS,CAACI,KAAK,CAACL,MAAP,CAA5B;AACA,UAAMc,WAAW,GAAG,IAAIC,aAAJ,CAAUH,IAAV,EAAgB,IAAII,cAAJ,CAAWH,UAAX,CAAhB,EAAwCR,KAAK,CAACY,QAA9C,CAApB;AACAjB,MAAAA,MAAM,CAACkB,IAAP,CAAYJ,WAAZ;AACD,KAJD,MAIO;AACL,UAAMK,SAAS,GAAG/D,oBAAoB,CAACiD,KAAK,CAACe,IAAP,CAAtC;AACA,UAAMd,QAAQ,GAAGF,gBAAgB,CAACC,KAAD,CAAjC;AACA,UAAMgB,UAAU,GAAG,IAAIN,aAAJ,CAAUH,IAAV,EAAgB,IAAIO,SAAJ,EAAhB,EAAiCd,KAAK,CAACY,QAAvC,EAAiDX,QAAjD,CAAnB;AACAN,MAAAA,MAAM,CAACkB,IAAP,CAAYG,UAAZ;AACD;AACF;;AAED,SAAOrB,MAAP;AACD","sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  Struct,\n  Field,\n  DataType,\n  Bool,\n  Float64,\n  Int32,\n  Float32,\n  Binary,\n  Utf8,\n  Int64,\n  Uint16,\n  Uint32,\n  Uint64,\n  Int8,\n  Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n  BOOLEAN: Bool,\n  INT32: Int32,\n  INT64: Float64,\n  INT96: Float64,\n  FLOAT: Float32,\n  DOUBLE: Float64,\n  BYTE_ARRAY: Binary,\n  FIXED_LEN_BYTE_ARRAY: Binary,\n  UTF8: Utf8,\n  DATE: Int32,\n  TIME_MILLIS: Int64,\n  TIME_MICROS: Int64,\n  TIMESTAMP_MILLIS: Int64,\n  TIMESTAMP_MICROS: Int64,\n  UINT_8: Int32,\n  UINT_16: Uint16,\n  UINT_32: Uint32,\n  UINT_64: Uint64,\n  INT_8: Int8,\n  INT_16: Int16,\n  INT_32: Int32,\n  INT_64: Int64,\n  JSON: Binary,\n  BSON: Binary,\n  // TODO check interal type\n  INTERVAL: Binary,\n  DECIMAL_INT32: Float32,\n  DECIMAL_INT64: Float64,\n  DECIMAL_BYTE_ARRAY: Float64,\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n  const fields = getFields(parquetSchema.schema);\n\n  // TODO add metadata if needed.\n  return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n  const metadata = new Map();\n\n  for (const key in field) {\n    if (key !== 'name') {\n      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata.set(key, value);\n    }\n  }\n\n  return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const childField = getFields(field.fields);\n      const nestedField = new Field(name, new Struct(childField), field.optional);\n      fields.push(nestedField);\n    } else {\n      const FieldType = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n"],"file":"convert-schema.js"}
+{"version":3,"file":"convert-schema.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","Bool","INT32","Int32","INT64","Float64","INT96","FLOAT","Float32","DOUBLE","BYTE_ARRAY","Binary","FIXED_LEN_BYTE_ARRAY","UTF8","Utf8","DATE","TIME_MILLIS","Int64","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","Uint16","UINT_32","Uint32","UINT_64","Uint64","INT_8","Int8","INT_16","Int16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","Schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","Field","Struct","optional","push","FieldType","type","arrowField"],"sources":["../../../src/lib/convert-schema.ts"],"sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  Struct,\n  Field,\n  DataType,\n  Bool,\n  Float64,\n  Int32,\n  Float32,\n  Binary,\n  Utf8,\n  Int64,\n  Uint16,\n  Uint32,\n  Uint64,\n  Int8,\n  Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n  BOOLEAN: Bool,\n  INT32: Int32,\n  INT64: Float64,\n  INT96: Float64,\n  FLOAT: Float32,\n  DOUBLE: Float64,\n  BYTE_ARRAY: Binary,\n  FIXED_LEN_BYTE_ARRAY: Binary,\n  UTF8: Utf8,\n  DATE: Int32,\n  TIME_MILLIS: Int64,\n  TIME_MICROS: Int64,\n  TIMESTAMP_MILLIS: Int64,\n  TIMESTAMP_MICROS: Int64,\n  UINT_8: Int32,\n  UINT_16: Uint16,\n  UINT_32: Uint32,\n  UINT_64: Uint64,\n  INT_8: Int8,\n  INT_16: Int16,\n  INT_32: Int32,\n  INT_64: Int64,\n  JSON: Binary,\n  BSON: Binary,\n  // TODO check interal type\n  INTERVAL: Binary,\n  DECIMAL_INT32: Float32,\n  DECIMAL_INT64: Float64,\n  DECIMAL_BYTE_ARRAY: Float64,\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n  const fields = getFields(parquetSchema.schema);\n\n  // TODO add metadata if needed.\n  return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n  const metadata = new Map();\n\n  for (const key in field) {\n    if (key !== 'name') {\n      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata.set(key, value);\n    }\n  }\n\n  return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const childField = getFields(field.fields);\n      const nestedField = new Field(name, new Struct(childField), field.optional);\n      fields.push(nestedField);\n    } else {\n      const FieldType = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n"],"mappings":";;;;;;;AAGA;AAmBO,IAAMA,oBAA8D,GAAG;EAC5EC,OAAO,EAAEC,YAAI;EACbC,KAAK,EAAEC,aAAK;EACZC,KAAK,EAAEC,eAAO;EACdC,KAAK,EAAED,eAAO;EACdE,KAAK,EAAEC,eAAO;EACdC,MAAM,EAAEJ,eAAO;EACfK,UAAU,EAAEC,cAAM;EAClBC,oBAAoB,EAAED,cAAM;EAC5BE,IAAI,EAAEC,YAAI;EACVC,IAAI,EAAEZ,aAAK;EACXa,WAAW,EAAEC,aAAK;EAClBC,WAAW,EAAED,aAAK;EAClBE,gBAAgB,EAAEF,aAAK;EACvBG,gBAAgB,EAAEH,aAAK;EACvBI,MAAM,EAAElB,aAAK;EACbmB,OAAO,EAAEC,cAAM;EACfC,OAAO,EAAEC,cAAM;EACfC,OAAO,EAAEC,cAAM;EACfC,KAAK,EAAEC,YAAI;EACXC,MAAM,EAAEC,aAAK;EACbC,MAAM,EAAE7B,aAAK;EACb8B,MAAM,EAAEhB,aAAK;EACbiB,IAAI,EAAEvB,cAAM;EACZwB,IAAI,EAAExB,cAAM;EAEZyB,QAAQ,EAAEzB,cAAM;EAChB0B,aAAa,EAAE7B,eAAO;EACtB8B,aAAa,EAAEjC,eAAO;EACtBkC,kBAAkB,EAAElC,eAAO;EAC3BmC,4BAA4B,EAAEnC;AAChC,CAAC;AAAC;AAEK,SAASoC,2BAA2B,CAACC,aAA4B,EAAU;EAChF,IAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAM,CAAC;;EAG9C,OAAO,IAAIC,cAAM,CAACH,MAAM,CAAC;AAC3B;AAEA,SAASI,gBAAgB,CAACC,KAAmB,EAAuB;EAClE,IAAMC,QAAQ,GAAG,IAAIC,GAAG,EAAE;EAE1B,KAAK,IAAMC,GAAG,IAAIH,KAAK,EAAE;IACvB,IAAIG,GAAG,KAAK,MAAM,EAAE;MAClB,IAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAG,CAAC,KAAK,QAAQ,GAAGjB,IAAI,CAACmB,SAAS,CAACL,KAAK,CAACG,GAAG,CAAC,CAAC,GAAGH,KAAK,CAACG,GAAG,CAAC;MACtFF,QAAQ,CAACK,GAAG,CAACH,GAAG,EAAEC,KAAK,CAAC;IAC1B;EACF;EAEA,OAAOH,QAAQ;AACjB;AAEA,SAASL,SAAS,CAACC,MAAuB,EAAW;EACnD,IAAMF,MAAe,GAAG,EAAE;EAE1B,KAAK,IAAMY,IAAI,IAAIV,MAAM,EAAE;IACzB,IAAMG,KAAK,GAAGH,MAAM,CAACU,IAAI,CAAC;IAE1B,IAAIP,KAAK,CAACL,MAAM,EAAE;MAChB,IAAMa,UAAU,GAAGZ,SAAS,CAACI,KAAK,CAACL,MAAM,CAAC;MAC1C,IAAMc,WAAW,GAAG,IAAIC,aAAK,CAACH,IAAI,EAAE,IAAII,cAAM,CAACH,UAAU,CAAC,EAAER,KAAK,CAACY,QAAQ,CAAC;MAC3EjB,MAAM,CAACkB,IAAI,CAACJ,WAAW,CAAC;IAC1B,CAAC,MAAM;MACL,IAAMK,SAAS,GAAG/D,oBAAoB,CAACiD,KAAK,CAACe,IAAI,CAAC;MAClD,IAAMd,QAAQ,GAAGF,gBAAgB,CAACC,KAAK,CAAC;MACxC,IAAMgB,UAAU,GAAG,IAAIN,aAAK,CAACH,IAAI,EAAE,IAAIO,SAAS,EAAE,EAAEd,KAAK,CAACY,QAAQ,EAAEX,QAAQ,CAAC;MAC7EN,MAAM,CAACkB,IAAI,CAACG,UAAU,CAAC;IACzB;EACF;EAEA,OAAOrB,MAAM;AACf"}

package/dist/es5/lib/parse-parquet.js

@@ -1,124 +1,89 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.parseParquet = parseParquet;
 exports.parseParquetFileInBatches = parseParquetFileInBatches;
-
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
 var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-
 var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
-
-var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
-
 var _parquetReader = require("../parquetjs/parser/parquet-reader");
-
+function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
+function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
 function parseParquet(_x3, _x4) {
   return _parseParquet.apply(this, arguments);
 }
-
 function _parseParquet() {
   _parseParquet = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(arrayBuffer, options) {
-    var blob, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, batch;
-
+    var blob, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch;
     return _regenerator.default.wrap(function _callee2$(_context2) {
       while (1) {
         switch (_context2.prev = _context2.next) {
           case 0:
             blob = new Blob([arrayBuffer]);
-            _iteratorNormalCompletion = true;
+            _iteratorAbruptCompletion = false;
             _didIteratorError = false;
             _context2.prev = 3;
-            _iterator = (0, _asyncIterator2.default)(parseParquetFileInBatches(blob, options));
-
+            _iterator = _asyncIterator(parseParquetFileInBatches(blob, options));
           case 5:
             _context2.next = 7;
             return _iterator.next();
-
           case 7:
-            _step = _context2.sent;
-            _iteratorNormalCompletion = _step.done;
-            _context2.next = 11;
-            return _step.value;
-
-          case 11:
-            _value = _context2.sent;
-
-            if (_iteratorNormalCompletion) {
-              _context2.next = 18;
+            if (!(_iteratorAbruptCompletion = !(_step = _context2.sent).done)) {
+              _context2.next = 13;
               break;
             }
-
-            batch = _value;
+            batch = _step.value;
             return _context2.abrupt("return", batch);
-
-          case 15:
-            _iteratorNormalCompletion = true;
+          case 10:
+            _iteratorAbruptCompletion = false;
             _context2.next = 5;
             break;
-
-          case 18:
-            _context2.next = 24;
+          case 13:
+            _context2.next = 19;
             break;
-
-          case 20:
-            _context2.prev = 20;
+          case 15:
+            _context2.prev = 15;
             _context2.t0 = _context2["catch"](3);
             _didIteratorError = true;
             _iteratorError = _context2.t0;
-
-          case 24:
-            _context2.prev = 24;
-            _context2.prev = 25;
-
-            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
-              _context2.next = 29;
+          case 19:
+            _context2.prev = 19;
+            _context2.prev = 20;
+            if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
+              _context2.next = 24;
               break;
             }
-
-            _context2.next = 29;
+            _context2.next = 24;
             return _iterator.return();
-
-          case 29:
-            _context2.prev = 29;
-
+          case 24:
+            _context2.prev = 24;
             if (!_didIteratorError) {
-              _context2.next = 32;
+              _context2.next = 27;
               break;
             }
-
             throw _iteratorError;
-
-          case 32:
-            return _context2.finish(29);
-
-          case 33:
+          case 27:
             return _context2.finish(24);
-
-          case 34:
+          case 28:
+            return _context2.finish(19);
+          case 29:
             return _context2.abrupt("return", null);
-
-          case 35:
+          case 30:
           case "end":
             return _context2.stop();
         }
       }
-    }, _callee2, null, [[3, 20, 24, 34], [25,, 29, 33]]);
+    }, _callee2, null, [[3, 15, 19, 29], [20,, 24, 28]]);
   }));
   return _parseParquet.apply(this, arguments);
 }
-
 function parseParquetFileInBatches(_x, _x2) {
   return _parseParquetFileInBatches.apply(this, arguments);
 }
-
 function _parseParquetFileInBatches() {
   _parseParquetFileInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(blob, options) {
     var reader, rows, cursor, record;
@@ -128,39 +93,31 @@ function _parseParquetFileInBatches() {
           case 0:
             _context.next = 2;
             return (0, _awaitAsyncGenerator2.default)(_parquetReader.ParquetReader.openBlob(blob));
-
           case 2:
             reader = _context.sent;
             rows = [];
             _context.prev = 4;
             cursor = reader.getCursor();
-
           case 6:
             _context.next = 8;
             return (0, _awaitAsyncGenerator2.default)(cursor.next());
-
           case 8:
             if (!(record = _context.sent)) {
               _context.next = 12;
               break;
             }
-
             rows.push(record);
             _context.next = 6;
             break;
-
           case 12:
             _context.prev = 12;
             _context.next = 15;
             return (0, _awaitAsyncGenerator2.default)(reader.close());
-
           case 15:
             return _context.finish(12);
-
           case 16:
             _context.next = 18;
             return rows;
-
           case 18:
           case "end":
             return _context.stop();

package/dist/es5/lib/parse-parquet.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-parquet.ts"],"names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","ParquetReader","openBlob","reader","rows","cursor","getCursor","next","record","push","close"],"mappings":";;;;;;;;;;;;;;;;;;;;AAGA;;SAEsBA,Y;;;;;4EAAf,kBAA4BC,WAA5B,EAAsDC,OAAtD;AAAA;;AAAA;AAAA;AAAA;AAAA;AACCC,YAAAA,IADD,GACQ,IAAIC,IAAJ,CAAS,CAACH,WAAD,CAAT,CADR;AAAA;AAAA;AAAA;AAAA,qDAEqBI,yBAAyB,CAACF,IAAD,EAAOD,OAAP,CAF9C;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAEYI,YAAAA,KAFZ;AAAA,8CAGIA,KAHJ;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA,8CAKE,IALF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAQgBD,yB;;;;;2FAAhB,iBAA0CF,IAA1C,EAAsDD,OAAtD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sDACgBK,6BAAcC,QAAd,CAAuBL,IAAvB,CADhB;;AAAA;AACCM,YAAAA,MADD;AAECC,YAAAA,IAFD,GAEiB,EAFjB;AAAA;AAIGC,YAAAA,MAJH,GAIYF,MAAM,CAACG,SAAP,EAJZ;;AAAA;AAAA;AAAA,sDAMoBD,MAAM,CAACE,IAAP,EANpB;;AAAA;AAAA,kBAMKC,MANL;AAAA;AAAA;AAAA;;AAODJ,YAAAA,IAAI,CAACK,IAAL,CAAUD,MAAV;AAPC;AAAA;;AAAA;AAAA;AAAA;AAAA,sDAUGL,MAAM,CAACO,KAAP,EAVH;;AAAA;AAAA;;AAAA;AAAA;AAYL,mBAAMN,IAAN;;AAZK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"file":"parse-parquet.js"}
+{"version":3,"file":"parse-parquet.js","names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","ParquetReader","openBlob","reader","rows","cursor","getCursor","next","record","push","close"],"sources":["../../../src/lib/parse-parquet.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"mappings":";;;;;;;;;;;;AAGA;AAAiE;AAAA;AAAA,SAE3CA,YAAY;EAAA;AAAA;AAAA;EAAA,0EAA3B,kBAA4BC,WAAwB,EAAEC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,yBAAyB,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAjDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,yBAAyB;EAAA;AAAA;AAAA;EAAA,yFAAzC,iBAA0CF,IAAU,EAAED,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YAAA;YAAA,0CACpEK,4BAAa,CAACC,QAAQ,CAACL,IAAI,CAAC;UAAA;YAA3CM,MAAM;YACNC,IAAa,GAAG,EAAE;YAAA;YAEhBC,MAAM,GAAGF,MAAM,CAACG,SAAS,EAAE;UAAA;YAAA;YAAA,0CAEVD,MAAM,CAACE,IAAI,EAAE;UAAA;YAAA,MAA5BC,MAAM;cAAA;cAAA;YAAA;YACZJ,IAAI,CAACK,IAAI,CAACD,MAAM,CAAC;YAAC;YAAA;UAAA;YAAA;YAAA;YAAA,0CAGdL,MAAM,CAACO,KAAK,EAAE;UAAA;YAAA;UAAA;YAAA;YAEtB,OAAMN,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACX;EAAA;AAAA"}

package/dist/es5/lib/read-array-buffer.js

@@ -1,20 +1,15 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.readArrayBuffer = readArrayBuffer;
-
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
 function readArrayBuffer(_x, _x2, _x3) {
   return _readArrayBuffer.apply(this, arguments);
 }
-
 function _readArrayBuffer() {
   _readArrayBuffer = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(file, start, length) {
     var slice;
@@ -26,21 +21,16 @@ function _readArrayBuffer() {
               _context.next = 5;
               break;
             }
-
             slice = file.slice(start, start + length);
             _context.next = 4;
             return slice.arrayBuffer();
-
           case 4:
             return _context.abrupt("return", _context.sent);
-
           case 5:
             _context.next = 7;
             return file.read(start, start + length);
-
           case 7:
             return _context.abrupt("return", _context.sent);
-
           case 8:
           case "end":
             return _context.stop();

package/dist/es5/lib/read-array-buffer.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/read-array-buffer.ts"],"names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"mappings":";;;;;;;;;;;;;SAEsBA,e;;;;;+EAAf,iBACLC,IADK,EAELC,KAFK,EAGLC,MAHK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,kBAKDF,IAAI,YAAYG,IALf;AAAA;AAAA;AAAA;;AAMGC,YAAAA,KANH,GAMWJ,IAAI,CAACI,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,CANX;AAAA;AAAA,mBAOUE,KAAK,CAACC,WAAN,EAPV;;AAAA;AAAA;;AAAA;AAAA;AAAA,mBASQL,IAAI,CAACM,IAAL,CAAUL,KAAV,EAAiBA,KAAK,GAAGC,MAAzB,CATR;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n  file: Blob | ArrayBuffer | any,\n  start: number,\n  length: number\n): Promise<ArrayBuffer> {\n  if (file instanceof Blob) {\n    const slice = file.slice(start, start + length);\n    return await slice.arrayBuffer();\n  }\n  return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n  return await new Promise((resolve, reject) => {\n    const fileReader = new FileReader();\n    fileReader.onload = (event: ProgressEvent<FileReader>) =>\n      resolve(event?.target?.result as ArrayBuffer);\n    // TODO - reject with a proper Error\n    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n    fileReader.readAsArrayBuffer(blob);\n  });\n}\n*/\n"],"file":"read-array-buffer.js"}
+{"version":3,"file":"read-array-buffer.js","names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"sources":["../../../src/lib/read-array-buffer.ts"],"sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n  file: Blob | ArrayBuffer | any,\n  start: number,\n  length: number\n): Promise<ArrayBuffer> {\n  if (file instanceof Blob) {\n    const slice = file.slice(start, start + length);\n    return await slice.arrayBuffer();\n  }\n  return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n  return await new Promise((resolve, reject) => {\n    const fileReader = new FileReader();\n    fileReader.onload = (event: ProgressEvent<FileReader>) =>\n      resolve(event?.target?.result as ArrayBuffer);\n    // TODO - reject with a proper Error\n    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n    fileReader.readAsArrayBuffer(blob);\n  });\n}\n*/\n"],"mappings":";;;;;;;;;SAEsBA,eAAe;EAAA;AAAA;AAAA;EAAA,6EAA9B,iBACLC,IAA8B,EAC9BC,KAAa,EACbC,MAAc;IAAA;IAAA;MAAA;QAAA;UAAA;YAAA,MAEVF,IAAI,YAAYG,IAAI;cAAA;cAAA;YAAA;YAChBC,KAAK,GAAGJ,IAAI,CAACI,KAAK,CAACH,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC;YAAA;YAAA,OAClCE,KAAK,CAACC,WAAW,EAAE;UAAA;YAAA;UAAA;YAAA;YAAA,OAErBL,IAAI,CAACM,IAAI,CAACL,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAC9C;EAAA;AAAA"}

package/dist/es5/lib/wasm/encode-parquet-wasm.js

@@ -1,29 +1,21 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.encode = encode;
 exports.tableToIPC = tableToIPC;
-
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
 var _apacheArrow = require("apache-arrow");
-
 var _loadWasm = require("./load-wasm");
-
 function encode(_x, _x2) {
   return _encode.apply(this, arguments);
 }
-
 function _encode() {
   _encode = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(table, options) {
     var _options$parquet;
-
     var wasmUrl, wasm, arrowIPCBytes, writerProperties, parquetBytes;
     return _regenerator.default.wrap(function _callee$(_context) {
       while (1) {
@@ -32,14 +24,12 @@ function _encode() {
             wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
             _context.next = 3;
             return (0, _loadWasm.loadWasm)(wasmUrl);
-
           case 3:
             wasm = _context.sent;
             arrowIPCBytes = tableToIPC(table);
             writerProperties = new wasm.WriterPropertiesBuilder().build();
             parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
             return _context.abrupt("return", parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset));
-
           case 8:
           case "end":
             return _context.stop();
@@ -49,7 +39,6 @@ function _encode() {
   }));
   return _encode.apply(this, arguments);
 }
-
 function tableToIPC(table) {
   return _apacheArrow.RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
 }

package/dist/es5/lib/wasm/encode-parquet-wasm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/lib/wasm/encode-parquet-wasm.ts"],"names":["encode","table","options","wasmUrl","parquet","wasm","arrowIPCBytes","tableToIPC","writerProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength","RecordBatchStreamWriter","writeAll","toUint8Array"],"mappings":";;;;;;;;;;;;;;AAGA;;AACA;;SAWsBA,M;;;;;sEAAf,iBAAsBC,KAAtB,EAAoCC,OAApC;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AACCC,YAAAA,OADD,GACWD,OADX,aACWA,OADX,2CACWA,OAAO,CAAEE,OADpB,qDACW,iBAAkBD,OAD7B;AAAA;AAAA,mBAEc,wBAASA,OAAT,CAFd;;AAAA;AAECE,YAAAA,IAFD;AAICC,YAAAA,aAJD,GAIiBC,UAAU,CAACN,KAAD,CAJ3B;AAMCO,YAAAA,gBAND,GAMoB,IAAIH,IAAI,CAACI,uBAAT,GAAmCC,KAAnC,EANpB;AAOCC,YAAAA,YAPD,GAOgBN,IAAI,CAACO,YAAL,CAAkBN,aAAlB,EAAiCE,gBAAjC,CAPhB;AAAA,6CAQEG,YAAY,CAACE,MAAb,CAAoBC,KAApB,CACLH,YAAY,CAACI,UADR,EAELJ,YAAY,CAACK,UAAb,GAA0BL,YAAY,CAACI,UAFlC,CARF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAsBA,SAASR,UAAT,CAAoBN,KAApB,EAA8C;AACnD,SAAOgB,qCAAwBC,QAAxB,CAAiCjB,KAAjC,EAAwCkB,YAAxC,CAAqD,IAArD,CAAP;AACD","sourcesContent":["import type {Table} from 'apache-arrow';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\n\nimport {RecordBatchStreamWriter} from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nexport type ParquetWriterOptions = WriterOptions & {\n  parquet?: {\n    wasmUrl?: string;\n  };\n};\n\n/**\n * Encode Arrow Table to Parquet buffer\n */\nexport async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arrowIPCBytes = tableToIPC(table);\n  // TODO: provide options for how to write table.\n  const writerProperties = new wasm.WriterPropertiesBuilder().build();\n  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);\n  return parquetBytes.buffer.slice(\n    parquetBytes.byteOffset,\n    parquetBytes.byteLength + parquetBytes.byteOffset\n  );\n}\n\n/**\n * Serialize a {@link Table} to the IPC format. This function is a convenience\n * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.\n * Opposite of {@link tableFromIPC}.\n *\n * @param table The Table to serialize.\n * @param type Whether to serialize the Table as a file or a stream.\n */\nexport function tableToIPC(table: Table): Uint8Array {\n  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);\n}\n"],"file":"encode-parquet-wasm.js"}
+{"version":3,"file":"encode-parquet-wasm.js","names":["encode","table","options","wasmUrl","parquet","loadWasm","wasm","arrowIPCBytes","tableToIPC","writerProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength","RecordBatchStreamWriter","writeAll","toUint8Array"],"sources":["../../../../src/lib/wasm/encode-parquet-wasm.ts"],"sourcesContent":["import type {Table} from 'apache-arrow';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\n\nimport {RecordBatchStreamWriter} from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nexport type ParquetWriterOptions = WriterOptions & {\n  parquet?: {\n    wasmUrl?: string;\n  };\n};\n\n/**\n * Encode Arrow Table to Parquet buffer\n */\nexport async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arrowIPCBytes = tableToIPC(table);\n  // TODO: provide options for how to write table.\n  const writerProperties = new wasm.WriterPropertiesBuilder().build();\n  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);\n  return parquetBytes.buffer.slice(\n    parquetBytes.byteOffset,\n    parquetBytes.byteLength + parquetBytes.byteOffset\n  );\n}\n\n/**\n * Serialize a {@link Table} to the IPC format. This function is a convenience\n * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.\n * Opposite of {@link tableFromIPC}.\n *\n * @param table The Table to serialize.\n * @param type Whether to serialize the Table as a file or a stream.\n */\nexport function tableToIPC(table: Table): Uint8Array {\n  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);\n}\n"],"mappings":";;;;;;;;;;AAGA;AACA;AAAqC,SAWfA,MAAM;EAAA;AAAA;AAAA;EAAA,oEAArB,iBAAsBC,KAAY,EAAEC,OAA8B;IAAA;IAAA;IAAA;MAAA;QAAA;UAAA;YACjEC,OAAO,GAAGD,OAAO,aAAPA,OAAO,2CAAPA,OAAO,CAAEE,OAAO,qDAAhB,iBAAkBD,OAAO;YAAA;YAAA,OACtB,IAAAE,kBAAQ,EAACF,OAAO,CAAC;UAAA;YAA9BG,IAAI;YAEJC,aAAa,GAAGC,UAAU,CAACP,KAAK,CAAC;YAEjCQ,gBAAgB,GAAG,IAAIH,IAAI,CAACI,uBAAuB,EAAE,CAACC,KAAK,EAAE;YAC7DC,YAAY,GAAGN,IAAI,CAACO,YAAY,CAACN,aAAa,EAAEE,gBAAgB,CAAC;YAAA,iCAChEG,YAAY,CAACE,MAAM,CAACC,KAAK,CAC9BH,YAAY,CAACI,UAAU,EACvBJ,YAAY,CAACK,UAAU,GAAGL,YAAY,CAACI,UAAU,CAClD;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACF;EAAA;AAAA;AAUM,SAASR,UAAU,CAACP,KAAY,EAAc;EACnD,OAAOiB,oCAAuB,CAACC,QAAQ,CAAClB,KAAK,CAAC,CAACmB,YAAY,CAAC,IAAI,CAAC;AACnE"}

package/dist/es5/lib/wasm/load-wasm/index.js

@@ -9,6 +9,5 @@ Object.defineProperty(exports, "loadWasm", {
     return _loadWasmNode.loadWasm;
   }
 });
-
 var _loadWasmNode = require("./load-wasm-node");
 //# sourceMappingURL=index.js.map

package/dist/es5/lib/wasm/load-wasm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA","sourcesContent":["export {loadWasm} from './load-wasm-node';\n"],"file":"index.js"}
+{"version":3,"file":"index.js","names":[],"sources":["../../../../../src/lib/wasm/load-wasm/index.ts"],"sourcesContent":["export {loadWasm} from './load-wasm-node';\n"],"mappings":";;;;;;;;;;;AAAA"}