@loaders.gl/parquet 3.0.12 → 3.1.0-alpha.4

package/dist/es5/lib/parse-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/parse-parquet.ts"],"names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","ParquetReader","openBlob","reader","rows","cursor","getCursor","next","record","push","close"],"mappings":";;;;;;;;;;;;;;;;;;;;AAGA;;SAEsBA,Y;;;;;4EAAf,kBAA4BC,WAA5B,EAAsDC,OAAtD;AAAA;;AAAA;AAAA;AAAA;AAAA;AACCC,YAAAA,IADD,GACQ,IAAIC,IAAJ,CAAS,CAACH,WAAD,CAAT,CADR;AAAA;AAAA;AAAA;AAAA,qDAEqBI,yBAAyB,CAACF,IAAD,EAAOD,OAAP,CAF9C;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAEYI,YAAAA,KAFZ;AAAA,8CAGIA,KAHJ;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA,8CAKE,IALF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAQgBD,yB;;;;;2FAAhB,iBAA0CF,IAA1C,EAAsDD,OAAtD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sDACgBK,6BAAcC,QAAd,CAAuBL,IAAvB,CADhB;;AAAA;AACCM,YAAAA,MADD;AAECC,YAAAA,IAFD,GAEiB,EAFjB;AAAA;AAIGC,YAAAA,MAJH,GAIYF,MAAM,CAACG,SAAP,EAJZ;;AAAA;AAAA;AAAA,sDAMoBD,MAAM,CAACE,IAAP,EANpB;;AAAA;AAAA,kBAMKC,MANL;AAAA;AAAA;AAAA;;AAODJ,YAAAA,IAAI,CAACK,IAAL,CAAUD,MAAV;AAPC;AAAA;;AAAA;AAAA;AAAA;AAAA,sDAUGL,MAAM,CAACO,KAAP,EAVH;;AAAA;AAAA;;AAAA;AAAA;AAYL,mBAAMN,IAAN;;AAZK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"file":"parse-parquet.js"}

package/dist/es5/lib/read-array-buffer.js

@@ -0,0 +1,53 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.readArrayBuffer = readArrayBuffer;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+function readArrayBuffer(_x, _x2, _x3) {
+  return _readArrayBuffer.apply(this, arguments);
+}
+
+function _readArrayBuffer() {
+  _readArrayBuffer = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(file, start, length) {
+    var slice;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            if (!(file instanceof Blob)) {
+              _context.next = 5;
+              break;
+            }
+
+            slice = file.slice(start, start + length);
+            _context.next = 4;
+            return slice.arrayBuffer();
+
+          case 4:
+            return _context.abrupt("return", _context.sent);
+
+          case 5:
+            _context.next = 7;
+            return file.read(start, start + length);
+
+          case 7:
+            return _context.abrupt("return", _context.sent);
+
+          case 8:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _readArrayBuffer.apply(this, arguments);
+}
+//# sourceMappingURL=read-array-buffer.js.map

package/dist/es5/lib/read-array-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/read-array-buffer.ts"],"names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"mappings":";;;;;;;;;;;;;SAEsBA,e;;;;;+EAAf,iBACLC,IADK,EAELC,KAFK,EAGLC,MAHK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,kBAKDF,IAAI,YAAYG,IALf;AAAA;AAAA;AAAA;;AAMGC,YAAAA,KANH,GAMWJ,IAAI,CAACI,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,CANX;AAAA;AAAA,mBAOUE,KAAK,CAACC,WAAN,EAPV;;AAAA;AAAA;;AAAA;AAAA;AAAA,mBASQL,IAAI,CAACM,IAAL,CAAUL,KAAV,EAAiBA,KAAK,GAAGC,MAAzB,CATR;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n  file: Blob | ArrayBuffer | any,\n  start: number,\n  length: number\n): Promise<ArrayBuffer> {\n  if (file instanceof Blob) {\n    const slice = file.slice(start, start + length);\n    return await slice.arrayBuffer();\n  }\n  return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n  return await new Promise((resolve, reject) => {\n    const fileReader = new FileReader();\n    fileReader.onload = (event: ProgressEvent<FileReader>) =>\n      resolve(event?.target?.result as ArrayBuffer);\n    // TODO - reject with a proper Error\n    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n    fileReader.readAsArrayBuffer(blob);\n  });\n}\n*/\n"],"file":"read-array-buffer.js"}

package/dist/es5/parquet-loader.js

@@ -1,32 +1,17 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.ParquetLoader = exports.ParquetWorkerLoader = void 0;
-
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _reader = require("./parquetjs/reader");
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-var VERSION = typeof "3.0.12" !== 'undefined' ? "3.0.12" : 'latest';
+exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
+var VERSION = typeof "3.1.0-alpha.4" !== 'undefined' ? "3.1.0-alpha.4" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',
     url: undefined
   }
 };
-var ParquetWorkerLoader = {
+var ParquetLoader = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -36,68 +21,10 @@ var ParquetWorkerLoader = {
   extensions: ['parquet'],
   mimeTypes: ['application/octet-stream'],
   binary: true,
+  tests: ['PAR1', 'PARE'],
   options: DEFAULT_PARQUET_LOADER_OPTIONS
 };
-exports.ParquetWorkerLoader = ParquetWorkerLoader;
-
-var ParquetLoader = _objectSpread(_objectSpread({}, ParquetWorkerLoader), {}, {
-  parse: parse
-});
-
 exports.ParquetLoader = ParquetLoader;
-
-function parse(_x, _x2) {
-  return _parse.apply(this, arguments);
-}
-
-function _parse() {
-  _parse = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(arrayBuffer, options) {
-    var reader, rows, cursor, record;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            _context.next = 2;
-            return _reader.ParquetReader.openArrayBuffer(arrayBuffer);
-
-          case 2:
-            reader = _context.sent;
-            rows = [];
-            _context.prev = 4;
-            cursor = reader.getCursor();
-
-          case 6:
-            _context.next = 8;
-            return cursor.next();
-
-          case 8:
-            if (!(record = _context.sent)) {
-              _context.next = 12;
-              break;
-            }
-
-            rows.push(record);
-            _context.next = 6;
-            break;
-
-          case 12:
-            _context.prev = 12;
-            _context.next = 15;
-            return reader.close();
-
-          case 15:
-            return _context.finish(12);
-
-          case 16:
-            return _context.abrupt("return", rows);
-
-          case 17:
-          case "end":
-            return _context.stop();
-        }
-      }
-    }, _callee, null, [[4,, 12, 16]]);
-  }));
-  return _parse.apply(this, arguments);
-}
+var _typecheckParquetLoader = ParquetLoader;
+exports._typecheckParquetLoader = _typecheckParquetLoader;
 //# sourceMappingURL=parquet-loader.js.map

package/dist/es5/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetWorkerLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","options","ParquetLoader","parse","arrayBuffer","ParquetReader","openArrayBuffer","reader","rows","cursor","getCursor","next","record","push","close"],"mappings":";;;;;;;;;;;;;;;AACA;;;;;;AAIA,IAAMA,OAAO,GAAG,oBAAuB,WAAvB,cAAmD,QAAnE;AASA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,IAAMC,mBAA2B,GAAG;AACzCC,EAAAA,IAAI,EAAE,gBADmC;AAEzCC,EAAAA,EAAE,EAAE,SAFqC;AAGzCC,EAAAA,MAAM,EAAE,SAHiC;AAIzCC,EAAAA,OAAO,EAAEV,OAJgC;AAKzCW,EAAAA,MAAM,EAAE,IALiC;AAMzCC,EAAAA,QAAQ,EAAE,OAN+B;AAOzCC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAP6B;AAQzCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CAR8B;AASzCC,EAAAA,MAAM,EAAE,IATiC;AAWzCC,EAAAA,OAAO,EAAEf;AAXgC,CAApC;;;AAeA,IAAMgB,aAA+B,mCACvCX,mBADuC;AAE1CY,EAAAA,KAAK,EAALA;AAF0C,EAArC;;;;SAKQA,K;;;;;qEAAf,iBAAqBC,WAArB,EAA+CH,OAA/C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBACuBI,sBAAcC,eAAd,CAA8BF,WAA9B,CADvB;;AAAA;AACQG,YAAAA,MADR;AAEQC,YAAAA,IAFR,GAEwB,EAFxB;AAAA;AAIUC,YAAAA,MAJV,GAImBF,MAAM,CAACG,SAAP,EAJnB;;AAAA;AAAA;AAAA,mBAM2BD,MAAM,CAACE,IAAP,EAN3B;;AAAA;AAAA,kBAMYC,MANZ;AAAA;AAAA;AAAA;;AAOMJ,YAAAA,IAAI,CAACK,IAAL,CAAUD,MAAV;AAPN;AAAA;;AAAA;AAAA;AAAA;AAAA,mBAUUL,MAAM,CAACO,KAAP,EAVV;;AAAA;AAAA;;AAAA;AAAA,6CAYSN,IAZT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {ParquetReader} from './parquetjs/reader';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWorkerLoader: Loader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  // tests: ['PARQUET'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader: LoaderWithParser = {\n  ...ParquetWorkerLoader,\n  parse\n};\n\nasync function parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openArrayBuffer(arrayBuffer);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  return rows;\n}\n"],"file":"parquet-loader.js"}
+{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,IAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;;AAcA,IAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}

package/dist/es5/parquet-writer.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetWriter = void 0;
-var VERSION = typeof "3.0.12" !== 'undefined' ? "3.0.12" : 'latest';
+var VERSION = typeof "3.1.0-alpha.4" !== 'undefined' ? "3.1.0-alpha.4" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {};
 var ParquetWriter = {
   name: 'Apache Parquet',

package/dist/es5/parquet-writer.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,oBAAuB,WAAvB,cAAmD,QAAnE;AAIA,IAAMC,8BAA8B,GAAG,EAAvC;AAEO,IAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAAU,EAAVA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
+{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAIA,IAAMC,8BAA8B,GAAG,EAAvC;AAEO,IAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAAU,EAAVA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}

package/dist/es5/parquetjs/codecs/dictionary.js

@@ -0,0 +1,30 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.decodeValues = decodeValues;
+exports.encodeValues = encodeValues;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _rle = require("./rle");
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function decodeValues(type, cursor, count, opts) {
+  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+  cursor.offset += 1;
+  return (0, _rle.decodeValues)(type, cursor, count, _objectSpread(_objectSpread({}, opts), {}, {
+    disableEnvelope: true
+  }));
+}
+
+function encodeValues(type, cursor, count, opts) {
+  throw new Error('Encode dictionary functionality is not supported');
+}
+//# sourceMappingURL=dictionary.js.map

package/dist/es5/parquetjs/codecs/dictionary.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":";;;;;;;;;;;;AAAA;;;;;;AAEO,SAASA,YAAT,CAAsBC,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtDA,EAAAA,IAAI,CAACC,QAAL,GAAgBH,MAAM,CAACI,MAAP,CAAcC,KAAd,CAAoBL,MAAM,CAACM,MAA3B,EAAmCN,MAAM,CAACM,MAAP,GAAgB,CAAnD,EAAsDC,QAAtD,CAA+D,CAA/D,CAAhB;AACAP,EAAAA,MAAM,CAACM,MAAP,IAAiB,CAAjB;AACA,SAAO,uBAAgBP,IAAhB,EAAsBC,MAAtB,EAA8BC,KAA9B,kCAAyCC,IAAzC;AAA+CM,IAAAA,eAAe,EAAE;AAAhE,KAAP;AACD;;AAEM,SAASC,YAAT,CAAsBV,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtD,QAAM,IAAIQ,KAAJ,CAAU,kDAAV,CAAN;AACD","sourcesContent":["import {decodeValues as decodeRleValues} from './rle';\n\nexport function decodeValues(type, cursor, count, opts) {\n  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);\n  cursor.offset += 1;\n  return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});\n}\n\nexport function encodeValues(type, cursor, count, opts) {\n  throw new Error('Encode dictionary functionality is not supported');\n}\n"],"file":"dictionary.js"}

package/dist/es5/parquetjs/codecs/index.js

@@ -14,6 +14,8 @@ var PLAIN = _interopRequireWildcard(require("./plain"));
 
 var RLE = _interopRequireWildcard(require("./rle"));
 
+var DICTIONARY = _interopRequireWildcard(require("./dictionary"));
+
 var _declare = require("./declare");
 
 Object.keys(_declare).forEach(function (key) {
@@ -40,6 +42,14 @@ var PARQUET_CODECS = {
   RLE: {
     encodeValues: RLE.encodeValues,
     decodeValues: RLE.decodeValues
+  },
+  PLAIN_DICTIONARY: {
+    encodeValues: DICTIONARY.encodeValues,
+    decodeValues: DICTIONARY.decodeValues
+  },
+  RLE_DICTIONARY: {
+    encodeValues: DICTIONARY.encodeValues,
+    decodeValues: DICTIONARY.decodeValues
   }
 };
 exports.PARQUET_CODECS = PARQUET_CODECS;

package/dist/es5/parquetjs/codecs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PARQUET_CODECS","PLAIN","encodeValues","decodeValues","RLE"],"mappings":";;;;;;;;;;;;AAGA;;AACA;;AAEA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;;;;AAEO,IAAMA,cAAqD,GAAG;AACnEC,EAAAA,KAAK,EAAE;AACLC,IAAAA,YAAY,EAAED,KAAK,CAACC,YADf;AAELC,IAAAA,YAAY,EAAEF,KAAK,CAACE;AAFf,GAD4D;AAKnEC,EAAAA,GAAG,EAAE;AACHF,IAAAA,YAAY,EAAEE,GAAG,CAACF,YADf;AAEHC,IAAAA,YAAY,EAAEC,GAAG,CAACD;AAFf;AAL8D,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n  PLAIN: {\n    encodeValues: PLAIN.encodeValues,\n    decodeValues: PLAIN.decodeValues\n  },\n  RLE: {\n    encodeValues: RLE.encodeValues,\n    decodeValues: RLE.decodeValues\n  }\n};\n"],"file":"index.js"}
+{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PARQUET_CODECS","PLAIN","encodeValues","decodeValues","RLE","PLAIN_DICTIONARY","DICTIONARY","RLE_DICTIONARY"],"mappings":";;;;;;;;;;;;AAGA;;AACA;;AACA;;AAEA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;;;;AAEO,IAAMA,cAAqD,GAAG;AACnEC,EAAAA,KAAK,EAAE;AACLC,IAAAA,YAAY,EAAED,KAAK,CAACC,YADf;AAELC,IAAAA,YAAY,EAAEF,KAAK,CAACE;AAFf,GAD4D;AAKnEC,EAAAA,GAAG,EAAE;AACHF,IAAAA,YAAY,EAAEE,GAAG,CAACF,YADf;AAEHC,IAAAA,YAAY,EAAEC,GAAG,CAACD;AAFf,GAL8D;AAUnEE,EAAAA,gBAAgB,EAAE;AAEhBH,IAAAA,YAAY,EAAEI,UAAU,CAACJ,YAFT;AAGhBC,IAAAA,YAAY,EAAEG,UAAU,CAACH;AAHT,GAViD;AAgBnEI,EAAAA,cAAc,EAAE;AAEdL,IAAAA,YAAY,EAAEI,UAAU,CAACJ,YAFX;AAGdC,IAAAA,YAAY,EAAEG,UAAU,CAACH;AAHX;AAhBmD,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\nimport * as DICTIONARY from './dictionary';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n  PLAIN: {\n    encodeValues: PLAIN.encodeValues,\n    decodeValues: PLAIN.decodeValues\n  },\n  RLE: {\n    encodeValues: RLE.encodeValues,\n    decodeValues: RLE.decodeValues\n  },\n  // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.\n  PLAIN_DICTIONARY: {\n    // @ts-ignore\n    encodeValues: DICTIONARY.encodeValues,\n    decodeValues: DICTIONARY.decodeValues\n  },\n  // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.\n  RLE_DICTIONARY: {\n    // @ts-ignore\n    encodeValues: DICTIONARY.encodeValues,\n    decodeValues: DICTIONARY.decodeValues\n  }\n};\n"],"file":"index.js"}

package/dist/es5/parquetjs/codecs/rle.js

@@ -134,7 +134,7 @@ function decodeRunRepeated(cursor, count, opts) {
   var value = 0;
 
   for (var i = 0; i < Math.ceil(bitWidth / 8); i++) {
-    value <<= 8;
+    value << 8;
     value += cursor.buffer[cursor.offset];
     cursor.offset += 1;
   }
@@ -166,7 +166,7 @@ function encodeRunRepeated(value, count, opts) {
 
   for (var i = 0; i < buf.length; i++) {
     buf.writeUInt8(value & 0xff, i);
-    value >>= 8;
+    value >> 8;
   }
 
   return Buffer.concat([Buffer.from(_varint.default.encode(count << 1)), buf]);

package/dist/es5/parquetjs/codecs/rle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAChD6B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AACA6B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    value <<= 8;\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    value >>= 8;\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
+{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}

package/dist/es5/parquetjs/compression.js

@@ -2,99 +2,170 @@
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 
-var _typeof = require("@babel/runtime/helpers/typeof");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+exports.preloadCompressions = preloadCompressions;
 exports.deflate = deflate;
+exports.decompress = decompress;
 exports.inflate = inflate;
 exports.PARQUET_COMPRESSION_METHODS = void 0;
 
-var Util = _interopRequireWildcard(require("./util"));
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
 
-var _zlib = _interopRequireDefault(require("zlib"));
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
 
-var _snappyjs = _interopRequireDefault(require("snappyjs"));
+var _compression = require("@loaders.gl/compression");
 
-function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+var _bufferUtils = require("./utils/buffer-utils");
 
-function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+var _decompress2 = _interopRequireDefault(require("brotli/decompress"));
 
-var brotli;
-var lzo;
-var lz4js;
-var PARQUET_COMPRESSION_METHODS = {
-  UNCOMPRESSED: {
-    deflate: deflate_identity,
-    inflate: inflate_identity
-  },
-  GZIP: {
-    deflate: deflate_gzip,
-    inflate: inflate_gzip
-  },
-  SNAPPY: {
-    deflate: deflate_snappy,
-    inflate: inflate_snappy
-  },
-  LZO: {
-    deflate: deflate_lzo,
-    inflate: inflate_lzo
-  },
-  BROTLI: {
-    deflate: deflate_brotli,
-    inflate: inflate_brotli
+var _lz4js = _interopRequireDefault(require("lz4js"));
+
+var _lzo = _interopRequireDefault(require("lzo"));
+
+var _zstdCodec = require("zstd-codec");
+
+var modules = {
+  brotli: {
+    decompress: _decompress2.default,
+    compress: function compress() {
+      throw new Error('brotli compress');
+    }
   },
-  LZ4: {
-    deflate: deflate_lz4,
-    inflate: inflate_lz4
-  }
+  lz4js: _lz4js.default,
+  lzo: _lzo.default,
+  'zstd-codec': _zstdCodec.ZstdCodec
+};
+var PARQUET_COMPRESSION_METHODS = {
+  UNCOMPRESSED: new _compression.NoCompression(),
+  GZIP: new _compression.GZipCompression(),
+  SNAPPY: new _compression.SnappyCompression(),
+  BROTLI: new _compression.BrotliCompression({
+    modules: modules
+  }),
+  LZ4: new _compression.LZ4Compression({
+    modules: modules
+  }),
+  LZ4_RAW: new _compression.LZ4Compression({
+    modules: modules
+  }),
+  LZO: new _compression.LZOCompression({
+    modules: modules
+  }),
+  ZSTD: new _compression.ZstdCompression({
+    modules: modules
+  })
 };
 exports.PARQUET_COMPRESSION_METHODS = PARQUET_COMPRESSION_METHODS;
 
-function deflate(method, value) {
-  if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw new Error("invalid compression method: ".concat(method));
-  }
-
-  return PARQUET_COMPRESSION_METHODS[method].deflate(value);
+function preloadCompressions(_x) {
+  return _preloadCompressions.apply(this, arguments);
 }
 
-function deflate_identity(value) {
-  return value;
+function _preloadCompressions() {
+  _preloadCompressions = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(options) {
+    var compressions;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            compressions = Object.values(PARQUET_COMPRESSION_METHODS);
+            _context.next = 3;
+            return Promise.all(compressions.map(function (compression) {
+              return compression.preload();
+            }));
+
+          case 3:
+            return _context.abrupt("return", _context.sent);
+
+          case 4:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _preloadCompressions.apply(this, arguments);
 }
 
-function deflate_gzip(value) {
-  return _zlib.default.gzipSync(value);
+function deflate(_x2, _x3) {
+  return _deflate.apply(this, arguments);
 }
 
-function deflate_snappy(value) {
-  return _snappyjs.default.compress(value);
+function _deflate() {
+  _deflate = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(method, value) {
+    var compression, inputArrayBuffer, compressedArrayBuffer;
+    return _regenerator.default.wrap(function _callee2$(_context2) {
+      while (1) {
+        switch (_context2.prev = _context2.next) {
+          case 0:
+            compression = PARQUET_COMPRESSION_METHODS[method];
+
+            if (compression) {
+              _context2.next = 3;
+              break;
+            }
+
+            throw new Error("parquet: invalid compression method: ".concat(method));
+
+          case 3:
+            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
+            _context2.next = 6;
+            return compression.compress(inputArrayBuffer);
+
+          case 6:
+            compressedArrayBuffer = _context2.sent;
+            return _context2.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
+
+          case 8:
+          case "end":
+            return _context2.stop();
+        }
+      }
+    }, _callee2);
+  }));
+  return _deflate.apply(this, arguments);
 }
 
-function deflate_lzo(value) {
-  lzo = lzo || Util.load('lzo');
-  return lzo.compress(value);
+function decompress(_x4, _x5, _x6) {
+  return _decompress.apply(this, arguments);
 }
 
-function deflate_brotli(value) {
-  brotli = brotli || Util.load('brotli');
-  var result = brotli.compress(value, {
-    mode: 0,
-    quality: 8,
-    lgwin: 22
-  });
-  return result ? Buffer.from(result) : Buffer.alloc(0);
-}
-
-function deflate_lz4(value) {
-  lz4js = lz4js || Util.load('lz4js');
-
-  try {
-    return Buffer.from(lz4js.compress(value));
-  } catch (err) {
-    throw err;
-  }
+function _decompress() {
+  _decompress = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(method, value, size) {
+    var compression, inputArrayBuffer, compressedArrayBuffer;
+    return _regenerator.default.wrap(function _callee3$(_context3) {
+      while (1) {
+        switch (_context3.prev = _context3.next) {
+          case 0:
+            compression = PARQUET_COMPRESSION_METHODS[method];
+
+            if (compression) {
+              _context3.next = 3;
+              break;
+            }
+
+            throw new Error("parquet: invalid compression method: ".concat(method));
+
+          case 3:
+            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
+            _context3.next = 6;
+            return compression.decompress(inputArrayBuffer, size);
+
+          case 6:
+            compressedArrayBuffer = _context3.sent;
+            return _context3.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
+
+          case 8:
+          case "end":
+            return _context3.stop();
+        }
+      }
+    }, _callee3);
+  }));
+  return _decompress.apply(this, arguments);
 }
 
 function inflate(method, value, size) {
@@ -104,41 +175,4 @@ function inflate(method, value, size) {
 
   return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
 }
-
-function inflate_identity(value) {
-  return value;
-}
-
-function inflate_gzip(value) {
-  return _zlib.default.gunzipSync(value);
-}
-
-function inflate_snappy(value) {
-  return _snappyjs.default.uncompress(value);
-}
-
-function inflate_lzo(value, size) {
-  lzo = lzo || Util.load('lzo');
-  return lzo.decompress(value, size);
-}
-
-function inflate_lz4(value, size) {
-  lz4js = lz4js || Util.load('lz4js');
-
-  try {
-    return Buffer.from(lz4js.decompress(value, size));
-  } catch (err) {
-    throw err;
-  }
-}
-
-function inflate_brotli(value) {
-  brotli = brotli || Util.load('brotli');
-
-  if (!value.length) {
-    return Buffer.alloc(0);
-  }
-
-  return Buffer.from(brotli.decompress(value));
-}
 //# sourceMappingURL=compression.js.map