@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.2

package/dist/esm/parquet-loader.js

@@ -1,9 +1,11 @@
 
-const VERSION = typeof "3.4.0-alpha.1" !== 'undefined' ? "3.4.0-alpha.1" : 'latest';
+const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',
-    url: undefined
+    url: undefined,
+    columnList: [],
+    geoparquet: true
   }
 };
 

package/dist/esm/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAS3E,MAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,kBAAkB;IACxBC,GAAG,EAAEC;EACP;AACF,CAAC;;AAGD,OAAO,MAAMC,aAAa,GAAG;EAC3BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEV,OAAO;EAChBW,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEhB;AACX,CAAC;AAED,OAAO,MAAMiB,uBAA+B,GAAGZ,aAAa"}
+{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","columnList","geoparquet","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n    columnList?: string[] | string[][];\n    geoparquet?: boolean;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined,\n    columnList: [],\n    geoparquet: true\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAW3E,MAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,kBAAkB;IACxBC,GAAG,EAAEC,SAAS;IACdC,UAAU,EAAE,EAAE;IACdC,UAAU,EAAE;EACd;AACF,CAAC;;AAGD,OAAO,MAAMC,aAAa,GAAG;EAC3BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEZ,OAAO;EAChBa,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAElB;AACX,CAAC;AAED,OAAO,MAAMmB,uBAA+B,GAAGZ,aAAa"}

package/dist/esm/parquet-wasm-loader.js

@@ -1,5 +1,5 @@
 
-const VERSION = typeof "3.4.0-alpha.1" !== 'undefined' ? "3.4.0-alpha.1" : 'latest';
+const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'arrow-table',

package/dist/esm/parquet-wasm-writer.js

@@ -1,6 +1,6 @@
 import { encode } from './lib/wasm/encode-parquet-wasm';
 
-const VERSION = typeof "3.4.0-alpha.1" !== 'undefined' ? "3.4.0-alpha.1" : 'latest';
+const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
 const DEFAULT_PARQUET_WRITER_OPTIONS = {
   parquet: {
     wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'

package/dist/esm/parquet-writer.js

@@ -1,5 +1,5 @@
 
-const VERSION = typeof "3.4.0-alpha.1" !== 'undefined' ? "3.4.0-alpha.1" : 'latest';
+const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
 export const ParquetWriter = {
   name: 'Apache Parquet',

package/dist/esm/parquetjs/compression.js

@@ -1,7 +1,16 @@
 
 
 import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, LZOCompression, LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
-import { toArrayBuffer, toBuffer } from './utils/buffer-utils';
+function toBuffer(arrayBuffer) {
+  return Buffer.from(arrayBuffer);
+}
+function toArrayBuffer(buffer) {
+  if (Buffer.isBuffer(buffer)) {
+    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
 
 import lz4js from 'lz4js';
 import lzo from 'lzo';

package/dist/esm/parquetjs/compression.js.map

@@ -1 +1 @@
-{"version":3,"file":"compression.js","names":["NoCompression","GZipCompression","SnappyCompression","BrotliCompression","LZOCompression","LZ4Compression","ZstdCompression","toArrayBuffer","toBuffer","lz4js","lzo","modules","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","GZIP","SNAPPY","BROTLI","LZ4","LZ4_RAW","LZO","ZSTD","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;AAIA,SAEEA,aAAa,EACbC,eAAe,EACfC,iBAAiB,EACjBC,iBAAiB,EACjBC,cAAc,EACdC,cAAc,EACdC,eAAe,QACV,yBAAyB;AAGhC,SAAQC,aAAa,EAAEC,QAAQ,QAAO,sBAAsB;;AAM5D,OAAOC,KAAK,MAAM,OAAO;AACzB,OAAOC,GAAG,MAAM,KAAK;;AAIrB,MAAMC,OAAO,GAAG;EAQdF,KAAK;EACLC;AAEF,CAAC;;AAGD,OAAO,MAAME,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIb,aAAa,EAAE;EACjCc,IAAI,EAAE,IAAIb,eAAe,EAAE;EAC3Bc,MAAM,EAAE,IAAIb,iBAAiB,EAAE;EAC/Bc,MAAM,EAAE,IAAIb,iBAAiB,CAAC;IAACQ;EAAO,CAAC,CAAC;EAExCM,GAAG,EAAE,IAAIZ,cAAc,CAAC;IAACM;EAAO,CAAC,CAAC;EAClCO,OAAO,EAAE,IAAIb,cAAc,CAAC;IAACM;EAAO,CAAC,CAAC;EACtCQ,GAAG,EAAE,IAAIf,cAAc,CAAC;IAACO;EAAO,CAAC,CAAC;EAClCS,IAAI,EAAE,IAAId,eAAe,CAAC;IAACK;EAAO,CAAC;AACrC,CAAC;;AAMD,OAAO,eAAeU,mBAAmB,CAACC,OAAyC,EAAE;EACnF,MAAMC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACb,2BAA2B,CAAC;EAC/D,OAAO,MAAMc,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAEC,WAAW,IAAKA,WAAW,CAACC,OAAO,EAAE,CAAC,CAAC;AACpF;;AAKA,OAAO,eAAeC,OAAO,CAACC,MAA0B,EAAEC,KAAa,EAAmB;EACxF,MAAMJ,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACQ,QAAQ,CAACF,gBAAgB,CAAC;EAC1E,OAAO3B,QAAQ,CAAC4B,qBAAqB,CAAC;AACxC;;AAKA,OAAO,eAAeE,UAAU,CAC9BN,MAA0B,EAC1BC,KAAa,EACbM,IAAY,EACK;EACjB,MAAMV,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACS,UAAU,CAACH,gBAAgB,EAAEI,IAAI,CAAC;EAClF,OAAO/B,QAAQ,CAAC4B,qBAAqB,CAAC;AACxC;;AAKA,OAAO,SAASI,OAAO,CAACR,MAA0B,EAAEC,KAAa,EAAEM,IAAY,EAAU;EACvF,IAAI,EAAEP,MAAM,IAAIpB,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAIsB,KAAK,uCAAgCF,MAAM,EAAG;EAC1D;EAEA,OAAOpB,2BAA2B,CAACoB,MAAM,CAAC,CAACQ,OAAO,CAACP,KAAK,EAAEM,IAAI,CAAC;AACjE"}
+{"version":3,"file":"compression.js","names":["NoCompression","GZipCompression","SnappyCompression","BrotliCompression","LZOCompression","LZ4Compression","ZstdCompression","toBuffer","arrayBuffer","Buffer","from","toArrayBuffer","buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","lz4js","lzo","modules","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","GZIP","SNAPPY","BROTLI","LZ4","LZ4_RAW","LZO","ZSTD","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\n\n/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */\nfunction toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n  return Buffer.from(arrayBuffer);\n}\n\nfunction toArrayBuffer(buffer: Buffer): ArrayBuffer {\n  // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n  if (Buffer.isBuffer(buffer)) {\n    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n    return typedArray.slice().buffer;\n  }\n  return buffer;\n}\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;AAIA,SAEEA,aAAa,EACbC,eAAe,EACfC,iBAAiB,EACjBC,iBAAiB,EACjBC,cAAc,EACdC,cAAc,EACdC,eAAe,QACV,yBAAyB;AAKhC,SAASC,QAAQ,CAACC,WAAwB,EAAU;EAClD,OAAOC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;AACjC;AAEA,SAASG,aAAa,CAACC,MAAc,EAAe;EAElD,IAAIH,MAAM,CAACI,QAAQ,CAACD,MAAM,CAAC,EAAE;IAC3B,MAAME,UAAU,GAAG,IAAIC,UAAU,CAACH,MAAM,CAACA,MAAM,EAAEA,MAAM,CAACI,UAAU,EAAEJ,MAAM,CAACK,MAAM,CAAC;IAClF,OAAOH,UAAU,CAACI,KAAK,EAAE,CAACN,MAAM;EAClC;EACA,OAAOA,MAAM;AACf;;AAMA,OAAOO,KAAK,MAAM,OAAO;AACzB,OAAOC,GAAG,MAAM,KAAK;;AAIrB,MAAMC,OAAO,GAAG;EAQdF,KAAK;EACLC;AAEF,CAAC;;AAGD,OAAO,MAAME,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIvB,aAAa,EAAE;EACjCwB,IAAI,EAAE,IAAIvB,eAAe,EAAE;EAC3BwB,MAAM,EAAE,IAAIvB,iBAAiB,EAAE;EAC/BwB,MAAM,EAAE,IAAIvB,iBAAiB,CAAC;IAACkB;EAAO,CAAC,CAAC;EAExCM,GAAG,EAAE,IAAItB,cAAc,CAAC;IAACgB;EAAO,CAAC,CAAC;EAClCO,OAAO,EAAE,IAAIvB,cAAc,CAAC;IAACgB;EAAO,CAAC,CAAC;EACtCQ,GAAG,EAAE,IAAIzB,cAAc,CAAC;IAACiB;EAAO,CAAC,CAAC;EAClCS,IAAI,EAAE,IAAIxB,eAAe,CAAC;IAACe;EAAO,CAAC;AACrC,CAAC;;AAMD,OAAO,eAAeU,mBAAmB,CAACC,OAAyC,EAAE;EACnF,MAAMC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACb,2BAA2B,CAAC;EAC/D,OAAO,MAAMc,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAEC,WAAW,IAAKA,WAAW,CAACC,OAAO,EAAE,CAAC,CAAC;AACpF;;AAKA,OAAO,eAAeC,OAAO,CAACC,MAA0B,EAAEC,KAAa,EAAmB;EACxF,MAAMJ,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAGlC,aAAa,CAACgC,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACQ,QAAQ,CAACF,gBAAgB,CAAC;EAC1E,OAAOtC,QAAQ,CAACuC,qBAAqB,CAAC;AACxC;;AAKA,OAAO,eAAeE,UAAU,CAC9BN,MAA0B,EAC1BC,KAAa,EACbM,IAAY,EACK;EACjB,MAAMV,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAGlC,aAAa,CAACgC,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACS,UAAU,CAACH,gBAAgB,EAAEI,IAAI,CAAC;EAClF,OAAO1C,QAAQ,CAACuC,qBAAqB,CAAC;AACxC;;AAKA,OAAO,SAASI,OAAO,CAACR,MAA0B,EAAEC,KAAa,EAAEM,IAAY,EAAU;EACvF,IAAI,EAAEP,MAAM,IAAIpB,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAIsB,KAAK,uCAAgCF,MAAM,EAAG;EAC1D;EAEA,OAAOpB,2BAA2B,CAACoB,MAAM,CAAC,CAACQ,OAAO,CAACP,KAAK,EAAEM,IAAI,CAAC;AACjE"}

package/dist/esm/parquetjs/encoder/{writer.js → parquet-encoder.js}

@@ -1,5 +1,5 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Transform } from 'stream';
+
 import { PARQUET_CODECS } from '../codecs';
 import * as Compression from '../compression';
 import * as Shred from '../schema/shred';
@@ -17,18 +17,16 @@ const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
 
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
-export class ParquetWriter {
+export class ParquetEncoder {
   static async openFile(schema, path, opts) {
     const outputStream = await osopen(path, opts);
-    return ParquetWriter.openStream(schema, outputStream, opts);
+    return ParquetEncoder.openStream(schema, outputStream, opts);
   }
 
-  static async openStream(schema, outputStream, opts) {
-    if (!opts) {
-      opts = {};
-    }
+  static async openStream(schema, outputStream) {
+    let opts = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
     const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-    return new ParquetWriter(schema, envelopeWriter, opts);
+    return new ParquetEncoder(schema, envelopeWriter, opts);
   }
   constructor(schema, envelopeWriter, opts) {
     _defineProperty(this, "schema", void 0);
@@ -150,34 +148,6 @@ export class ParquetEnvelopeWriter {
   }
 }
 
-export class ParquetTransformer extends Transform {
-  constructor(schema) {
-    let opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
-    super({
-      objectMode: true
-    });
-    _defineProperty(this, "writer", void 0);
-    const writeProxy = function (t) {
-      return async function (b) {
-        t.push(b);
-      };
-    }(this);
-    this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts), opts);
-  }
-
-  _transform(row, encoding, callback) {
-    if (row) {
-      return this.writer.appendRow(row).then(callback);
-    }
-    callback();
-    return Promise.resolve();
-  }
-
-  async _flush(callback) {
-    await this.writer.close(callback);
-  }
-}
-
 function encodeValues(type, encoding, values, opts) {
   if (!(encoding in PARQUET_CODECS)) {
     throw new Error("invalid encoding: ".concat(encoding));
@@ -388,4 +358,4 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
   return footerEncoded;
 }
-//# sourceMappingURL=writer.js.map
+//# sourceMappingURL=parquet-encoder.js.map

package/dist/esm/parquetjs/encoder/parquet-encoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-encoder.js","names":["PARQUET_CODECS","Compression","Shred","ColumnChunk","ColumnMetaData","CompressionCodec","ConvertedType","DataPageHeader","DataPageHeaderV2","Encoding","FieldRepetitionType","FileMetaData","KeyValue","PageHeader","PageType","RowGroup","SchemaElement","Type","osopen","oswrite","osclose","getBitWidth","serializeThrift","Int64","PARQUET_MAGIC","PARQUET_VERSION","PARQUET_DEFAULT_PAGE_SIZE","PARQUET_DEFAULT_ROW_GROUP_SIZE","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","ParquetEncoder","openFile","schema","path","opts","outputStream","openStream","envelopeWriter","ParquetEnvelopeWriter","constructor","rowBuffer","rowGroupSize","closed","userMetadata","writeHeader","err","close","appendRow","row","Error","shredRecord","rowCount","callback","writeFooter","setMetadata","key","value","String","setRowGroupSize","cnt","setPageSize","writeFn","bind","undefined","closeFn","fileOffset","write","offset","rowGroups","pageSize","useDataPageV2","Boolean","writeSection","buf","length","Buffer","from","writeRowGroup","records","rgroup","encodeRowGroup","baseOffset","push","metadata","body","encodeFooter","encodeValues","type","encoding","values","encodeDataPage","column","data","rLevelsBuf","alloc","rLevelMax","rlevels","bitWidth","dLevelsBuf","dLevelMax","dlevels","valuesBuf","primitiveType","typeLength","dataBuf","concat","compressedBuf","deflate","compression","header","DATA_PAGE","data_page_header","num_values","count","definition_level_encoding","repetition_level_encoding","uncompressed_page_size","compressed_page_size","headerBuf","page","headerSize","encodeDataPageV2","disableEnvelope","DATA_PAGE_V2","data_page_header_v2","num_nulls","num_rows","definition_levels_byte_length","repetition_levels_byte_length","is_compressed","encodeColumnChunk","buffer","columnData","join","pageBuf","total_uncompressed_size","total_compressed_size","result","path_in_schema","data_page_offset","encodings","codec","metadataOffset","columns","total_byte_size","field","fieldList","isNested","cchunkData","cchunk","file_offset","meta_data","Number","version","created_by","row_groups","key_value_metadata","kv","schemaRoot","name","num_children","Object","keys","fields","relt","repetitionType","schemaElem","repetition_type","fieldCount","originalType","converted_type","type_length","metadataEncoded","footerEncoded","copy","writeUInt32LE"],"sources":["../../../../src/parquetjs/encoder/parquet-encoder.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {stream} from '@loaders.gl/loader-utils';\nimport {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport * as Compression from '../compression';\nimport {\n  ParquetBuffer,\n  ParquetCodec,\n  ParquetData,\n  ParquetField,\n  PrimitiveType\n} from '../schema/declare';\nimport {ParquetSchema} from '../schema/schema';\nimport * as Shred from '../schema/shred';\nimport {\n  ColumnChunk,\n  ColumnMetaData,\n  CompressionCodec,\n  ConvertedType,\n  DataPageHeader,\n  DataPageHeaderV2,\n  Encoding,\n  FieldRepetitionType,\n  FileMetaData,\n  KeyValue,\n  PageHeader,\n  PageType,\n  RowGroup,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {osopen, oswrite, osclose} from '../utils/file-utils';\nimport {getBitWidth, serializeThrift} from '../utils/read-utils';\nimport Int64 from 'node-int64';\n\n/**\n * Parquet File Magic String\n */\nconst PARQUET_MAGIC = 'PAR1';\n\n/**\n * Parquet File Format Version\n */\nconst PARQUET_VERSION = 1;\n\n/**\n * Default Page and Row Group sizes\n */\nconst PARQUET_DEFAULT_PAGE_SIZE = 8192;\nconst PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;\n\n/**\n * Repetition and Definition Level Encoding\n */\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\nexport interface ParquetEncoderOptions {\n  baseOffset?: number;\n  rowGroupSize?: number;\n  pageSize?: number;\n  useDataPageV2?: boolean;\n\n  // Write Stream Options\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\n/**\n * Write a parquet file to an output stream. The ParquetEncoder will perform\n * buffering/batching for performance, so close() must be called after all rows\n * are written.\n */\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nexport class ParquetEncoder<T> {\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified file\n   */\n  static async openFile<T>(\n    schema: ParquetSchema,\n    path: string,\n    opts?: ParquetEncoderOptions\n  ): Promise<ParquetEncoder<T>> {\n    const outputStream = await osopen(path, opts);\n    return ParquetEncoder.openStream(schema, outputStream, opts);\n  }\n\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified stream\n   */\n  static async openStream<T>(\n    schema: ParquetSchema,\n    outputStream: stream.Writable,\n    opts: ParquetEncoderOptions = {}\n  ): Promise<ParquetEncoder<T>> {\n    const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);\n    return new ParquetEncoder(schema, envelopeWriter, opts);\n  }\n\n  public schema: ParquetSchema;\n  public envelopeWriter: ParquetEnvelopeWriter;\n  public rowBuffer: ParquetBuffer;\n  public rowGroupSize: number;\n  public closed: boolean;\n  public userMetadata: Record<string, string>;\n\n  /**\n   * Create a new buffered parquet writer for a given envelope writer\n   */\n  constructor(\n    schema: ParquetSchema,\n    envelopeWriter: ParquetEnvelopeWriter,\n    opts: ParquetEncoderOptions\n  ) {\n    this.schema = schema;\n    this.envelopeWriter = envelopeWriter;\n    // @ts-ignore Row buffer typings...\n    this.rowBuffer = {};\n    this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;\n    this.closed = false;\n    this.userMetadata = {};\n\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.writeHeader();\n  }\n\n  async writeHeader(): Promise<void> {\n    // TODO - better not mess with promises in the constructor\n    try {\n      await this.envelopeWriter.writeHeader();\n    } catch (err) {\n      await this.envelopeWriter.close();\n      throw err;\n    }\n  }\n\n  /**\n   * Append a single row to the parquet file. Rows are buffered in memory until\n   * rowGroupSize rows are in the buffer or close() is called\n   */\n  async appendRow<T>(row: T): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n    Shred.shredRecord(this.schema, row, this.rowBuffer);\n    if (this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n  }\n\n  /**\n   * Finish writing the parquet file and commit the footer to disk. This method\n   * MUST be called after you are finished adding rows. You must not call this\n   * method twice on the same object or add any rows after the close() method has\n   * been called\n   */\n  async close(callback?: () => void): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n\n    this.closed = true;\n\n    if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n\n    await this.envelopeWriter.writeFooter(this.userMetadata);\n    await this.envelopeWriter.close();\n    // this.envelopeWriter = null;\n\n    if (callback) {\n      callback();\n    }\n  }\n\n  /**\n   * Add key<>value metadata to the file\n   */\n  setMetadata(key: string, value: string): void {\n    // TODO: value to be any, obj -> JSON\n    this.userMetadata[String(key)] = String(value);\n  }\n\n  /**\n   * Set the parquet row group size. This values controls the maximum number\n   * of rows that are buffered in memory at any given time as well as the number\n   * of rows that are co-located on disk. A higher value is generally better for\n   * read-time I/O performance at the tradeoff of write-time memory usage.\n   */\n  setRowGroupSize(cnt: number): void {\n    this.rowGroupSize = cnt;\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.envelopeWriter.setPageSize(cnt);\n  }\n}\n\n/**\n * Create a parquet file from a schema and a number of row groups. This class\n * performs direct, unbuffered writes to the underlying output stream and is\n * intendend for advanced and internal users; the writeXXX methods must be\n * called in the correct order to produce a valid file.\n */\nexport class ParquetEnvelopeWriter {\n  /**\n   * Create a new parquet envelope writer that writes to the specified stream\n   */\n  static async openStream(\n    schema: ParquetSchema,\n    outputStream: stream.Writable,\n    opts: ParquetEncoderOptions\n  ): Promise<ParquetEnvelopeWriter> {\n    const writeFn = oswrite.bind(undefined, outputStream);\n    const closeFn = osclose.bind(undefined, outputStream);\n    return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);\n  }\n\n  public schema: ParquetSchema;\n  public write: (buf: Buffer) => Promise<void>;\n  public close: () => Promise<void>;\n  public offset: number;\n  public rowCount: number;\n  public rowGroups: RowGroup[];\n  public pageSize: number;\n  public useDataPageV2: boolean;\n\n  constructor(\n    schema: ParquetSchema,\n    writeFn: (buf: Buffer) => Promise<void>,\n    closeFn: () => Promise<void>,\n    fileOffset: number,\n    opts: ParquetEncoderOptions\n  ) {\n    this.schema = schema;\n    this.write = writeFn;\n    this.close = closeFn;\n    this.offset = fileOffset;\n    this.rowCount = 0;\n    this.rowGroups = [];\n    this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;\n    this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;\n  }\n\n  writeSection(buf: Buffer): Promise<void> {\n    this.offset += buf.length;\n    return this.write(buf);\n  }\n\n  /**\n   * Encode the parquet file header\n   */\n  writeHeader(): Promise<void> {\n    return this.writeSection(Buffer.from(PARQUET_MAGIC));\n  }\n\n  /**\n   * Encode a parquet row group. The records object should be created using the\n   * shredRecord method\n   */\n  async writeRowGroup(records: ParquetBuffer): Promise<void> {\n    const rgroup = await encodeRowGroup(this.schema, records, {\n      baseOffset: this.offset,\n      pageSize: this.pageSize,\n      useDataPageV2: this.useDataPageV2\n    });\n\n    this.rowCount += records.rowCount;\n    this.rowGroups.push(rgroup.metadata);\n    return await this.writeSection(rgroup.body);\n  }\n\n  /**\n   * Write the parquet file footer\n   */\n  writeFooter(userMetadata: Record<string, string>): Promise<void> {\n    if (!userMetadata) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      userMetadata = {};\n    }\n\n    return this.writeSection(\n      encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)\n    );\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.pageSize = cnt;\n  }\n}\n\n/**\n * Create a parquet transform stream\nexport class ParquetTransformer<T> extends stream.Transform {\n  public writer: ParquetEncoder<T>;\n\n  constructor(schema: ParquetSchema, opts: ParquetEncoderOptions = {}) {\n    super({objectMode: true});\n\n    const writeProxy = (function (t: ParquetTransformer<any>) {\n      return async function (b: any): Promise<void> {\n        t.push(b);\n      };\n    })(this);\n\n    this.writer = new ParquetEncoder(\n      schema,\n      new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),\n      opts\n    );\n  }\n\n  // tslint:disable-next-line:function-name\n  _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {\n    if (row) {\n      return this.writer.appendRow(row).then(callback);\n    }\n    callback();\n    return Promise.resolve();\n  }\n\n  // tslint:disable-next-line:function-name\n  async _flush(callback: (val?: any) => void) {\n    await this.writer.close(callback);\n  }\n}\n */\n\n/**\n * Encode a consecutive array of data using one of the parquet encodings\n */\nfunction encodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  values: any[],\n  opts: ParquetCodecOptions\n) {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].encodeValues(type, values, opts);\n}\n\n/**\n * Encode a parquet data page\n */\nasync function encodeDataPage(\n  column: ParquetField,\n  data: ParquetData\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, dataBuf);\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE,\n    data_page_header: new DataPageHeader({\n      num_values: data.count,\n      encoding: Encoding[column.encoding!] as any,\n      definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],\n      repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]\n    }),\n    uncompressed_page_size: dataBuf.length,\n    compressed_page_size: compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, compressedBuf]);\n\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode a parquet data page (v2)\n */\nasync function encodeDataPageV2(\n  column: ParquetField,\n  data: ParquetData,\n  rowCount: number\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);\n\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE_V2,\n    data_page_header_v2: new DataPageHeaderV2({\n      num_values: data.count,\n      num_nulls: data.count - data.values.length,\n      num_rows: rowCount,\n      encoding: Encoding[column.encoding!] as any,\n      definition_levels_byte_length: dLevelsBuf.length,\n      repetition_levels_byte_length: rLevelsBuf.length,\n      is_compressed: column.compression !== 'UNCOMPRESSED'\n    }),\n    uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,\n    compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode an array of values into a parquet column chunk\n */\nasync function encodeColumnChunk(\n  column: ParquetField,\n  buffer: ParquetBuffer,\n  offset: number,\n  opts: ParquetEncoderOptions\n): Promise<{\n  body: Buffer;\n  metadata: ColumnMetaData;\n  metadataOffset: number;\n}> {\n  const data = buffer.columnData[column.path.join()];\n  const baseOffset = (opts.baseOffset || 0) + offset;\n  /* encode data page(s) */\n  // const pages: Buffer[] = [];\n  let pageBuf: Buffer;\n  // tslint:disable-next-line:variable-name\n  let total_uncompressed_size = 0;\n  // tslint:disable-next-line:variable-name\n  let total_compressed_size = 0;\n  {\n    const result = opts.useDataPageV2\n      ? await encodeDataPageV2(column, data, buffer.rowCount)\n      : await encodeDataPage(column, data);\n    // pages.push(result.page);\n    pageBuf = result.page;\n    total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;\n    total_compressed_size += result.header.compressed_page_size + result.headerSize;\n  }\n\n  // const pagesBuf = Buffer.concat(pages);\n  // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;\n\n  /* prepare metadata header */\n  const metadata = new ColumnMetaData({\n    path_in_schema: column.path,\n    num_values: data.count,\n    data_page_offset: baseOffset,\n    encodings: [],\n    total_uncompressed_size, //  : pagesBuf.length,\n    total_compressed_size,\n    type: Type[column.primitiveType!],\n    codec: CompressionCodec[column.compression!]\n  });\n\n  /* list encodings */\n  metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);\n  metadata.encodings.push(Encoding[column.encoding!]);\n\n  /* concat metadata header and data pages */\n  const metadataOffset = baseOffset + pageBuf.length;\n  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);\n  return {body, metadata, metadataOffset};\n}\n\n/**\n * Encode a list of column values into a parquet row group\n */\nasync function encodeRowGroup(\n  schema: ParquetSchema,\n  data: ParquetBuffer,\n  opts: ParquetEncoderOptions\n): Promise<{\n  body: Buffer;\n  metadata: RowGroup;\n}> {\n  const metadata = new RowGroup({\n    num_rows: data.rowCount,\n    columns: [],\n    total_byte_size: 0\n  });\n\n  let body = Buffer.alloc(0);\n  for (const field of schema.fieldList) {\n    if (field.isNested) {\n      continue; // eslint-disable-line no-continue\n    }\n\n    const cchunkData = await encodeColumnChunk(field, data, body.length, opts);\n\n    const cchunk = new ColumnChunk({\n      file_offset: cchunkData.metadataOffset,\n      meta_data: cchunkData.metadata\n    });\n\n    metadata.columns.push(cchunk);\n    metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);\n\n    body = Buffer.concat([body, cchunkData.body]);\n  }\n\n  return {body, metadata};\n}\n\n/**\n * Encode a parquet file metadata footer\n */\nfunction encodeFooter(\n  schema: ParquetSchema,\n  rowCount: number,\n  rowGroups: RowGroup[],\n  userMetadata: Record<string, string>\n): Buffer {\n  const metadata = new FileMetaData({\n    version: PARQUET_VERSION,\n    created_by: 'parquets',\n    num_rows: rowCount,\n    row_groups: rowGroups,\n    schema: [],\n    key_value_metadata: []\n  });\n\n  for (const key in userMetadata) {\n    const kv = new KeyValue({\n      key,\n      value: userMetadata[key]\n    });\n    metadata.key_value_metadata?.push?.(kv);\n  }\n\n  {\n    const schemaRoot = new SchemaElement({\n      name: 'root',\n      num_children: Object.keys(schema.fields).length\n    });\n    metadata.schema.push(schemaRoot);\n  }\n\n  for (const field of schema.fieldList) {\n    const relt = FieldRepetitionType[field.repetitionType];\n    const schemaElem = new SchemaElement({\n      name: field.name,\n      repetition_type: relt as any\n    });\n\n    if (field.isNested) {\n      schemaElem.num_children = field.fieldCount;\n    } else {\n      schemaElem.type = Type[field.primitiveType!] as Type;\n    }\n\n    if (field.originalType) {\n      schemaElem.converted_type = ConvertedType[field.originalType] as ConvertedType;\n    }\n\n    schemaElem.type_length = field.typeLength;\n\n    metadata.schema.push(schemaElem);\n  }\n\n  const metadataEncoded = serializeThrift(metadata);\n  const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);\n  metadataEncoded.copy(footerEncoded);\n  footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);\n  footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);\n  return footerEncoded;\n}\n"],"mappings":";;AAGA,SAA6BA,cAAc,QAAO,WAAW;AAC7D,OAAO,KAAKC,WAAW,MAAM,gBAAgB;AAS7C,OAAO,KAAKC,KAAK,MAAM,iBAAiB;AACxC,SACEC,WAAW,EACXC,cAAc,EACdC,gBAAgB,EAChBC,aAAa,EACbC,cAAc,EACdC,gBAAgB,EAChBC,QAAQ,EACRC,mBAAmB,EACnBC,YAAY,EACZC,QAAQ,EACRC,UAAU,EACVC,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,MAAM,EAAEC,OAAO,EAAEC,OAAO,QAAO,qBAAqB;AAC5D,SAAQC,WAAW,EAAEC,eAAe,QAAO,qBAAqB;AAChE,OAAOC,KAAK,MAAM,YAAY;;AAK9B,MAAMC,aAAa,GAAG,MAAM;;AAK5B,MAAMC,eAAe,GAAG,CAAC;;AAKzB,MAAMC,yBAAyB,GAAG,IAAI;AACtC,MAAMC,8BAA8B,GAAG,IAAI;;AAK3C,MAAMC,kBAAkB,GAAG,OAAO;AAClC,MAAMC,sBAAsB,GAAG,KAAK;AAuBpC,OAAO,MAAMC,cAAc,CAAI;EAK7B,aAAaC,QAAQ,CACnBC,MAAqB,EACrBC,IAAY,EACZC,IAA4B,EACA;IAC5B,MAAMC,YAAY,GAAG,MAAMjB,MAAM,CAACe,IAAI,EAAEC,IAAI,CAAC;IAC7C,OAAOJ,cAAc,CAACM,UAAU,CAACJ,MAAM,EAAEG,YAAY,EAAED,IAAI,CAAC;EAC9D;;EAMA,aAAaE,UAAU,CACrBJ,MAAqB,EACrBG,YAA6B,EAED;IAAA,IAD5BD,IAA2B,uEAAG,CAAC,CAAC;IAEhC,MAAMG,cAAc,GAAG,MAAMC,qBAAqB,CAACF,UAAU,CAACJ,MAAM,EAAEG,YAAY,EAAED,IAAI,CAAC;IACzF,OAAO,IAAIJ,cAAc,CAACE,MAAM,EAAEK,cAAc,EAAEH,IAAI,CAAC;EACzD;EAYAK,WAAW,CACTP,MAAqB,EACrBK,cAAqC,EACrCH,IAA2B,EAC3B;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACA,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACK,cAAc,GAAGA,cAAc;IAEpC,IAAI,CAACG,SAAS,GAAG,CAAC,CAAC;IACnB,IAAI,CAACC,YAAY,GAAGP,IAAI,CAACO,YAAY,IAAId,8BAA8B;IACvE,IAAI,CAACe,MAAM,GAAG,KAAK;IACnB,IAAI,CAACC,YAAY,GAAG,CAAC,CAAC;;IAGtB,IAAI,CAACC,WAAW,EAAE;EACpB;EAEA,MAAMA,WAAW,GAAkB;IAEjC,IAAI;MACF,MAAM,IAAI,CAACP,cAAc,CAACO,WAAW,EAAE;IACzC,CAAC,CAAC,OAAOC,GAAG,EAAE;MACZ,MAAM,IAAI,CAACR,cAAc,CAACS,KAAK,EAAE;MACjC,MAAMD,GAAG;IACX;EACF;;EAMA,MAAME,SAAS,CAAIC,GAAM,EAAiB;IACxC,IAAI,IAAI,CAACN,MAAM,EAAE;MACf,MAAM,IAAIO,KAAK,CAAC,mBAAmB,CAAC;IACtC;IACA/C,KAAK,CAACgD,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEgB,GAAG,EAAE,IAAI,CAACR,SAAS,CAAC;IACnD,IAAI,IAAI,CAACA,SAAS,CAACW,QAAQ,IAAI,IAAI,CAACV,YAAY,EAAE;MAEhD,IAAI,CAACD,SAAS,GAAG,CAAC,CAAC;IACrB;EACF;;EAQA,MAAMM,KAAK,CAACM,QAAqB,EAAiB;IAChD,IAAI,IAAI,CAACV,MAAM,EAAE;MACf,MAAM,IAAIO,KAAK,CAAC,mBAAmB,CAAC;IACtC;IAEA,IAAI,CAACP,MAAM,GAAG,IAAI;IAElB,IAAI,IAAI,CAACF,SAAS,CAACW,QAAQ,GAAG,CAAC,IAAI,IAAI,CAACX,SAAS,CAACW,QAAQ,IAAI,IAAI,CAACV,YAAY,EAAE;MAE/E,IAAI,CAACD,SAAS,GAAG,CAAC,CAAC;IACrB;IAEA,MAAM,IAAI,CAACH,cAAc,CAACgB,WAAW,CAAC,IAAI,CAACV,YAAY,CAAC;IACxD,MAAM,IAAI,CAACN,cAAc,CAACS,KAAK,EAAE;;IAGjC,IAAIM,QAAQ,EAAE;MACZA,QAAQ,EAAE;IACZ;EACF;;EAKAE,WAAW,CAACC,GAAW,EAAEC,KAAa,EAAQ;IAE5C,IAAI,CAACb,YAAY,CAACc,MAAM,CAACF,GAAG,CAAC,CAAC,GAAGE,MAAM,CAACD,KAAK,CAAC;EAChD;;EAQAE,eAAe,CAACC,GAAW,EAAQ;IACjC,IAAI,CAAClB,YAAY,GAAGkB,GAAG;EACzB;;EAMAC,WAAW,CAACD,GAAW,EAAQ;IAC7B,IAAI,CAACtB,cAAc,CAACuB,WAAW,CAACD,GAAG,CAAC;EACtC;AACF;;AAQA,OAAO,MAAMrB,qBAAqB,CAAC;EAIjC,aAAaF,UAAU,CACrBJ,MAAqB,EACrBG,YAA6B,EAC7BD,IAA2B,EACK;IAChC,MAAM2B,OAAO,GAAG1C,OAAO,CAAC2C,IAAI,CAACC,SAAS,EAAE5B,YAAY,CAAC;IACrD,MAAM6B,OAAO,GAAG5C,OAAO,CAAC0C,IAAI,CAACC,SAAS,EAAE5B,YAAY,CAAC;IACrD,OAAO,IAAIG,qBAAqB,CAACN,MAAM,EAAE6B,OAAO,EAAEG,OAAO,EAAE,CAAC,EAAE9B,IAAI,CAAC;EACrE;EAWAK,WAAW,CACTP,MAAqB,EACrB6B,OAAuC,EACvCG,OAA4B,EAC5BC,UAAkB,EAClB/B,IAA2B,EAC3B;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACA,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACkC,KAAK,GAAGL,OAAO;IACpB,IAAI,CAACf,KAAK,GAAGkB,OAAO;IACpB,IAAI,CAACG,MAAM,GAAGF,UAAU;IACxB,IAAI,CAACd,QAAQ,GAAG,CAAC;IACjB,IAAI,CAACiB,SAAS,GAAG,EAAE;IACnB,IAAI,CAACC,QAAQ,GAAGnC,IAAI,CAACmC,QAAQ,IAAI3C,yBAAyB;IAC1D,IAAI,CAAC4C,aAAa,GAAG,eAAe,IAAIpC,IAAI,GAAGqC,OAAO,CAACrC,IAAI,CAACoC,aAAa,CAAC,GAAG,KAAK;EACpF;EAEAE,YAAY,CAACC,GAAW,EAAiB;IACvC,IAAI,CAACN,MAAM,IAAIM,GAAG,CAACC,MAAM;IACzB,OAAO,IAAI,CAACR,KAAK,CAACO,GAAG,CAAC;EACxB;;EAKA7B,WAAW,GAAkB;IAC3B,OAAO,IAAI,CAAC4B,YAAY,CAACG,MAAM,CAACC,IAAI,CAACpD,aAAa,CAAC,CAAC;EACtD;;EAMA,MAAMqD,aAAa,CAACC,OAAsB,EAAiB;IACzD,MAAMC,MAAM,GAAG,MAAMC,cAAc,CAAC,IAAI,CAAChD,MAAM,EAAE8C,OAAO,EAAE;MACxDG,UAAU,EAAE,IAAI,CAACd,MAAM;MACvBE,QAAQ,EAAE,IAAI,CAACA,QAAQ;MACvBC,aAAa,EAAE,IAAI,CAACA;IACtB,CAAC,CAAC;IAEF,IAAI,CAACnB,QAAQ,IAAI2B,OAAO,CAAC3B,QAAQ;IACjC,IAAI,CAACiB,SAAS,CAACc,IAAI,CAACH,MAAM,CAACI,QAAQ,CAAC;IACpC,OAAO,MAAM,IAAI,CAACX,YAAY,CAACO,MAAM,CAACK,IAAI,CAAC;EAC7C;;EAKA/B,WAAW,CAACV,YAAoC,EAAiB;IAC/D,IAAI,CAACA,YAAY,EAAE;MAEjBA,YAAY,GAAG,CAAC,CAAC;IACnB;IAEA,OAAO,IAAI,CAAC6B,YAAY,CACtBa,YAAY,CAAC,IAAI,CAACrD,MAAM,EAAE,IAAI,CAACmB,QAAQ,EAAE,IAAI,CAACiB,SAAS,EAAEzB,YAAY,CAAC,CACvE;EACH;;EAMAiB,WAAW,CAACD,GAAW,EAAQ;IAC7B,IAAI,CAACU,QAAQ,GAAGV,GAAG;EACrB;AACF;;AA0CA,SAAS2B,YAAY,CACnBC,IAAmB,EACnBC,QAAsB,EACtBC,MAAa,EACbvD,IAAyB,EACzB;EACA,IAAI,EAAEsD,QAAQ,IAAIxF,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIiD,KAAK,6BAAsBuC,QAAQ,EAAG;EAClD;EACA,OAAOxF,cAAc,CAACwF,QAAQ,CAAC,CAACF,YAAY,CAACC,IAAI,EAAEE,MAAM,EAAEvD,IAAI,CAAC;AAClE;;AAKA,eAAewD,cAAc,CAC3BC,MAAoB,EACpBC,IAAiB,EAKhB;EAED,IAAIC,UAAU,GAAGlB,MAAM,CAACmB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACI,SAAS,GAAG,CAAC,EAAE;IACxBF,UAAU,GAAGP,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAE+D,IAAI,CAACI,OAAO,EAAE;MAClFC,QAAQ,EAAE5E,WAAW,CAACsE,MAAM,CAACI,SAAS;IAExC,CAAC,CAAC;EACJ;;EAEA,IAAIG,UAAU,GAAGvB,MAAM,CAACmB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IACxBD,UAAU,GAAGZ,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAE+D,IAAI,CAACQ,OAAO,EAAE;MAClFH,QAAQ,EAAE5E,WAAW,CAACsE,MAAM,CAACQ,SAAS;IAExC,CAAC,CAAC;EACJ;;EAGA,MAAME,SAAS,GAAGf,YAAY,CAACK,MAAM,CAACW,aAAa,EAAGX,MAAM,CAACH,QAAQ,EAAGI,IAAI,CAACH,MAAM,EAAE;IACnFc,UAAU,EAAEZ,MAAM,CAACY,UAAU;IAC7BN,QAAQ,EAAEN,MAAM,CAACY;EACnB,CAAC,CAAC;EAEF,MAAMC,OAAO,GAAG7B,MAAM,CAAC8B,MAAM,CAAC,CAACZ,UAAU,EAAEK,UAAU,EAAEG,SAAS,CAAC,CAAC;;EAGlE,MAAMK,aAAa,GAAG,MAAMzG,WAAW,CAAC0G,OAAO,CAAChB,MAAM,CAACiB,WAAW,EAAGJ,OAAO,CAAC;;EAG7E,MAAMK,MAAM,GAAG,IAAIhG,UAAU,CAAC;IAC5B0E,IAAI,EAAEzE,QAAQ,CAACgG,SAAS;IACxBC,gBAAgB,EAAE,IAAIxG,cAAc,CAAC;MACnCyG,UAAU,EAAEpB,IAAI,CAACqB,KAAK;MACtBzB,QAAQ,EAAE/E,QAAQ,CAACkF,MAAM,CAACH,QAAQ,CAAS;MAC3C0B,yBAAyB,EAAEzG,QAAQ,CAACoB,sBAAsB,CAAC;MAC3DsF,yBAAyB,EAAE1G,QAAQ,CAACoB,sBAAsB;IAC5D,CAAC,CAAC;;IACFuF,sBAAsB,EAAEZ,OAAO,CAAC9B,MAAM;IACtC2C,oBAAoB,EAAEX,aAAa,CAAChC;EACtC,CAAC,CAAC;;EAGF,MAAM4C,SAAS,GAAGhG,eAAe,CAACuF,MAAM,CAAC;EACzC,MAAMU,IAAI,GAAG5C,MAAM,CAAC8B,MAAM,CAAC,CAACa,SAAS,EAAEZ,aAAa,CAAC,CAAC;EAEtD,OAAO;IAACG,MAAM;IAAEW,UAAU,EAAEF,SAAS,CAAC5C,MAAM;IAAE6C;EAAI,CAAC;AACrD;;AAKA,eAAeE,gBAAgB,CAC7B9B,MAAoB,EACpBC,IAAiB,EACjBzC,QAAgB,EAKf;EAED,MAAMkD,SAAS,GAAGf,YAAY,CAACK,MAAM,CAACW,aAAa,EAAGX,MAAM,CAACH,QAAQ,EAAGI,IAAI,CAACH,MAAM,EAAE;IACnFc,UAAU,EAAEZ,MAAM,CAACY,UAAU;IAC7BN,QAAQ,EAAEN,MAAM,CAACY;EACnB,CAAC,CAAC;;EAGF,MAAMG,aAAa,GAAG,MAAMzG,WAAW,CAAC0G,OAAO,CAAChB,MAAM,CAACiB,WAAW,EAAGP,SAAS,CAAC;;EAG/E,IAAIR,UAAU,GAAGlB,MAAM,CAACmB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACI,SAAS,GAAG,CAAC,EAAE;IACxBF,UAAU,GAAGP,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAE+D,IAAI,CAACI,OAAO,EAAE;MAClFC,QAAQ,EAAE5E,WAAW,CAACsE,MAAM,CAACI,SAAS,CAAC;MACvC2B,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ;EAEA,IAAIxB,UAAU,GAAGvB,MAAM,CAACmB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IACxBD,UAAU,GAAGZ,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAE+D,IAAI,CAACQ,OAAO,EAAE;MAClFH,QAAQ,EAAE5E,WAAW,CAACsE,MAAM,CAACQ,SAAS,CAAC;MACvCuB,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ;;EAGA,MAAMb,MAAM,GAAG,IAAIhG,UAAU,CAAC;IAC5B0E,IAAI,EAAEzE,QAAQ,CAAC6G,YAAY;IAC3BC,mBAAmB,EAAE,IAAIpH,gBAAgB,CAAC;MACxCwG,UAAU,EAAEpB,IAAI,CAACqB,KAAK;MACtBY,SAAS,EAAEjC,IAAI,CAACqB,KAAK,GAAGrB,IAAI,CAACH,MAAM,CAACf,MAAM;MAC1CoD,QAAQ,EAAE3E,QAAQ;MAClBqC,QAAQ,EAAE/E,QAAQ,CAACkF,MAAM,CAACH,QAAQ,CAAS;MAC3CuC,6BAA6B,EAAE7B,UAAU,CAACxB,MAAM;MAChDsD,6BAA6B,EAAEnC,UAAU,CAACnB,MAAM;MAChDuD,aAAa,EAAEtC,MAAM,CAACiB,WAAW,KAAK;IACxC,CAAC,CAAC;IACFQ,sBAAsB,EAAEvB,UAAU,CAACnB,MAAM,GAAGwB,UAAU,CAACxB,MAAM,GAAG2B,SAAS,CAAC3B,MAAM;IAChF2C,oBAAoB,EAAExB,UAAU,CAACnB,MAAM,GAAGwB,UAAU,CAACxB,MAAM,GAAGgC,aAAa,CAAChC;EAC9E,CAAC,CAAC;;EAGF,MAAM4C,SAAS,GAAGhG,eAAe,CAACuF,MAAM,CAAC;EACzC,MAAMU,IAAI,GAAG5C,MAAM,CAAC8B,MAAM,CAAC,CAACa,SAAS,EAAEzB,UAAU,EAAEK,UAAU,EAAEQ,aAAa,CAAC,CAAC;EAC9E,OAAO;IAACG,MAAM;IAAEW,UAAU,EAAEF,SAAS,CAAC5C,MAAM;IAAE6C;EAAI,CAAC;AACrD;;AAKA,eAAeW,iBAAiB,CAC9BvC,MAAoB,EACpBwC,MAAqB,EACrBhE,MAAc,EACdjC,IAA2B,EAK1B;EACD,MAAM0D,IAAI,GAAGuC,MAAM,CAACC,UAAU,CAACzC,MAAM,CAAC1D,IAAI,CAACoG,IAAI,EAAE,CAAC;EAClD,MAAMpD,UAAU,GAAG,CAAC/C,IAAI,CAAC+C,UAAU,IAAI,CAAC,IAAId,MAAM;EAGlD,IAAImE,OAAe;EAEnB,IAAIC,uBAAuB,GAAG,CAAC;EAE/B,IAAIC,qBAAqB,GAAG,CAAC;EAC7B;IACE,MAAMC,MAAM,GAAGvG,IAAI,CAACoC,aAAa,GAC7B,MAAMmD,gBAAgB,CAAC9B,MAAM,EAAEC,IAAI,EAAEuC,MAAM,CAAChF,QAAQ,CAAC,GACrD,MAAMuC,cAAc,CAACC,MAAM,EAAEC,IAAI,CAAC;IAEtC0C,OAAO,GAAGG,MAAM,CAAClB,IAAI;IACrBgB,uBAAuB,IAAIE,MAAM,CAAC5B,MAAM,CAACO,sBAAsB,GAAGqB,MAAM,CAACjB,UAAU;IACnFgB,qBAAqB,IAAIC,MAAM,CAAC5B,MAAM,CAACQ,oBAAoB,GAAGoB,MAAM,CAACjB,UAAU;EACjF;;EAMA,MAAMrC,QAAQ,GAAG,IAAI/E,cAAc,CAAC;IAClCsI,cAAc,EAAE/C,MAAM,CAAC1D,IAAI;IAC3B+E,UAAU,EAAEpB,IAAI,CAACqB,KAAK;IACtB0B,gBAAgB,EAAE1D,UAAU;IAC5B2D,SAAS,EAAE,EAAE;IACbL,uBAAuB;IACvBC,qBAAqB;IACrBjD,IAAI,EAAEtE,IAAI,CAAC0E,MAAM,CAACW,aAAa,CAAE;IACjCuC,KAAK,EAAExI,gBAAgB,CAACsF,MAAM,CAACiB,WAAW;EAC5C,CAAC,CAAC;;EAGFzB,QAAQ,CAACyD,SAAS,CAAC1D,IAAI,CAACzE,QAAQ,CAACoB,sBAAsB,CAAC,CAAC;EACzDsD,QAAQ,CAACyD,SAAS,CAAC1D,IAAI,CAACzE,QAAQ,CAACkF,MAAM,CAACH,QAAQ,CAAE,CAAC;;EAGnD,MAAMsD,cAAc,GAAG7D,UAAU,GAAGqD,OAAO,CAAC5D,MAAM;EAClD,MAAMU,IAAI,GAAGT,MAAM,CAAC8B,MAAM,CAAC,CAAC6B,OAAO,EAAEhH,eAAe,CAAC6D,QAAQ,CAAC,CAAC,CAAC;EAChE,OAAO;IAACC,IAAI;IAAED,QAAQ;IAAE2D;EAAc,CAAC;AACzC;;AAKA,eAAe9D,cAAc,CAC3BhD,MAAqB,EACrB4D,IAAmB,EACnB1D,IAA2B,EAI1B;EACD,MAAMiD,QAAQ,GAAG,IAAIpE,QAAQ,CAAC;IAC5B+G,QAAQ,EAAElC,IAAI,CAACzC,QAAQ;IACvB4F,OAAO,EAAE,EAAE;IACXC,eAAe,EAAE;EACnB,CAAC,CAAC;EAEF,IAAI5D,IAAI,GAAGT,MAAM,CAACmB,KAAK,CAAC,CAAC,CAAC;EAC1B,KAAK,MAAMmD,KAAK,IAAIjH,MAAM,CAACkH,SAAS,EAAE;IACpC,IAAID,KAAK,CAACE,QAAQ,EAAE;MAClB;IACF;;IAEA,MAAMC,UAAU,GAAG,MAAMlB,iBAAiB,CAACe,KAAK,EAAErD,IAAI,EAAER,IAAI,CAACV,MAAM,EAAExC,IAAI,CAAC;IAE1E,MAAMmH,MAAM,GAAG,IAAIlJ,WAAW,CAAC;MAC7BmJ,WAAW,EAAEF,UAAU,CAACN,cAAc;MACtCS,SAAS,EAAEH,UAAU,CAACjE;IACxB,CAAC,CAAC;IAEFA,QAAQ,CAAC4D,OAAO,CAAC7D,IAAI,CAACmE,MAAM,CAAC;IAC7BlE,QAAQ,CAAC6D,eAAe,GAAG,IAAIzH,KAAK,CAACiI,MAAM,CAACrE,QAAQ,CAAC6D,eAAe,CAAC,GAAGI,UAAU,CAAChE,IAAI,CAACV,MAAM,CAAC;IAE/FU,IAAI,GAAGT,MAAM,CAAC8B,MAAM,CAAC,CAACrB,IAAI,EAAEgE,UAAU,CAAChE,IAAI,CAAC,CAAC;EAC/C;EAEA,OAAO;IAACA,IAAI;IAAED;EAAQ,CAAC;AACzB;;AAKA,SAASE,YAAY,CACnBrD,MAAqB,EACrBmB,QAAgB,EAChBiB,SAAqB,EACrBzB,YAAoC,EAC5B;EACR,MAAMwC,QAAQ,GAAG,IAAIxE,YAAY,CAAC;IAChC8I,OAAO,EAAEhI,eAAe;IACxBiI,UAAU,EAAE,UAAU;IACtB5B,QAAQ,EAAE3E,QAAQ;IAClBwG,UAAU,EAAEvF,SAAS;IACrBpC,MAAM,EAAE,EAAE;IACV4H,kBAAkB,EAAE;EACtB,CAAC,CAAC;EAEF,KAAK,MAAMrG,GAAG,IAAIZ,YAAY,EAAE;IAAA;IAC9B,MAAMkH,EAAE,GAAG,IAAIjJ,QAAQ,CAAC;MACtB2C,GAAG;MACHC,KAAK,EAAEb,YAAY,CAACY,GAAG;IACzB,CAAC,CAAC;IACF,yBAAA4B,QAAQ,CAACyE,kBAAkB,oFAA3B,iDAA6B1E,IAAI,2DAAjC,oDAAoC2E,EAAE,CAAC;EACzC;EAEA;IACE,MAAMC,UAAU,GAAG,IAAI9I,aAAa,CAAC;MACnC+I,IAAI,EAAE,MAAM;MACZC,YAAY,EAAEC,MAAM,CAACC,IAAI,CAAClI,MAAM,CAACmI,MAAM,CAAC,CAACzF;IAC3C,CAAC,CAAC;IACFS,QAAQ,CAACnD,MAAM,CAACkD,IAAI,CAAC4E,UAAU,CAAC;EAClC;EAEA,KAAK,MAAMb,KAAK,IAAIjH,MAAM,CAACkH,SAAS,EAAE;IACpC,MAAMkB,IAAI,GAAG1J,mBAAmB,CAACuI,KAAK,CAACoB,cAAc,CAAC;IACtD,MAAMC,UAAU,GAAG,IAAItJ,aAAa,CAAC;MACnC+I,IAAI,EAAEd,KAAK,CAACc,IAAI;MAChBQ,eAAe,EAAEH;IACnB,CAAC,CAAC;IAEF,IAAInB,KAAK,CAACE,QAAQ,EAAE;MAClBmB,UAAU,CAACN,YAAY,GAAGf,KAAK,CAACuB,UAAU;IAC5C,CAAC,MAAM;MACLF,UAAU,CAAC/E,IAAI,GAAGtE,IAAI,CAACgI,KAAK,CAAC3C,aAAa,CAAU;IACtD;IAEA,IAAI2C,KAAK,CAACwB,YAAY,EAAE;MACtBH,UAAU,CAACI,cAAc,GAAGpK,aAAa,CAAC2I,KAAK,CAACwB,YAAY,CAAkB;IAChF;IAEAH,UAAU,CAACK,WAAW,GAAG1B,KAAK,CAAC1C,UAAU;IAEzCpB,QAAQ,CAACnD,MAAM,CAACkD,IAAI,CAACoF,UAAU,CAAC;EAClC;EAEA,MAAMM,eAAe,GAAGtJ,eAAe,CAAC6D,QAAQ,CAAC;EACjD,MAAM0F,aAAa,GAAGlG,MAAM,CAACmB,KAAK,CAAC8E,eAAe,CAAClG,MAAM,GAAG,CAAC,CAAC;EAC9DkG,eAAe,CAACE,IAAI,CAACD,aAAa,CAAC;EACnCA,aAAa,CAACE,aAAa,CAACH,eAAe,CAAClG,MAAM,EAAEkG,eAAe,CAAClG,MAAM,CAAC;EAC3EmG,aAAa,CAAC3G,KAAK,CAAC1C,aAAa,EAAEoJ,eAAe,CAAClG,MAAM,GAAG,CAAC,CAAC;EAC9D,OAAOmG,aAAa;AACtB"}

package/dist/esm/parquetjs/parser/parquet-reader.js

@@ -1,103 +1,189 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-let _Symbol$asyncIterator;
-import { ParquetEnvelopeReader } from './parquet-envelope-reader';
+
 import { ParquetSchema } from '../schema/schema';
-import { ParquetCursor } from './parquet-cursor';
-import { PARQUET_VERSION } from '../../constants';
 import { decodeSchema } from './decoders';
+import { materializeRecords } from '../schema/shred';
+import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from '../../constants';
+import { CompressionCodec, Type } from '../parquet-thrift';
+import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from '../utils/read-utils';
+import { decodeDataPages, decodePage } from './decoders';
+const DEFAULT_PROPS = {
+  defaultDictionarySize: 1e6
+};
 
-_Symbol$asyncIterator = Symbol.asyncIterator;
 export class ParquetReader {
-  static async openBlob(blob) {
-    const readFn = async (start, length) => {
-      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
-      return Buffer.from(arrayBuffer);
+  constructor(file, props) {
+    _defineProperty(this, "props", void 0);
+    _defineProperty(this, "file", void 0);
+    _defineProperty(this, "metadata", null);
+    this.file = file;
+    this.props = {
+      ...DEFAULT_PROPS,
+      ...props
     };
-    const closeFn = async () => {};
-    const size = blob.size;
-    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
+  }
+  close() {
+    this.file.close();
   }
 
-  static async openArrayBuffer(arrayBuffer) {
-    const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
-    const closeFn = async () => {};
-    const size = arrayBuffer.byteLength;
-    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
+  async *rowIterator(props) {
+    for await (const rows of this.rowBatchIterator(props)) {
+      for (const row of rows) {
+        yield row;
+      }
     }
   }
-  static async openBuffer(buffer) {
-    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
+
+  async *rowBatchIterator(props) {
+    const schema = await this.getSchema();
+    for await (const rowGroup of this.rowGroupIterator(props)) {
+      yield materializeRecords(schema, rowGroup);
     }
   }
-  constructor(metadata, envelopeReader) {
-    _defineProperty(this, "metadata", void 0);
-    _defineProperty(this, "envelopeReader", void 0);
-    _defineProperty(this, "schema", void 0);
-    if (metadata.version !== PARQUET_VERSION) {
-      throw new Error('invalid parquet version');
+
+  async *rowGroupIterator(props) {
+    const columnList = ((props === null || props === void 0 ? void 0 : props.columnList) || []).map(x => Array.isArray(x) ? x : [x]);
+    const metadata = await this.getFileMetadata();
+    const schema = await this.getSchema();
+    const rowGroupCount = (metadata === null || metadata === void 0 ? void 0 : metadata.row_groups.length) || 0;
+    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {
+      const rowGroup = await this.readRowGroup(schema, metadata.row_groups[rowGroupIndex], columnList);
+      yield rowGroup;
     }
-    this.metadata = metadata;
-    this.envelopeReader = envelopeReader;
-    const root = this.metadata.schema[0];
+  }
+  async getRowCount() {
+    const metadata = await this.getFileMetadata();
+    return Number(metadata.num_rows);
+  }
+  async getSchema() {
+    const metadata = await this.getFileMetadata();
+    const root = metadata.schema[0];
     const {
-      schema
-    } = decodeSchema(this.metadata.schema, 1, root.num_children);
-    this.schema = new ParquetSchema(schema);
+      schema: schemaDefinition
+    } = decodeSchema(metadata.schema, 1, root.num_children);
+    const schema = new ParquetSchema(schemaDefinition);
+    return schema;
   }
 
-  async close() {
-    await this.envelopeReader.close();
+  async getSchemaMetadata() {
+    const metadata = await this.getFileMetadata();
+    const md = {};
+    for (const kv of metadata.key_value_metadata) {
+      md[kv.key] = kv.value;
+    }
+    return md;
   }
-
-  getCursor(columnList) {
-    if (!columnList) {
-      columnList = [];
+  async getFileMetadata() {
+    if (!this.metadata) {
+      await this.readHeader();
+      this.metadata = this.readFooter();
     }
+    return this.metadata;
+  }
 
-    columnList = columnList.map(x => Array.isArray(x) ? x : [x]);
-    return new ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
+  async readHeader() {
+    const buffer = await this.file.read(0, PARQUET_MAGIC.length);
+    const magic = buffer.toString();
+    switch (magic) {
+      case PARQUET_MAGIC:
+        break;
+      case PARQUET_MAGIC_ENCRYPTED:
+        throw new Error('Encrypted parquet file not supported');
+      default:
+        throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
+    }
   }
 
-  getRowCount() {
-    return Number(this.metadata.num_rows);
+  async readFooter() {
+    const trailerLen = PARQUET_MAGIC.length + 4;
+    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const magic = trailerBuf.slice(4).toString();
+    if (magic !== PARQUET_MAGIC) {
+      throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
+    }
+    const metadataSize = trailerBuf.readUInt32LE(0);
+    const metadataOffset = this.file.size - metadataSize - trailerLen;
+    if (metadataOffset < PARQUET_MAGIC.length) {
+      throw new Error("Invalid metadata size ".concat(metadataOffset));
+    }
+    const metadataBuf = await this.file.read(metadataOffset, metadataSize);
+    const {
+      metadata
+    } = decodeFileMetadata(metadataBuf);
+    return metadata;
   }
 
-  getSchema() {
-    return this.schema;
+  async readRowGroup(schema, rowGroup, columnList) {
+    const buffer = {
+      rowCount: Number(rowGroup.num_rows),
+      columnData: {}
+    };
+    for (const colChunk of rowGroup.columns) {
+      const colMetadata = colChunk.meta_data;
+      const colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
+      if (columnList.length > 0 && fieldIndexOf(columnList, colKey) < 0) {
+        continue;
+      }
+
+      buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
+    }
+    return buffer;
   }
 
-  getMetadata() {
-    const md = {};
-    for (const kv of this.metadata.key_value_metadata) {
-      md[kv.key] = kv.value;
+  async readColumnChunk(schema, colChunk) {
+    var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
+    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
+      throw new Error('external references are not supported');
     }
-    return md;
+    const field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
+    const type = getThriftEnum(Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
+    if (type !== field.primitiveType) {
+      throw new Error("chunk type not matching schema: ".concat(type));
+    }
+    const compression = getThriftEnum(CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
+    const pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
+    let pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
+    if (!colChunk.file_path) {
+      var _colChunk$meta_data6;
+      pagesSize = Math.min(this.file.size - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
+    }
+    const options = {
+      type,
+      rLevelMax: field.rLevelMax,
+      dLevelMax: field.dLevelMax,
+      compression,
+      column: field,
+      numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
+      dictionary: []
+    };
+    let dictionary;
+    const dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
+    if (dictionaryPageOffset) {
+      const dictionaryOffset = Number(dictionaryPageOffset);
+      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+    }
+    dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
+    const pagesBuf = await this.file.read(pagesOffset, pagesSize);
+    return await decodeDataPages(pagesBuf, {
+      ...options,
+      dictionary
+    });
   }
 
-  [_Symbol$asyncIterator]() {
-    return this.getCursor()[Symbol.asyncIterator]();
+  async getDictionary(dictionaryPageOffset, options, pagesOffset) {
+    if (dictionaryPageOffset === 0) {
+
+      return [];
+    }
+    const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);
+    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const cursor = {
+      buffer: pagesBuf,
+      offset: 0,
+      size: pagesBuf.length
+    };
+    const decodedPage = await decodePage(cursor, options);
+    return decodedPage.dictionary;
   }
 }
 //# sourceMappingURL=parquet-reader.js.map

package/dist/esm/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["ParquetEnvelopeReader","ParquetSchema","ParquetCursor","PARQUET_VERSION","decodeSchema","Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openBuffer","buffer","constructor","version","Error","root","schema","num_children","getCursor","columnList","map","x","Array","isArray","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"mappings":";;AACA,SAAQA,qBAAqB,QAAO,2BAA2B;AAE/D,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,eAAe,QAAO,iBAAiB;AAC/C,SAAQC,YAAY,QAAO,YAAY;;AAAC,wBAyJrCC,MAAM,CAACC,aAAa;AAhJvB,OAAO,MAAMC,aAAa,CAAgC;EAIxD,aAAaC,QAAQ,CAAIC,IAAU,EAA6B;IAC9D,MAAMC,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAK;MACtD,MAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAK,CAACH,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC,CAACC,WAAW,EAAE;MACzE,OAAOE,MAAM,CAACC,IAAI,CAACH,WAAW,CAAC;IACjC,CAAC;IACD,MAAMI,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGT,IAAI,CAACS,IAAI;IACtB,MAAMC,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;;EAKA,aAAaE,eAAe,CAAIZ,WAAwB,EAA6B;IACnF,MAAMH,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAKG,MAAM,CAACC,IAAI,CAACH,WAAW,EAAEF,KAAK,EAAEC,MAAM,CAAC;IAC/F,MAAMK,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGL,WAAW,CAACa,UAAU;IACnC,MAAMP,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAEA,aAAaI,UAAU,CAAIC,MAAc,EAA6B;IACpE,MAAMT,cAAc,GAAG,MAAMnB,qBAAqB,CAAC2B,UAAU,CAACC,MAAM,CAAC;IACrE,IAAI;MACF,MAAMT,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAAIc,QAAQ,EAAEF,cAAc,CAAC;IACvD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAYAM,WAAW,CAACR,QAAsB,EAAEF,cAAqC,EAAE;IAAA;IAAA;IAAA;IACzE,IAAIE,QAAQ,CAACS,OAAO,KAAK3B,eAAe,EAAE;MACxC,MAAM,IAAI4B,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IAEA,IAAI,CAACV,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACF,cAAc,GAAGA,cAAc;IACpC,MAAMa,IAAI,GAAG,IAAI,CAACX,QAAQ,CAACY,MAAM,CAAC,CAAC,CAAC;IACpC,MAAM;MAACA;IAAM,CAAC,GAAG7B,YAAY,CAAC,IAAI,CAACiB,QAAQ,CAACY,MAAM,EAAE,CAAC,EAAED,IAAI,CAACE,YAAY,CAAE;IAC1E,IAAI,CAACD,MAAM,GAAG,IAAIhC,aAAa,CAACgC,MAAM,CAAC;EACzC;;EAMA,MAAMT,KAAK,GAAkB;IAC3B,MAAM,IAAI,CAACL,cAAc,CAACK,KAAK,EAAE;EAGnC;;EAeAW,SAAS,CAACC,UAAkC,EAA6B;IACvE,IAAI,CAACA,UAAU,EAAE;MAEfA,UAAU,GAAG,EAAE;IACjB;;IAGAA,UAAU,GAAGA,UAAU,CAACC,GAAG,CAAEC,CAAC,IAAMC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAE,CAAC;IAEhE,OAAO,IAAIpC,aAAa,CACtB,IAAI,CAACmB,QAAQ,EACb,IAAI,CAACF,cAAc,EACnB,IAAI,CAACc,MAAM,EACXG,UAAU,CACX;EACH;;EAMAK,WAAW,GAAW;IACpB,OAAOC,MAAM,CAAC,IAAI,CAACrB,QAAQ,CAACsB,QAAQ,CAAC;EACvC;;EAKAC,SAAS,GAAkB;IACzB,OAAO,IAAI,CAACX,MAAM;EACpB;;EAKAY,WAAW,GAA2B;IACpC,MAAMC,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAI,IAAI,CAAC1B,QAAQ,CAAC2B,kBAAkB,EAAG;MAClDF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;;EAMA,0BAA2C;IACzC,OAAO,IAAI,CAACX,SAAS,EAAE,CAAC9B,MAAM,CAACC,aAAa,CAAC,EAAE;EACjD;AACF"}
+{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRecords","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRecords(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,kBAAkB,QAAO,iBAAiB;AAElD,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAW,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA;IAAA;IAAA,kCAFnB,IAAI;IAG3C,IAAI,CAACD,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAC,KAAK,GAAS;IAEZ,IAAI,CAACF,IAAI,CAACE,KAAK,EAAE;EACnB;;EAKA,OAAOC,WAAW,CAACF,KAA6B,EAAE;IAChD,WAAW,MAAMG,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACJ,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMK,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;;EAGA,OAAOD,gBAAgB,CAACJ,KAA6B,EAAE;IACrD,MAAMM,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACT,KAAK,CAAC,EAAE;MACzD,MAAMf,kBAAkB,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;;EAGA,OAAOC,gBAAgB,CAACT,KAA6B,EAAE;IAErD,MAAMU,UAAsB,GAAG,CAAC,CAAAV,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEU,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC,CAC3B;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAW,GAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAAS,GAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG1C,YAAY,CAAC+B,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAY,CAAE;IACvF,MAAMrB,MAAM,GAAG,IAAIvB,aAAa,CAAC2C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;;EAMA,MAAMsB,iBAAiB,GAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAe,GAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,EAAE;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,EAAE;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;;EAKA,MAAMmB,UAAU,GAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACrC,IAAI,CAACsC,IAAI,CAAC,CAAC,EAAEnD,aAAa,CAACiC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAKpD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIqD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,uCAAgCF,KAAK,OAAI;IAAC;EAE/D;;EAGA,MAAMH,UAAU,GAA0B;IACxC,MAAMM,UAAU,GAAGvD,aAAa,CAACiC,MAAM,GAAG,CAAC;IAC3C,MAAMuB,UAAU,GAAG,MAAM,IAAI,CAAC3C,IAAI,CAACsC,IAAI,CAAC,IAAI,CAACtC,IAAI,CAAC4C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMH,KAAK,GAAGI,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACL,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAKpD,aAAa,EAAE;MAC3B,MAAM,IAAIsD,KAAK,6CAAqCF,KAAK,OAAI;IAC/D;IAEA,MAAMO,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAChD,IAAI,CAAC4C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG7D,aAAa,CAACiC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,iCAA0BO,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACjD,IAAI,CAACsC,IAAI,CAACU,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC9B;IAAQ,CAAC,GAAGzB,kBAAkB,CAAC0D,WAAW,CAAC;IAClD,OAAOjC,QAAQ;EACjB;;EAGA,MAAMM,YAAY,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACE;IACxB,MAAM0B,MAAqB,GAAG;MAC5Ba,QAAQ,EAAE1B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC0B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI3C,QAAQ,CAAC4C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI9C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI3B,YAAY,CAACkB,UAAU,EAAE6C,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;;MACAnB,MAAM,CAACc,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACpD,MAAM,EAAE6C,QAAQ,CAAC;IAClF;IACA,OAAOf,MAAM;EACf;;EAKA,MAAMsB,eAAe,CAACpD,MAAqB,EAAE6C,QAAqB,EAAwB;IAAA;IACxF,IAAIA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAInB,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAMqB,KAAK,GAAGvD,MAAM,CAACwD,SAAS,wBAACX,QAAQ,CAACG,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;IACnE,MAAMO,IAAmB,GAAGxE,aAAa,CAACF,IAAI,0BAAE8D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBS,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIxB,KAAK,2CAAoCuB,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAG1E,aAAa,CACnDH,gBAAgB,0BAChB+D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBY,KAAK,CACnB;IAER,MAAMC,WAAW,GAAG5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAG9C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAE;IAElE,IAAI,CAACnB,QAAQ,CAACQ,SAAS,EAAE;MAAA;MACvBU,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGwB,WAAW,EAC5B5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMG,OAAuB,GAAG;MAC9BV,IAAI;MACJW,SAAS,EAAEb,KAAK,CAACa,SAAS;MAC1BC,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BV,WAAW;MACXW,MAAM,EAAEf,KAAK;MACbgB,SAAS,0BAAE1B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBwB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAG7B,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEG,SAAS,yDAAnB,qBAAqB2B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAG3D,MAAM,CAACyD,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEN,WAAW,CAAC;IAC/E;IAEAY,UAAU,GAAG,uBAAAN,OAAO,CAACM,UAAU,gDAAlB,oBAAoB5D,MAAM,GAAGsD,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC8B,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAM5E,eAAe,CAAC2F,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;;EASA,MAAMI,aAAa,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBN,WAAmB,EACA;IACnB,IAAIa,oBAAoB,KAAK,CAAC,EAAE;;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGqC,oBAAoB,EACrC,IAAI,CAAChF,KAAK,CAACJ,qBAAqB,CACjC;IACD,MAAMwF,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC2C,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAClD,MAAM,EAAEgD,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAE5C,IAAI,EAAEyC,QAAQ,CAACjE;IAAM,CAAC;IACnE,MAAMqE,WAAW,GAAG,MAAM9F,UAAU,CAAC4F,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}

package/dist/esm/parquetjs/schema/declare.js

@@ -1,6 +1,7 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 
 export class ParquetBuffer {
+
   constructor() {
     let rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
     let columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};