@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

package/dist/es5/parquetjs/compression.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["modules","lz4js","lzo","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","LZO","LZOCompression","ZSTD","ZstdCompression","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compress","compressedArrayBuffer","decompress","size","inflate"],"mappings":";;;;;;;;;;;;;;;;;AAIA;;AAYA;;AAMA;;AACA;;AAIA,IAAMA,OAAO,GAAG;AAQdC,EAAAA,KAAK,EAALA,cARc;AASdC,EAAAA,GAAG,EAAHA;AATc,CAAhB;AAcO,IAAMC,2BAAoE,GAAG;AAClFC,EAAAA,YAAY,EAAE,IAAIC,0BAAJ,EADoE;AAElFC,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,EAF4E;AAGlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,EAH0E;AAIlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,CAAsB;AAACX,IAAAA,OAAO,EAAPA;AAAD,GAAtB,CAJ0E;AAMlFY,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAACb,IAAAA,OAAO,EAAPA;AAAD,GAAnB,CAN6E;AAOlFc,EAAAA,OAAO,EAAE,IAAID,2BAAJ,CAAmB;AAACb,IAAAA,OAAO,EAAPA;AAAD,GAAnB,CAPyE;AAQlFe,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAAChB,IAAAA,OAAO,EAAPA;AAAD,GAAnB,CAR6E;AASlFiB,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,CAAoB;AAAClB,IAAAA,OAAO,EAAPA;AAAD,GAApB;AAT4E,CAA7E;;;SAgBemB,mB;;;;;mFAAf,iBAAmCC,OAAnC;AAAA;AAAA;AAAA;AAAA;AAAA;AACCC,YAAAA,YADD,GACgBC,MAAM,CAACC,MAAP,CAAcpB,2BAAd,CADhB;AAAA;AAAA,mBAEQqB,OAAO,CAACC,GAAR,CAAYJ,YAAY,CAACK,GAAb,CAAiB,UAACC,WAAD;AAAA,qBAAiBA,WAAW,CAACC,OAAZ,EAAjB;AAAA,aAAjB,CAAZ,CAFR;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAQeC,O;;;;;uEAAf,kBAAuBC,MAAvB,EAAmDC,KAAnD;AAAA;AAAA;AAAA;AAAA;AAAA;AACCJ,YAAAA,WADD,GACexB,2BAA2B,CAAC2B,MAAD,CAD1C;;AAAA,gBAEAH,WAFA;AAAA;AAAA;AAAA;;AAAA,kBAGG,IAAIK,KAAJ,gDAAkDF,MAAlD,EAHH;;AAAA;AAKCG,YAAAA,gBALD,GAKoB,gCAAcF,KAAd,CALpB;AAAA;AAAA,mBAM+BJ,WAAW,CAACO,QAAZ,CAAqBD,gBAArB,CAN/B;;AAAA;AAMCE,YAAAA,qBAND;AAAA,8CAOE,2BAASA,qBAAT,CAPF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAaeC,U;;;;;0EAAf,kBACLN,MADK,EAELC,KAFK,EAGLM,IAHK;AAAA;AAAA;AAAA;AAAA;AAAA;AAKCV,YAAAA,WALD,GAKexB,2BAA2B,CAAC2B,MAAD,CAL1C;;AAAA,gBAMAH,WANA;AAAA;AAAA;AAAA;;AAAA,kBAOG,IAAIK,KAAJ,gDAAkDF,MAAlD,EAPH;;AAAA;AASCG,YAAAA,gBATD,GASoB,gCAAcF,KAAd,CATpB;AAAA;AAAA,mBAU+BJ,WAAW,CAACS,UAAZ,CAAuBH,gBAAvB,EAAyCI,IAAzC,CAV/B;;AAAA;AAUCF,YAAAA,qBAVD;AAAA,8CAWE,2BAASA,qBAAT,CAXF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAiBA,SAASG,OAAT,CAAiBR,MAAjB,EAA6CC,KAA7C,EAA4DM,IAA5D,EAAkF;AACvF,MAAI,EAAEP,MAAM,IAAI3B,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAI6B,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAO3B,2BAA2B,CAAC2B,MAAD,CAA3B,CAAoCQ,OAApC,CAA4CP,KAA5C,EAAmDM,IAAnD,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"file":"compression.js"}