@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/dist/parquetjs/compression.js

@@ -4,178 +4,132 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 // Forked from https://github.com/ironSource/parquetjs under MIT license
-import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, 
-// LZOCompression,
-LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
-import { registerJSModules } from '@loaders.gl/loader-utils';
-/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
-function toBuffer(arrayBuffer) {
-    return Buffer.from(arrayBuffer);
-}
-function toArrayBuffer(buffer) {
-    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-    if (Buffer.isBuffer(buffer)) {
-        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-        return typedArray.slice().buffer;
-    }
-    return buffer;
-}
-// TODO switch to worker compression to avoid bundling...
-// import brotli from 'brotli'; - brotli has problems with decompress in browsers
-// import brotliDecompress from 'brotli/decompress';
-import lz4js from 'lz4js';
-// import lzo from 'lzo';
-// import {ZstdCodec} from 'zstd-codec';
-// Inject large dependencies through Compression constructor options
-const modules = {
-    // brotli has problems with decompress in browsers
-    // brotli: {
-    //   decompress: brotliDecompress,
-    //   compress: () => {
-    //     throw new Error('brotli compress');
-    //   }
-    // },
-    lz4js
-    // lzo
-    // 'zstd-codec': ZstdCodec
-};
+import { decompressWithNativeDecompressionStream } from '@loaders.gl/compression/native-decompression';
+import { getJSModuleOrNull, registerJSModules } from '@loaders.gl/loader-utils';
+import { toArrayBuffer, toUint8Array } from "./utils/binary-utils.js";
 /**
  * See https://github.com/apache/parquet-format/blob/master/Compression.md
  */
-// @ts-expect-error
 export const PARQUET_COMPRESSION_METHODS = {
-    UNCOMPRESSED: new NoCompression(),
-    GZIP: new GZipCompression(),
-    SNAPPY: new SnappyCompression(),
-    BROTLI: new BrotliCompression({ modules }),
-    // TODO: Understand difference between LZ4 and LZ4_RAW
-    LZ4: new LZ4Compression({ modules }),
-    LZ4_RAW: new LZ4Compression({ modules }),
-    //
-    // LZO: new LZOCompression({modules}),
-    ZSTD: new ZstdCompression({ modules })
+    UNCOMPRESSED: true,
+    GZIP: true,
+    SNAPPY: true,
+    BROTLI: true,
+    // TODO: Understand difference between LZ4 and LZ4_RAW.
+    LZ4: true,
+    LZ4_RAW: true,
+    ZSTD: true
 };
+/** Native formats available to asynchronous Parquet page decompression. */
+const PARQUET_NATIVE_DECOMPRESSION_FORMATS = {
+    GZIP: 'gzip',
+    BROTLI: 'brotli',
+    ZSTD: 'zstd'
+};
+/** Lazily constructed codec-backed compression implementations. */
+const compressionPromises = {};
 /**
- * Register compressions that have big external libraries
- * @param options.modules External library dependencies
+ * Registers optional codec modules without eagerly loading codec-backed implementations.
+ *
+ * @param options.modules External library dependencies.
  */
 export async function preloadCompressions(options) {
     registerJSModules(options?.modules);
-    const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-    return await Promise.all(compressions.map((compression) => compression.preload(options?.modules)));
 }
 /**
  * Deflate a value using compression method `method`
  */
 export async function deflate(method, value) {
-    const compression = PARQUET_COMPRESSION_METHODS[method];
-    if (!compression) {
+    if (!(method in PARQUET_COMPRESSION_METHODS)) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
+    if (method === 'UNCOMPRESSED') {
+        return value;
+    }
+    const compression = await getParquetCompression(method);
     const inputArrayBuffer = toArrayBuffer(value);
     const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
-    return toBuffer(compressedArrayBuffer);
+    return toUint8Array(compressedArrayBuffer);
 }
 /**
  * Inflate a value using compression method `method`
  */
 export async function decompress(method, value, size) {
-    const compression = PARQUET_COMPRESSION_METHODS[method];
-    if (!compression) {
+    if (!(method in PARQUET_COMPRESSION_METHODS)) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
     const inputArrayBuffer = toArrayBuffer(value);
+    if (method === 'UNCOMPRESSED') {
+        return toUint8Array(inputArrayBuffer);
+    }
+    const nativeFormat = PARQUET_NATIVE_DECOMPRESSION_FORMATS[method];
+    if (nativeFormat && shouldUseNativeDecompressionStream(method)) {
+        const nativeOutput = await decompressWithNativeDecompressionStream(inputArrayBuffer, nativeFormat);
+        if (nativeOutput) {
+            return toUint8Array(nativeOutput);
+        }
+    }
+    const compression = await getParquetCompression(method);
     const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
-    return toBuffer(compressedArrayBuffer);
+    return toUint8Array(compressedArrayBuffer);
 }
-/*
- * Inflate a value using compression method `method`
+/**
+ * Returns whether a native stream can take precedence for one Parquet compression method.
+ *
+ * @param method Parquet compression method.
+ * @returns Whether no explicitly registered codec should take precedence.
  */
-export function inflate(method, value, size) {
-    if (!(method in PARQUET_COMPRESSION_METHODS)) {
-        throw new Error(`invalid compression method: ${method}`);
+function shouldUseNativeDecompressionStream(method) {
+    if (method === 'BROTLI') {
+        return !getJSModuleOrNull('brotli');
     }
-    // @ts-ignore
-    return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
-}
-/*
-function deflate_identity(value: Buffer): Buffer {
-  return value;
-}
-
-function deflate_gzip(value: Buffer): Buffer {
-  return zlib.gzipSync(value);
-}
-
-function deflate_snappy(value: Buffer): Buffer {
-  return snappyjs.compress(value);
-}
-
-function deflate_lzo(value: Buffer): Buffer {
-  lzo = lzo || Util.load('lzo');
-  return lzo.compress(value);
-}
-
-function deflate_brotli(value: Buffer): Buffer {
-  brotli = brotli || Util.load('brotli');
-  const result = brotli.compress(value, {
-    mode: 0,
-    quality: 8,
-    lgwin: 22
-  });
-  return result ? Buffer.from(result) : Buffer.alloc(0);
-}
-
-function deflate_lz4(value: Buffer): Buffer {
-  lz4js = lz4js || Util.load('lz4js');
-  try {
-    // let result = Buffer.alloc(lz4js.encodeBound(value.length));
-    // const compressedSize = lz4.encodeBlock(value, result);
-    // // remove unnecessary bytes
-    // result = result.slice(0, compressedSize);
-    // return result;
-    return Buffer.from(lz4js.compress(value));
-  } catch (err) {
-    throw err;
-  }
-}
-function inflate_identity(value: Buffer): Buffer {
-  return value;
-}
-
-function inflate_gzip(value: Buffer): Buffer {
-  return zlib.gunzipSync(value);
-}
-
-function inflate_snappy(value: Buffer): Buffer {
-  return snappyjs.uncompress(value);
-}
-
-function inflate_lzo(value: Buffer, size: number): Buffer {
-  lzo = lzo || Util.load('lzo');
-  return lzo.decompress(value, size);
+    if (method === 'ZSTD') {
+        return !getJSModuleOrNull('zstd-codec');
+    }
+    return true;
 }
-
-function inflate_lz4(value: Buffer, size: number): Buffer {
-  lz4js = lz4js || Util.load('lz4js');
-  try {
-    // let result = Buffer.alloc(size);
-    // const uncompressedSize = lz4js.decodeBlock(value, result);
-    // // remove unnecessary bytes
-    // result = result.slice(0, uncompressedSize);
-    // return result;
-    return Buffer.from(lz4js.decompress(value, size));
-  } catch (err) {
-    throw err;
-  }
+/**
+ * Loads one codec-backed implementation only after native decompression is unavailable.
+ *
+ * @param method Parquet compression method.
+ * @returns Codec-backed compression implementation.
+ */
+async function getParquetCompression(method) {
+    compressionPromises[method] ||= createParquetCompression(method);
+    return await compressionPromises[method];
 }
-
-function inflate_brotli(value: Buffer): Buffer {
-  brotli = brotli || Util.load('brotli');
-  if (!value.length) {
-    return Buffer.alloc(0);
-  }
-  return Buffer.from(brotli.decompress(value));
+/**
+ * Creates one lazily loaded codec-backed Parquet compression implementation.
+ *
+ * @param method Parquet compression method.
+ * @returns Codec-backed compression implementation.
+ */
+async function createParquetCompression(method) {
+    switch (method) {
+        case 'GZIP': {
+            const { GZipCompression } = await import('@loaders.gl/compression/gzip-compression');
+            return new GZipCompression();
+        }
+        case 'SNAPPY': {
+            const { SnappyCompression } = await import('@loaders.gl/compression/snappy-compression');
+            return new SnappyCompression();
+        }
+        case 'BROTLI': {
+            const { BrotliCompression } = await import('@loaders.gl/compression/brotli-compression');
+            return new BrotliCompression();
+        }
+        case 'LZ4':
+        case 'LZ4_RAW': {
+            const { LZ4Compression } = await import('@loaders.gl/compression/lz4-compression');
+            const lz4js = getJSModuleOrNull('lz4js') || (await import('lz4js')).default;
+            return new LZ4Compression({ modules: { lz4js } });
+        }
+        case 'ZSTD': {
+            const { ZstdCompression } = await import('@loaders.gl/compression/zstd-compression');
+            return new ZstdCompression();
+        }
+        default:
+            throw new Error(`parquet: invalid compression method: ${method}`);
+    }
 }
-*/
 //# sourceMappingURL=compression.js.map

package/dist/parquetjs/compression.js.map

@@ -1 +1 @@
-{"version":3,"file":"compression.js","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AACpE,wEAAwE;AAExE,OAAO,EAEL,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,iBAAiB;AACjB,kBAAkB;AAClB,cAAc,EACd,eAAe,EAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAI3D,sGAAsG;AACtG,SAAS,QAAQ,CAAC,WAAwB;IACxC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,qFAAqF;IACrF,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACnF,OAAO,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yDAAyD;AAEzD,iFAAiF;AACjF,oDAAoD;AACpD,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,yBAAyB;AACzB,wCAAwC;AAExC,oEAAoE;AACpE,MAAM,OAAO,GAAG;IACd,kDAAkD;IAClD,YAAY;IACZ,kCAAkC;IAClC,sBAAsB;IACtB,0CAA0C;IAC1C,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,0BAA0B;CAC3B,CAAC;AAEF;;GAEG;AACH,mBAAmB;AACnB,MAAM,CAAC,MAAM,2BAA2B,GAA4C;IAClF,YAAY,EAAE,IAAI,aAAa,EAAE;IACjC,IAAI,EAAE,IAAI,eAAe,EAAE;IAC3B,MAAM,EAAE,IAAI,iBAAiB,EAAE;IAC/B,MAAM,EAAE,IAAI,iBAAiB,CAAC,EAAC,OAAO,EAAC,CAAC;IACxC,sDAAsD;IACtD,GAAG,EAAE,IAAI,cAAc,CAAC,EAAC,OAAO,EAAC,CAAC;IAClC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAC,OAAO,EAAC,CAAC;IACtC,EAAE;IACF,sCAAsC;IACtC,IAAI,EAAE,IAAI,eAAe,CAAC,EAAC,OAAO,EAAC,CAAC;CACrC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAA0C;IAClF,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAChE,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAA0B,EAAE,KAAa;IACrE,MAAM,WAAW,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,qBAAqB,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAC3E,OAAO,QAAQ,CAAC,qBAAqB,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAA0B,EAC1B,KAAa,EACb,IAAY;IAEZ,MAAM,WAAW,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,qBAAqB,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IACnF,OAAO,QAAQ,CAAC,qBAAqB,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,MAA0B,EAAE,KAAa,EAAE,IAAY;IAC7E,IAAI,CAAC,CAAC,MAAM,IAAI,2BAA2B,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,aAAa;IACb,OAAO,2BAA2B,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA+EE"}
+{"version":3,"file":"compression.js","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AACpE,wEAAwE;AAGxE,OAAO,EACL,uCAAuC,EAExC,MAAM,8CAA8C,CAAC;AACtD,OAAO,EAAC,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAG9E,OAAO,EAAC,aAAa,EAAE,YAAY,EAAC,gCAA6B;AAEjE;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAA8C;IACpF,YAAY,EAAE,IAAI;IAClB,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,IAAI;IACZ,MAAM,EAAE,IAAI;IACZ,uDAAuD;IACvD,GAAG,EAAE,IAAI;IACT,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,IAAI;CACX,CAAC;AAEF,2EAA2E;AAC3E,MAAM,oCAAoC,GAEtC;IACF,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,mEAAmE;AACnE,MAAM,mBAAmB,GAA8D,EAAE,CAAC;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAA0C;IAClF,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAA0B,EAAE,KAAiB;IACzE,IAAI,CAAC,CAAC,MAAM,IAAI,2BAA2B,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,qBAAqB,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAC3E,OAAO,YAAY,CAAC,qBAAqB,CAAC,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAA0B,EAC1B,KAAiB,EACjB,IAAY;IAEZ,IAAI,CAAC,CAAC,MAAM,IAAI,2BAA2B,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;QAC9B,OAAO,YAAY,CAAC,gBAAgB,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,YAAY,GAAG,oCAAoC,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,YAAY,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,MAAM,uCAAuC,CAChE,gBAAgB,EAChB,YAAY,CACb,CAAC;QACF,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,YAAY,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,qBAAqB,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IACnF,OAAO,YAAY,CAAC,qBAAqB,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,SAAS,kCAAkC,CAAC,MAA0B;IACpE,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,qBAAqB,CAAC,MAA0B;IAC7D,mBAAmB,CAAC,MAAM,CAAC,KAAK,wBAAwB,CAAC,MAAM,CAAC,CAAC;IACjE,OAAO,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,wBAAwB,CAAC,MAA0B;IAChE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,EAAC,eAAe,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;YACnF,OAAO,IAAI,eAAe,EAAE,CAAC;QAC/B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAC,iBAAiB,EAAC,GAAG,MAAM,MAAM,CAAC,4CAA4C,CAAC,CAAC;YACvF,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACjC,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAC,iBAAiB,EAAC,GAAG,MAAM,MAAM,CAAC,4CAA4C,CAAC,CAAC;YACvF,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACjC,CAAC;QACD,KAAK,KAAK,CAAC;QACX,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAC,cAAc,EAAC,GAAG,MAAM,MAAM,CAAC,yCAAyC,CAAC,CAAC;YACjF,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;YAC5E,OAAO,IAAI,cAAc,CAAC,EAAC,OAAO,EAAE,EAAC,KAAK,EAAC,EAAC,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,EAAC,eAAe,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;YACnF,OAAO,IAAI,eAAe,EAAE,CAAC;QAC/B,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}

package/dist/parquetjs/encoder/dictionary-planner.d.ts

@@ -0,0 +1,17 @@
+import type { ParquetField } from "../schema/declare.js";
+/** Dictionary selection policy for one Parquet column chunk. */
+export type ParquetDictionaryPolicy = boolean | 'auto';
+/** A chunk-wide dictionary and the index stream replacing its non-null values. */
+export interface ParquetDictionaryPlan {
+    /** Unique physical values written to the dictionary page. */
+    readonly values: unknown[];
+    /** Dictionary index corresponding to every non-null column value. */
+    readonly indices: number[];
+    /** PLAIN-encoded dictionary page payload. */
+    readonly encodedValues: Uint8Array;
+    /** Bit width used by RLE_DICTIONARY index pages. */
+    readonly bitWidth: number;
+}
+/** Builds a forced or size-beneficial dictionary plan for a complete column chunk. */
+export declare function planDictionary(column: ParquetField, values: readonly unknown[], policy: ParquetDictionaryPolicy, dictionaryPageSizeLimit: number): ParquetDictionaryPlan | undefined;
+//# sourceMappingURL=dictionary-planner.d.ts.map

package/dist/parquetjs/encoder/dictionary-planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary-planner.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/dictionary-planner.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,6BAA0B;AAEpD,gEAAgE;AAChE,MAAM,MAAM,uBAAuB,GAAG,OAAO,GAAG,MAAM,CAAC;AAEvD,kFAAkF;AAClF,MAAM,WAAW,qBAAqB;IACpC,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,6CAA6C;IAC7C,QAAQ,CAAC,aAAa,EAAE,UAAU,CAAC;IACnC,oDAAoD;IACpD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,sFAAsF;AACtF,wBAAgB,cAAc,CAC5B,MAAM,EAAE,YAAY,EACpB,MAAM,EAAE,SAAS,OAAO,EAAE,EAC1B,MAAM,EAAE,uBAAuB,EAC/B,uBAAuB,EAAE,MAAM,GAC9B,qBAAqB,GAAG,SAAS,CAiDnC"}

package/dist/parquetjs/encoder/dictionary-planner.js

@@ -0,0 +1,64 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { PARQUET_CODECS } from "../codecs/index.js";
+/** Builds a forced or size-beneficial dictionary plan for a complete column chunk. */
+export function planDictionary(column, values, policy, dictionaryPageSizeLimit) {
+    if (!Number.isInteger(dictionaryPageSizeLimit) || dictionaryPageSizeLimit <= 0) {
+        throw new Error(`Parquet dictionary page size limit must be a positive integer, received ${dictionaryPageSizeLimit}`);
+    }
+    if (policy === false || values.length === 0 || isDictionaryEncoding(column.encoding)) {
+        return undefined;
+    }
+    const dictionaryValues = [];
+    const dictionaryIndices = new Map();
+    const indices = new Array(values.length);
+    for (let valueIndex = 0; valueIndex < values.length; valueIndex++) {
+        const value = values[valueIndex];
+        const key = getDictionaryKey(value);
+        let dictionaryIndex = dictionaryIndices.get(key);
+        if (dictionaryIndex === undefined) {
+            dictionaryIndex = dictionaryValues.length;
+            dictionaryIndices.set(key, dictionaryIndex);
+            dictionaryValues.push(value);
+        }
+        indices[valueIndex] = dictionaryIndex;
+    }
+    const encodedValues = PARQUET_CODECS.PLAIN.encodeValues(column.primitiveType, dictionaryValues, {
+        typeLength: column.typeLength
+    });
+    if (encodedValues.length > dictionaryPageSizeLimit) {
+        return undefined;
+    }
+    const bitWidth = dictionaryValues.length <= 1 ? 0 : Math.ceil(Math.log2(dictionaryValues.length));
+    if (policy === 'auto') {
+        const encodedIndices = PARQUET_CODECS.RLE_DICTIONARY.encodeValues(column.primitiveType, indices, { bitWidth });
+        const encodedPrimaryValues = PARQUET_CODECS[column.encoding].encodeValues(column.primitiveType, values, { typeLength: column.typeLength, bitWidth: column.typeLength });
+        if (encodedValues.length + encodedIndices.length >= encodedPrimaryValues.length) {
+            return undefined;
+        }
+    }
+    return { values: dictionaryValues, indices, encodedValues, bitWidth };
+}
+/** Returns a stable content key for one physical Parquet value. */
+function getDictionaryKey(value) {
+    if (value instanceof Uint8Array) {
+        let key = `bytes:${value.length}:`;
+        for (const byte of value) {
+            key += String.fromCharCode(byte);
+        }
+        return key;
+    }
+    if (typeof value === 'number') {
+        if (Number.isNaN(value))
+            return 'number:NaN';
+        if (Object.is(value, -0))
+            return 'number:-0';
+    }
+    return `${typeof value}:${String(value)}`;
+}
+/** Returns whether the selected primary encoding already requires a dictionary page. */
+function isDictionaryEncoding(encoding) {
+    return encoding === 'PLAIN_DICTIONARY' || encoding === 'RLE_DICTIONARY';
+}
+//# sourceMappingURL=dictionary-planner.js.map

package/dist/parquetjs/encoder/dictionary-planner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary-planner.js","sourceRoot":"","sources":["../../../src/parquetjs/encoder/dictionary-planner.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,cAAc,EAAC,2BAAwB;AAkB/C,sFAAsF;AACtF,MAAM,UAAU,cAAc,CAC5B,MAAoB,EACpB,MAA0B,EAC1B,MAA+B,EAC/B,uBAA+B;IAE/B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,IAAI,uBAAuB,IAAI,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CACb,2EAA2E,uBAAuB,EAAE,CACrG,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,oBAAoB,CAAC,MAAM,CAAC,QAAS,CAAC,EAAE,CAAC;QACtF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,gBAAgB,GAAc,EAAE,CAAC;IACvC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAS,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAClC,eAAe,GAAG,gBAAgB,CAAC,MAAM,CAAC;YAC1C,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAC5C,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,CAAC,UAAU,CAAC,GAAG,eAAe,CAAC;IACxC,CAAC;IAED,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,aAAc,EAAE,gBAAgB,EAAE;QAC/F,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IACH,IAAI,aAAa,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QACnD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClG,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC,YAAY,CAC/D,MAAM,CAAC,aAAc,EACrB,OAAO,EACP,EAAC,QAAQ,EAAC,CACX,CAAC;QACF,MAAM,oBAAoB,GAAG,cAAc,CAAC,MAAM,CAAC,QAAS,CAAC,CAAC,YAAY,CACxE,MAAM,CAAC,aAAc,EACrB,MAAmB,EACnB,EAAC,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAC,CAC7D,CAAC;QACF,IAAI,aAAa,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,IAAI,oBAAoB,CAAC,MAAM,EAAE,CAAC;YAChF,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAC,CAAC;AACtE,CAAC;AAED,mEAAmE;AACnE,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,IAAI,GAAG,GAAG,SAAS,KAAK,CAAC,MAAM,GAAG,CAAC;QACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;YAAE,OAAO,YAAY,CAAC;QAC7C,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,WAAW,CAAC;IAC/C,CAAC;IACD,OAAO,GAAG,OAAO,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC5C,CAAC;AAED,wFAAwF;AACxF,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,QAAQ,KAAK,kBAAkB,IAAI,QAAQ,KAAK,gBAAgB,CAAC;AAC1E,CAAC"}

package/dist/parquetjs/encoder/page-planner.d.ts

@@ -0,0 +1,11 @@
+import type { ParquetColumnChunk, ParquetField } from "../schema/declare.js";
+/** One row-aligned slice of a shredded column prepared for page encoding. */
+export interface PlannedParquetPage {
+    /** Shredded levels and non-null values contained in this page. */
+    readonly data: ParquetColumnChunk;
+    /** Number of top-level rows represented by the page. */
+    readonly rowCount: number;
+}
+/** Splits a shredded column into target-sized pages without splitting a top-level row. */
+export declare function planColumnPages(column: ParquetField, data: ParquetColumnChunk, rowCount: number, pageSize: number): PlannedParquetPage[];
+//# sourceMappingURL=page-planner.d.ts.map

package/dist/parquetjs/encoder/page-planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-planner.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/page-planner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,kBAAkB,EAAE,YAAY,EAAC,6BAA0B;AAExE,6EAA6E;AAC7E,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;IAClC,wDAAwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,0FAA0F;AAC1F,wBAAgB,eAAe,CAC7B,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,kBAAkB,EACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,kBAAkB,EAAE,CA8DtB"}

package/dist/parquetjs/encoder/page-planner.js

@@ -0,0 +1,60 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/** Splits a shredded column into target-sized pages without splitting a top-level row. */
+export function planColumnPages(column, data, rowCount, pageSize) {
+    if (!Number.isInteger(pageSize) || pageSize <= 0) {
+        throw new Error(`Parquet page size must be a positive integer, received ${pageSize}`);
+    }
+    if (data.rlevels.length < data.count || data.dlevels.length < data.count) {
+        throw new Error(`Invalid shredded column ${column.key}: level buffers are shorter than count`);
+    }
+    if (rowCount === 0) {
+        return [];
+    }
+    const rowStarts = [0];
+    for (let levelIndex = 1; levelIndex < data.count; levelIndex++) {
+        if (data.rlevels[levelIndex] === 0) {
+            rowStarts.push(levelIndex);
+        }
+    }
+    rowStarts.push(data.count);
+    if (rowStarts.length !== rowCount + 1) {
+        throw new Error(`Invalid shredded column ${column.key}: found ${rowStarts.length - 1} rows, expected ${rowCount}`);
+    }
+    const pages = [];
+    let pageStartRow = 0;
+    let valueStart = 0;
+    while (pageStartRow < rowCount) {
+        const levelStart = rowStarts[pageStartRow];
+        let pageEndRow = pageStartRow + 1;
+        while (pageEndRow < rowCount && rowStarts[pageEndRow + 1] - levelStart <= pageSize) {
+            pageEndRow++;
+        }
+        const levelEnd = rowStarts[pageEndRow];
+        let valueCount = 0;
+        for (let levelIndex = levelStart; levelIndex < levelEnd; levelIndex++) {
+            if (data.dlevels[levelIndex] === column.dLevelMax) {
+                valueCount++;
+            }
+        }
+        const valueEnd = valueStart + valueCount;
+        pages.push({
+            rowCount: pageEndRow - pageStartRow,
+            data: {
+                rlevels: data.rlevels.slice(levelStart, levelEnd),
+                dlevels: data.dlevels.slice(levelStart, levelEnd),
+                values: data.values.slice(valueStart, valueEnd),
+                count: levelEnd - levelStart,
+                pageHeaders: []
+            }
+        });
+        pageStartRow = pageEndRow;
+        valueStart = valueEnd;
+    }
+    if (valueStart !== data.values.length) {
+        throw new Error(`Invalid shredded column ${column.key}: consumed ${valueStart} values, expected ${data.values.length}`);
+    }
+    return pages;
+}
+//# sourceMappingURL=page-planner.js.map

package/dist/parquetjs/encoder/page-planner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-planner.js","sourceRoot":"","sources":["../../../src/parquetjs/encoder/page-planner.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAYpC,0FAA0F;AAC1F,MAAM,UAAU,eAAe,CAC7B,MAAoB,EACpB,IAAwB,EACxB,QAAgB,EAChB,QAAgB;IAEhB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,0DAA0D,QAAQ,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,wCAAwC,CAAC,CAAC;IACjG,CAAC;IACD,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;QAC/D,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,2BAA2B,MAAM,CAAC,GAAG,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,mBAAmB,QAAQ,EAAE,CAClG,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,YAAY,GAAG,QAAQ,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;QAC3C,IAAI,UAAU,GAAG,YAAY,GAAG,CAAC,CAAC;QAClC,OAAO,UAAU,GAAG,QAAQ,IAAI,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,UAAU,IAAI,QAAQ,EAAE,CAAC;YACnF,UAAU,EAAE,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,IAAI,UAAU,GAAG,UAAU,EAAE,UAAU,GAAG,QAAQ,EAAE,UAAU,EAAE,EAAE,CAAC;YACtE,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;gBAClD,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,QAAQ,EAAE,UAAU,GAAG,YAAY;YACnC,IAAI,EAAE;gBACJ,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACjD,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACjD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBAC/C,KAAK,EAAE,QAAQ,GAAG,UAAU;gBAC5B,WAAW,EAAE,EAAE;aAChB;SACF,CAAC,CAAC;QACH,YAAY,GAAG,UAAU,CAAC;QAC1B,UAAU,GAAG,QAAQ,CAAC;IACxB,CAAC;IAED,IAAI,UAAU,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,2BAA2B,MAAM,CAAC,GAAG,cAAc,UAAU,qBAAqB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CACvG,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/parquetjs/encoder/parquet-encoder.d.ts

@@ -2,11 +2,15 @@ import { stream } from '@loaders.gl/loader-utils';
 import { ParquetRowGroup, ParquetRow } from "../schema/declare.js";
 import { ParquetSchema } from "../schema/schema.js";
 import { RowGroup } from "../parquet-thrift/index.js";
+import { type ParquetDictionaryPolicy } from "./dictionary-planner.js";
 export interface ParquetEncoderOptions {
     baseOffset?: number;
     rowGroupSize?: number;
     pageSize?: number;
     useDataPageV2?: boolean;
+    dictionary?: ParquetDictionaryPolicy;
+    columnDictionaries?: Record<string, ParquetDictionaryPolicy>;
+    dictionaryPageSizeLimit?: number;
     flags?: string;
     encoding?: string;
     fd?: number;
@@ -19,7 +23,7 @@ export interface ParquetEncoderOptions {
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
-export declare class ParquetEncoder<T> {
+export declare class ParquetEncoder<_T> {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
@@ -82,15 +86,18 @@ export declare class ParquetEnvelopeWriter {
      */
     static openStream(schema: ParquetSchema, outputStream: stream.Writable, opts: ParquetEncoderOptions): Promise<ParquetEnvelopeWriter>;
     schema: ParquetSchema;
-    write: (buf: Buffer) => Promise<void>;
+    write: (buf: Uint8Array) => Promise<void>;
     close: () => Promise<void>;
     offset: number;
     rowCount: number;
     rowGroups: RowGroup[];
     pageSize: number;
     useDataPageV2: boolean;
-    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetEncoderOptions);
-    writeSection(buf: Buffer): Promise<void>;
+    dictionary: ParquetDictionaryPolicy;
+    columnDictionaries: Record<string, ParquetDictionaryPolicy>;
+    dictionaryPageSizeLimit: number;
+    constructor(schema: ParquetSchema, writeFn: (buf: Uint8Array) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetEncoderOptions);
+    writeSection(buf: Uint8Array): Promise<void>;
     /**
      * Encode the parquet file header
      */

package/dist/parquetjs/encoder/parquet-encoder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,eAAe,EAKf,UAAU,EACX,6BAA0B;AAC3B,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,OAAO,EAaL,QAAQ,EAGT,mCAAgC;AA2BjC,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,CAAC;IAC3B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,eAAe,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAcvB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAY7B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAY5D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}
+{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,eAAe,EAMf,UAAU,EACX,6BAA0B;AAC3B,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,OAAO,EAgCL,QAAQ,EAST,mCAAgC;AAMjC,OAAO,EAAiB,KAAK,uBAAuB,EAAC,gCAA6B;AAyBlF,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAGjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,EAAE;IAC5B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,eAAe,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAavB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,uBAAuB,CAAC;IACpC,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IAC5D,uBAAuB,EAAE,MAAM,CAAC;gBAGrC,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,EAC3C,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAe7B,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5C;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAe5D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}