@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/parquetjs/compression.js

@@ -0,0 +1,168 @@
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+/* eslint-disable camelcase */
+// Forked from https://github.com/ironSource/parquetjs under MIT license
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inflate = exports.decompress = exports.deflate = exports.preloadCompressions = exports.PARQUET_COMPRESSION_METHODS = void 0;
+const compression_1 = require("@loaders.gl/compression");
+const buffer_utils_1 = require("./utils/buffer-utils");
+// TODO switch to worker compression to avoid bundling...
+// import brotli from 'brotli'; - brotli has problems with decompress in browsers
+// import brotliDecompress from 'brotli/decompress';
+const lz4js_1 = __importDefault(require("lz4js"));
+const lzo_1 = __importDefault(require("lzo"));
+// import {ZstdCodec} from 'zstd-codec';
+// Inject large dependencies through Compression constructor options
+const modules = {
+    // brotli has problems with decompress in browsers
+    // brotli: {
+    //   decompress: brotliDecompress,
+    //   compress: () => {
+    //     throw new Error('brotli compress');
+    //   }
+    // },
+    lz4js: lz4js_1.default,
+    lzo: lzo_1.default
+    // 'zstd-codec': ZstdCodec
+};
+// See https://github.com/apache/parquet-format/blob/master/Compression.md
+exports.PARQUET_COMPRESSION_METHODS = {
+    UNCOMPRESSED: new compression_1.NoCompression(),
+    GZIP: new compression_1.GZipCompression(),
+    SNAPPY: new compression_1.SnappyCompression(),
+    BROTLI: new compression_1.BrotliCompression({ modules }),
+    // TODO: Understand difference between LZ4 and LZ4_RAW
+    LZ4: new compression_1.LZ4Compression({ modules }),
+    LZ4_RAW: new compression_1.LZ4Compression({ modules }),
+    LZO: new compression_1.LZOCompression({ modules }),
+    ZSTD: new compression_1.ZstdCompression({ modules })
+};
+/**
+ * Register compressions that have big external libraries
+ * @param options.modules External library dependencies
+ */
+async function preloadCompressions(options) {
+    const compressions = Object.values(exports.PARQUET_COMPRESSION_METHODS);
+    return await Promise.all(compressions.map((compression) => compression.preload()));
+}
+exports.preloadCompressions = preloadCompressions;
+/**
+ * Deflate a value using compression method `method`
+ */
+async function deflate(method, value) {
+    const compression = exports.PARQUET_COMPRESSION_METHODS[method];
+    if (!compression) {
+        throw new Error(`parquet: invalid compression method: ${method}`);
+    }
+    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
+    const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
+    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
+}
+exports.deflate = deflate;
+/**
+ * Inflate a value using compression method `method`
+ */
+async function decompress(method, value, size) {
+    const compression = exports.PARQUET_COMPRESSION_METHODS[method];
+    if (!compression) {
+        throw new Error(`parquet: invalid compression method: ${method}`);
+    }
+    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
+    const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
+    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
+}
+exports.decompress = decompress;
+/*
+ * Inflate a value using compression method `method`
+ */
+function inflate(method, value, size) {
+    if (!(method in exports.PARQUET_COMPRESSION_METHODS)) {
+        throw new Error(`invalid compression method: ${method}`);
+    }
+    // @ts-ignore
+    return exports.PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
+}
+exports.inflate = inflate;
+/*
+function deflate_identity(value: Buffer): Buffer {
+  return value;
+}
+
+function deflate_gzip(value: Buffer): Buffer {
+  return zlib.gzipSync(value);
+}
+
+function deflate_snappy(value: Buffer): Buffer {
+  return snappyjs.compress(value);
+}
+
+function deflate_lzo(value: Buffer): Buffer {
+  lzo = lzo || Util.load('lzo');
+  return lzo.compress(value);
+}
+
+function deflate_brotli(value: Buffer): Buffer {
+  brotli = brotli || Util.load('brotli');
+  const result = brotli.compress(value, {
+    mode: 0,
+    quality: 8,
+    lgwin: 22
+  });
+  return result ? Buffer.from(result) : Buffer.alloc(0);
+}
+
+function deflate_lz4(value: Buffer): Buffer {
+  lz4js = lz4js || Util.load('lz4js');
+  try {
+    // let result = Buffer.alloc(lz4js.encodeBound(value.length));
+    // const compressedSize = lz4.encodeBlock(value, result);
+    // // remove unnecessary bytes
+    // result = result.slice(0, compressedSize);
+    // return result;
+    return Buffer.from(lz4js.compress(value));
+  } catch (err) {
+    throw err;
+  }
+}
+function inflate_identity(value: Buffer): Buffer {
+  return value;
+}
+
+function inflate_gzip(value: Buffer): Buffer {
+  return zlib.gunzipSync(value);
+}
+
+function inflate_snappy(value: Buffer): Buffer {
+  return snappyjs.uncompress(value);
+}
+
+function inflate_lzo(value: Buffer, size: number): Buffer {
+  lzo = lzo || Util.load('lzo');
+  return lzo.decompress(value, size);
+}
+
+function inflate_lz4(value: Buffer, size: number): Buffer {
+  lz4js = lz4js || Util.load('lz4js');
+  try {
+    // let result = Buffer.alloc(size);
+    // const uncompressedSize = lz4js.decodeBlock(value, result);
+    // // remove unnecessary bytes
+    // result = result.slice(0, uncompressedSize);
+    // return result;
+    return Buffer.from(lz4js.decompress(value, size));
+  } catch (err) {
+    throw err;
+  }
+}
+
+function inflate_brotli(value: Buffer): Buffer {
+  brotli = brotli || Util.load('brotli');
+  if (!value.length) {
+    return Buffer.alloc(0);
+  }
+  return Buffer.from(brotli.decompress(value));
+}
+*/

package/dist/parquetjs/encoder/writer.d.ts

@@ -0,0 +1,123 @@
+/// <reference types="node" />
+import { Transform, Writable } from 'stream';
+import { ParquetBuffer } from '../schema/declare';
+import { ParquetSchema } from '../schema/schema';
+import { RowGroup } from '../parquet-thrift';
+export interface ParquetWriterOptions {
+    baseOffset?: number;
+    rowGroupSize?: number;
+    pageSize?: number;
+    useDataPageV2?: boolean;
+    flags?: string;
+    encoding?: string;
+    fd?: number;
+    mode?: number;
+    autoClose?: boolean;
+    start?: number;
+}
+/**
+ * Write a parquet file to an output stream. The ParquetWriter will perform
+ * buffering/batching for performance, so close() must be called after all rows
+ * are written.
+ */
+export declare class ParquetWriter<T> {
+    /**
+     * Convenience method to create a new buffered parquet writer that writes to
+     * the specified file
+     */
+    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    /**
+     * Convenience method to create a new buffered parquet writer that writes to
+     * the specified stream
+     */
+    static openStream<T>(schema: ParquetSchema, outputStream: Writable, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    schema: ParquetSchema;
+    envelopeWriter: ParquetEnvelopeWriter;
+    rowBuffer: ParquetBuffer;
+    rowGroupSize: number;
+    closed: boolean;
+    userMetadata: Record<string, string>;
+    /**
+     * Create a new buffered parquet writer for a given envelope writer
+     */
+    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetWriterOptions);
+    writeHeader(): Promise<void>;
+    /**
+     * Append a single row to the parquet file. Rows are buffered in memory until
+     * rowGroupSize rows are in the buffer or close() is called
+     */
+    appendRow<T>(row: T): Promise<void>;
+    /**
+     * Finish writing the parquet file and commit the footer to disk. This method
+     * MUST be called after you are finished adding rows. You must not call this
+     * method twice on the same object or add any rows after the close() method has
+     * been called
+     */
+    close(callback?: () => void): Promise<void>;
+    /**
+     * Add key<>value metadata to the file
+     */
+    setMetadata(key: string, value: string): void;
+    /**
+     * Set the parquet row group size. This values controls the maximum number
+     * of rows that are buffered in memory at any given time as well as the number
+     * of rows that are co-located on disk. A higher value is generally better for
+     * read-time I/O performance at the tradeoff of write-time memory usage.
+     */
+    setRowGroupSize(cnt: number): void;
+    /**
+     * Set the parquet data page size. The data page size controls the maximum
+     * number of column values that are written to disk as a consecutive array
+     */
+    setPageSize(cnt: number): void;
+}
+/**
+ * Create a parquet file from a schema and a number of row groups. This class
+ * performs direct, unbuffered writes to the underlying output stream and is
+ * intendend for advanced and internal users; the writeXXX methods must be
+ * called in the correct order to produce a valid file.
+ */
+export declare class ParquetEnvelopeWriter {
+    /**
+     * Create a new parquet envelope writer that writes to the specified stream
+     */
+    static openStream(schema: ParquetSchema, outputStream: Writable, opts: ParquetWriterOptions): Promise<ParquetEnvelopeWriter>;
+    schema: ParquetSchema;
+    write: (buf: Buffer) => Promise<void>;
+    close: () => Promise<void>;
+    offset: number;
+    rowCount: number;
+    rowGroups: RowGroup[];
+    pageSize: number;
+    useDataPageV2: boolean;
+    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetWriterOptions);
+    writeSection(buf: Buffer): Promise<void>;
+    /**
+     * Encode the parquet file header
+     */
+    writeHeader(): Promise<void>;
+    /**
+     * Encode a parquet row group. The records object should be created using the
+     * shredRecord method
+     */
+    writeRowGroup(records: ParquetBuffer): Promise<void>;
+    /**
+     * Write the parquet file footer
+     */
+    writeFooter(userMetadata: Record<string, string>): Promise<void>;
+    /**
+     * Set the parquet data page size. The data page size controls the maximum
+     * number of column values that are written to disk as a consecutive array
+     */
+    setPageSize(cnt: number): void;
+}
+/**
+ * Create a parquet transform stream
+ */
+export declare class ParquetTransformer<T> extends Transform {
+    writer: ParquetWriter<T>;
+    constructor(schema: ParquetSchema, opts?: ParquetWriterOptions);
+    _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void>;
+    _flush(callback: (val?: any) => void): Promise<void>;
+}
+//# sourceMappingURL=writer.d.ts.map

package/dist/parquetjs/encoder/writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/writer.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAG3C,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,aAAa,CAAC,CAAC;IAC1B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAK5B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAWrB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,oBAAoB;IActB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,oBAAoB;IAY5B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;GAEG;AACH,qBAAa,kBAAkB,CAAC,CAAC,CAAE,SAAQ,SAAS;IAC3C,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;gBAEpB,MAAM,EAAE,aAAa,EAAE,IAAI,GAAE,oBAAyB;IAiBlE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9E,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI;CAG3C"}