@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.7

package/dist/parquetjs/codecs/rle.js

@@ -1,151 +1,145 @@
-import varint from 'varint';
-export function encodeValues(type, values, opts) {
-  if (!('bitWidth' in opts)) {
-    throw new Error('bitWidth is required');
-  }
-
-  switch (type) {
-    case 'BOOLEAN':
-    case 'INT32':
-    case 'INT64':
-      values = values.map(x => parseInt(x, 10));
-      break;
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
-
-  let buf = Buffer.alloc(0);
-  let run = [];
-  let repeats = 0;
-
-  for (let i = 0; i < values.length; i++) {
-    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
-      if (run.length) {
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeValues = exports.encodeValues = void 0;
+const varint_1 = __importDefault(require("varint"));
+// eslint-disable-next-line max-statements, complexity
+function encodeValues(type, values, opts) {
+    if (!('bitWidth' in opts)) {
+        throw new Error('bitWidth is required');
+    }
+    switch (type) {
+        case 'BOOLEAN':
+        case 'INT32':
+        case 'INT64':
+            // tslint:disable-next-line:no-parameter-reassignment
+            values = values.map((x) => parseInt(x, 10));
+            break;
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
+    let buf = Buffer.alloc(0);
+    let run = [];
+    let repeats = 0;
+    for (let i = 0; i < values.length; i++) {
+        // If we are at the beginning of a run and the next value is same we start
+        // collecting repeated values
+        if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
+            // If we have any data in runs we need to encode them
+            if (run.length) {
+                buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+                run = [];
+            }
+            repeats = 1;
+        }
+        else if (repeats > 0 && values[i] === values[i - 1]) {
+            repeats += 1;
+        }
+        else {
+            // If values changes we need to post any previous repeated values
+            if (repeats) {
+                buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
+                repeats = 0;
+            }
+            run.push(values[i]);
+        }
+    }
+    if (repeats) {
+        buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
+    }
+    else if (run.length) {
         buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
-        run = [];
-      }
-
-      repeats = 1;
-    } else if (repeats > 0 && values[i] === values[i - 1]) {
-      repeats += 1;
-    } else {
-      if (repeats) {
-        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
-        repeats = 0;
-      }
-
-      run.push(values[i]);
-    }
-  }
-
-  if (repeats) {
-    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
-  } else if (run.length) {
-    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
-  }
-
-  if (opts.disableEnvelope) {
-    return buf;
-  }
-
-  const envelope = Buffer.alloc(buf.length + 4);
-  envelope.writeUInt32LE(buf.length, undefined);
-  buf.copy(envelope, 4);
-  return envelope;
+    }
+    if (opts.disableEnvelope) {
+        return buf;
+    }
+    const envelope = Buffer.alloc(buf.length + 4);
+    envelope.writeUInt32LE(buf.length, undefined);
+    buf.copy(envelope, 4);
+    return envelope;
 }
-export function decodeValues(type, cursor, count, opts) {
-  if (!('bitWidth' in opts)) {
-    throw new Error('bitWidth is required');
-  }
-
-  if (!opts.disableEnvelope) {
-    cursor.offset += 4;
-  }
-
-  let values = [];
-
-  while (values.length < count) {
-    const header = varint.decode(cursor.buffer, cursor.offset);
-    cursor.offset += varint.encodingLength(header);
-
-    if (header & 1) {
-      const count = (header >> 1) * 8;
-      values.push(...decodeRunBitpacked(cursor, count, opts));
-    } else {
-      const count = header >> 1;
-      values.push(...decodeRunRepeated(cursor, count, opts));
-    }
-  }
-
-  values = values.slice(0, count);
-
-  if (values.length !== count) {
-    throw new Error('invalid RLE encoding');
-  }
-
-  return values;
+exports.encodeValues = encodeValues;
+function decodeValues(type, cursor, count, opts) {
+    if (!('bitWidth' in opts)) {
+        throw new Error('bitWidth is required');
+    }
+    if (!opts.disableEnvelope) {
+        cursor.offset += 4;
+    }
+    let values = [];
+    while (values.length < count) {
+        const header = varint_1.default.decode(cursor.buffer, cursor.offset);
+        cursor.offset += varint_1.default.encodingLength(header);
+        if (header & 1) {
+            const count = (header >> 1) * 8;
+            values.push(...decodeRunBitpacked(cursor, count, opts));
+        }
+        else {
+            const count = header >> 1;
+            values.push(...decodeRunRepeated(cursor, count, opts));
+        }
+    }
+    values = values.slice(0, count);
+    if (values.length !== count) {
+        throw new Error('invalid RLE encoding');
+    }
+    return values;
 }
-
+exports.decodeValues = decodeValues;
 function decodeRunBitpacked(cursor, count, opts) {
-  const bitWidth = opts.bitWidth;
-
-  if (count % 8 !== 0) {
-    throw new Error('must be a multiple of 8');
-  }
-
-  const values = new Array(count).fill(0);
-
-  for (let b = 0; b < bitWidth * count; b++) {
-    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
-      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
-    }
-  }
-
-  cursor.offset += bitWidth * (count / 8);
-  return values;
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    if (count % 8 !== 0) {
+        throw new Error('must be a multiple of 8');
+    }
+    // tslint:disable-next-line:prefer-array-literal
+    const values = new Array(count).fill(0);
+    for (let b = 0; b < bitWidth * count; b++) {
+        if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {
+            values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
+        }
+    }
+    cursor.offset += bitWidth * (count / 8);
+    return values;
 }
-
 function decodeRunRepeated(cursor, count, opts) {
-  const bitWidth = opts.bitWidth;
-  let value = 0;
-
-  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
-    value << 8;
-    value += cursor.buffer[cursor.offset];
-    cursor.offset += 1;
-  }
-
-  return new Array(count).fill(value);
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    let value = 0;
+    for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
+        // eslint-disable-next-line
+        value << 8; //  TODO - this looks wrong
+        value += cursor.buffer[cursor.offset];
+        cursor.offset += 1;
+    }
+    // tslint:disable-next-line:prefer-array-literal
+    return new Array(count).fill(value);
 }
-
 function encodeRunBitpacked(values, opts) {
-  const bitWidth = opts.bitWidth;
-
-  for (let i = 0; i < values.length % 8; i++) {
-    values.push(0);
-  }
-
-  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
-
-  for (let b = 0; b < bitWidth * values.length; b++) {
-    if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
-      buf[Math.floor(b / 8)] |= 1 << b % 8;
-    }
-  }
-
-  return Buffer.concat([Buffer.from(varint.encode(values.length / 8 << 1 | 1)), buf]);
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    for (let i = 0; i < values.length % 8; i++) {
+        values.push(0);
+    }
+    const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
+    for (let b = 0; b < bitWidth * values.length; b++) {
+        if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {
+            buf[Math.floor(b / 8)] |= 1 << b % 8;
+        }
+    }
+    return Buffer.concat([Buffer.from(varint_1.default.encode(((values.length / 8) << 1) | 1)), buf]);
 }
-
 function encodeRunRepeated(value, count, opts) {
-  const bitWidth = opts.bitWidth;
-  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
-
-  for (let i = 0; i < buf.length; i++) {
-    buf.writeUInt8(value & 0xff, i);
-    value >> 8;
-  }
-
-  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
+    for (let i = 0; i < buf.length; i++) {
+        buf.writeUInt8(value & 0xff, i);
+        // eslint-disable-next-line
+        value >> 8; //  TODO - this looks wrong
+    }
+    return Buffer.concat([Buffer.from(varint_1.default.encode(count << 1)), buf]);
 }
-//# sourceMappingURL=rle.js.map

package/dist/parquetjs/compression.d.ts

@@ -1,6 +1,9 @@
 /// <reference types="node" />
 import { Compression } from '@loaders.gl/compression';
 import { ParquetCompression } from './schema/declare';
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
 export declare const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression>;
 /**
  * Register compressions that have big external libraries

package/dist/parquetjs/compression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AA0BpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAsCpD;;GAEG;AAEH,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAW/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}

package/dist/parquetjs/compression.js

@@ -1,62 +1,183 @@
-import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, LZOCompression, LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
-import { toArrayBuffer, toBuffer } from './utils/buffer-utils';
-import lz4js from 'lz4js';
-import lzo from 'lzo';
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+/* eslint-disable camelcase */
+// Forked from https://github.com/ironSource/parquetjs under MIT license
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inflate = exports.decompress = exports.deflate = exports.preloadCompressions = exports.PARQUET_COMPRESSION_METHODS = void 0;
+const compression_1 = require("@loaders.gl/compression");
+/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
+function toBuffer(arrayBuffer) {
+    return Buffer.from(arrayBuffer);
+}
+function toArrayBuffer(buffer) {
+    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+    if (Buffer.isBuffer(buffer)) {
+        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+        return typedArray.slice().buffer;
+    }
+    return buffer;
+}
+// TODO switch to worker compression to avoid bundling...
+// import brotli from 'brotli'; - brotli has problems with decompress in browsers
+// import brotliDecompress from 'brotli/decompress';
+const lz4js_1 = __importDefault(require("lz4js"));
+// import lzo from 'lzo';
+// import {ZstdCodec} from 'zstd-codec';
+// Inject large dependencies through Compression constructor options
 const modules = {
-  lz4js,
-  lzo
+    // brotli has problems with decompress in browsers
+    // brotli: {
+    //   decompress: brotliDecompress,
+    //   compress: () => {
+    //     throw new Error('brotli compress');
+    //   }
+    // },
+    lz4js: lz4js_1.default
+    // lzo
+    // 'zstd-codec': ZstdCodec
 };
-export const PARQUET_COMPRESSION_METHODS = {
-  UNCOMPRESSED: new NoCompression(),
-  GZIP: new GZipCompression(),
-  SNAPPY: new SnappyCompression(),
-  BROTLI: new BrotliCompression({
-    modules
-  }),
-  LZ4: new LZ4Compression({
-    modules
-  }),
-  LZ4_RAW: new LZ4Compression({
-    modules
-  }),
-  LZO: new LZOCompression({
-    modules
-  }),
-  ZSTD: new ZstdCompression({
-    modules
-  })
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
+// @ts-expect-error
+exports.PARQUET_COMPRESSION_METHODS = {
+    UNCOMPRESSED: new compression_1.NoCompression(),
+    GZIP: new compression_1.GZipCompression(),
+    SNAPPY: new compression_1.SnappyCompression(),
+    BROTLI: new compression_1.BrotliCompression({ modules }),
+    // TODO: Understand difference between LZ4 and LZ4_RAW
+    LZ4: new compression_1.LZ4Compression({ modules }),
+    LZ4_RAW: new compression_1.LZ4Compression({ modules }),
+    //
+    // LZO: new LZOCompression({modules}),
+    ZSTD: new compression_1.ZstdCompression({ modules })
 };
-export async function preloadCompressions(options) {
-  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-  return await Promise.all(compressions.map(compression => compression.preload()));
+/**
+ * Register compressions that have big external libraries
+ * @param options.modules External library dependencies
+ */
+async function preloadCompressions(options) {
+    const compressions = Object.values(exports.PARQUET_COMPRESSION_METHODS);
+    return await Promise.all(compressions.map((compression) => compression.preload()));
+}
+exports.preloadCompressions = preloadCompressions;
+/**
+ * Deflate a value using compression method `method`
+ */
+async function deflate(method, value) {
+    const compression = exports.PARQUET_COMPRESSION_METHODS[method];
+    if (!compression) {
+        throw new Error(`parquet: invalid compression method: ${method}`);
+    }
+    const inputArrayBuffer = toArrayBuffer(value);
+    const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
+    return toBuffer(compressedArrayBuffer);
+}
+exports.deflate = deflate;
+/**
+ * Inflate a value using compression method `method`
+ */
+async function decompress(method, value, size) {
+    const compression = exports.PARQUET_COMPRESSION_METHODS[method];
+    if (!compression) {
+        throw new Error(`parquet: invalid compression method: ${method}`);
+    }
+    const inputArrayBuffer = toArrayBuffer(value);
+    const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
+    return toBuffer(compressedArrayBuffer);
+}
+exports.decompress = decompress;
+/*
+ * Inflate a value using compression method `method`
+ */
+function inflate(method, value, size) {
+    if (!(method in exports.PARQUET_COMPRESSION_METHODS)) {
+        throw new Error(`invalid compression method: ${method}`);
+    }
+    // @ts-ignore
+    return exports.PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
+}
+exports.inflate = inflate;
+/*
+function deflate_identity(value: Buffer): Buffer {
+  return value;
 }
-export async function deflate(method, value) {
-  const compression = PARQUET_COMPRESSION_METHODS[method];
 
-  if (!compression) {
-    throw new Error("parquet: invalid compression method: ".concat(method));
-  }
+function deflate_gzip(value: Buffer): Buffer {
+  return zlib.gzipSync(value);
+}
+
+function deflate_snappy(value: Buffer): Buffer {
+  return snappyjs.compress(value);
+}
 
-  const inputArrayBuffer = toArrayBuffer(value);
-  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
-  return toBuffer(compressedArrayBuffer);
+function deflate_lzo(value: Buffer): Buffer {
+  lzo = lzo || Util.load('lzo');
+  return lzo.compress(value);
 }
-export async function decompress(method, value, size) {
-  const compression = PARQUET_COMPRESSION_METHODS[method];
 
-  if (!compression) {
-    throw new Error("parquet: invalid compression method: ".concat(method));
+function deflate_brotli(value: Buffer): Buffer {
+  brotli = brotli || Util.load('brotli');
+  const result = brotli.compress(value, {
+    mode: 0,
+    quality: 8,
+    lgwin: 22
+  });
+  return result ? Buffer.from(result) : Buffer.alloc(0);
+}
+
+function deflate_lz4(value: Buffer): Buffer {
+  lz4js = lz4js || Util.load('lz4js');
+  try {
+    // let result = Buffer.alloc(lz4js.encodeBound(value.length));
+    // const compressedSize = lz4.encodeBlock(value, result);
+    // // remove unnecessary bytes
+    // result = result.slice(0, compressedSize);
+    // return result;
+    return Buffer.from(lz4js.compress(value));
+  } catch (err) {
+    throw err;
   }
+}
+function inflate_identity(value: Buffer): Buffer {
+  return value;
+}
+
+function inflate_gzip(value: Buffer): Buffer {
+  return zlib.gunzipSync(value);
+}
 
-  const inputArrayBuffer = toArrayBuffer(value);
-  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
-  return toBuffer(compressedArrayBuffer);
+function inflate_snappy(value: Buffer): Buffer {
+  return snappyjs.uncompress(value);
 }
-export function inflate(method, value, size) {
-  if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw new Error("invalid compression method: ".concat(method));
+
+function inflate_lzo(value: Buffer, size: number): Buffer {
+  lzo = lzo || Util.load('lzo');
+  return lzo.decompress(value, size);
+}
+
+function inflate_lz4(value: Buffer, size: number): Buffer {
+  lz4js = lz4js || Util.load('lz4js');
+  try {
+    // let result = Buffer.alloc(size);
+    // const uncompressedSize = lz4js.decodeBlock(value, result);
+    // // remove unnecessary bytes
+    // result = result.slice(0, uncompressedSize);
+    // return result;
+    return Buffer.from(lz4js.decompress(value, size));
+  } catch (err) {
+    throw err;
   }
+}
 
-  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
+function inflate_brotli(value: Buffer): Buffer {
+  brotli = brotli || Util.load('brotli');
+  if (!value.length) {
+    return Buffer.alloc(0);
+  }
+  return Buffer.from(brotli.decompress(value));
 }
-//# sourceMappingURL=compression.js.map
+*/

package/dist/parquetjs/encoder/{writer.d.ts → parquet-encoder.d.ts}

@@ -1,9 +1,10 @@
 /// <reference types="node" />
-import { Transform, Writable } from 'stream';
-import { ParquetBuffer } from '../schema/declare';
+/// <reference types="node" />
+import { stream } from '@loaders.gl/loader-utils';
+import { ParquetRowGroup, ParquetRow } from '../schema/declare';
 import { ParquetSchema } from '../schema/schema';
 import { RowGroup } from '../parquet-thrift';
-export interface ParquetWriterOptions {
+export interface ParquetEncoderOptions {
     baseOffset?: number;
     rowGroupSize?: number;
     pageSize?: number;
@@ -16,37 +17,37 @@ export interface ParquetWriterOptions {
     start?: number;
 }
 /**
- * Write a parquet file to an output stream. The ParquetWriter will perform
+ * Write a parquet file to an output stream. The ParquetEncoder will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
-export declare class ParquetWriter<T> {
+export declare class ParquetEncoder<T> {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
      */
-    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified stream
      */
-    static openStream<T>(schema: ParquetSchema, outputStream: Writable, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    static openStream<T>(schema: ParquetSchema, outputStream: stream.Writable, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
     schema: ParquetSchema;
     envelopeWriter: ParquetEnvelopeWriter;
-    rowBuffer: ParquetBuffer;
+    rowBuffer: ParquetRowGroup;
     rowGroupSize: number;
     closed: boolean;
     userMetadata: Record<string, string>;
     /**
      * Create a new buffered parquet writer for a given envelope writer
      */
-    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetWriterOptions);
+    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetEncoderOptions);
     writeHeader(): Promise<void>;
     /**
      * Append a single row to the parquet file. Rows are buffered in memory until
      * rowGroupSize rows are in the buffer or close() is called
      */
-    appendRow<T>(row: T): Promise<void>;
+    appendRow<T extends ParquetRow>(row: T): Promise<void>;
     /**
      * Finish writing the parquet file and commit the footer to disk. This method
      * MUST be called after you are finished adding rows. You must not call this
@@ -81,7 +82,7 @@ export declare class ParquetEnvelopeWriter {
     /**
      * Create a new parquet envelope writer that writes to the specified stream
      */
-    static openStream(schema: ParquetSchema, outputStream: Writable, opts: ParquetWriterOptions): Promise<ParquetEnvelopeWriter>;
+    static openStream(schema: ParquetSchema, outputStream: stream.Writable, opts: ParquetEncoderOptions): Promise<ParquetEnvelopeWriter>;
     schema: ParquetSchema;
     write: (buf: Buffer) => Promise<void>;
     close: () => Promise<void>;
@@ -90,7 +91,7 @@ export declare class ParquetEnvelopeWriter {
     rowGroups: RowGroup[];
     pageSize: number;
     useDataPageV2: boolean;
-    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetWriterOptions);
+    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetEncoderOptions);
     writeSection(buf: Buffer): Promise<void>;
     /**
      * Encode the parquet file header
@@ -100,7 +101,7 @@ export declare class ParquetEnvelopeWriter {
      * Encode a parquet row group. The records object should be created using the
      * shredRecord method
      */
-    writeRowGroup(records: ParquetBuffer): Promise<void>;
+    writeRowGroup(records: ParquetRowGroup): Promise<void>;
     /**
      * Write the parquet file footer
      */
@@ -111,13 +112,4 @@ export declare class ParquetEnvelopeWriter {
      */
     setPageSize(cnt: number): void;
 }
-/**
- * Create a parquet transform stream
- */
-export declare class ParquetTransformer<T> extends Transform {
-    writer: ParquetWriter<T>;
-    constructor(schema: ParquetSchema, opts?: ParquetWriterOptions);
-    _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void>;
-    _flush(callback: (val?: any) => void): Promise<void>;
-}
-//# sourceMappingURL=writer.d.ts.map
+//# sourceMappingURL=parquet-encoder.d.ts.map

package/dist/parquetjs/encoder/parquet-encoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,eAAe,EAKf,UAAU,EACX,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,CAAC;IAC3B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,eAAe,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAcvB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAY7B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAY5D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}