@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/parquet-writer.d.ts

@@ -0,0 +1,4 @@
+import type { Writer } from '@loaders.gl/loader-utils';
+export declare type ParquetWriterOptions = {};
+export declare const ParquetWriter: Writer;
+//# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,oBAAY,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAU3B,CAAC"}

package/dist/parquet-writer.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetWriter = void 0;
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+const DEFAULT_PARQUET_LOADER_OPTIONS = {};
+exports.ParquetWriter = {
+    name: 'Apache Parquet',
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    extensions: ['parquet'],
+    mimeTypes: ['application/octet-stream'],
+    encodeSync,
+    binary: true,
+    options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+function encodeSync(data, options) {
+    return new ArrayBuffer(0);
+}

package/dist/parquetjs/codecs/declare.d.ts

@@ -0,0 +1,17 @@
+/// <reference types="node" />
+import { PrimitiveType } from '../schema/declare';
+export interface CursorBuffer {
+    buffer: Buffer;
+    offset: number;
+    size?: number;
+}
+export interface ParquetCodecOptions {
+    bitWidth?: number;
+    disableEnvelope?: boolean;
+    typeLength?: number;
+}
+export interface ParquetCodecKit {
+    encodeValues(type: PrimitiveType, values: any[], opts?: ParquetCodecOptions): Buffer;
+    decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, opts: ParquetCodecOptions): any[];
+}
+//# sourceMappingURL=declare.d.ts.map

package/dist/parquetjs/codecs/declare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/declare.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AAEhD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,MAAM,CAAC;IACrF,YAAY,CACV,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,GAAG,EAAE,CAAC;CACV"}

package/dist/parquetjs/codecs/declare.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/parquetjs/codecs/dictionary.d.ts

@@ -0,0 +1,3 @@
+export declare function decodeValues(type: any, cursor: any, count: any, opts: any): number[];
+export declare function encodeValues(type: any, cursor: any, count: any, opts: any): void;
+//# sourceMappingURL=dictionary.d.ts.map

package/dist/parquetjs/codecs/dictionary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/dictionary.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,CAAC,IAAI,KAAA,EAAE,MAAM,KAAA,EAAE,KAAK,KAAA,EAAE,IAAI,KAAA,YAIrD;AAED,wBAAgB,YAAY,CAAC,IAAI,KAAA,EAAE,MAAM,KAAA,EAAE,KAAK,KAAA,EAAE,IAAI,KAAA,QAErD"}

package/dist/parquetjs/codecs/dictionary.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encodeValues = exports.decodeValues = void 0;
+const rle_1 = require("./rle");
+function decodeValues(type, cursor, count, opts) {
+    opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+    cursor.offset += 1;
+    return (0, rle_1.decodeValues)(type, cursor, count, { ...opts, disableEnvelope: true });
+}
+exports.decodeValues = decodeValues;
+function encodeValues(type, cursor, count, opts) {
+    throw new Error('Encode dictionary functionality is not supported');
+}
+exports.encodeValues = encodeValues;

package/dist/parquetjs/codecs/index.d.ts

@@ -0,0 +1,5 @@
+import type { ParquetCodec } from '../schema/declare';
+import type { ParquetCodecKit } from './declare';
+export * from './declare';
+export declare const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit>;
+//# sourceMappingURL=index.d.ts.map

package/dist/parquetjs/codecs/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAK/C,cAAc,WAAW,CAAC;AAE1B,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,CAqBhE,CAAC"}

package/dist/parquetjs/codecs/index.js

@@ -0,0 +1,51 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PARQUET_CODECS = void 0;
+const PLAIN = __importStar(require("./plain"));
+const RLE = __importStar(require("./rle"));
+const DICTIONARY = __importStar(require("./dictionary"));
+__exportStar(require("./declare"), exports);
+exports.PARQUET_CODECS = {
+    PLAIN: {
+        encodeValues: PLAIN.encodeValues,
+        decodeValues: PLAIN.decodeValues
+    },
+    RLE: {
+        encodeValues: RLE.encodeValues,
+        decodeValues: RLE.decodeValues
+    },
+    // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.
+    PLAIN_DICTIONARY: {
+        // @ts-ignore
+        encodeValues: DICTIONARY.encodeValues,
+        decodeValues: DICTIONARY.decodeValues
+    },
+    // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.
+    RLE_DICTIONARY: {
+        // @ts-ignore
+        encodeValues: DICTIONARY.encodeValues,
+        decodeValues: DICTIONARY.decodeValues
+    }
+};

package/dist/parquetjs/codecs/plain.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="node" />
+import type { PrimitiveType } from '../schema/declare';
+import type { CursorBuffer, ParquetCodecOptions } from './declare';
+export declare function encodeValues(type: PrimitiveType, values: any[], opts: ParquetCodecOptions): Buffer;
+export declare function decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, opts: ParquetCodecOptions): any[];
+//# sourceMappingURL=plain.d.ts.map

package/dist/parquetjs/codecs/plain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plain.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/plain.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAGjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CAqBR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,GAAG,EAAE,CAqBP"}

package/dist/parquetjs/codecs/plain.js

@@ -0,0 +1,211 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeValues = exports.encodeValues = void 0;
+const int53_1 = __importDefault(require("int53"));
+function encodeValues(type, values, opts) {
+    switch (type) {
+        case 'BOOLEAN':
+            return encodeValues_BOOLEAN(values);
+        case 'INT32':
+            return encodeValues_INT32(values);
+        case 'INT64':
+            return encodeValues_INT64(values);
+        case 'INT96':
+            return encodeValues_INT96(values);
+        case 'FLOAT':
+            return encodeValues_FLOAT(values);
+        case 'DOUBLE':
+            return encodeValues_DOUBLE(values);
+        case 'BYTE_ARRAY':
+            return encodeValues_BYTE_ARRAY(values);
+        case 'FIXED_LEN_BYTE_ARRAY':
+            return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
+}
+exports.encodeValues = encodeValues;
+function decodeValues(type, cursor, count, opts) {
+    switch (type) {
+        case 'BOOLEAN':
+            return decodeValues_BOOLEAN(cursor, count);
+        case 'INT32':
+            return decodeValues_INT32(cursor, count);
+        case 'INT64':
+            return decodeValues_INT64(cursor, count);
+        case 'INT96':
+            return decodeValues_INT96(cursor, count);
+        case 'FLOAT':
+            return decodeValues_FLOAT(cursor, count);
+        case 'DOUBLE':
+            return decodeValues_DOUBLE(cursor, count);
+        case 'BYTE_ARRAY':
+            return decodeValues_BYTE_ARRAY(cursor, count);
+        case 'FIXED_LEN_BYTE_ARRAY':
+            return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
+}
+exports.decodeValues = decodeValues;
+function encodeValues_BOOLEAN(values) {
+    const buf = Buffer.alloc(Math.ceil(values.length / 8));
+    buf.fill(0);
+    for (let i = 0; i < values.length; i++) {
+        if (values[i]) {
+            buf[Math.floor(i / 8)] |= 1 << i % 8;
+        }
+    }
+    return buf;
+}
+function decodeValues_BOOLEAN(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
+        values.push((b & (1 << i % 8)) > 0);
+    }
+    cursor.offset += Math.ceil(count / 8);
+    return values;
+}
+function encodeValues_INT32(values) {
+    const buf = Buffer.alloc(4 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeInt32LE(values[i], i * 4);
+    }
+    return buf;
+}
+function decodeValues_INT32(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readInt32LE(cursor.offset));
+        cursor.offset += 4;
+    }
+    return values;
+}
+function encodeValues_INT64(values) {
+    const buf = Buffer.alloc(8 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        int53_1.default.writeInt64LE(values[i], buf, i * 8);
+    }
+    return buf;
+}
+function decodeValues_INT64(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(int53_1.default.readInt64LE(cursor.buffer, cursor.offset));
+        cursor.offset += 8;
+    }
+    return values;
+}
+function encodeValues_INT96(values) {
+    const buf = Buffer.alloc(12 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        if (values[i] >= 0) {
+            int53_1.default.writeInt64LE(values[i], buf, i * 12);
+            buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
+        }
+        else {
+            int53_1.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
+            buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
+        }
+    }
+    return buf;
+}
+function decodeValues_INT96(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const low = int53_1.default.readInt64LE(cursor.buffer, cursor.offset);
+        const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
+        if (high === 0xffffffff) {
+            values.push(~-low + 1); // truncate to 64 actual precision
+        }
+        else {
+            values.push(low); // truncate to 64 actual precision
+        }
+        cursor.offset += 12;
+    }
+    return values;
+}
+function encodeValues_FLOAT(values) {
+    const buf = Buffer.alloc(4 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeFloatLE(values[i], i * 4);
+    }
+    return buf;
+}
+function decodeValues_FLOAT(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readFloatLE(cursor.offset));
+        cursor.offset += 4;
+    }
+    return values;
+}
+function encodeValues_DOUBLE(values) {
+    const buf = Buffer.alloc(8 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeDoubleLE(values[i], i * 8);
+    }
+    return buf;
+}
+function decodeValues_DOUBLE(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readDoubleLE(cursor.offset));
+        cursor.offset += 8;
+    }
+    return values;
+}
+function encodeValues_BYTE_ARRAY(values) {
+    // tslint:disable-next-line:variable-name
+    let buf_len = 0;
+    for (let i = 0; i < values.length; i++) {
+        values[i] = Buffer.from(values[i]);
+        buf_len += 4 + values[i].length;
+    }
+    const buf = Buffer.alloc(buf_len);
+    // tslint:disable-next-line:variable-name
+    let buf_pos = 0;
+    for (let i = 0; i < values.length; i++) {
+        buf.writeUInt32LE(values[i].length, buf_pos);
+        values[i].copy(buf, buf_pos + 4);
+        buf_pos += 4 + values[i].length;
+    }
+    return buf;
+}
+function decodeValues_BYTE_ARRAY(cursor, count) {
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const len = cursor.buffer.readUInt32LE(cursor.offset);
+        cursor.offset += 4;
+        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
+        cursor.offset += len;
+    }
+    return values;
+}
+function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
+    if (!opts.typeLength) {
+        throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
+    }
+    for (let i = 0; i < values.length; i++) {
+        values[i] = Buffer.from(values[i]);
+        if (values[i].length !== opts.typeLength) {
+            throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
+        }
+    }
+    return Buffer.concat(values);
+}
+function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
+    const values = [];
+    if (!opts.typeLength) {
+        throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
+    }
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
+        cursor.offset += opts.typeLength;
+    }
+    return values;
+}

package/dist/parquetjs/codecs/rle.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="node" />
+import type { PrimitiveType } from '../schema/declare';
+import type { CursorBuffer, ParquetCodecOptions } from './declare';
+export declare function encodeValues(type: PrimitiveType, values: any[], opts: ParquetCodecOptions): Buffer;
+export declare function decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, opts: ParquetCodecOptions): number[];
+//# sourceMappingURL=rle.d.ts.map

package/dist/parquetjs/codecs/rle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA0DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CA4BV"}

package/dist/parquetjs/codecs/rle.js

@@ -0,0 +1,145 @@
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeValues = exports.encodeValues = void 0;
+const varint_1 = __importDefault(require("varint"));
+// eslint-disable-next-line max-statements, complexity
+function encodeValues(type, values, opts) {
+    if (!('bitWidth' in opts)) {
+        throw new Error('bitWidth is required');
+    }
+    switch (type) {
+        case 'BOOLEAN':
+        case 'INT32':
+        case 'INT64':
+            // tslint:disable-next-line:no-parameter-reassignment
+            values = values.map((x) => parseInt(x, 10));
+            break;
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
+    let buf = Buffer.alloc(0);
+    let run = [];
+    let repeats = 0;
+    for (let i = 0; i < values.length; i++) {
+        // If we are at the beginning of a run and the next value is same we start
+        // collecting repeated values
+        if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
+            // If we have any data in runs we need to encode them
+            if (run.length) {
+                buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+                run = [];
+            }
+            repeats = 1;
+        }
+        else if (repeats > 0 && values[i] === values[i - 1]) {
+            repeats += 1;
+        }
+        else {
+            // If values changes we need to post any previous repeated values
+            if (repeats) {
+                buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
+                repeats = 0;
+            }
+            run.push(values[i]);
+        }
+    }
+    if (repeats) {
+        buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
+    }
+    else if (run.length) {
+        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+    }
+    if (opts.disableEnvelope) {
+        return buf;
+    }
+    const envelope = Buffer.alloc(buf.length + 4);
+    envelope.writeUInt32LE(buf.length, undefined);
+    buf.copy(envelope, 4);
+    return envelope;
+}
+exports.encodeValues = encodeValues;
+function decodeValues(type, cursor, count, opts) {
+    if (!('bitWidth' in opts)) {
+        throw new Error('bitWidth is required');
+    }
+    if (!opts.disableEnvelope) {
+        cursor.offset += 4;
+    }
+    let values = [];
+    while (values.length < count) {
+        const header = varint_1.default.decode(cursor.buffer, cursor.offset);
+        cursor.offset += varint_1.default.encodingLength(header);
+        if (header & 1) {
+            const count = (header >> 1) * 8;
+            values.push(...decodeRunBitpacked(cursor, count, opts));
+        }
+        else {
+            const count = header >> 1;
+            values.push(...decodeRunRepeated(cursor, count, opts));
+        }
+    }
+    values = values.slice(0, count);
+    if (values.length !== count) {
+        throw new Error('invalid RLE encoding');
+    }
+    return values;
+}
+exports.decodeValues = decodeValues;
+function decodeRunBitpacked(cursor, count, opts) {
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    if (count % 8 !== 0) {
+        throw new Error('must be a multiple of 8');
+    }
+    // tslint:disable-next-line:prefer-array-literal
+    const values = new Array(count).fill(0);
+    for (let b = 0; b < bitWidth * count; b++) {
+        if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {
+            values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
+        }
+    }
+    cursor.offset += bitWidth * (count / 8);
+    return values;
+}
+function decodeRunRepeated(cursor, count, opts) {
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    let value = 0;
+    for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
+        // eslint-disable-next-line
+        value << 8; //  TODO - this looks wrong
+        value += cursor.buffer[cursor.offset];
+        cursor.offset += 1;
+    }
+    // tslint:disable-next-line:prefer-array-literal
+    return new Array(count).fill(value);
+}
+function encodeRunBitpacked(values, opts) {
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    for (let i = 0; i < values.length % 8; i++) {
+        values.push(0);
+    }
+    const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
+    for (let b = 0; b < bitWidth * values.length; b++) {
+        if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {
+            buf[Math.floor(b / 8)] |= 1 << b % 8;
+        }
+    }
+    return Buffer.concat([Buffer.from(varint_1.default.encode(((values.length / 8) << 1) | 1)), buf]);
+}
+function encodeRunRepeated(value, count, opts) {
+    // @ts-ignore
+    const bitWidth = opts.bitWidth;
+    const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
+    for (let i = 0; i < buf.length; i++) {
+        buf.writeUInt8(value & 0xff, i);
+        // eslint-disable-next-line
+        value >> 8; //  TODO - this looks wrong
+    }
+    return Buffer.concat([Buffer.from(varint_1.default.encode(count << 1)), buf]);
+}

package/dist/parquetjs/compression.d.ts

@@ -0,0 +1,23 @@
+/// <reference types="node" />
+import { Compression } from '@loaders.gl/compression';
+import { ParquetCompression } from './schema/declare';
+export declare const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression>;
+/**
+ * Register compressions that have big external libraries
+ * @param options.modules External library dependencies
+ */
+export declare function preloadCompressions(options?: {
+    modules: {
+        [key: string]: any;
+    };
+}): Promise<void[]>;
+/**
+ * Deflate a value using compression method `method`
+ */
+export declare function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer>;
+/**
+ * Inflate a value using compression method `method`
+ */
+export declare function decompress(method: ParquetCompression, value: Buffer, size: number): Promise<Buffer>;
+export declare function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer;
+//# sourceMappingURL=compression.d.ts.map

package/dist/parquetjs/compression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AA0BpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}