@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/dist/parquet-writer.d.ts

@@ -0,0 +1,10 @@
+import type { WriterOptions } from '@loaders.gl/loader-utils';
+/** Public options for the wasm-backed Parquet writer. */
+export type ParquetWriterOptions = WriterOptions & {
+    parquet?: {
+        wasmUrl?: string;
+    };
+};
+/** Plain-row Parquet writer that converts tables to Arrow before wasm encoding. */
+export declare const ParquetWriter: any;
+//# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAoB,MAAM,0BAA0B,CAAC;AAU/E,yDAAyD;AACzD,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,mFAAmF;AACnF,eAAO,MAAM,aAAa,KAmBqD,CAAC"}

package/dist/parquet-writer.js

@@ -0,0 +1,27 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { convertTable, deduceTableSchema } from '@loaders.gl/schema-utils';
+import { encodeArrowToParquet } from "./lib/encoders/encode-arrow-to-parquet.js";
+import { ensureGeoParquetMetadataOnArrowTable } from "./lib/geo/geospatial-metadata.js";
+import { normalizeParquetOptions } from "./lib/utils/normalize-parquet-options.js";
+import { ParquetFormat } from "./parquet-format.js";
+import { VERSION } from "./lib/constants.js";
+/** Plain-row Parquet writer that converts tables to Arrow before wasm encoding. */
+export const ParquetWriter = {
+    ...ParquetFormat,
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    options: {
+        parquet: {}
+    },
+    encode(table, options) {
+        const arrowTable = table.shape === 'arrow-table'
+            ? table
+            : convertTable({ ...table, schema: table.schema || deduceTableSchema(table) }, 'arrow-table');
+        const parquetOptions = normalizeParquetOptions(options, ParquetWriter.options.parquet);
+        return encodeArrowToParquet(ensureGeoParquetMetadataOnArrowTable(arrowTable), parquetOptions);
+    }
+};
+//# sourceMappingURL=parquet-writer.js.map

package/dist/parquet-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-writer.js","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,YAAY,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEzE,OAAO,EAAC,oBAAoB,EAAC,kDAA+C;AAC5E,OAAO,EAAC,oCAAoC,EAAC,yCAAsC;AACnF,OAAO,EAAC,uBAAuB,EAAC,iDAA8C;AAC9E,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,OAAO,EAAC,2BAAwB;AASxC,mFAAmF;AACnF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,GAAG,aAAa;IAChB,EAAE,EAAE,SAAS;IACb,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE;QACP,OAAO,EAAE,EAAE;KACZ;IACD,MAAM,CAAC,KAAY,EAAE,OAA8B;QACjD,MAAM,UAAU,GACd,KAAK,CAAC,KAAK,KAAK,aAAa;YAC3B,CAAC,CAAE,KAAoB;YACvB,CAAC,CAAE,YAAY,CACX,EAAC,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAC,EAC5D,aAAa,CACC,CAAC;QACvB,MAAM,cAAc,GAAG,uBAAuB,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACvF,OAAO,oBAAoB,CAAC,oCAAoC,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC,CAAC;IAChG,CAAC;CAC4E,CAAC"}

package/dist/parquetjs/codecs/byte-stream-split.d.ts

@@ -0,0 +1,9 @@
+import type { PrimitiveType } from "../schema/declare.js";
+import { type CursorBuffer, type ParquetCodecOptions, type ParquetValueBuffer } from "./declare.js";
+/** Returns whether a physical Parquet type supports BYTE_STREAM_SPLIT. */
+export declare function isByteStreamSplitType(type: PrimitiveType): boolean;
+/** Encodes fixed-width physical values using Parquet BYTE_STREAM_SPLIT. */
+export declare function encodeValues(type: PrimitiveType, values: any[], options: ParquetCodecOptions): Uint8Array;
+/** Decodes Parquet BYTE_STREAM_SPLIT values into an optional column destination. */
+export declare function decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, options: ParquetCodecOptions): ParquetValueBuffer;
+//# sourceMappingURL=byte-stream-split.d.ts.map

package/dist/parquetjs/codecs/byte-stream-split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"byte-stream-split.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/byte-stream-split.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AAErD,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,qBAAkB;AAcnB,0EAA0E;AAC1E,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAElE;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,OAAO,EAAE,mBAAmB,GAC3B,UAAU,CAYZ;AAED,oFAAoF;AACpF,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,mBAAmB,GAC3B,kBAAkB,CAqDpB"}

package/dist/parquetjs/codecs/byte-stream-split.js

@@ -0,0 +1,159 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { copyUint8Array, toUint8Array } from "../utils/binary-utils.js";
+import { getParquetValueOutput } from "./declare.js";
+/** Physical Parquet types accepted by BYTE_STREAM_SPLIT. */
+const BYTE_STREAM_SPLIT_TYPES = new Set([
+    'INT32',
+    'INT64',
+    'FLOAT',
+    'DOUBLE',
+    'FIXED_LEN_BYTE_ARRAY'
+]);
+/** Whether typed-array storage uses the little-endian layout required by Parquet. */
+const IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
+/** Returns whether a physical Parquet type supports BYTE_STREAM_SPLIT. */
+export function isByteStreamSplitType(type) {
+    return BYTE_STREAM_SPLIT_TYPES.has(type);
+}
+/** Encodes fixed-width physical values using Parquet BYTE_STREAM_SPLIT. */
+export function encodeValues(type, values, options) {
+    const byteWidth = getByteWidth(type, options);
+    const contiguousBytes = encodeContiguousValues(type, values, byteWidth);
+    const encodedBytes = new Uint8Array(contiguousBytes.length);
+    for (let byteIndex = 0; byteIndex < byteWidth; byteIndex++) {
+        const streamOffset = byteIndex * values.length;
+        for (let valueIndex = 0; valueIndex < values.length; valueIndex++) {
+            encodedBytes[streamOffset + valueIndex] = contiguousBytes[valueIndex * byteWidth + byteIndex];
+        }
+    }
+    return encodedBytes;
+}
+/** Decodes Parquet BYTE_STREAM_SPLIT values into an optional column destination. */
+export function decodeValues(type, cursor, count, options) {
+    const byteWidth = getByteWidth(type, options);
+    const { output, outputOffset } = getParquetValueOutput(options, count);
+    const directOutputBytes = getDirectOutputBytes(type, output, outputOffset, count, options.int64AsBigInt);
+    const contiguousBytes = decodeContiguousBytes(cursor, count, byteWidth, directOutputBytes);
+    if (directOutputBytes) {
+        return output;
+    }
+    if (type === 'FIXED_LEN_BYTE_ARRAY') {
+        for (let valueIndex = 0; valueIndex < count; valueIndex++) {
+            const value = contiguousBytes.subarray(valueIndex * byteWidth, (valueIndex + 1) * byteWidth);
+            output[outputOffset + valueIndex] = options.retainByteArrayViews
+                ? value
+                : copyUint8Array(value);
+        }
+        return output;
+    }
+    const dataView = new DataView(contiguousBytes.buffer, contiguousBytes.byteOffset, contiguousBytes.byteLength);
+    for (let valueIndex = 0; valueIndex < count; valueIndex++) {
+        const byteOffset = valueIndex * byteWidth;
+        switch (type) {
+            case 'INT32':
+                output[outputOffset + valueIndex] = dataView.getInt32(byteOffset, true);
+                break;
+            case 'INT64': {
+                const value = dataView.getBigInt64(byteOffset, true);
+                output[outputOffset + valueIndex] = options.int64AsBigInt ? value : Number(value);
+                break;
+            }
+            case 'FLOAT':
+                output[outputOffset + valueIndex] = dataView.getFloat32(byteOffset, true);
+                break;
+            case 'DOUBLE':
+                output[outputOffset + valueIndex] = dataView.getFloat64(byteOffset, true);
+                break;
+            default:
+                throw new Error(`BYTE_STREAM_SPLIT does not support ${type}`);
+        }
+    }
+    return output;
+}
+/** Returns the fixed byte width for one BYTE_STREAM_SPLIT physical type. */
+function getByteWidth(type, options) {
+    switch (type) {
+        case 'INT32':
+        case 'FLOAT':
+            return 4;
+        case 'INT64':
+        case 'DOUBLE':
+            return 8;
+        case 'FIXED_LEN_BYTE_ARRAY':
+            if (!options.typeLength || options.typeLength < 1) {
+                throw new Error('missing option: typeLength (required for BYTE_STREAM_SPLIT FIXED_LEN_BYTE_ARRAY)');
+            }
+            return options.typeLength;
+        default:
+            throw new Error(`BYTE_STREAM_SPLIT does not support ${type}`);
+    }
+}
+/** Serializes values in the ordinary little-endian physical layout before transposition. */
+function encodeContiguousValues(type, values, byteWidth) {
+    const contiguousBytes = new Uint8Array(values.length * byteWidth);
+    const dataView = new DataView(contiguousBytes.buffer);
+    for (let valueIndex = 0; valueIndex < values.length; valueIndex++) {
+        const byteOffset = valueIndex * byteWidth;
+        switch (type) {
+            case 'INT32':
+                dataView.setInt32(byteOffset, values[valueIndex], true);
+                break;
+            case 'INT64':
+                dataView.setBigInt64(byteOffset, BigInt(values[valueIndex]), true);
+                break;
+            case 'FLOAT':
+                dataView.setFloat32(byteOffset, values[valueIndex], true);
+                break;
+            case 'DOUBLE':
+                dataView.setFloat64(byteOffset, values[valueIndex], true);
+                break;
+            case 'FIXED_LEN_BYTE_ARRAY': {
+                const value = toUint8Array(values[valueIndex]);
+                if (value.byteLength !== byteWidth) {
+                    throw new Error(`invalid BYTE_STREAM_SPLIT FIXED_LEN_BYTE_ARRAY width: expected ${byteWidth}, received ${value.byteLength}`);
+                }
+                contiguousBytes.set(value, byteOffset);
+                break;
+            }
+            default:
+                throw new Error(`BYTE_STREAM_SPLIT does not support ${type}`);
+        }
+    }
+    return contiguousBytes;
+}
+/** Reassembles the encoded byte streams into ordinary contiguous physical values. */
+function decodeContiguousBytes(cursor, count, byteWidth, output) {
+    const encodedByteLength = count * byteWidth;
+    const cursorEnd = cursor.offset + encodedByteLength;
+    if (cursorEnd > (cursor.size ?? cursor.buffer.length)) {
+        throw new Error(`Invalid BYTE_STREAM_SPLIT payload: expected ${encodedByteLength} bytes, received ${Math.max(0, (cursor.size ?? cursor.buffer.length) - cursor.offset)}`);
+    }
+    const contiguousBytes = output || new Uint8Array(encodedByteLength);
+    for (let byteIndex = 0; byteIndex < byteWidth; byteIndex++) {
+        const streamOffset = cursor.offset + byteIndex * count;
+        for (let valueIndex = 0; valueIndex < count; valueIndex++) {
+            contiguousBytes[valueIndex * byteWidth + byteIndex] =
+                cursor.buffer[streamOffset + valueIndex];
+        }
+    }
+    cursor.offset = cursorEnd;
+    return contiguousBytes;
+}
+/** Returns a direct little-endian byte view for a compatible typed column destination. */
+function getDirectOutputBytes(type, output, outputOffset, count, int64AsBigInt) {
+    if (!IS_LITTLE_ENDIAN) {
+        return undefined;
+    }
+    const outputMatchesType = (type === 'INT32' && output instanceof Int32Array) ||
+        (type === 'INT64' && int64AsBigInt && output instanceof BigInt64Array) ||
+        (type === 'FLOAT' && output instanceof Float32Array) ||
+        (type === 'DOUBLE' && output instanceof Float64Array);
+    if (!outputMatchesType) {
+        return undefined;
+    }
+    const bytesPerElement = output.BYTES_PER_ELEMENT;
+    return new Uint8Array(output.buffer, output.byteOffset + outputOffset * bytesPerElement, count * bytesPerElement);
+}
+//# sourceMappingURL=byte-stream-split.js.map

package/dist/parquetjs/codecs/byte-stream-split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"byte-stream-split.js","sourceRoot":"","sources":["../../../src/parquetjs/codecs/byte-stream-split.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,cAAc,EAAE,YAAY,EAAC,iCAA8B;AACnE,OAAO,EACL,qBAAqB,EAItB,qBAAkB;AAEnB,4DAA4D;AAC5D,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAgB;IACrD,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;IACR,sBAAsB;CACvB,CAAC,CAAC;AAEH,qFAAqF;AACrF,MAAM,gBAAgB,GAAG,IAAI,UAAU,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAE9E,0EAA0E;AAC1E,MAAM,UAAU,qBAAqB,CAAC,IAAmB;IACvD,OAAO,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,YAAY,CAC1B,IAAmB,EACnB,MAAa,EACb,OAA4B;IAE5B,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACxE,MAAM,YAAY,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAE5D,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAC/C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;YAClE,YAAY,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,eAAe,CAAC,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,YAAY,CAC1B,IAAmB,EACnB,MAAoB,EACpB,KAAa,EACb,OAA4B;IAE5B,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,EAAC,MAAM,EAAE,YAAY,EAAC,GAAG,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,oBAAoB,CAC5C,IAAI,EACJ,MAAM,EACN,YAAY,EACZ,KAAK,EACL,OAAO,CAAC,aAAa,CACtB,CAAC;IACF,MAAM,eAAe,GAAG,qBAAqB,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAE3F,IAAI,iBAAiB,EAAE,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,IAAI,KAAK,sBAAsB,EAAE,CAAC;QACpC,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;YAC7F,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,oBAAoB;gBAC9D,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAC3B,eAAe,CAAC,MAAM,EACtB,eAAe,CAAC,UAAU,EAC1B,eAAe,CAAC,UAAU,CAC3B,CAAC;IACF,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;QAC1D,MAAM,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;QAC1C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACxE,MAAM;YACR,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACrD,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClF,MAAM;YACR,CAAC;YACD,KAAK,OAAO;gBACV,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBAC1E,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBAC1E,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,4EAA4E;AAC5E,SAAS,YAAY,CAAC,IAAmB,EAAE,OAA4B;IACrE,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,OAAO,CAAC;QACb,KAAK,OAAO;YACV,OAAO,CAAC,CAAC;QACX,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,CAAC,CAAC;QACX,KAAK,sBAAsB;YACzB,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CACb,kFAAkF,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,UAAU,CAAC;QAC5B;YACE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,SAAS,sBAAsB,CAAC,IAAmB,EAAE,MAAa,EAAE,SAAiB;IACnF,MAAM,eAAe,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAEtD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QAClE,MAAM,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;QAC1C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;gBACxD,MAAM;YACR,KAAK,OAAO;gBACV,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBACnE,MAAM;YACR,KAAK,OAAO;gBACV,QAAQ,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC1D,MAAM;YACR,KAAK,QAAQ;gBACX,QAAQ,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC1D,MAAM;YACR,KAAK,sBAAsB,CAAC,CAAC,CAAC;gBAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC/C,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CACb,kEAAkE,SAAS,cAAc,KAAK,CAAC,UAAU,EAAE,CAC5G,CAAC;gBACJ,CAAC;gBACD,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBACvC,MAAM;YACR,CAAC;YACD;gBACE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,qFAAqF;AACrF,SAAS,qBAAqB,CAC5B,MAAoB,EACpB,KAAa,EACb,SAAiB,EACjB,MAAmB;IAEnB,MAAM,iBAAiB,GAAG,KAAK,GAAG,SAAS,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,iBAAiB,CAAC;IACpD,IAAI,SAAS,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,+CAA+C,iBAAiB,oBAAoB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CACzJ,CAAC;IACJ,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,IAAI,IAAI,UAAU,CAAC,iBAAiB,CAAC,CAAC;IACpE,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;QACvD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;YAC1D,eAAe,CAAC,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;gBACjD,MAAM,CAAC,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,0FAA0F;AAC1F,SAAS,oBAAoB,CAC3B,IAAmB,EACnB,MAA0B,EAC1B,YAAoB,EACpB,KAAa,EACb,aAAkC;IAElC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,iBAAiB,GACrB,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,YAAY,UAAU,CAAC;QAClD,CAAC,IAAI,KAAK,OAAO,IAAI,aAAa,IAAI,MAAM,YAAY,aAAa,CAAC;QACtE,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,YAAY,YAAY,CAAC;QACpD,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,YAAY,YAAY,CAAC,CAAC;IACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,eAAe,GAAI,MAAiD,CAAC,iBAAiB,CAAC;IAC7F,OAAO,IAAI,UAAU,CACnB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,GAAG,YAAY,GAAG,eAAe,EAClD,KAAK,GAAG,eAAe,CACxB,CAAC;AACJ,CAAC"}

package/dist/parquetjs/codecs/declare.d.ts

@@ -1,16 +1,48 @@
 import { PrimitiveType } from "../schema/declare.js";
+/** Mutable destination accepted by Parquet value decoders. */
+export type ParquetValueBuffer = unknown[] | Uint8Array | Uint16Array | Uint32Array | Int32Array | BigInt64Array | Float32Array | Float64Array;
+/** Mutable Arrow-compatible destination for compact physical BYTE_ARRAY values. */
+export type ParquetByteArrayOutput = {
+    /** Contiguous bytes copied from decoded physical values. */
+    data: Uint8Array;
+    /** Arrow-compatible offsets for compact non-null physical values. */
+    valueOffsets: Int32Array;
+    /** Number of bytes written to `data`. */
+    byteLength: number;
+};
+/** Ensures a compact byte destination can accept an additional physical value. */
+export declare function reserveParquetByteArrayOutput(output: ParquetByteArrayOutput, additionalByteLength: number): void;
 export interface CursorBuffer {
-    buffer: Buffer;
+    buffer: Uint8Array;
     offset: number;
     size?: number;
 }
 export interface ParquetCodecOptions {
     bitWidth?: number;
     disableEnvelope?: boolean;
+    /** Retain byte arrays as views into the decoded page buffer. */
+    retainByteArrayViews?: boolean;
     typeLength?: number;
+    /** Optional destination that lets a codec avoid allocating a page-local values array. */
+    output?: ParquetValueBuffer;
+    /** First element in `output` written by the codec. */
+    outputOffset?: number;
+    /** Optional dictionary resolved while decoding dictionary indices. */
+    dictionary?: readonly unknown[];
+    /** Optional compact destination used by PLAIN BYTE_ARRAY decoding. */
+    byteArrayOutput?: ParquetByteArrayOutput;
+    /** Preserve decoded INT64 values as bigint instead of converting them to number. */
+    int64AsBigInt?: boolean;
+    /** Decode legacy INT96 physical values as epoch nanoseconds instead of raw numbers. */
+    int96AsTimestamp?: boolean;
 }
 export interface ParquetCodecKit {
-    encodeValues(type: PrimitiveType, values: any[], opts?: ParquetCodecOptions): Buffer;
-    decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, opts: ParquetCodecOptions): any[];
+    encodeValues(type: PrimitiveType, values: any[], opts?: ParquetCodecOptions): Uint8Array;
+    decodeValues(type: PrimitiveType, cursor: CursorBuffer, count: number, opts: ParquetCodecOptions): ParquetValueBuffer;
 }
+/** Allocates a page-local output or returns the caller-provided column destination. */
+export declare function getParquetValueOutput(options: ParquetCodecOptions, count: number): {
+    output: ParquetValueBuffer;
+    outputOffset: number;
+};
 //# sourceMappingURL=declare.d.ts.map

package/dist/parquetjs/codecs/declare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/declare.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,aAAa,EAAC,6BAA0B;AAEhD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,MAAM,CAAC;IACrF,YAAY,CACV,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,GAAG,EAAE,CAAC;CACV"}
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/declare.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,aAAa,EAAC,6BAA0B;AAEhD,8DAA8D;AAC9D,MAAM,MAAM,kBAAkB,GAC1B,OAAO,EAAE,GACT,UAAU,GACV,WAAW,GACX,WAAW,GACX,UAAU,GACV,aAAa,GACb,YAAY,GACZ,YAAY,CAAC;AAEjB,mFAAmF;AACnF,MAAM,MAAM,sBAAsB,GAAG;IACnC,4DAA4D;IAC5D,IAAI,EAAE,UAAU,CAAC;IACjB,qEAAqE;IACrE,YAAY,EAAE,UAAU,CAAC;IACzB,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,kFAAkF;AAClF,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,sBAAsB,EAC9B,oBAAoB,EAAE,MAAM,GAC3B,IAAI,CAON;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gEAAgE;IAChE,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,UAAU,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAChC,sEAAsE;IACtE,eAAe,CAAC,EAAE,sBAAsB,CAAC;IACzC,oFAAoF;IACpF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IACzF,YAAY,CACV,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,kBAAkB,CAAC;CACvB;AAED,uFAAuF;AACvF,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,EAAE,MAAM,GACZ;IAAC,MAAM,EAAE,kBAAkB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,CAKpD"}

package/dist/parquetjs/codecs/declare.js

@@ -3,5 +3,21 @@
 // Copyright (c) vis.gl contributors
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
-export {};
+/** Ensures a compact byte destination can accept an additional physical value. */
+export function reserveParquetByteArrayOutput(output, additionalByteLength) {
+    const requiredByteLength = output.byteLength + additionalByteLength;
+    if (requiredByteLength <= output.data.byteLength)
+        return;
+    const nextByteLength = Math.max(requiredByteLength, output.data.byteLength * 2, 1024);
+    const data = new Uint8Array(nextByteLength);
+    data.set(output.data.subarray(0, output.byteLength));
+    output.data = data;
+}
+/** Allocates a page-local output or returns the caller-provided column destination. */
+export function getParquetValueOutput(options, count) {
+    return {
+        output: options.output || new Array(count),
+        outputOffset: options.outputOffset || 0
+    };
+}
 //# sourceMappingURL=declare.js.map

package/dist/parquetjs/codecs/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","sourceRoot":"","sources":["../../../src/parquetjs/codecs/declare.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE"}
+{"version":3,"file":"declare.js","sourceRoot":"","sources":["../../../src/parquetjs/codecs/declare.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAyBpE,kFAAkF;AAClF,MAAM,UAAU,6BAA6B,CAC3C,MAA8B,EAC9B,oBAA4B;IAE5B,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,GAAG,oBAAoB,CAAC;IACpE,IAAI,kBAAkB,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU;QAAE,OAAO;IACzD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,cAAc,CAAC,CAAC;IAC5C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;AACrB,CAAC;AAsCD,uFAAuF;AACvF,MAAM,UAAU,qBAAqB,CACnC,OAA4B,EAC5B,KAAa;IAEb,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,KAAK,CAAU,KAAK,CAAC;QACnD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC;KACxC,CAAC;AACJ,CAAC"}

package/dist/parquetjs/codecs/delta.d.ts

@@ -0,0 +1,17 @@
+import type { PrimitiveType } from "../schema/declare.js";
+import { type CursorBuffer, type ParquetCodecOptions, type ParquetValueBuffer } from "./declare.js";
+/** Encodes INT32 or INT64 values using Parquet DELTA_BINARY_PACKED. */
+export declare function encodeDeltaBinaryPackedValues(type: PrimitiveType, values: unknown[], _options: ParquetCodecOptions): Uint8Array;
+/** Encodes BYTE_ARRAY values as delta-packed lengths followed by their contiguous payloads. */
+export declare function encodeDeltaLengthByteArrayValues(type: PrimitiveType, values: unknown[], _options: ParquetCodecOptions): Uint8Array;
+/** Encodes byte arrays as delta-packed prefix lengths and delta-length suffixes. */
+export declare function encodeDeltaByteArrayValues(type: PrimitiveType, values: unknown[], _options: ParquetCodecOptions): Uint8Array;
+/** Returns whether an encoding is valid for a physical Parquet type. */
+export declare function isDeltaEncodingType(encoding: string, type: PrimitiveType): boolean;
+/** Decodes one Parquet delta encoding based on its registered codec name. */
+export declare function decodeDeltaBinaryPackedValues(type: PrimitiveType, cursor: CursorBuffer, count: number, options: ParquetCodecOptions): ParquetValueBuffer;
+/** Decodes lengths with delta packing followed by contiguous byte-array payloads. */
+export declare function decodeDeltaLengthByteArrayValues(type: PrimitiveType, cursor: CursorBuffer, count: number, options: ParquetCodecOptions): ParquetValueBuffer;
+/** Decodes prefix lengths and delta-length suffixes into complete byte-array values. */
+export declare function decodeDeltaByteArrayValues(type: PrimitiveType, cursor: CursorBuffer, count: number, options: ParquetCodecOptions): ParquetValueBuffer;
+//# sourceMappingURL=delta.d.ts.map

package/dist/parquetjs/codecs/delta.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delta.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/delta.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AAErD,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,qBAAkB;AAQnB,uEAAuE;AACvE,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,OAAO,EAAE,EACjB,QAAQ,EAAE,mBAAmB,GAC5B,UAAU,CAoDZ;AAED,+FAA+F;AAC/F,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,OAAO,EAAE,EACjB,QAAQ,EAAE,mBAAmB,GAC5B,UAAU,CAWZ;AAED,oFAAoF;AACpF,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,OAAO,EAAE,EACjB,QAAQ,EAAE,mBAAmB,GAC5B,UAAU,CAgBZ;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAWlF;AA8ED,6EAA6E;AAC7E,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,mBAAmB,GAC3B,kBAAkB,CAkCpB;AA+fD,qFAAqF;AACrF,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,mBAAmB,GAC3B,kBAAkB,CAYpB;AAiCD,wFAAwF;AACxF,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,mBAAmB,GAC3B,kBAAkB,CAkCpB"}