@loaders.gl/parquet 4.2.0-alpha.4 → 4.2.0-alpha.5

package/dist/parquetjs/parser/decoders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,YAAY,EAAsC,MAAM,WAAW,CAAC;AAC5E,OAAO,EAML,aAAa,EAEd,MAAM,mBAAmB,CAAC;AAK3B;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAmD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}
+{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAmD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}

package/dist/parquetjs/parser/decoders.js

@@ -3,267 +3,331 @@ import { ConvertedType, Encoding, FieldRepetitionType, PageType, Type } from "..
 import { decompress } from "../compression.js";
 import { PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING } from "../../lib/constants.js";
 import { decodePageHeader, getThriftEnum, getBitWidth } from "../utils/read-utils.js";
+/**
+ * Decode data pages
+ * @param buffer - input data
+ * @param column - parquet column
+ * @param compression - compression type
+ * @returns parquet data page data
+ */
 export async function decodeDataPages(buffer, context) {
-  const cursor = {
-    buffer,
-    offset: 0,
-    size: buffer.length
-  };
-  const data = {
-    rlevels: [],
-    dlevels: [],
-    values: [],
-    pageHeaders: [],
-    count: 0
-  };
-  let dictionary = context.dictionary || [];
-  while (cursor.offset < cursor.size && (!context.numValues || data.dlevels.length < Number(context.numValues))) {
-    const page = await decodePage(cursor, context);
-    if (page.dictionary) {
-      dictionary = page.dictionary;
-      continue;
-    }
-    if (dictionary.length) {
-      page.values = page.values.map(value => dictionary[value]);
-    }
-    for (let index = 0; index < page.rlevels.length; index++) {
-      data.rlevels.push(page.rlevels[index]);
-      data.dlevels.push(page.dlevels[index]);
-      const value = page.values[index];
-      if (value !== undefined) {
-        data.values.push(value);
-      }
+    const cursor = {
+        buffer,
+        offset: 0,
+        size: buffer.length
+    };
+    const data = {
+        rlevels: [],
+        dlevels: [],
+        values: [],
+        pageHeaders: [],
+        count: 0
+    };
+    let dictionary = context.dictionary || [];
+    while (
+    // @ts-ignore size can be undefined
+    cursor.offset < cursor.size &&
+        (!context.numValues || data.dlevels.length < Number(context.numValues))) {
+        // Looks like we have to decode these in sequence due to cursor updates?
+        const page = await decodePage(cursor, context);
+        if (page.dictionary) {
+            dictionary = page.dictionary;
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        if (dictionary.length) {
+            // eslint-disable-next-line no-loop-func
+            page.values = page.values.map((value) => dictionary[value]);
+        }
+        for (let index = 0; index < page.rlevels.length; index++) {
+            data.rlevels.push(page.rlevels[index]);
+            data.dlevels.push(page.dlevels[index]);
+            const value = page.values[index];
+            if (value !== undefined) {
+                data.values.push(value);
+            }
+        }
+        data.count += page.count;
+        data.pageHeaders.push(page.pageHeader);
     }
-    data.count += page.count;
-    data.pageHeaders.push(page.pageHeader);
-  }
-  return data;
+    return data;
 }
+/**
+ * Decode parquet page based on page type
+ * @param cursor
+ * @param context
+ */
 export async function decodePage(cursor, context) {
-  let page;
-  const {
-    pageHeader,
-    length
-  } = decodePageHeader(cursor.buffer, cursor.offset);
-  cursor.offset += length;
-  const pageType = getThriftEnum(PageType, pageHeader.type);
-  switch (pageType) {
-    case 'DATA_PAGE':
-      page = await decodeDataPage(cursor, pageHeader, context);
-      break;
-    case 'DATA_PAGE_V2':
-      page = await decodeDataPageV2(cursor, pageHeader, context);
-      break;
-    case 'DICTIONARY_PAGE':
-      page = {
-        dictionary: await decodeDictionaryPage(cursor, pageHeader, context),
-        pageHeader
-      };
-      break;
-    default:
-      throw new Error(`invalid page type: ${pageType}`);
-  }
-  return page;
+    let page;
+    const { pageHeader, length } = decodePageHeader(cursor.buffer, cursor.offset);
+    cursor.offset += length;
+    const pageType = getThriftEnum(PageType, pageHeader.type);
+    switch (pageType) {
+        case 'DATA_PAGE':
+            page = await decodeDataPage(cursor, pageHeader, context);
+            break;
+        case 'DATA_PAGE_V2':
+            page = await decodeDataPageV2(cursor, pageHeader, context);
+            break;
+        case 'DICTIONARY_PAGE':
+            page = {
+                dictionary: await decodeDictionaryPage(cursor, pageHeader, context),
+                pageHeader
+            };
+            break;
+        default:
+            throw new Error(`invalid page type: ${pageType}`);
+    }
+    return page;
 }
+/**
+ * Decode parquet schema
+ * @param schemaElements input schema elements data
+ * @param offset offset to read from
+ * @param len length of data
+ * @returns result.offset
+ *   result.next - offset at the end of function
+ *   result.schema - schema read from the input data
+ * @todo output offset is the same as input - possibly excess output field
+ */
 export function decodeSchema(schemaElements, offset, len) {
-  const schema = {};
-  let next = offset;
-  for (let i = 0; i < len; i++) {
-    const schemaElement = schemaElements[next];
-    const repetitionType = next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
-    let optional = false;
-    let repeated = false;
-    switch (repetitionType) {
-      case 'REQUIRED':
-        break;
-      case 'OPTIONAL':
-        optional = true;
-        break;
-      case 'REPEATED':
-        repeated = true;
-        break;
-      default:
-        throw new Error('parquet: unknown repetition type');
-    }
-    if (schemaElement.num_children > 0) {
-      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
-      next = res.next;
-      schema[schemaElement.name] = {
-        optional,
-        repeated,
-        fields: res.schema
-      };
-    } else {
-      const type = getThriftEnum(Type, schemaElement.type);
-      let logicalType = type;
-      if (schemaElement.converted_type) {
-        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);
-      }
-      switch (logicalType) {
-        case 'DECIMAL':
-          logicalType = `${logicalType}_${type}`;
-          break;
-        default:
-      }
-      schema[schemaElement.name] = {
-        type: logicalType,
-        typeLength: schemaElement.type_length,
-        presision: schemaElement.precision,
-        scale: schemaElement.scale,
-        optional,
-        repeated
-      };
-      next++;
+    const schema = {};
+    let next = offset;
+    for (let i = 0; i < len; i++) {
+        const schemaElement = schemaElements[next];
+        const repetitionType = next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
+        let optional = false;
+        let repeated = false;
+        switch (repetitionType) {
+            case 'REQUIRED':
+                break;
+            case 'OPTIONAL':
+                optional = true;
+                break;
+            case 'REPEATED':
+                repeated = true;
+                break;
+            default:
+                throw new Error('parquet: unknown repetition type');
+        }
+        if (schemaElement.num_children > 0) {
+            const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
+            next = res.next;
+            schema[schemaElement.name] = {
+                // type: undefined,
+                optional,
+                repeated,
+                fields: res.schema
+            };
+        }
+        else {
+            const type = getThriftEnum(Type, schemaElement.type);
+            let logicalType = type;
+            if (schemaElement.converted_type) {
+                logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);
+            }
+            switch (logicalType) {
+                case 'DECIMAL':
+                    logicalType = `${logicalType}_${type}`;
+                    break;
+                default:
+            }
+            schema[schemaElement.name] = {
+                type: logicalType,
+                typeLength: schemaElement.type_length,
+                presision: schemaElement.precision,
+                scale: schemaElement.scale,
+                optional,
+                repeated
+            };
+            next++;
+        }
     }
-  }
-  return {
-    schema,
-    offset,
-    next
-  };
+    return { schema, offset, next };
 }
+/**
+ * Decode a consecutive array of data using one of the parquet encodings
+ */
 function decodeValues(type, encoding, cursor, count, opts) {
-  if (!(encoding in PARQUET_CODECS)) {
-    throw new Error(`invalid encoding: ${encoding}`);
-  }
-  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
+    if (!(encoding in PARQUET_CODECS)) {
+        throw new Error(`invalid encoding: ${encoding}`);
+    }
+    return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
 }
+/**
+ * Do decoding of parquet dataPage from column chunk
+ * @param cursor
+ * @param header
+ * @param options
+ */
 async function decodeDataPage(cursor, header, context) {
-  var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
-  const cursorEnd = cursor.offset + header.compressed_page_size;
-  const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
-  let dataCursor = cursor;
-  if (context.compression !== 'UNCOMPRESSED') {
-    const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
-    dataCursor = {
-      buffer: valuesBuf,
-      offset: 0,
-      size: valuesBuf.length
-    };
-    cursor.offset = cursorEnd;
-  }
-  const rLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
-  let rLevels = new Array(valueCount);
-  if (context.column.rLevelMax > 0) {
-    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
-      bitWidth: getBitWidth(context.column.rLevelMax),
-      disableEnvelope: false
-    });
-  } else {
-    rLevels.fill(0);
-  }
-  const dLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
-  let dLevels = new Array(valueCount);
-  if (context.column.dLevelMax > 0) {
-    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
-      bitWidth: getBitWidth(context.column.dLevelMax),
-      disableEnvelope: false
-    });
-  } else {
-    dLevels.fill(0);
-  }
-  let valueCountNonNull = 0;
-  for (const dlvl of dLevels) {
-    if (dlvl === context.column.dLevelMax) {
-      valueCountNonNull++;
+    const cursorEnd = cursor.offset + header.compressed_page_size;
+    const valueCount = header.data_page_header?.num_values;
+    /* uncompress page */
+    let dataCursor = cursor;
+    if (context.compression !== 'UNCOMPRESSED') {
+        const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
+        dataCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    /* read repetition levels */
+    const rLevelEncoding = getThriftEnum(Encoding, header.data_page_header?.repetition_level_encoding);
+    // tslint:disable-next-line:prefer-array-literal
+    let rLevels = new Array(valueCount);
+    if (context.column.rLevelMax > 0) {
+        rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
+            bitWidth: getBitWidth(context.column.rLevelMax),
+            disableEnvelope: false
+            // column: opts.column
+        });
+    }
+    else {
+        rLevels.fill(0);
+    }
+    /* read definition levels */
+    const dLevelEncoding = getThriftEnum(Encoding, header.data_page_header?.definition_level_encoding);
+    // tslint:disable-next-line:prefer-array-literal
+    let dLevels = new Array(valueCount);
+    if (context.column.dLevelMax > 0) {
+        dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
+            bitWidth: getBitWidth(context.column.dLevelMax),
+            disableEnvelope: false
+            // column: opts.column
+        });
+    }
+    else {
+        dLevels.fill(0);
     }
-  }
-  const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
-  const decodeOptions = {
-    typeLength: context.column.typeLength,
-    bitWidth: context.column.typeLength
-  };
-  const values = decodeValues(context.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
-  return {
-    dlevels: dLevels,
-    rlevels: rLevels,
-    values,
-    count: valueCount,
-    pageHeader: header
-  };
+    let valueCountNonNull = 0;
+    for (const dlvl of dLevels) {
+        if (dlvl === context.column.dLevelMax) {
+            valueCountNonNull++;
+        }
+    }
+    /* read values */
+    const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding);
+    const decodeOptions = {
+        typeLength: context.column.typeLength,
+        bitWidth: context.column.typeLength
+    };
+    const values = decodeValues(context.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
+    return {
+        dlevels: dLevels,
+        rlevels: rLevels,
+        values,
+        count: valueCount,
+        pageHeader: header
+    };
 }
+/**
+ * Do decoding of parquet dataPage in version 2 from column chunk
+ * @param cursor
+ * @param header
+ * @param opts
+ * @returns
+ */
 async function decodeDataPageV2(cursor, header, context) {
-  var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
-  const cursorEnd = cursor.offset + header.compressed_page_size;
-  const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
-  const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
-  const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
-  let rLevels = new Array(valueCount);
-  if (context.column.rLevelMax > 0) {
-    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
-      bitWidth: getBitWidth(context.column.rLevelMax),
-      disableEnvelope: true
-    });
-  } else {
-    rLevels.fill(0);
-  }
-  let dLevels = new Array(valueCount);
-  if (context.column.dLevelMax > 0) {
-    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
-      bitWidth: getBitWidth(context.column.dLevelMax),
-      disableEnvelope: true
-    });
-  } else {
-    dLevels.fill(0);
-  }
-  let valuesBufCursor = cursor;
-  if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
-    const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
-    valuesBufCursor = {
-      buffer: valuesBuf,
-      offset: 0,
-      size: valuesBuf.length
+    const cursorEnd = cursor.offset + header.compressed_page_size;
+    const valueCount = header.data_page_header_v2?.num_values;
+    // @ts-ignore
+    const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;
+    const valueEncoding = getThriftEnum(Encoding, header.data_page_header_v2?.encoding);
+    /* read repetition levels */
+    // tslint:disable-next-line:prefer-array-literal
+    let rLevels = new Array(valueCount);
+    if (context.column.rLevelMax > 0) {
+        rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
+            bitWidth: getBitWidth(context.column.rLevelMax),
+            disableEnvelope: true
+        });
+    }
+    else {
+        rLevels.fill(0);
+    }
+    /* read definition levels */
+    // tslint:disable-next-line:prefer-array-literal
+    let dLevels = new Array(valueCount);
+    if (context.column.dLevelMax > 0) {
+        dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
+            bitWidth: getBitWidth(context.column.dLevelMax),
+            disableEnvelope: true
+        });
+    }
+    else {
+        dLevels.fill(0);
+    }
+    /* read values */
+    let valuesBufCursor = cursor;
+    if (header.data_page_header_v2?.is_compressed) {
+        const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
+        valuesBufCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    const decodeOptions = {
+        typeLength: context.column.typeLength,
+        bitWidth: context.column.typeLength
+    };
+    const values = decodeValues(context.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
+    return {
+        dlevels: dLevels,
+        rlevels: rLevels,
+        values,
+        count: valueCount,
+        pageHeader: header
     };
-    cursor.offset = cursorEnd;
-  }
-  const decodeOptions = {
-    typeLength: context.column.typeLength,
-    bitWidth: context.column.typeLength
-  };
-  const values = decodeValues(context.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
-  return {
-    dlevels: dLevels,
-    rlevels: rLevels,
-    values,
-    count: valueCount,
-    pageHeader: header
-  };
 }
+/**
+ * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.
+ * @param cursor
+ * @param pageHeader
+ * @param context
+ */
 async function decodeDictionaryPage(cursor, pageHeader, context) {
-  var _pageHeader$dictionar;
-  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
-  let dictCursor = {
-    offset: 0,
-    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
-    size: cursorEnd - cursor.offset
-  };
-  cursor.offset = cursorEnd;
-  if (context.compression !== 'UNCOMPRESSED') {
-    const valuesBuf = await decompress(context.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
-    dictCursor = {
-      buffer: valuesBuf,
-      offset: 0,
-      size: valuesBuf.length
+    const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
+    let dictCursor = {
+        offset: 0,
+        buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
+        size: cursorEnd - cursor.offset
     };
     cursor.offset = cursorEnd;
-  }
-  const numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
-  const decodedDictionaryValues = decodeValues(context.column.primitiveType, context.column.encoding, dictCursor, numValues, context);
-  let values;
-  if (context !== null && context !== void 0 && context.preserveBinary) {
-    values = decodedDictionaryValues.map(d => preserveBinary(d));
-  } else {
-    values = decodedDictionaryValues.map(d => d.toString());
-  }
-  return values;
+    if (context.compression !== 'UNCOMPRESSED') {
+        const valuesBuf = await decompress(context.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
+        dictCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
+    const decodedDictionaryValues = decodeValues(context.column.primitiveType, context.column.encoding, dictCursor, numValues, 
+    // TODO - this looks wrong?
+    context);
+    // Makes it look a little easier
+    let values;
+    if (context?.preserveBinary) {
+        values = decodedDictionaryValues.map((d) => preserveBinary(d));
+    }
+    else {
+        values = decodedDictionaryValues.map((d) => d.toString());
+    }
+    return values;
 }
 function preserveBinary(d) {
-  if (ArrayBuffer.isView(d)) {
-    return d;
-  }
-  if (Buffer.isBuffer(d)) {
-    return d.buffer.slice(d.byteOffset, d.byteLength);
-  }
-  return d.toString();
+    if (ArrayBuffer.isView(d)) {
+        return d;
+    }
+    // Convert to ArrayBuffer
+    if (Buffer.isBuffer(d)) {
+        return d.buffer.slice(d.byteOffset, d.byteLength);
+    }
+    return d.toString();
 }
-//# sourceMappingURL=decoders.js.map

package/dist/parquetjs/parser/parquet-reader.d.ts

@@ -1,7 +1,7 @@
 import type { ReadableFile } from '@loaders.gl/loader-utils';
-import { ParquetSchema } from '../schema/schema';
-import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
-import { ParquetRowGroup, ParquetColumnChunk, ParquetReaderContext } from '../schema/declare';
+import { ParquetSchema } from "../schema/schema.js";
+import { ColumnChunk, FileMetaData, RowGroup } from "../parquet-thrift/index.js";
+import { ParquetRowGroup, ParquetColumnChunk, ParquetReaderContext } from "../schema/declare.js";
 export type ParquetReaderProps = {
     defaultDictionarySize?: number;
     preserveBinary?: boolean;

package/dist/parquetjs/parser/parquet-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAG/C;IAEF,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAqB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAcjC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAyBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC;IAgB3B;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuDhG;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;CAwBrB"}
+{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,mCAAgC;AACpG,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EACrB,6BAA0B;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAG/C;IAEF,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAqB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAcjC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAyBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC;IAgB3B;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuDhG;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;CAwBrB"}