@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/dist/lib/parquet-predicate.js

@@ -0,0 +1,300 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { encodeUtf8 } from "../parquetjs/utils/binary-utils.js";
+/** Returns the unique top-level columns referenced by a predicate. */
+export function getParquetPredicateColumns(predicate) {
+    const columns = new Set();
+    visitParquetPredicate(predicate, child => {
+        const property = getPredicateProperty(child);
+        if (property) {
+            columns.add(property.property);
+        }
+    });
+    return [...columns];
+}
+/** Copies a predicate tree and mutable scalar values for read-scoped option snapshots. */
+export function copyParquetPredicate(predicate) {
+    if (isParquetLogicalPredicate(predicate)) {
+        return {
+            op: predicate.op,
+            args: predicate.args.map(copyParquetPredicate)
+        };
+    }
+    if (isParquetNotPredicate(predicate)) {
+        return {
+            op: 'not',
+            args: [copyParquetPredicate(predicate.args[0])]
+        };
+    }
+    const property = copyPredicateProperty(predicate.args[0]);
+    if (predicate.op === 'isNull') {
+        return { op: 'isNull', args: [property] };
+    }
+    if (predicate.op === 'in') {
+        return {
+            op: 'in',
+            args: [property, predicate.args[1].map(copyPredicateValue)]
+        };
+    }
+    return {
+        op: predicate.op,
+        args: [property, copyPredicateValue(predicate.args[1])]
+    };
+}
+/** Validates a predicate and every referenced top-level column. */
+export function validateParquetPredicate(predicate, availableColumns) {
+    visitParquetPredicate(predicate, child => {
+        if (isParquetLogicalPredicate(child)) {
+            if (child.args.length < 2) {
+                throw new Error(`Parquet predicate ${child.op} requires at least two child predicates`);
+            }
+            return;
+        }
+        if (isParquetNotPredicate(child)) {
+            return;
+        }
+        const property = child.args[0];
+        if (!availableColumns.has(property.property)) {
+            throw new Error(`Parquet predicate column not found: ${property.property}`);
+        }
+        if (child.op === 'in' && child.args[1].length === 0) {
+            throw new Error('Parquet predicate in requires at least one value');
+        }
+    });
+}
+/** Conservatively determines whether a row group can contain a matching row. */
+export function canParquetRowGroupMatch(predicate, rowGroup) {
+    if (isParquetLogicalPredicate(predicate)) {
+        return predicate.op === 'and'
+            ? predicate.args.every(child => canParquetRowGroupMatch(child, rowGroup))
+            : predicate.args.some(child => canParquetRowGroupMatch(child, rowGroup));
+    }
+    if (isParquetNotPredicate(predicate)) {
+        const child = predicate.args[0];
+        if (child.op !== 'isNull') {
+            return true;
+        }
+        const column = getRowGroupColumn(rowGroup, child.args[0].property);
+        return !column?.statistics || column.statistics.nullCount !== rowGroup.rowCount;
+    }
+    const column = getRowGroupColumn(rowGroup, predicate.args[0].property);
+    const statistics = column?.statistics;
+    if (!statistics) {
+        return true;
+    }
+    return canParquetStatisticsMatch(predicate, statistics, rowGroup.rowCount);
+}
+/** Conservatively determines whether one predicate leaf can match supplied statistics. */
+export function canParquetStatisticsMatch(predicate, statistics, rowCount) {
+    if (statistics.nullCount === rowCount) {
+        return predicate.op === 'isNull';
+    }
+    if (predicate.op === 'isNull') {
+        return statistics.nullCount !== 0;
+    }
+    const minimum = statistics.minIsExact === false ? undefined : statistics.min;
+    const maximum = statistics.maxIsExact === false ? undefined : statistics.max;
+    if (predicate.op === 'in') {
+        return predicate.args[1].some(value => canComparisonMatch('=', value, minimum, maximum));
+    }
+    return canComparisonMatch(predicate.op, predicate.args[1], minimum, maximum);
+}
+/** Returns exact source row indexes matching a predicate. */
+export function filterParquetRowIndices(predicate, columns, rowCount) {
+    if (!predicate) {
+        return Array.from({ length: rowCount }, (_, index) => index);
+    }
+    const rowIndices = [];
+    for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+        if (evaluateParquetPredicate(predicate, columns, rowIndex) === true) {
+            rowIndices.push(rowIndex);
+        }
+    }
+    return rowIndices;
+}
+/** Gathers selected source rows without constructing row objects. */
+export function gatherParquetColumns(columns, rowIndices, selectedColumns) {
+    const gatheredColumns = {};
+    for (const [name, column] of Object.entries(columns)) {
+        if (selectedColumns && !selectedColumns.has(name)) {
+            continue;
+        }
+        gatheredColumns[name] = rowIndices.map(rowIndex => getColumnValue(column, rowIndex));
+    }
+    return gatheredColumns;
+}
+/** Applies an exact predicate to one materialized row. */
+function evaluateParquetPredicate(predicate, columns, rowIndex) {
+    if (isParquetLogicalPredicate(predicate)) {
+        const results = predicate.args.map(child => evaluateParquetPredicate(child, columns, rowIndex));
+        if (predicate.op === 'and') {
+            return results.includes(false) ? false : results.includes(null) ? null : true;
+        }
+        return results.includes(true) ? true : results.includes(null) ? null : false;
+    }
+    if (isParquetNotPredicate(predicate)) {
+        const result = evaluateParquetPredicate(predicate.args[0], columns, rowIndex);
+        return result === null ? null : !result;
+    }
+    const value = getColumnValue(columns[predicate.args[0].property], rowIndex);
+    if (predicate.op === 'isNull') {
+        return value === null || value === undefined;
+    }
+    if (value === null || value === undefined) {
+        return null;
+    }
+    if (predicate.op === 'in') {
+        return predicate.args[1].some(candidate => comparePredicateValues(value, candidate) === 0);
+    }
+    const comparison = comparePredicateValues(value, predicate.args[1]);
+    switch (predicate.op) {
+        case '=':
+            return comparison === 0;
+        case '<>':
+            return comparison !== 0;
+        case '<':
+            return comparison < 0;
+        case '<=':
+            return comparison <= 0;
+        case '>':
+            return comparison > 0;
+        case '>=':
+            return comparison >= 0;
+    }
+}
+/** Returns whether footer min/max leave any possible match for one comparison. */
+function canComparisonMatch(operator, value, minimum, maximum) {
+    try {
+        switch (operator) {
+            case '=':
+                return !((minimum !== undefined && compareStatisticsValues(value, minimum) < 0) ||
+                    (maximum !== undefined && compareStatisticsValues(value, maximum) > 0));
+            case '<>':
+                return !(minimum !== undefined &&
+                    maximum !== undefined &&
+                    compareStatisticsValues(value, minimum) === 0 &&
+                    compareStatisticsValues(value, maximum) === 0);
+            case '<':
+                return minimum === undefined || compareStatisticsValues(minimum, value) < 0;
+            case '<=':
+                return minimum === undefined || compareStatisticsValues(minimum, value) <= 0;
+            case '>':
+                return maximum === undefined || compareStatisticsValues(maximum, value) > 0;
+            case '>=':
+                return maximum === undefined || compareStatisticsValues(maximum, value) >= 0;
+        }
+    }
+    catch {
+        return true;
+    }
+}
+/** Compares statistics using Parquet's unsigned UTF-8 byte ordering for strings. */
+function compareStatisticsValues(left, right) {
+    if (typeof left === 'string' && typeof right === 'string') {
+        return compareUint8Arrays(encodeUtf8(left), encodeUtf8(right));
+    }
+    return comparePredicateValues(left, right);
+}
+/** Compares predicate-compatible scalar values and rejects incomparable types. */
+function comparePredicateValues(left, right) {
+    if (left instanceof Uint8Array && right instanceof Uint8Array) {
+        return compareUint8Arrays(left, right);
+    }
+    if (left instanceof Uint8Array || right instanceof Uint8Array) {
+        throw new Error('Parquet binary predicates require binary values on both sides');
+    }
+    if (left instanceof Date || right instanceof Date) {
+        if (!(left instanceof Date) || !(right instanceof Date)) {
+            throw new Error('Parquet date predicates require date values on both sides');
+        }
+        const leftTime = left.getTime();
+        const rightTime = right.getTime();
+        if (!Number.isFinite(leftTime) || !Number.isFinite(rightTime)) {
+            throw new Error('Invalid dates cannot be ordered by a Parquet predicate');
+        }
+        return leftTime < rightTime ? -1 : leftTime > rightTime ? 1 : 0;
+    }
+    const leftType = typeof left;
+    const rightType = typeof right;
+    const bothNumeric = (leftType === 'number' || leftType === 'bigint') &&
+        (rightType === 'number' || rightType === 'bigint');
+    if (!bothNumeric && leftType !== rightType) {
+        throw new Error(`Parquet predicate cannot compare ${leftType} with ${rightType}`);
+    }
+    if (!bothNumeric && leftType !== 'string' && leftType !== 'boolean') {
+        throw new Error(`Parquet predicate does not support ${leftType} values`);
+    }
+    const comparableLeft = left;
+    const comparableRight = right;
+    if ((typeof comparableLeft === 'number' && !Number.isFinite(comparableLeft)) ||
+        (typeof comparableRight === 'number' && !Number.isFinite(comparableRight))) {
+        throw new Error('Non-finite numbers cannot be ordered by a Parquet predicate');
+    }
+    if (comparableLeft < comparableRight) {
+        return -1;
+    }
+    if (comparableLeft > comparableRight) {
+        return 1;
+    }
+    return 0;
+}
+/** Compares two byte sequences using unsigned lexicographic ordering. */
+function compareUint8Arrays(left, right) {
+    const length = Math.min(left.length, right.length);
+    for (let index = 0; index < length; index++) {
+        if (left[index] !== right[index]) {
+            return left[index] < right[index] ? -1 : 1;
+        }
+    }
+    return Math.sign(left.length - right.length);
+}
+/** Reads one value from an array-like materialized Parquet column. */
+function getColumnValue(column, rowIndex) {
+    if (!column) {
+        throw new Error('Parquet predicate column was not decoded');
+    }
+    return column[rowIndex];
+}
+/** Copies mutable binary and date predicate values. */
+function copyPredicateValue(value) {
+    if (value instanceof Uint8Array) {
+        return value.slice();
+    }
+    if (value instanceof Date) {
+        return new Date(value.getTime());
+    }
+    return value;
+}
+/** Copies a predicate property reference. */
+function copyPredicateProperty(property) {
+    return { property: property.property };
+}
+/** Returns the property reference from a leaf predicate. */
+function getPredicateProperty(predicate) {
+    return isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)
+        ? undefined
+        : predicate.args[0];
+}
+/** Returns one top-level column chunk from normalized row-group metadata. */
+function getRowGroupColumn(rowGroup, property) {
+    return rowGroup.columns.find(columnChunk => columnChunk.path.length === 1 && columnChunk.path[0] === property);
+}
+/** Visits every predicate node depth-first. */
+function visitParquetPredicate(predicate, visit) {
+    visit(predicate);
+    if (isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)) {
+        for (const child of predicate.args) {
+            visitParquetPredicate(child, visit);
+        }
+    }
+}
+/** Returns whether a predicate combines multiple child predicates. */
+function isParquetLogicalPredicate(predicate) {
+    return predicate.op === 'and' || predicate.op === 'or';
+}
+/** Returns whether a predicate negates one child predicate. */
+function isParquetNotPredicate(predicate) {
+    return predicate.op === 'not';
+}
+//# sourceMappingURL=parquet-predicate.js.map

package/dist/lib/parquet-predicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-predicate.js","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAepC,OAAO,EAAC,UAAU,EAAC,2CAAwC;AAE3D,sEAAsE;AACtE,MAAM,UAAU,0BAA0B,CAAC,SAA2B;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,oBAAoB,CAAC,SAA2B;IAC9D,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC;SAC/C,CAAC;IACJ,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,CAAC,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;SAChD,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAC,CAAC;IAC1C,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SAC5D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,wBAAwB,CACtC,SAA2B,EAC3B,gBAAqC;IAErC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,IAAI,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,CAAC,EAAE,yCAAyC,CAAC,CAAC;YAC1F,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,uBAAuB,CACrC,SAA2B,EAC3B,QAAiC;IAEjC,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK;YAC3B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,KAAK,QAAQ,CAAC,QAAQ,CAAC;IAClF,CAAC;IAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,yBAAyB,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,SAAiF,EACjF,UAAwC,EACxC,QAAgB;IAEhB,IAAI,UAAU,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC;IACnC,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,UAAU,CAAC,SAAS,KAAK,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,uBAAuB,CACrC,SAAuC,EACvC,OAAkC,EAClC,QAAgB;IAEhB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;QACvD,IAAI,wBAAwB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YACpE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,oBAAoB,CAClC,OAAkC,EAClC,UAA6B,EAC7B,eAAqC;IAErC,MAAM,eAAe,GAA8B,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QACD,eAAe,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,0DAA0D;AAC1D,SAAS,wBAAwB,CAC/B,SAA2B,EAC3B,OAAkC,EAClC,QAAgB;IAEhB,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChG,IAAI,SAAS,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5E,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,QAAQ,SAAS,CAAC,EAAE,EAAE,CAAC;QACrB,KAAK,GAAG;YACN,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,IAAI;YACP,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;QACzB,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,kFAAkF;AAClF,SAAS,kBAAkB,CACzB,QAA8C,EAC9C,KAA4B,EAC5B,OAAgB,EAChB,OAAgB;IAEhB,IAAI,CAAC;QACH,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,GAAG;gBACN,OAAO,CAAC,CACN,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtE,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;YACJ,KAAK,IAAI;gBACP,OAAO,CAAC,CACN,OAAO,KAAK,SAAS;oBACrB,OAAO,KAAK,SAAS;oBACrB,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;oBAC7C,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAC9C,CAAC;YACJ,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/E,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,uBAAuB,CAAC,IAAa,EAAE,KAAc;IAC5D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,sBAAsB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAED,kFAAkF;AAClF,SAAS,sBAAsB,CAAC,IAAa,EAAE,KAAc;IAC3D,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,IAAI,YAAY,IAAI,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC;IAC/B,MAAM,WAAW,GACf,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;QAChD,CAAC,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,SAAS,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,SAAS,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,cAAc,GAAG,IAA0C,CAAC;IAClE,MAAM,eAAe,GAAG,KAA2C,CAAC;IACpE,IACE,CAAC,OAAO,cAAc,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACxE,CAAC,OAAO,eAAe,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,EAC1E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC,CAAC;IACZ,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yEAAyE;AACzE,SAAS,kBAAkB,CAAC,IAAgB,EAAE,KAAiB;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc,CAAC,MAA6B,EAAE,QAAgB;IACrE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,OAAQ,MAA6B,CAAC,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED,uDAAuD;AACvD,SAAS,kBAAkB,CAAC,KAA4B;IACtD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6CAA6C;AAC7C,SAAS,qBAAqB,CAAC,QAA4B;IACzD,OAAO,EAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAC,CAAC;AACvC,CAAC;AAED,4DAA4D;AAC5D,SAAS,oBAAoB,CAAC,SAA2B;IACvD,OAAO,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAC7E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED,6EAA6E;AAC7E,SAAS,iBAAiB,CAAC,QAAiC,EAAE,QAAgB;IAC5E,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAC1B,WAAW,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,CACjF,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,SAAS,qBAAqB,CAC5B,SAA2B,EAC3B,KAA4C;IAE5C,KAAK,CAAC,SAAS,CAAC,CAAC;IACjB,IAAI,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,SAAS,yBAAyB,CAChC,SAA2B;IAE3B,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,qBAAqB,CAAC,SAA2B;IACxD,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC;AAChC,CAAC"}

package/dist/lib/parquet-source-worker-client.d.ts

@@ -0,0 +1,6 @@
+import type { ParquetSourceWorkerInput, ParquetSourceWorkerOptions, ParquetSourceWorkerResult } from "./parquet-source-worker-types.js";
+/** Returns whether this runtime and option set can decode Parquet source rows on a worker. */
+export declare function canDecodeParquetSourceOnWorker(options: ParquetSourceWorkerOptions): boolean;
+/** Sends selected compressed Parquet chunks to a worker and returns transferable Arrow buffers. */
+export declare function decodeParquetSourceRowGroupOnWorker(input: ParquetSourceWorkerInput, options: ParquetSourceWorkerOptions): Promise<ParquetSourceWorkerResult>;
+//# sourceMappingURL=parquet-source-worker-client.d.ts.map

package/dist/lib/parquet-source-worker-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-client.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-client.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EACV,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EAC1B,yCAAsC;AAUvC,8FAA8F;AAC9F,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAE3F;AAED,mGAAmG;AACnG,wBAAsB,mCAAmC,CACvD,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,yBAAyB,CAAC,CAMpC"}

package/dist/lib/parquet-source-worker-client.js

@@ -0,0 +1,22 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { canParseWithWorker, isBrowser, parseWithWorker } from '@loaders.gl/loader-utils';
+import { PARQUET_LOADER_BASE } from "../parquet-loader-base.js";
+import { PARQUET_SOURCE_WORKER_URL } from "../parquet-source-worker-url.js";
+/** Private worker descriptor using the package-local selective source worker bundle. */
+const PARQUET_SOURCE_WORKER = {
+    ...PARQUET_LOADER_BASE,
+    id: 'parquet-source',
+    name: 'ParquetSource',
+    worker: PARQUET_SOURCE_WORKER_URL
+};
+/** Returns whether this runtime and option set can decode Parquet source rows on a worker. */
+export function canDecodeParquetSourceOnWorker(options) {
+    return isBrowser && canParseWithWorker(PARQUET_SOURCE_WORKER, options);
+}
+/** Sends selected compressed Parquet chunks to a worker and returns transferable Arrow buffers. */
+export async function decodeParquetSourceRowGroupOnWorker(input, options) {
+    return (await parseWithWorker(PARQUET_SOURCE_WORKER, input, options));
+}
+//# sourceMappingURL=parquet-source-worker-client.js.map

package/dist/lib/parquet-source-worker-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-client.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-client.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,kBAAkB,EAAE,SAAS,EAAE,eAAe,EAAC,MAAM,0BAA0B,CAAC;AAExF,OAAO,EAAC,mBAAmB,EAAC,kCAA+B;AAC3D,OAAO,EAAC,yBAAyB,EAAC,wCAAqC;AAOvE,wFAAwF;AACxF,MAAM,qBAAqB,GAAW;IACpC,GAAG,mBAAmB;IACtB,EAAE,EAAE,gBAAgB;IACpB,IAAI,EAAE,eAAe;IACrB,MAAM,EAAE,yBAAyB;CAClC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,UAAU,8BAA8B,CAAC,OAAmC;IAChF,OAAO,SAAS,IAAI,kBAAkB,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;AACzE,CAAC;AAED,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,KAA+B,EAC/B,OAAmC;IAEnC,OAAO,CAAC,MAAM,eAAe,CAC3B,qBAAqB,EACrB,KAAK,EACL,OAAO,CACR,CAA8B,CAAC;AAClC,CAAC"}

package/dist/lib/parquet-source-worker-decoder.d.ts

@@ -0,0 +1,4 @@
+import type { ParquetSourceWorkerInput, ParquetSourceWorkerResult } from "./parquet-source-worker-types.js";
+/** Decodes one transferred row group entirely inside the worker. */
+export declare function decodeParquetSourceWorkerInput(input: ParquetSourceWorkerInput): Promise<ParquetSourceWorkerResult>;
+//# sourceMappingURL=parquet-source-worker-decoder.d.ts.map

package/dist/lib/parquet-source-worker-decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-decoder.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAEV,wBAAwB,EACxB,yBAAyB,EAC1B,yCAAsC;AAEvC,oEAAoE;AACpE,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,yBAAyB,CAAC,CAwEpC"}

package/dist/lib/parquet-source-worker-decoder.js

@@ -0,0 +1,115 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { dehydrateArrowTable } from '@loaders.gl/arrow/transport';
+import { convertTable } from '@loaders.gl/schema-utils';
+import { preloadCompressions } from "../parquetjs/compression.js";
+import { filterParquetRowIndices, gatherParquetColumns } from "./parquet-predicate.js";
+import { ParquetReader } from "../parquetjs/parser/parquet-reader.js";
+import { ParquetSchema } from "../parquetjs/schema/schema.js";
+import { ParquetSourceWorkerFile } from "./parquet-source-worker-file.js";
+/** Decodes one transferred row group entirely inside the worker. */
+export async function decodeParquetSourceWorkerInput(input) {
+    await preloadCompressions();
+    const file = new ParquetSourceWorkerFile(input.fileByteLength, input.ranges);
+    const schema = new ParquetSchema(input.schemaDefinition);
+    const reader = new ParquetReader(file, { preserveBinary: input.preserveBinary });
+    const rowGroup = createWorkerRowGroup(input);
+    const decodeStartTime = getCurrentTime();
+    const decodedRowGroups = input.pagePlan
+        ? await Promise.all(input.pagePlan.rowRanges.map(rowRange => reader.readRowGroupRange(schema, rowGroup, [], rowRange, input.pagePlan.pageLocations)))
+        : [await reader.readRowGroup(schema, rowGroup, [])];
+    const columns = concatenateMaterializedColumns(decodedRowGroups.map(decodedRowGroup => schema.materializeColumns(decodedRowGroup)));
+    const sourceRowIndices = input.pagePlan
+        ? input.pagePlan.rowRanges.flatMap(rowRange => Array.from({ length: rowRange.end - rowRange.start }, (_, index) => rowRange.start + index))
+        : undefined;
+    const sourceRowCount = sourceRowIndices?.length ?? input.rowCount;
+    const localRowIndices = input.predicate
+        ? filterParquetRowIndices(input.predicate, columns, sourceRowCount)
+        : undefined;
+    const rowIndices = localRowIndices?.map(rowIndex => sourceRowIndices?.[rowIndex] ?? rowIndex);
+    const decodeDurationMs = getCurrentTime() - decodeStartTime;
+    const conversionStartTime = getCurrentTime();
+    const batches = [];
+    const outputColumns = new Set(input.projectedSchema.fields.map(field => field.name));
+    const outputRowCount = rowIndices?.length ?? sourceRowCount;
+    for (let outputRowOffset = 0; outputRowOffset < outputRowCount; outputRowOffset += input.batchSize) {
+        const rowCount = Math.min(input.batchSize, outputRowCount - outputRowOffset);
+        const batchRowIndices = rowIndices?.slice(outputRowOffset, outputRowOffset + rowCount);
+        const batchLocalRowIndices = localRowIndices?.slice(outputRowOffset, outputRowOffset + rowCount);
+        const rowGroupRowOffset = batchRowIndices?.[0] ?? outputRowOffset;
+        const batchColumns = batchLocalRowIndices
+            ? gatherParquetColumns(columns, batchLocalRowIndices, outputColumns)
+            : sliceColumns(columns, outputRowOffset, outputRowOffset + rowCount);
+        const arrowTable = convertTable({
+            shape: 'columnar-table',
+            schema: input.projectedSchema,
+            data: batchColumns
+        }, 'arrow-table');
+        batches.push({
+            rowGroupRowOffset,
+            rowCount,
+            rowGroupRowIndices: batchRowIndices,
+            arrowTable: dehydrateArrowTable(arrowTable.data)
+        });
+    }
+    const arrowConversionDurationMs = getCurrentTime() - conversionStartTime;
+    return {
+        sourceRowCount,
+        rowCount: outputRowCount,
+        batches,
+        decodeDurationMs,
+        arrowConversionDurationMs
+    };
+}
+/** Concatenates materialized page-range fragments without constructing row objects. */
+function concatenateMaterializedColumns(fragments) {
+    const columns = {};
+    for (const fragment of fragments) {
+        for (const [name, values] of Object.entries(fragment)) {
+            columns[name] ||= [];
+            const destination = columns[name];
+            for (let index = 0; index < values.length; index++) {
+                destination.push(values[index]);
+            }
+        }
+    }
+    return columns;
+}
+/** Returns a contiguous row slice of every decoded column without constructing row objects. */
+function sliceColumns(columns, start, end) {
+    const slicedColumns = {};
+    for (const [name, column] of Object.entries(columns)) {
+        const slice = column.slice;
+        slicedColumns[name] = slice
+            ? slice.call(column, start, end)
+            : Array.prototype.slice.call(column, start, end);
+    }
+    return slicedColumns;
+}
+/** Reconstructs the minimal Thrift-compatible row-group object used by the decoder. */
+function createWorkerRowGroup(input) {
+    return {
+        num_rows: input.rowCount,
+        total_byte_size: input.uncompressedByteLength,
+        columns: input.columnChunks.map(columnChunk => ({
+            file_path: columnChunk.filePath,
+            meta_data: {
+                type: columnChunk.physicalType,
+                codec: columnChunk.compressionCodec,
+                path_in_schema: columnChunk.path,
+                num_values: columnChunk.valueCount,
+                total_compressed_size: columnChunk.compressedByteLength,
+                total_uncompressed_size: columnChunk.uncompressedByteLength,
+                data_page_offset: columnChunk.dataPageOffset,
+                dictionary_page_offset: columnChunk.dictionaryPageOffset,
+                encodings: []
+            }
+        }))
+    };
+}
+/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
+function getCurrentTime() {
+    return globalThis.performance?.now() ?? Date.now();
+}
+//# sourceMappingURL=parquet-source-worker-decoder.js.map

package/dist/lib/parquet-source-worker-decoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-decoder.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,mBAAmB,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAC,mBAAmB,EAAC,oCAAiC;AAC7D,OAAO,EAAC,uBAAuB,EAAE,oBAAoB,EAAC,+BAA4B;AAElF,OAAO,EAAC,aAAa,EAAC,8CAA2C;AACjE,OAAO,EAAC,aAAa,EAAC,sCAAmC;AACzD,OAAO,EAAC,uBAAuB,EAAC,wCAAqC;AAOrE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,KAA+B;IAE/B,MAAM,mBAAmB,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,uBAAuB,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE,EAAC,cAAc,EAAE,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,cAAc,EAAE,CAAC;IACzC,MAAM,gBAAgB,GAAG,KAAK,CAAC,QAAQ;QACrC,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACtC,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAS,CAAC,aAAa,CAAC,CACxF,CACF;QACH,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,8BAA8B,CAC5C,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC,CACpF,CAAC;IACF,MAAM,gBAAgB,GAAG,KAAK,CAAC,QAAQ;QACrC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAC1C,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,CAC1F;QACH,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,GAAG,gBAAgB,EAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC;IAClE,MAAM,eAAe,GAAG,KAAK,CAAC,SAAS;QACrC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,CAAC;QACnE,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,UAAU,GAAG,eAAe,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC;IAC9F,MAAM,gBAAgB,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC;IAE5D,MAAM,mBAAmB,GAAG,cAAc,EAAE,CAAC;IAC7C,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,UAAU,EAAE,MAAM,IAAI,cAAc,CAAC;IAC5D,KACE,IAAI,eAAe,GAAG,CAAC,EACvB,eAAe,GAAG,cAAc,EAChC,eAAe,IAAI,KAAK,CAAC,SAAS,EAClC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC;QAC7E,MAAM,eAAe,GAAG,UAAU,EAAE,KAAK,CAAC,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvF,MAAM,oBAAoB,GAAG,eAAe,EAAE,KAAK,CACjD,eAAe,EACf,eAAe,GAAG,QAAQ,CAC3B,CAAC;QACF,MAAM,iBAAiB,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC;QAClE,MAAM,YAAY,GAAG,oBAAoB;YACvC,CAAC,CAAC,oBAAoB,CAAC,OAAO,EAAE,oBAAoB,EAAE,aAAa,CAAC;YACpE,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,YAAY,CAC7B;YACE,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,KAAK,CAAC,eAAe;YAC7B,IAAI,EAAE,YAAY;SACnB,EACD,aAAa,CACd,CAAC;QACF,OAAO,CAAC,IAAI,CAAC;YACX,iBAAiB;YACjB,QAAQ;YACR,kBAAkB,EAAE,eAAe;YACnC,UAAU,EAAE,mBAAmB,CAAC,UAAU,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,yBAAyB,GAAG,cAAc,EAAE,GAAG,mBAAmB,CAAC;IACzE,OAAO;QACL,cAAc;QACd,QAAQ,EAAE,cAAc;QACxB,OAAO;QACP,gBAAgB;QAChB,yBAAyB;KAC1B,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,SAAS,8BAA8B,CACrC,SAA+C;IAE/C,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAClC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBACnD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAoC,CAAC;AAC9C,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CACnB,OAAkC,EAClC,KAAa,EACb,GAAW;IAEX,MAAM,aAAa,GAA8B,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAI,MAA0E,CAAC,KAAK,CAAC;QAChG,aAAa,CAAC,IAAI,CAAC,GAAG,KAAK;YACzB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC;YAChC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,uFAAuF;AACvF,SAAS,oBAAoB,CAAC,KAA+B;IAC3D,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,eAAe,EAAE,KAAK,CAAC,sBAAsB;QAC7C,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC9C,SAAS,EAAE,WAAW,CAAC,QAAQ;YAC/B,SAAS,EAAE;gBACT,IAAI,EAAE,WAAW,CAAC,YAAY;gBAC9B,KAAK,EAAE,WAAW,CAAC,gBAAgB;gBACnC,cAAc,EAAE,WAAW,CAAC,IAAI;gBAChC,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,qBAAqB,EAAE,WAAW,CAAC,oBAAoB;gBACvD,uBAAuB,EAAE,WAAW,CAAC,sBAAsB;gBAC3D,gBAAgB,EAAE,WAAW,CAAC,cAAc;gBAC5C,sBAAsB,EAAE,WAAW,CAAC,oBAAoB;gBACxD,SAAS,EAAE,EAAE;aACd;SACF,CAAC,CAAC;KACmB,CAAC;AAC3B,CAAC;AAED,sFAAsF;AACtF,SAAS,cAAc;IACrB,OAAO,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AACrD,CAAC"}

package/dist/lib/parquet-source-worker-file.d.ts

@@ -0,0 +1,24 @@
+import type { ReadableFile, Stat } from '@loaders.gl/loader-utils';
+import type { ParquetSourceWorkerRange } from "./parquet-source-worker-types.js";
+/** Readable file backed only by selected transferred ranges from the source object. */
+export declare class ParquetSourceWorkerFile implements ReadableFile {
+    /** Synthetic worker-local file handle. */
+    readonly handle = "parquet-source-worker";
+    /** Synthetic worker-local URL. */
+    readonly url = "";
+    /** Total source byte length. */
+    readonly size: number;
+    /** Total source byte length as a bigint. */
+    readonly bigsize: bigint;
+    /** Selected ranges available to decoder reads. */
+    private readonly ranges;
+    /** Creates a virtual file over transferred selected ranges. */
+    constructor(fileByteLength: number, ranges: ParquetSourceWorkerRange[]);
+    /** Releases this no-op worker-local virtual file. */
+    close(): Promise<void>;
+    /** Returns the original source byte length. */
+    stat(): Promise<Stat>;
+    /** Reads one decoder-requested slice from a transferred selected range. */
+    read(start?: number | bigint, length?: number): Promise<ArrayBuffer>;
+}
+//# sourceMappingURL=parquet-source-worker-file.d.ts.map

package/dist/lib/parquet-source-worker-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-file.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-file.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAE,IAAI,EAAC,MAAM,0BAA0B,CAAC;AAEjE,OAAO,KAAK,EAAC,wBAAwB,EAAC,yCAAsC;AAE5E,uFAAuF;AACvF,qBAAa,uBAAwB,YAAW,YAAY;IAC1D,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,2BAA2B;IAC1C,kCAAkC;IAClC,QAAQ,CAAC,GAAG,MAAM;IAClB,gCAAgC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IAEpD,+DAA+D;gBACnD,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,EAAE;IAMtE,qDAAqD;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAE5B,+CAA+C;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,2EAA2E;IACrE,IAAI,CACR,KAAK,GAAE,MAAM,GAAG,MAAU,EAC1B,MAAM,GAAE,MAAkC,GACzC,OAAO,CAAC,WAAW,CAAC;CAexB"}

package/dist/lib/parquet-source-worker-file.js

@@ -0,0 +1,40 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/** Readable file backed only by selected transferred ranges from the source object. */
+export class ParquetSourceWorkerFile {
+    /** Synthetic worker-local file handle. */
+    handle = 'parquet-source-worker';
+    /** Synthetic worker-local URL. */
+    url = '';
+    /** Total source byte length. */
+    size;
+    /** Total source byte length as a bigint. */
+    bigsize;
+    /** Selected ranges available to decoder reads. */
+    ranges;
+    /** Creates a virtual file over transferred selected ranges. */
+    constructor(fileByteLength, ranges) {
+        this.size = fileByteLength;
+        this.bigsize = BigInt(fileByteLength);
+        this.ranges = ranges;
+    }
+    /** Releases this no-op worker-local virtual file. */
+    async close() { }
+    /** Returns the original source byte length. */
+    async stat() {
+        return { size: this.size, bigsize: this.bigsize, isDirectory: false };
+    }
+    /** Reads one decoder-requested slice from a transferred selected range. */
+    async read(start = 0, length = this.size - Number(start)) {
+        const offset = Number(start);
+        const range = this.ranges.find(candidate => offset >= candidate.offset &&
+            offset + length <= candidate.offset + candidate.data.byteLength);
+        if (!range) {
+            throw new Error(`Parquet worker requested unavailable byte range ${offset}-${offset + length - 1}`);
+        }
+        const relativeOffset = offset - range.offset;
+        return range.data.slice(relativeOffset, relativeOffset + length);
+    }
+}
+//# sourceMappingURL=parquet-source-worker-file.js.map

package/dist/lib/parquet-source-worker-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-file.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-file.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAMpC,uFAAuF;AACvF,MAAM,OAAO,uBAAuB;IAClC,0CAA0C;IACjC,MAAM,GAAG,uBAAuB,CAAC;IAC1C,kCAAkC;IACzB,GAAG,GAAG,EAAE,CAAC;IAClB,gCAAgC;IACvB,IAAI,CAAS;IACtB,4CAA4C;IACnC,OAAO,CAAS;IACzB,kDAAkD;IACjC,MAAM,CAA6B;IAEpD,+DAA+D;IAC/D,YAAY,cAAsB,EAAE,MAAkC;QACpE,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,KAAK,KAAmB,CAAC;IAE/B,+CAA+C;IAC/C,KAAK,CAAC,IAAI;QACR,OAAO,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAC,CAAC;IACtE,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CACR,QAAyB,CAAC,EAC1B,SAAiB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;QAE1C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAC5B,SAAS,CAAC,EAAE,CACV,MAAM,IAAI,SAAS,CAAC,MAAM;YAC1B,MAAM,GAAG,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,CAClE,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,mDAAmD,MAAM,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CACnF,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,cAAc,GAAG,MAAM,CAAC,CAAC;IACnE,CAAC;CACF"}