@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/dist/lib/parquet-predicate.js

@@ -0,0 +1,328 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { encodeUtf8 } from "../parquetjs/utils/binary-utils.js";
+/** Returns the unique top-level columns referenced by a predicate. */
+export function getParquetPredicateColumns(predicate) {
+    return [...new Set(getParquetPredicatePaths(predicate).map(path => path[0]))];
+}
+/** Returns the unique physical schema paths referenced by a predicate. */
+export function getParquetPredicatePaths(predicate) {
+    const paths = new Map();
+    visitParquetPredicate(predicate, child => {
+        const property = getPredicateProperty(child);
+        if (property) {
+            const path = [...getParquetPredicatePath(property)];
+            paths.set(path.join('\0'), path);
+        }
+    });
+    return [...paths.values()];
+}
+/** Copies a predicate tree and mutable scalar values for read-scoped option snapshots. */
+export function copyParquetPredicate(predicate) {
+    if (isParquetLogicalPredicate(predicate)) {
+        return {
+            op: predicate.op,
+            args: predicate.args.map(copyParquetPredicate)
+        };
+    }
+    if (isParquetNotPredicate(predicate)) {
+        return {
+            op: 'not',
+            args: [copyParquetPredicate(predicate.args[0])]
+        };
+    }
+    const property = copyPredicateProperty(predicate.args[0]);
+    if (predicate.op === 'isNull') {
+        return { op: 'isNull', args: [property] };
+    }
+    if (predicate.op === 'in') {
+        return {
+            op: 'in',
+            args: [property, predicate.args[1].map(copyPredicateValue)]
+        };
+    }
+    return {
+        op: predicate.op,
+        args: [property, copyPredicateValue(predicate.args[1])]
+    };
+}
+/** Validates a predicate and every referenced top-level column. */
+export function validateParquetPredicate(predicate, availableColumns) {
+    visitParquetPredicate(predicate, child => {
+        if (isParquetLogicalPredicate(child)) {
+            if (child.args.length < 2) {
+                throw new Error(`Parquet predicate ${child.op} requires at least two child predicates`);
+            }
+            return;
+        }
+        if (isParquetNotPredicate(child)) {
+            return;
+        }
+        const property = child.args[0];
+        const path = getParquetPredicatePath(property);
+        if (!path.length || path.some(pathComponent => !pathComponent)) {
+            throw new Error('Parquet predicate property path must contain non-empty strings');
+        }
+        if (!availableColumns.has(path[0])) {
+            throw new Error(`Parquet predicate column not found: ${path.join('.')}`);
+        }
+        if (child.op === 'in' && child.args[1].length === 0) {
+            throw new Error('Parquet predicate in requires at least one value');
+        }
+    });
+}
+/** Conservatively determines whether a row group can contain a matching row. */
+export function canParquetRowGroupMatch(predicate, rowGroup) {
+    if (isParquetLogicalPredicate(predicate)) {
+        return predicate.op === 'and'
+            ? predicate.args.every(child => canParquetRowGroupMatch(child, rowGroup))
+            : predicate.args.some(child => canParquetRowGroupMatch(child, rowGroup));
+    }
+    if (isParquetNotPredicate(predicate)) {
+        const child = predicate.args[0];
+        if (child.op !== 'isNull') {
+            return true;
+        }
+        const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(child.args[0]));
+        return !column?.statistics || column.statistics.nullCount !== rowGroup.rowCount;
+    }
+    const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(predicate.args[0]));
+    const statistics = column?.statistics;
+    if (!statistics) {
+        return true;
+    }
+    return canParquetStatisticsMatch(predicate, statistics, rowGroup.rowCount);
+}
+/** Conservatively determines whether one predicate leaf can match supplied statistics. */
+export function canParquetStatisticsMatch(predicate, statistics, rowCount) {
+    if (statistics.nullCount === rowCount) {
+        return predicate.op === 'isNull';
+    }
+    if (predicate.op === 'isNull') {
+        return statistics.nullCount !== 0;
+    }
+    const minimum = statistics.minIsExact === false ? undefined : statistics.min;
+    const maximum = statistics.maxIsExact === false ? undefined : statistics.max;
+    if (predicate.op === 'in') {
+        return predicate.args[1].some(value => canComparisonMatch('=', value, minimum, maximum));
+    }
+    return canComparisonMatch(predicate.op, predicate.args[1], minimum, maximum);
+}
+/** Returns exact source row indexes matching a predicate. */
+export function filterParquetRowIndices(predicate, columns, rowCount) {
+    if (!predicate) {
+        return Array.from({ length: rowCount }, (_, index) => index);
+    }
+    const rowIndices = [];
+    for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+        if (evaluateParquetPredicate(predicate, columns, rowIndex) === true) {
+            rowIndices.push(rowIndex);
+        }
+    }
+    return rowIndices;
+}
+/** Gathers selected source rows without constructing row objects. */
+export function gatherParquetColumns(columns, rowIndices, selectedColumns) {
+    const gatheredColumns = {};
+    for (const [name, column] of Object.entries(columns)) {
+        if (selectedColumns && !selectedColumns.has(name)) {
+            continue;
+        }
+        gatheredColumns[name] = rowIndices.map(rowIndex => column[rowIndex]);
+    }
+    return gatheredColumns;
+}
+/** Applies an exact predicate to one materialized row. */
+function evaluateParquetPredicate(predicate, columns, rowIndex) {
+    if (isParquetLogicalPredicate(predicate)) {
+        const results = predicate.args.map(child => evaluateParquetPredicate(child, columns, rowIndex));
+        if (predicate.op === 'and') {
+            return results.includes(false) ? false : results.includes(null) ? null : true;
+        }
+        return results.includes(true) ? true : results.includes(null) ? null : false;
+    }
+    if (isParquetNotPredicate(predicate)) {
+        const result = evaluateParquetPredicate(predicate.args[0], columns, rowIndex);
+        return result === null ? null : !result;
+    }
+    const value = getColumnValue(columns, getParquetPredicatePath(predicate.args[0]), rowIndex);
+    if (predicate.op === 'isNull') {
+        return value === null || value === undefined;
+    }
+    if (value === null || value === undefined) {
+        return null;
+    }
+    if (predicate.op === 'in') {
+        return predicate.args[1].some(candidate => comparePredicateValues(value, candidate) === 0);
+    }
+    const comparison = comparePredicateValues(value, predicate.args[1]);
+    switch (predicate.op) {
+        case '=':
+            return comparison === 0;
+        case '<>':
+            return comparison !== 0;
+        case '<':
+            return comparison < 0;
+        case '<=':
+            return comparison <= 0;
+        case '>':
+            return comparison > 0;
+        case '>=':
+            return comparison >= 0;
+    }
+}
+/** Returns whether footer min/max leave any possible match for one comparison. */
+function canComparisonMatch(operator, value, minimum, maximum) {
+    try {
+        switch (operator) {
+            case '=':
+                return !((minimum !== undefined && compareStatisticsValues(value, minimum) < 0) ||
+                    (maximum !== undefined && compareStatisticsValues(value, maximum) > 0));
+            case '<>':
+                return !(minimum !== undefined &&
+                    maximum !== undefined &&
+                    compareStatisticsValues(value, minimum) === 0 &&
+                    compareStatisticsValues(value, maximum) === 0);
+            case '<':
+                return minimum === undefined || compareStatisticsValues(minimum, value) < 0;
+            case '<=':
+                return minimum === undefined || compareStatisticsValues(minimum, value) <= 0;
+            case '>':
+                return maximum === undefined || compareStatisticsValues(maximum, value) > 0;
+            case '>=':
+                return maximum === undefined || compareStatisticsValues(maximum, value) >= 0;
+        }
+    }
+    catch {
+        return true;
+    }
+}
+/** Compares statistics using Parquet's unsigned UTF-8 byte ordering for strings. */
+function compareStatisticsValues(left, right) {
+    if (typeof left === 'string' && typeof right === 'string') {
+        return compareUint8Arrays(encodeUtf8(left), encodeUtf8(right));
+    }
+    return comparePredicateValues(left, right);
+}
+/** Compares predicate-compatible scalar values and rejects incomparable types. */
+function comparePredicateValues(left, right) {
+    if (left instanceof Uint8Array && right instanceof Uint8Array) {
+        return compareUint8Arrays(left, right);
+    }
+    if (left instanceof Uint8Array || right instanceof Uint8Array) {
+        throw new Error('Parquet binary predicates require binary values on both sides');
+    }
+    if (left instanceof Date || right instanceof Date) {
+        if (!(left instanceof Date) || !(right instanceof Date)) {
+            throw new Error('Parquet date predicates require date values on both sides');
+        }
+        const leftTime = left.getTime();
+        const rightTime = right.getTime();
+        if (!Number.isFinite(leftTime) || !Number.isFinite(rightTime)) {
+            throw new Error('Invalid dates cannot be ordered by a Parquet predicate');
+        }
+        return leftTime < rightTime ? -1 : leftTime > rightTime ? 1 : 0;
+    }
+    const leftType = typeof left;
+    const rightType = typeof right;
+    const bothNumeric = (leftType === 'number' || leftType === 'bigint') &&
+        (rightType === 'number' || rightType === 'bigint');
+    if (!bothNumeric && leftType !== rightType) {
+        throw new Error(`Parquet predicate cannot compare ${leftType} with ${rightType}`);
+    }
+    if (!bothNumeric && leftType !== 'string' && leftType !== 'boolean') {
+        throw new Error(`Parquet predicate does not support ${leftType} values`);
+    }
+    const comparableLeft = left;
+    const comparableRight = right;
+    if ((typeof comparableLeft === 'number' && !Number.isFinite(comparableLeft)) ||
+        (typeof comparableRight === 'number' && !Number.isFinite(comparableRight))) {
+        throw new Error('Non-finite numbers cannot be ordered by a Parquet predicate');
+    }
+    if (comparableLeft < comparableRight) {
+        return -1;
+    }
+    if (comparableLeft > comparableRight) {
+        return 1;
+    }
+    return 0;
+}
+/** Compares two byte sequences using unsigned lexicographic ordering. */
+function compareUint8Arrays(left, right) {
+    const length = Math.min(left.length, right.length);
+    for (let index = 0; index < length; index++) {
+        if (left[index] !== right[index]) {
+            return left[index] < right[index] ? -1 : 1;
+        }
+    }
+    return Math.sign(left.length - right.length);
+}
+/** Reads one value from an array-like materialized Parquet column. */
+function getColumnValue(columns, path, rowIndex) {
+    const column = columns[path[0]];
+    if (!column) {
+        throw new Error('Parquet predicate column was not decoded');
+    }
+    let value = column[rowIndex];
+    for (let pathIndex = 1; pathIndex < path.length; pathIndex++) {
+        if (value === null || value === undefined || typeof value !== 'object') {
+            return undefined;
+        }
+        const property = path[pathIndex];
+        value =
+            typeof value.get === 'function'
+                ? value.get(property)
+                : value[property];
+    }
+    return value;
+}
+/** Copies mutable binary and date predicate values. */
+function copyPredicateValue(value) {
+    if (value instanceof Uint8Array) {
+        return value.slice();
+    }
+    if (value instanceof Date) {
+        return new Date(value.getTime());
+    }
+    return value;
+}
+/** Copies a predicate property reference. */
+function copyPredicateProperty(property) {
+    return {
+        property: Array.isArray(property.property) ? [...property.property] : property.property
+    };
+}
+/** Returns one predicate property as an explicit Parquet schema path. */
+export function getParquetPredicatePath(property) {
+    return Array.isArray(property.property) ? property.property : [property.property];
+}
+/** Returns the property reference from a leaf predicate. */
+function getPredicateProperty(predicate) {
+    return isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)
+        ? undefined
+        : predicate.args[0];
+}
+/** Returns one top-level column chunk from normalized row-group metadata. */
+function getRowGroupColumn(rowGroup, path) {
+    return rowGroup.columns.find(columnChunk => columnChunk.path.length === path.length &&
+        columnChunk.path.every((pathComponent, index) => pathComponent === path[index]));
+}
+/** Visits every predicate node depth-first. */
+function visitParquetPredicate(predicate, visit) {
+    visit(predicate);
+    if (isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)) {
+        for (const child of predicate.args) {
+            visitParquetPredicate(child, visit);
+        }
+    }
+}
+/** Returns whether a predicate combines multiple child predicates. */
+function isParquetLogicalPredicate(predicate) {
+    return predicate.op === 'and' || predicate.op === 'or';
+}
+/** Returns whether a predicate negates one child predicate. */
+function isParquetNotPredicate(predicate) {
+    return predicate.op === 'not';
+}
+//# sourceMappingURL=parquet-predicate.js.map

package/dist/lib/parquet-predicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-predicate.js","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAgBpC,OAAO,EAAC,UAAU,EAAC,2CAAwC;AAE3D,sEAAsE;AACtE,MAAM,UAAU,0BAA0B,CAAC,SAA2B;IACpE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,wBAAwB,CAAC,SAA2B;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,CAAC,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpD,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,oBAAoB,CAAC,SAA2B;IAC9D,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC;SAC/C,CAAC;IACJ,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,CAAC,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;SAChD,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAC,CAAC;IAC1C,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SAC5D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,wBAAwB,CACtC,SAA2B,EAC3B,gBAAqC;IAErC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,IAAI,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,CAAC,EAAE,yCAAyC,CAAC,CAAC;YAC1F,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,uBAAuB,CACrC,SAA2B,EAC3B,QAAiC;IAEjC,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK;YAC3B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,KAAK,QAAQ,CAAC,QAAQ,CAAC;IAClF,CAAC;IAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,yBAAyB,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,SAAiF,EACjF,UAAwC,EACxC,QAAgB;IAEhB,IAAI,UAAU,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC;IACnC,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,UAAU,CAAC,SAAS,KAAK,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,uBAAuB,CACrC,SAAuC,EACvC,OAAkC,EAClC,QAAgB;IAEhB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;QACvD,IAAI,wBAAwB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YACpE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,oBAAoB,CAClC,OAAkC,EAClC,UAA6B,EAC7B,eAAqC;IAErC,MAAM,eAAe,GAA8B,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QACD,eAAe,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,MAA6B,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,0DAA0D;AAC1D,SAAS,wBAAwB,CAC/B,SAA2B,EAC3B,OAAkC,EAClC,QAAgB;IAEhB,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChG,IAAI,SAAS,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5F,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,QAAQ,SAAS,CAAC,EAAE,EAAE,CAAC;QACrB,KAAK,GAAG;YACN,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,IAAI;YACP,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;QACzB,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,kFAAkF;AAClF,SAAS,kBAAkB,CACzB,QAA8C,EAC9C,KAA4B,EAC5B,OAAgB,EAChB,OAAgB;IAEhB,IAAI,CAAC;QACH,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,GAAG;gBACN,OAAO,CAAC,CACN,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtE,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;YACJ,KAAK,IAAI;gBACP,OAAO,CAAC,CACN,OAAO,KAAK,SAAS;oBACrB,OAAO,KAAK,SAAS;oBACrB,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;oBAC7C,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAC9C,CAAC;YACJ,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/E,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,uBAAuB,CAAC,IAAa,EAAE,KAAc;IAC5D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,sBAAsB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAED,kFAAkF;AAClF,SAAS,sBAAsB,CAAC,IAAa,EAAE,KAAc;IAC3D,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,IAAI,YAAY,IAAI,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC;IAC/B,MAAM,WAAW,GACf,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;QAChD,CAAC,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,SAAS,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,SAAS,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,cAAc,GAAG,IAA0C,CAAC;IAClE,MAAM,eAAe,GAAG,KAA2C,CAAC;IACpE,IACE,CAAC,OAAO,cAAc,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACxE,CAAC,OAAO,eAAe,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,EAC1E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC,CAAC;IACZ,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yEAAyE;AACzE,SAAS,kBAAkB,CAAC,IAAgB,EAAE,KAAiB;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc,CACrB,OAAkC,EAClC,IAAuB,EACvB,QAAgB;IAEhB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,KAAK,GAAI,MAA6B,CAAC,QAAQ,CAAC,CAAC;IACrD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QAC7D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,KAAK;YACH,OAAQ,KAAyB,CAAC,GAAG,KAAK,UAAU;gBAClD,CAAC,CAAE,KAA0C,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC3D,CAAC,CAAE,KAAiC,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,uDAAuD;AACvD,SAAS,kBAAkB,CAAC,KAA4B;IACtD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6CAA6C;AAC7C,SAAS,qBAAqB,CAAC,QAAkC;IAC/D,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ;KACxF,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,uBAAuB,CAAC,QAAkC;IACxE,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAkB,CAAC,CAAC;AAC9F,CAAC;AAED,4DAA4D;AAC5D,SAAS,oBAAoB,CAAC,SAA2B;IACvD,OAAO,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAC7E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED,6EAA6E;AAC7E,SAAS,iBAAiB,CAAC,QAAiC,EAAE,IAAuB;IACnF,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAC1B,WAAW,CAAC,EAAE,CACZ,WAAW,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;QACvC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAClF,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,SAAS,qBAAqB,CAC5B,SAA2B,EAC3B,KAA4C;IAE5C,KAAK,CAAC,SAAS,CAAC,CAAC;IACjB,IAAI,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,SAAS,yBAAyB,CAChC,SAA2B;IAE3B,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,qBAAqB,CAAC,SAA2B;IACxD,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC;AAChC,CAAC"}

package/dist/lib/parquet-source-worker-client.d.ts

@@ -0,0 +1,6 @@
+import type { ParquetSourceWorkerInput, ParquetSourceWorkerOptions, ParquetSourceWorkerResult } from "./parquet-source-worker-types.js";
+/** Returns whether this runtime and option set can decode Parquet source rows on a worker. */
+export declare function canDecodeParquetSourceOnWorker(options: ParquetSourceWorkerOptions): boolean;
+/** Sends selected compressed Parquet chunks to a worker and returns transferable Arrow buffers. */
+export declare function decodeParquetSourceRowGroupOnWorker(input: ParquetSourceWorkerInput, options: ParquetSourceWorkerOptions): Promise<ParquetSourceWorkerResult>;
+//# sourceMappingURL=parquet-source-worker-client.d.ts.map

package/dist/lib/parquet-source-worker-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-client.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-client.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EAC1B,yCAAsC;AAWvC,8FAA8F;AAC9F,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAE3F;AAED,mGAAmG;AACnG,wBAAsB,mCAAmC,CACvD,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,yBAAyB,CAAC,CAMpC"}

package/dist/lib/parquet-source-worker-client.js

@@ -0,0 +1,24 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { canParseWithWorker, isBrowser, parseWithWorker } from '@loaders.gl/loader-utils';
+import { PARQUET_LOADER_BASE } from "../parquet-loader-base.js";
+import { PARQUET_SOURCE_WORKER_LOAD_WORKER } from "../parquet-source-worker-factory.js";
+import { PARQUET_SOURCE_WORKER_URL } from "../parquet-source-worker-url.js";
+/** Private worker descriptor using the package-local selective source worker bundle. */
+const PARQUET_SOURCE_WORKER = {
+    ...PARQUET_LOADER_BASE,
+    id: 'parquet-source',
+    name: 'ParquetSource',
+    worker: PARQUET_SOURCE_WORKER_URL,
+    loadWorker: PARQUET_SOURCE_WORKER_LOAD_WORKER
+};
+/** Returns whether this runtime and option set can decode Parquet source rows on a worker. */
+export function canDecodeParquetSourceOnWorker(options) {
+    return isBrowser && canParseWithWorker(PARQUET_SOURCE_WORKER, options);
+}
+/** Sends selected compressed Parquet chunks to a worker and returns transferable Arrow buffers. */
+export async function decodeParquetSourceRowGroupOnWorker(input, options) {
+    return (await parseWithWorker(PARQUET_SOURCE_WORKER, input, options));
+}
+//# sourceMappingURL=parquet-source-worker-client.js.map

package/dist/lib/parquet-source-worker-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-client.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-client.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,kBAAkB,EAAE,SAAS,EAAE,eAAe,EAAC,MAAM,0BAA0B,CAAC;AAExF,OAAO,EAAC,mBAAmB,EAAC,kCAA+B;AAC3D,OAAO,EAAC,iCAAiC,EAAC,4CAAyC;AACnF,OAAO,EAAC,yBAAyB,EAAC,wCAAqC;AAOvE,wFAAwF;AACxF,MAAM,qBAAqB,GAAW;IACpC,GAAG,mBAAmB;IACtB,EAAE,EAAE,gBAAgB;IACpB,IAAI,EAAE,eAAe;IACrB,MAAM,EAAE,yBAAyB;IACjC,UAAU,EAAE,iCAAiC;CAC9C,CAAC;AAEF,8FAA8F;AAC9F,MAAM,UAAU,8BAA8B,CAAC,OAAmC;IAChF,OAAO,SAAS,IAAI,kBAAkB,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;AACzE,CAAC;AAED,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,KAA+B,EAC/B,OAAmC;IAEnC,OAAO,CAAC,MAAM,eAAe,CAC3B,qBAAqB,EACrB,KAAK,EACL,OAAO,CACR,CAA8B,CAAC;AAClC,CAAC"}

package/dist/lib/parquet-source-worker-decoder.d.ts

@@ -0,0 +1,7 @@
+import type { ParquetKeyRetriever } from "./parquet-encryption.js";
+import type { ParquetSourceWorkerColumnChunk, ParquetSourceWorkerInput, ParquetSourceWorkerResult } from "./parquet-source-worker-types.js";
+/** Decodes one transferred row group entirely inside the worker. */
+export declare function decodeParquetSourceWorkerInput(input: ParquetSourceWorkerInput): Promise<ParquetSourceWorkerResult>;
+/** Creates a worker-local key retriever from caller-resolved transferable keys. */
+export declare function createWorkerKeyRetriever(columnChunks: readonly ParquetSourceWorkerColumnChunk[], aadPrefixByteLength: number, fileUniqueByteLength: number): ParquetKeyRetriever;
+//# sourceMappingURL=parquet-source-worker-decoder.d.ts.map

package/dist/lib/parquet-source-worker-decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-decoder.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAC,mBAAmB,EAAC,gCAA6B;AAC9D,OAAO,KAAK,EAEV,8BAA8B,EAC9B,wBAAwB,EACxB,yBAAyB,EAC1B,yCAAsC;AAEvC,oEAAoE;AACpE,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,yBAAyB,CAAC,CAsGpC;AAyED,mFAAmF;AACnF,wBAAgB,wBAAwB,CACtC,YAAY,EAAE,SAAS,8BAA8B,EAAE,EACvD,mBAAmB,EAAE,MAAM,EAC3B,oBAAoB,EAAE,MAAM,GAC3B,mBAAmB,CAkBrB"}

package/dist/lib/parquet-source-worker-decoder.js

@@ -0,0 +1,181 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { dehydrateArrowTable } from '@loaders.gl/arrow/transport';
+import { convertTable } from '@loaders.gl/schema-utils';
+import { preloadCompressions } from "../parquetjs/compression.js";
+import { filterParquetRowIndices, gatherParquetColumns } from "./parquet-predicate.js";
+import { ParquetReader } from "../parquetjs/parser/parquet-reader.js";
+import { ParquetSchema } from "../parquetjs/schema/schema.js";
+import { ParquetSourceWorkerFile } from "./parquet-source-worker-file.js";
+/** Decodes one transferred row group entirely inside the worker. */
+export async function decodeParquetSourceWorkerInput(input) {
+    await preloadCompressions();
+    const file = new ParquetSourceWorkerFile(input.fileByteLength, input.ranges);
+    const schema = new ParquetSchema(input.schemaDefinition);
+    const keyRetriever = input.encryption
+        ? createWorkerKeyRetriever(input.columnChunks, input.encryption.aadPrefix?.byteLength ?? 0, input.encryption.fileUnique.byteLength)
+        : undefined;
+    const reader = new ParquetReader(file, {
+        preserveBinary: input.preserveBinary,
+        int96AsTimestamp: input.int96AsTimestamp,
+        verifyPageChecksums: input.verifyPageChecksums,
+        encryptionContext: input.encryption
+            ? {
+                algorithm: input.encryption.algorithm,
+                aadPrefix: input.encryption.aadPrefix
+                    ? new Uint8Array(input.encryption.aadPrefix)
+                    : undefined,
+                fileUnique: new Uint8Array(input.encryption.fileUnique)
+            }
+            : undefined,
+        keyRetriever
+    });
+    const rowGroup = createWorkerRowGroup(input);
+    const decodeStartTime = getCurrentTime();
+    let decodedRowGroups;
+    try {
+        decodedRowGroups = input.pagePlan
+            ? await Promise.all(input.pagePlan.rowRanges.map(rowRange => reader.readRowGroupRange(schema, rowGroup, [], rowRange, input.pagePlan.pageLocations)))
+            : [await reader.readRowGroup(schema, rowGroup, [])];
+    }
+    catch (error) {
+        throw new Error(`Parquet worker decode failed: ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`);
+    }
+    const columns = concatenateMaterializedColumns(decodedRowGroups.map(decodedRowGroup => schema.materializeColumns(decodedRowGroup)));
+    const sourceRowIndices = input.pagePlan
+        ? input.pagePlan.rowRanges.flatMap(rowRange => Array.from({ length: rowRange.end - rowRange.start }, (_, index) => rowRange.start + index))
+        : undefined;
+    const sourceRowCount = sourceRowIndices?.length ?? input.rowCount;
+    const localRowIndices = input.predicate
+        ? filterParquetRowIndices(input.predicate, columns, sourceRowCount)
+        : undefined;
+    const rowIndices = localRowIndices?.map(rowIndex => sourceRowIndices?.[rowIndex] ?? rowIndex);
+    const decodeDurationMs = getCurrentTime() - decodeStartTime;
+    const conversionStartTime = getCurrentTime();
+    const batches = [];
+    const outputColumns = new Set(input.projectedSchema.fields.map(field => field.name));
+    const outputRowCount = rowIndices?.length ?? sourceRowCount;
+    for (let outputRowOffset = 0; outputRowOffset < outputRowCount; outputRowOffset += input.batchSize) {
+        const rowCount = Math.min(input.batchSize, outputRowCount - outputRowOffset);
+        const batchRowIndices = rowIndices?.slice(outputRowOffset, outputRowOffset + rowCount);
+        const batchLocalRowIndices = localRowIndices?.slice(outputRowOffset, outputRowOffset + rowCount);
+        const rowGroupRowOffset = batchRowIndices?.[0] ?? outputRowOffset;
+        const batchColumns = batchLocalRowIndices
+            ? gatherParquetColumns(columns, batchLocalRowIndices, outputColumns)
+            : sliceColumns(columns, outputRowOffset, outputRowOffset + rowCount);
+        const arrowTable = convertTable({
+            shape: 'columnar-table',
+            schema: input.projectedSchema,
+            data: batchColumns
+        }, 'arrow-table');
+        batches.push({
+            rowGroupRowOffset,
+            rowCount,
+            rowGroupRowIndices: batchRowIndices,
+            arrowTable: dehydrateArrowTable(arrowTable.data)
+        });
+    }
+    const arrowConversionDurationMs = getCurrentTime() - conversionStartTime;
+    return {
+        sourceRowCount,
+        rowCount: outputRowCount,
+        batches,
+        decodeDurationMs,
+        arrowConversionDurationMs
+    };
+}
+/** Concatenates materialized page-range fragments without constructing row objects. */
+function concatenateMaterializedColumns(fragments) {
+    const columns = {};
+    for (const fragment of fragments) {
+        for (const [name, values] of Object.entries(fragment)) {
+            columns[name] ||= [];
+            const destination = columns[name];
+            for (let index = 0; index < values.length; index++) {
+                destination.push(values[index]);
+            }
+        }
+    }
+    return columns;
+}
+/** Returns a contiguous row slice of every decoded column without constructing row objects. */
+function sliceColumns(columns, start, end) {
+    const slicedColumns = {};
+    for (const [name, column] of Object.entries(columns)) {
+        const slice = column.slice;
+        slicedColumns[name] = slice
+            ? slice.call(column, start, end)
+            : Array.prototype.slice.call(column, start, end);
+    }
+    return slicedColumns;
+}
+/** Reconstructs the minimal Thrift-compatible row-group object used by the decoder. */
+function createWorkerRowGroup(input) {
+    return {
+        num_rows: input.rowCount,
+        total_byte_size: input.uncompressedByteLength,
+        columns: input.columnChunks.map((columnChunk, index) => {
+            const result = {
+                file_path: columnChunk.filePath,
+                parquetColumnOrdinal: columnChunk.columnOrdinal ?? index,
+                meta_data: {
+                    type: columnChunk.physicalType,
+                    codec: columnChunk.compressionCodec,
+                    path_in_schema: columnChunk.path,
+                    num_values: columnChunk.valueCount,
+                    total_compressed_size: columnChunk.compressedByteLength,
+                    total_uncompressed_size: columnChunk.uncompressedByteLength,
+                    data_page_offset: columnChunk.dataPageOffset,
+                    dictionary_page_offset: columnChunk.dictionaryPageOffset,
+                    encodings: []
+                }
+            };
+            if (columnChunk.encrypted) {
+                result.crypto_metadata = (columnChunk.keyMetadata
+                    ? {
+                        ENCRYPTION_WITH_COLUMN_KEY: {
+                            path_in_schema: columnChunk.path,
+                            key_metadata: new Uint8Array(columnChunk.keyMetadata)
+                        }
+                    }
+                    : {
+                        ENCRYPTION_WITH_FOOTER_KEY: {}
+                    });
+            }
+            return result;
+        })
+    };
+}
+/** Creates a worker-local key retriever from caller-resolved transferable keys. */
+export function createWorkerKeyRetriever(columnChunks, aadPrefixByteLength, fileUniqueByteLength) {
+    return (keyMetadata, context) => {
+        const columnOrdinal = getColumnOrdinalFromAad(context.aad, aadPrefixByteLength, fileUniqueByteLength);
+        const metadataMatches = columnChunks.filter(columnChunk => equalBytes(keyMetadata, columnChunk.keyMetadata && new Uint8Array(columnChunk.keyMetadata)));
+        const matchingColumn = metadataMatches.find(columnChunk => columnChunk.columnOrdinal === columnOrdinal) ??
+            (metadataMatches.length === 1 ? metadataMatches[0] : undefined);
+        if (!matchingColumn?.keyMaterial) {
+            throw new Error('Encrypted Parquet worker key was not transferred for the selected column');
+        }
+        return new Uint8Array(matchingColumn.keyMaterial);
+    };
+}
+/** Extracts the physical column ordinal encoded in a page-module AAD suffix. */
+function getColumnOrdinalFromAad(aad, aadPrefixByteLength, fileUniqueByteLength) {
+    const columnOrdinalOffset = aadPrefixByteLength + fileUniqueByteLength + 3;
+    if (columnOrdinalOffset + 2 > aad.byteLength)
+        return undefined;
+    return new DataView(aad.buffer, aad.byteOffset + columnOrdinalOffset, 2).getInt16(0, true);
+}
+/** Compares optional key metadata without exposing key material in structured-clone state. */
+function equalBytes(firstBytes, secondBytes) {
+    if (!firstBytes || !secondBytes)
+        return !firstBytes && !secondBytes;
+    return (firstBytes.length === secondBytes.length &&
+        firstBytes.every((value, index) => value === secondBytes[index]));
+}
+/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
+function getCurrentTime() {
+    return globalThis.performance?.now() ?? Date.now();
+}
+//# sourceMappingURL=parquet-source-worker-decoder.js.map

package/dist/lib/parquet-source-worker-decoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-decoder.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,mBAAmB,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAC,mBAAmB,EAAC,oCAAiC;AAC7D,OAAO,EAAC,uBAAuB,EAAE,oBAAoB,EAAC,+BAA4B;AAElF,OAAO,EAAC,aAAa,EAAC,8CAA2C;AACjE,OAAO,EAAC,aAAa,EAAC,sCAAmC;AACzD,OAAO,EAAC,uBAAuB,EAAC,wCAAqC;AASrE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,KAA+B;IAE/B,MAAM,mBAAmB,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,uBAAuB,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,UAAU;QACnC,CAAC,CAAC,wBAAwB,CACtB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,EAC3C,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,CACvC;QACH,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,iBAAiB,EAAE,KAAK,CAAC,UAAU;YACjC,CAAC,CAAC;gBACE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS;gBACrC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS;oBACnC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC5C,CAAC,CAAC,SAAS;gBACb,UAAU,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;aACxD;YACH,CAAC,CAAC,SAAS;QACb,YAAY;KACb,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,cAAc,EAAE,CAAC;IACzC,IAAI,gBAAgB,CAAC;IACrB,IAAI,CAAC;QACH,gBAAgB,GAAG,KAAK,CAAC,QAAQ;YAC/B,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACtC,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAS,CAAC,aAAa,CAAC,CACxF,CACF;YACH,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,iCACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAC3E,EAAE,CACH,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,8BAA8B,CAC5C,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC,CACpF,CAAC;IACF,MAAM,gBAAgB,GAAG,KAAK,CAAC,QAAQ;QACrC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAC1C,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,CAC1F;QACH,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,GAAG,gBAAgB,EAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC;IAClE,MAAM,eAAe,GAAG,KAAK,CAAC,SAAS;QACrC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,CAAC;QACnE,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,UAAU,GAAG,eAAe,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC;IAC9F,MAAM,gBAAgB,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC;IAE5D,MAAM,mBAAmB,GAAG,cAAc,EAAE,CAAC;IAC7C,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,UAAU,EAAE,MAAM,IAAI,cAAc,CAAC;IAC5D,KACE,IAAI,eAAe,GAAG,CAAC,EACvB,eAAe,GAAG,cAAc,EAChC,eAAe,IAAI,KAAK,CAAC,SAAS,EAClC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC;QAC7E,MAAM,eAAe,GAAG,UAAU,EAAE,KAAK,CAAC,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvF,MAAM,oBAAoB,GAAG,eAAe,EAAE,KAAK,CACjD,eAAe,EACf,eAAe,GAAG,QAAQ,CAC3B,CAAC;QACF,MAAM,iBAAiB,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC;QAClE,MAAM,YAAY,GAAG,oBAAoB;YACvC,CAAC,CAAC,oBAAoB,CAAC,OAAO,EAAE,oBAAoB,EAAE,aAAa,CAAC;YACpE,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,YAAY,CAC7B;YACE,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,KAAK,CAAC,eAAe;YAC7B,IAAI,EAAE,YAAY;SACnB,EACD,aAAa,CACd,CAAC;QACF,OAAO,CAAC,IAAI,CAAC;YACX,iBAAiB;YACjB,QAAQ;YACR,kBAAkB,EAAE,eAAe;YACnC,UAAU,EAAE,mBAAmB,CAAC,UAAU,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,yBAAyB,GAAG,cAAc,EAAE,GAAG,mBAAmB,CAAC;IACzE,OAAO;QACL,cAAc;QACd,QAAQ,EAAE,cAAc;QACxB,OAAO;QACP,gBAAgB;QAChB,yBAAyB;KAC1B,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,SAAS,8BAA8B,CACrC,SAA+C;IAE/C,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAClC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBACnD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAoC,CAAC;AAC9C,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CACnB,OAAkC,EAClC,KAAa,EACb,GAAW;IAEX,MAAM,aAAa,GAA8B,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAI,MAA0E,CAAC,KAAK,CAAC;QAChG,aAAa,CAAC,IAAI,CAAC,GAAG,KAAK;YACzB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC;YAChC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,uFAAuF;AACvF,SAAS,oBAAoB,CAAC,KAA+B;IAC3D,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,eAAe,EAAE,KAAK,CAAC,sBAAsB;QAC7C,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE;YACrD,MAAM,MAAM,GAAG;gBACb,SAAS,EAAE,WAAW,CAAC,QAAQ;gBAC/B,oBAAoB,EAAE,WAAW,CAAC,aAAa,IAAI,KAAK;gBACxD,SAAS,EAAE;oBACT,IAAI,EAAE,WAAW,CAAC,YAAY;oBAC9B,KAAK,EAAE,WAAW,CAAC,gBAAgB;oBACnC,cAAc,EAAE,WAAW,CAAC,IAAI;oBAChC,UAAU,EAAE,WAAW,CAAC,UAAU;oBAClC,qBAAqB,EAAE,WAAW,CAAC,oBAAoB;oBACvD,uBAAuB,EAAE,WAAW,CAAC,sBAAsB;oBAC3D,gBAAgB,EAAE,WAAW,CAAC,cAAc;oBAC5C,sBAAsB,EAAE,WAAW,CAAC,oBAAoB;oBACxD,SAAS,EAAE,EAAE;iBACd;aACwC,CAAC;YAC5C,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1B,MAAM,CAAC,eAAe,GAAG,CAAC,WAAW,CAAC,WAAW;oBAC/C,CAAC,CAAC;wBACE,0BAA0B,EAAE;4BAC1B,cAAc,EAAE,WAAW,CAAC,IAAI;4BAChC,YAAY,EAAE,IAAI,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC;yBACtD;qBACF;oBACH,CAAC,CAAC;wBACE,0BAA0B,EAAE,EAAE;qBAC/B,CAA8D,CAAC;YACtE,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;KACoB,CAAC;AAC3B,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,wBAAwB,CACtC,YAAuD,EACvD,mBAA2B,EAC3B,oBAA4B;IAE5B,OAAO,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE;QAC9B,MAAM,aAAa,GAAG,uBAAuB,CAC3C,OAAO,CAAC,GAAG,EACX,mBAAmB,EACnB,oBAAoB,CACrB,CAAC;QACF,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CACxD,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC,WAAW,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAC5F,CAAC;QACF,MAAM,cAAc,GAClB,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,WAAW,CAAC,aAAa,KAAK,aAAa,CAAC;YAChF,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,IAAI,UAAU,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACpD,CAAC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,SAAS,uBAAuB,CAC9B,GAAe,EACf,mBAA2B,EAC3B,oBAA4B;IAE5B,MAAM,mBAAmB,GAAG,mBAAmB,GAAG,oBAAoB,GAAG,CAAC,CAAC;IAC3E,IAAI,mBAAmB,GAAG,CAAC,GAAG,GAAG,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAC/D,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,GAAG,mBAAmB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,8FAA8F;AAC9F,SAAS,UAAU,CACjB,UAAkC,EAClC,WAAmC;IAEnC,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;QAAE,OAAO,CAAC,UAAU,IAAI,CAAC,WAAW,CAAC;IACpE,OAAO,CACL,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM;QACxC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,SAAS,cAAc;IACrB,OAAO,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AACrD,CAAC"}

package/dist/lib/parquet-source-worker-file.d.ts

@@ -0,0 +1,24 @@
+import type { ReadableFile, Stat } from '@loaders.gl/loader-utils';
+import type { ParquetSourceWorkerRange } from "./parquet-source-worker-types.js";
+/** Readable file backed only by selected transferred ranges from the source object. */
+export declare class ParquetSourceWorkerFile implements ReadableFile {
+    /** Synthetic worker-local file handle. */
+    readonly handle = "parquet-source-worker";
+    /** Synthetic worker-local URL. */
+    readonly url = "";
+    /** Total source byte length. */
+    readonly size: number;
+    /** Total source byte length as a bigint. */
+    readonly bigsize: bigint;
+    /** Selected ranges available to decoder reads. */
+    private readonly ranges;
+    /** Creates a virtual file over transferred selected ranges. */
+    constructor(fileByteLength: number, ranges: ParquetSourceWorkerRange[]);
+    /** Releases this no-op worker-local virtual file. */
+    close(): Promise<void>;
+    /** Returns the original source byte length. */
+    stat(): Promise<Stat>;
+    /** Reads one decoder-requested slice from a transferred selected range. */
+    read(start?: number | bigint, length?: number): Promise<ArrayBuffer>;
+}
+//# sourceMappingURL=parquet-source-worker-file.d.ts.map

package/dist/lib/parquet-source-worker-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-worker-file.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-file.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAE,IAAI,EAAC,MAAM,0BAA0B,CAAC;AAEjE,OAAO,KAAK,EAAC,wBAAwB,EAAC,yCAAsC;AAE5E,uFAAuF;AACvF,qBAAa,uBAAwB,YAAW,YAAY;IAC1D,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,2BAA2B;IAC1C,kCAAkC;IAClC,QAAQ,CAAC,GAAG,MAAM;IAClB,gCAAgC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IAEpD,+DAA+D;gBACnD,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,EAAE;IAMtE,qDAAqD;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAE5B,+CAA+C;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,2EAA2E;IACrE,IAAI,CACR,KAAK,GAAE,MAAM,GAAG,MAAU,EAC1B,MAAM,GAAE,MAAkC,GACzC,OAAO,CAAC,WAAW,CAAC;CAexB"}