npm - @gscdump/engine - Versions diffs - 2.1.1 → 2.3.0 - Mend

@@ -1,147 +0,0 @@
-import { getPhysicalColumns } from "./schema.mjs";
-import { parquetSchema } from "./metadata.mjs";
-import { bloomEligibleColumns, readBloomFilter } from "./bloom.mjs";
-import { canSkipRowGroup } from "./filter.mjs";
-const runLimit = 1 << 21;
-function parquetPlan({ metadata, rowStart = 0, rowEnd = Infinity, columns, filter, filterStrict = true, useOffsetIndex = false, bloomFiltersByGroup, schemaElements }) {
-	if (!metadata) throw new Error("parquetPlan requires metadata");
-	const groups = [];
-	const fetches = [];
-	const indexes = [];
-	const physicalColumns = getPhysicalColumns(parquetSchema(metadata));
-	let groupStart = 0;
-	let rgIdx = 0;
-	for (const rowGroup of metadata.row_groups) {
-		const groupRows = Number(rowGroup.num_rows);
-		const groupEnd = groupStart + groupRows;
-		const bloomFilters = bloomFiltersByGroup?.[rgIdx];
-		if (groupRows > 0 && groupEnd > rowStart && groupStart < rowEnd && !canSkipRowGroup({
-			rowGroup,
-			physicalColumns,
-			filter,
-			strict: filterStrict,
-			bloomFilters,
-			schemaElements
-		})) {
-			const chunks = [];
-			let groupStartByte = Infinity;
-			let groupEndByte = -Infinity;
-			for (const chunk of rowGroup.columns) {
-				const meta = chunk.meta_data;
-				if (chunk.file_path) throw new Error("parquet file_path not supported");
-				if (!meta) throw new Error("parquet column metadata is undefined");
-				if (!columns || columns.includes(meta.path_in_schema[0])) {
-					const columnOffset = meta.dictionary_page_offset || meta.data_page_offset;
-					const startByte = Number(columnOffset);
-					const endByte = Number(columnOffset + meta.total_compressed_size);
-					if (startByte < groupStartByte) groupStartByte = startByte;
-					if (endByte > groupEndByte) groupEndByte = endByte;
-					if (useOffsetIndex && chunk.offset_index_offset && chunk.offset_index_length && (rowStart > groupStart || rowEnd < groupEnd)) {
-						const offsetIndexStart = Number(chunk.offset_index_offset);
-						chunks.push({
-							columnMetadata: meta,
-							offsetIndex: {
-								startByte: offsetIndexStart,
-								endByte: offsetIndexStart + chunk.offset_index_length
-							},
-							range: {
-								startByte,
-								endByte
-							}
-						});
-					} else chunks.push({
-						columnMetadata: meta,
-						range: {
-							startByte,
-							endByte
-						}
-					});
-				}
-			}
-			const selectStart = Math.max(rowStart - groupStart, 0);
-			const selectEnd = Math.min(rowEnd - groupStart, groupRows);
-			groups.push({
-				chunks,
-				rowGroup,
-				groupStart,
-				groupRows,
-				selectStart,
-				selectEnd
-			});
-			let run;
-			for (const chunk of chunks) if ("offsetIndex" in chunk) indexes.push(chunk.offsetIndex);
-			else {
-				const { range } = chunk;
-				if (columns) fetches.push(range);
-				else if (run && range.endByte - run.startByte <= runLimit) run.endByte = range.endByte;
-				else {
-					if (run) fetches.push(run);
-					run = { ...range };
-				}
-			}
-			if (run) fetches.push(run);
-		}
-		groupStart = groupEnd;
-		rgIdx++;
-	}
-	if (!isFinite(rowEnd)) rowEnd = groupStart;
-	fetches.push(...indexes);
-	return {
-		metadata,
-		rowStart,
-		rowEnd,
-		columns,
-		fetches,
-		groups
-	};
-}
-async function prefetchBloomFilters({ file, metadata, filter, filterStrict = true }) {
-	const result = metadata.row_groups.map(() => ({}));
-	const eligibleCols = bloomEligibleColumns(filter);
-	if (eligibleCols.size === 0) return result;
-	const physicalColumns = getPhysicalColumns(parquetSchema(metadata));
-	const tasks = [];
-	metadata.row_groups.forEach((rowGroup, rgIdx) => {
-		if (canSkipRowGroup({
-			rowGroup,
-			physicalColumns,
-			filter,
-			strict: filterStrict
-		})) return;
-		for (const colName of eligibleCols) {
-			const columnIdx = physicalColumns.indexOf(colName);
-			if (columnIdx === -1) continue;
-			const meta = rowGroup.columns[columnIdx]?.meta_data;
-			if (!meta?.bloom_filter_offset || !meta.bloom_filter_length) continue;
-			const start = Number(meta.bloom_filter_offset);
-			const end = start + meta.bloom_filter_length;
-			tasks.push((async () => {
-				const buffer = await file.slice(start, end);
-				const bloom = readBloomFilter({
-					view: new DataView(buffer),
-					offset: 0
-				});
-				if (bloom) result[rgIdx][colName] = bloom;
-			})());
-		}
-	});
-	if (tasks.length) await Promise.all(tasks);
-	return result;
-}
-function prefetchAsyncBuffer(file, { fetches }) {
-	const promises = fetches.map(({ startByte, endByte }) => file.slice(startByte, endByte));
-	return {
-		byteLength: file.byteLength,
-		slice(start, end = file.byteLength) {
-			const index = fetches.findIndex(({ startByte, endByte }) => startByte <= start && end <= endByte);
-			if (index < 0) return file.slice(start, end);
-			if (fetches[index].startByte !== start || fetches[index].endByte !== end) {
-				const startOffset = start - fetches[index].startByte;
-				const endOffset = end - fetches[index].startByte;
-				if (promises[index] instanceof Promise) return promises[index].then((buffer) => buffer.slice(startOffset, endOffset));
-				else return promises[index].slice(startOffset, endOffset);
-			} else return promises[index];
-		}
-	};
-}
-export { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters };

@@ -1,10 +0,0 @@
-import { AsyncBuffer as AsyncBuffer$1, ParquetQueryFilter as ParquetQueryFilter$1 } from "../src/types.mjs";
-import "./query.mjs";
-import "./snappy.mjs";
-import "./indexes.mjs";
-import "./metadata.mjs";
-import "./read.mjs";
-import "./utils.mjs";
-type AsyncBuffer = AsyncBuffer$1;
-type ParquetQueryFilter = ParquetQueryFilter$1;
-export { AsyncBuffer, ParquetQueryFilter };

@@ -1,3 +0,0 @@
-import "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
-import "./sigv4.mjs";
-export {};

@@ -1,5 +0,0 @@
-import "../../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
-import "../utils.mjs";
-import "../schema.mjs";
-import "./conversions.mjs";
-export {};

@@ -1,8 +1,8 @@
 import "../utils.mjs";
 import "./partition.mjs";
-import "./manifest.mjs";
 import "./stats.mjs";
 import "./snapshot.mjs";
+import "./manifest.mjs";
 import "./parquet.mjs";
 import "./sort.mjs";
 import "./stage.mjs";

@@ -3,9 +3,9 @@ import "../puffin/puffin.mjs";
 import "../utils.mjs";
 import "../fetch.mjs";
 import "../schema.mjs";
+import "./snapshot.mjs";
 import "../delete.mjs";
 import "./manifest.mjs";
-import "./snapshot.mjs";
 import "./stage.mjs";
 import "./stage-position-delete.mjs";
 export {};

	@@ -1,2 +0,0 @@
1	- import "../../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
2	- export {};

	@@ -1,2 +0,0 @@
1	- import "../../../../hyparquet@1.26.2/node_modules/hyparquet/types/node.mjs";
2	- import "../src/types.mjs";

	@@ -1,2 +0,0 @@
1	- import "../../src/types.mjs";
2	- import "../../../../../squirreling@0.15.0/node_modules/squirreling/src/index.mjs";

@gscdump/engine 2.1.1 → 2.3.0