@gscdump/engine 3.2.0 → 3.4.0

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/scan.mjs

@@ -0,0 +1,74 @@
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
+import "./utils.mjs";
+import { columnsNeededForFilter } from "./filter.mjs";
+import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters, prefetchPageIndexes } from "./plan.mjs";
+import { readRowGroup } from "./rowgroup.mjs";
+async function prepareParquetRead(options) {
+	const prepared = await prepareParquetOptions(options);
+	return {
+		options: prepared,
+		plan: parquetPlan(prepared)
+	};
+}
+async function prepareParquetOptions(options) {
+	const metadata = options.metadata ?? await parquetMetadataAsync(options.file, options);
+	const schemaColumns = parquetSchema(metadata).children.map((child) => child.element.name);
+	const missingFilterColumns = columnsNeededForFilter(options.filter).filter((column) => !schemaColumns.includes(column));
+	if (missingFilterColumns.length) throw new Error(`parquet filter columns not found: ${missingFilterColumns.join(", ")}`);
+	if (options.columns) {
+		const missingColumns = options.columns.filter((column) => !schemaColumns.includes(column));
+		if (missingColumns.length) throw new Error(`parquet column not found: ${missingColumns[0]}`);
+	}
+	let prepared = {
+		...options,
+		metadata
+	};
+	prepared = await withBloomFilters(prepared);
+	prepared = await withPageIndexes(prepared);
+	return prepared;
+}
+function readParquetPlan(options, plan) {
+	const readOptions = {
+		...options,
+		file: prefetchAsyncBuffer(options.file, plan)
+	};
+	return plan.groups.map((group) => readRowGroup(readOptions, plan, group));
+}
+async function withBloomFilters(options) {
+	if (!options.useBloomFilters || !options.filter || !options.metadata) return options;
+	const schemaTree = parquetSchema(options.metadata);
+	const schemaElements = {};
+	for (const child of schemaTree.children) schemaElements[child.element.name] = child.element;
+	const bloomFiltersByGroup = await prefetchBloomFilters({
+		file: options.file,
+		metadata: options.metadata,
+		filter: options.filter,
+		filterStrict: options.filterStrict
+	});
+	return {
+		...options,
+		bloomFiltersByGroup,
+		schemaElements
+	};
+}
+async function withPageIndexes(options) {
+	if (!options.usePageIndex || !options.filter || !options.metadata) return options;
+	const { pageRangesByGroup, pageLocationsByGroup } = await prefetchPageIndexes({
+		file: options.file,
+		metadata: options.metadata,
+		filter: options.filter,
+		filterStrict: options.filterStrict,
+		rowStart: options.rowStart,
+		rowEnd: options.rowEnd,
+		columns: options.columns,
+		bloomFiltersByGroup: options.bloomFiltersByGroup,
+		schemaElements: options.schemaElements,
+		parsers: options.parsers
+	});
+	return {
+		...options,
+		pageRangesByGroup,
+		pageLocationsByGroup
+	};
+}
+export { prepareParquetRead, readParquetPlan };

package/dist/node_modules/.pnpm/{hyparquet@1.28.2 → hyparquet@1.29.2}/node_modules/hyparquet/src/types.d.mts

@@ -23,12 +23,4 @@ type ParquetQueryOperator = {
   $nin?: ParquetQueryValue[];
   $not?: ParquetQueryOperator;
 };
-/**
- * File-like object that can read slices of a file asynchronously.
- */
-interface AsyncBuffer {
-  byteLength: number;
-  slice(start: number, end?: number): Awaitable<ArrayBuffer>;
-}
-type Awaitable<T> = T | Promise<T>;
-export { AsyncBuffer, Awaitable, ParquetQueryFilter, ParquetQueryOperator, ParquetQueryValue };
+export { ParquetQueryFilter, ParquetQueryOperator, ParquetQueryValue };

package/dist/node_modules/.pnpm/{hyparquet@1.28.2 → hyparquet@1.29.2}/node_modules/hyparquet/types/index.d.mts

@@ -1,11 +1,11 @@
-import { AsyncBuffer as AsyncBuffer$1, ParquetQueryFilter as ParquetQueryFilter$1 } from "../src/types.mjs";
+import { ParquetQueryFilter as ParquetQueryFilter$1 } from "../src/types.mjs";
 import "./indexes.mjs";
 import "./metadata.mjs";
 import "./read.mjs";
+import "./scan.mjs";
 import "./query.mjs";
 import "./snappy.mjs";
 import "./utils.mjs";
-type AsyncBuffer = AsyncBuffer$1;
 type ParquetQueryFilter = ParquetQueryFilter$1;
 /**
  * Explicitly export types for use in downstream typescript projects through
@@ -47,8 +47,12 @@ type ParquetQueryFilter = ParquetQueryFilter$1;
  * @typedef {import('../src/types.d.ts').ColumnData} ColumnData
  * @typedef {import('../src/types.d.ts').SubColumnData} SubColumnData
  * @typedef {import('../src/types.d.ts').ParquetReadOptions} ParquetReadOptions
+ * @typedef {import('../src/types.d.ts').ParquetRowRange} ParquetRowRange
+ * @typedef {import('../src/types.d.ts').ParquetScan} ParquetScan
+ * @typedef {import('../src/types.d.ts').ParquetScanColumnOptions} ParquetScanColumnOptions
+ * @typedef {import('../src/types.d.ts').ParquetScanOptions} ParquetScanOptions
  * @typedef {import('../src/types.d.ts').MetadataOptions} MetadataOptions
  * @typedef {import('../src/types.d.ts').ParquetParsers} ParquetParsers
  * @typedef {import('../src/types.d.ts').ParquetQueryFilter} ParquetQueryFilter
  */
-export { AsyncBuffer, ParquetQueryFilter };
+export { ParquetQueryFilter };

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.d.mts

@@ -0,0 +1,3 @@
+import "../src/types.mjs";
+import { ParquetQueryFilter } from "./index.mjs";
+export { ParquetQueryFilter };

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/scan.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/utils.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,4 +1,4 @@
-import "../../../../../hyparquet@1.28.2/node_modules/hyparquet/src/convert.mjs";
+import "../../../../../hyparquet@1.29.1/node_modules/hyparquet/src/convert.mjs";
 import "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import "./avro.metadata.mjs";
 export {};

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/fetch.mjs

@@ -1,5 +1,5 @@
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
 import "../../../../hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import "./avro/avro.metadata.mjs";
 import "./avro/avro.read.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/index.mjs

@@ -3,11 +3,11 @@ import "./metadata.mjs";
 import "./catalog/rest.mjs";
 import "./create.mjs";
 import "./manifest.mjs";
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import { icebergCreateTable } from "./write/write.mjs";
 import "./catalog/file.mjs";
 import "./s3.mjs";
-import "../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
 export { icebergCreateTable };

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/src/read.mjs

@@ -0,0 +1,11 @@
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/filter.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
+import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
+import "./utils.mjs";
+import "./fetch.mjs";
+import "./metadata.mjs";
+import "./delete.mjs";
+import "./manifest.mjs";
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+export {};

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/sql/icebergDataSource.mjs

@@ -1,8 +1,8 @@
 import "../fetch.mjs";
 import "../metadata.mjs";
 import "../manifest.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "../read.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "../prune.mjs";
 import "./whereFilter.mjs";
 export {};

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/sql/icebergQuery.mjs

@@ -1,5 +1,5 @@
 import "../fetch.mjs";
 import "../catalog/loadTable.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "./icebergDataSource.mjs";
 export {};

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/types.d.mts

@@ -1,5 +1,5 @@
-import { AsyncBuffer } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/types/index.mjs";
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/types/node.mjs";
+import { AsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/index.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
 import { Writer } from "../../../../hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/index.mjs";
 import "../../../../hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/node.mjs";
 interface WriterOptions {

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/write/rewrite.mjs

@@ -1,4 +1,6 @@
+import "../../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
 import "../utils.mjs";
+import "../fetch.mjs";
 import "./partition.mjs";
 import "./stats.mjs";
 import "./snapshot.mjs";
@@ -6,5 +8,6 @@ import "./manifest.mjs";
 import "./parquet.mjs";
 import "./sort.mjs";
 import "./stage.mjs";
+import "../manifest.mjs";
 import "../read.mjs";
 export {};

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/create.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/fetch.d.mts

@@ -0,0 +1,2 @@
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/types/index.d.mts

@@ -4,10 +4,10 @@ import "./catalog/file.mjs";
 import "./catalog/rest.mjs";
 import "./fetch.mjs";
 import "./s3.mjs";
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import "./metadata.mjs";
 import "./manifest.mjs";
-import "../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
 export { icebergCreateTable };

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/manifest.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/metadata.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/read.d.mts

@@ -0,0 +1,3 @@
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
+import "../src/types.mjs";
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/s3.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/sql/icebergDataSource.d.mts

@@ -0,0 +1,2 @@
+import "../../src/types.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/sql/icebergQuery.d.mts

@@ -0,0 +1,2 @@
+import "../../src/types.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/ast.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/backend/batchAdapters.mjs

@@ -0,0 +1,3 @@
+import "./batch.mjs";
+import "../execute/yield.mjs";
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/execute/aggregates.mjs

@@ -1,6 +1,7 @@
+import "./yield.mjs";
+import "../backend/dataSource.mjs";
 import "../expression/alias.mjs";
 import "./utils.mjs";
-import "./yield.mjs";
 import "../expression/evaluate.mjs";
 import "./accumulator.mjs";
 import "./scanColumn.mjs";

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/execute/batchResults.mjs

@@ -0,0 +1,3 @@
+import "../backend/batchAdapters.mjs";
+import "./utils.mjs";
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/execute/batches.mjs

@@ -0,0 +1,4 @@
+import "../backend/batch.mjs";
+import "./yield.mjs";
+import "./utils.mjs";
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/execute/execute.mjs

@@ -1,15 +1,21 @@
+import "../backend/batch.mjs";
+import "./yield.mjs";
+import "../backend/batchAdapters.mjs";
 import "../backend/dataSource.mjs";
 import "../expression/alias.mjs";
 import "./utils.mjs";
-import "./yield.mjs";
 import "../validation/tables.mjs";
+import "../expression/batch.mjs";
 import "../expression/evaluate.mjs";
 import "../parse/parse.mjs";
 import "../plan/columns.mjs";
 import "../plan/plan.mjs";
 import "./scanColumn.mjs";
 import "./sort.mjs";
+import "./rowScope.mjs";
 import "./aggregates.mjs";
+import "./batchResults.mjs";
+import "./batches.mjs";
 import "./join.mjs";
 import "./window.mjs";
 export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/execute/rowScope.mjs

@@ -0,0 +1,2 @@
+import "../plan/columns.mjs";
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/execute/scanColumn.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/execute/streamingAggregate.mjs

@@ -1,9 +1,12 @@
+import "../backend/batch.mjs";
+import "./yield.mjs";
 import "../expression/alias.mjs";
 import "./utils.mjs";
-import "./yield.mjs";
 import "../validation/functions.mjs";
+import "../expression/batch.mjs";
 import "../expression/evaluate.mjs";
 import "../plan/columns.mjs";
 import "./accumulator.mjs";
 import "./sort.mjs";
+import "./rowScope.mjs";
 export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/execute/utils.mjs

@@ -0,0 +1,2 @@
+import "../backend/batchAdapters.mjs";
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/alias.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/batch.mjs

@@ -0,0 +1,10 @@
+import "../backend/batch.mjs";
+import "../execute/yield.mjs";
+import "../execute/utils.mjs";
+import "../validation/functions.mjs";
+import "../validation/tables.mjs";
+import "./binary.mjs";
+import "./regexp.mjs";
+import "./scalar.mjs";
+import "./strings.mjs";
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/binary.mjs

@@ -0,0 +1,2 @@
+import "../execute/utils.mjs";
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/date.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/expression/evaluate.mjs

@@ -1,15 +1,16 @@
+import "../execute/yield.mjs";
 import "./alias.mjs";
 import "../execute/utils.mjs";
-import "../execute/yield.mjs";
 import "../validation/parseErrors.mjs";
 import "../validation/functions.mjs";
 import "../validation/executionErrors.mjs";
 import "../validation/tables.mjs";
 import "./binary.mjs";
+import "./regexp.mjs";
 import "./date.mjs";
+import "./scalar.mjs";
+import "./strings.mjs";
 import "./math.mjs";
-import "./regexp.mjs";
 import "../spatial/spatial.mjs";
-import "./strings.mjs";
 import "../execute/execute.mjs";
 export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/math.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/expression/scalar.mjs

@@ -0,0 +1,4 @@
+import "../execute/utils.mjs";
+import "../validation/executionErrors.mjs";
+import "./date.mjs";
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/index.mjs

@@ -1,3 +1,5 @@
+import "./backend/batch.mjs";
+import "./backend/batchAdapters.mjs";
 import "./backend/dataSource.mjs";
 import "./expression/alias.mjs";
 import "./execute/utils.mjs";

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/parse/extractTables.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/plan/columns.mjs

@@ -1,3 +1,4 @@
+import "../backend/dataSource.mjs";
 import "../expression/alias.mjs";
 import "../parse/parse.mjs";
 export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/spatial/wkt.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/squirreling@0.16.2/node_modules/squirreling/src/validation/keywords.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/validation/tables.mjs

@@ -1,2 +1,3 @@
+import "../backend/dataSource.mjs";
 import "./executionErrors.mjs";
 export {};

package/dist/storage.d.mts

@@ -1,6 +1,6 @@
 import { CompactionThresholds } from "./compaction.mjs";
-import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/types/index.mjs";
-import "./node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/types/node.mjs";
+import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/index.mjs";
+import "./node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 interface WriteCtx extends TenantCtx {

package/dist/vendor/hysnappy-purejs.mjs

@@ -1,4 +1,4 @@
-import { snappyUncompress as snappyUncompress$1 } from "../node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/snappy.mjs";
+import { snappyUncompress as snappyUncompress$1 } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/snappy.mjs";
 function decode(input, outputLength) {
 	const output = new Uint8Array(outputLength);
 	snappyUncompress$1(input, output);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "3.2.0",
+  "version": "3.4.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,23 +189,23 @@
     }
   },
   "dependencies": {
+    "@gscdump/contracts": "^3.4.0",
+    "@gscdump/lakehouse": "^3.4.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "proper-lockfile": "^4.1.2",
-    "@gscdump/lakehouse": "^3.2.0",
-    "gscdump": "^3.2.0",
-    "@gscdump/contracts": "^3.2.0"
+    "gscdump": "^3.4.0",
+    "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
-    "@types/node": "^26.2.0",
+    "@types/node": "^26.4.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
-    "hyparquet": "^1.28.2",
+    "hyparquet": "^1.29.2",
     "hyparquet-writer": "^0.16.6",
-    "icebird": "^0.8.22",
+    "icebird": "^0.8.26",
     "tsx": "^4.23.12",
     "unstorage": "^1.17.5",
-    "vitest": "^4.1.10"
+    "vitest": "^4.1.11"
   },
   "scripts": {
     "dev": "obuild --stub",

package/dist/node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs

@@ -1,128 +0,0 @@
-import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
-import { concat } from "./utils.mjs";
-import { columnsNeededForFilter, matchFilter } from "./filter.mjs";
-import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters, prefetchPageIndexes } from "./plan.mjs";
-import { assembleAsync, asyncGroupToRows, readRowGroup } from "./rowgroup.mjs";
-async function parquetRead(options) {
-	options.metadata ??= await parquetMetadataAsync(options.file, options);
-	const { rowStart = 0, rowEnd, columns, onChunk, onComplete, rowFormat, filter, filterStrict = true } = options;
-	if (filter && rowFormat !== "object") throw new Error("parquet filter requires rowFormat: \"object\"");
-	const filterColumns = columnsNeededForFilter(filter);
-	if (filterColumns.length) {
-		const schemaColumns = parquetSchema(options.metadata).children.map((c) => c.element.name);
-		const missingColumns = filterColumns.filter((c) => !schemaColumns.includes(c));
-		if (missingColumns.length) throw new Error(`parquet filter columns not found: ${missingColumns.join(", ")}`);
-	}
-	let readColumns = columns;
-	let requiresProjection = false;
-	if (columns && filter) {
-		const missingFilterColumns = filterColumns.filter((c) => !columns.includes(c));
-		if (missingFilterColumns.length) {
-			readColumns = [...columns, ...missingFilterColumns];
-			requiresProjection = true;
-		}
-	}
-	let readOptions = readColumns !== columns ? {
-		...options,
-		columns: readColumns
-	} : options;
-	readOptions = await withBloomFilters(readOptions);
-	readOptions = await withPageIndexes(readOptions);
-	const asyncGroups = parquetReadAsync(readOptions);
-	if (!onComplete && !onChunk) {
-		await awaitAllColumns(asyncGroups);
-		return;
-	}
-	const schemaTree = parquetSchema(options.metadata);
-	const assembled = asyncGroups.map((arg) => assembleAsync(arg, schemaTree, options.parsers));
-	if (onChunk) for (const asyncGroup of assembled) for (const asyncColumn of asyncGroup.asyncColumns) asyncColumn.data.then(({ data, skipped }) => {
-		let rowStart = asyncGroup.groupStart + skipped;
-		for (const columnData of data) {
-			onChunk({
-				columnName: asyncColumn.pathInSchema[0],
-				columnData,
-				rowStart,
-				rowEnd: rowStart + columnData.length
-			});
-			rowStart += columnData.length;
-		}
-	}, () => {});
-	if (onComplete) {
-		await awaitAllColumns(assembled);
-		const rows = [];
-		for (const asyncGroup of assembled) {
-			const selectStart = asyncGroup.selectStart ?? Math.max(rowStart - asyncGroup.groupStart, 0);
-			const selectEnd = asyncGroup.selectEnd ?? Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
-			const groupData = rowFormat === "object" ? await asyncGroupToRows(asyncGroup, selectStart, selectEnd, readColumns, "object") : await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, "array");
-			if (filter) {
-				for (const row of groupData) if (matchFilter(row, filter, filterStrict)) {
-					if (requiresProjection && columns) {
-						for (const col of filterColumns) if (!columns.includes(col)) delete row[col];
-					}
-					rows.push(row);
-				}
-			} else concat(rows, groupData);
-		}
-		onComplete(rows);
-	} else await awaitAllColumns(assembled);
-}
-async function awaitAllColumns(asyncGroups) {
-	const all = asyncGroups.flatMap((g) => g.asyncColumns.map((c) => c.data));
-	const failed = (await Promise.allSettled(all)).find((r) => r.status === "rejected");
-	if (failed) throw failed.reason;
-}
-function parquetReadAsync(options) {
-	if (!options.metadata) throw new Error("parquet requires metadata");
-	const plan = parquetPlan(options);
-	options.file = prefetchAsyncBuffer(options.file, plan);
-	return plan.groups.map((groupPlan) => readRowGroup(options, plan, groupPlan));
-}
-async function withBloomFilters(options) {
-	if (!options.useBloomFilters) return options;
-	if (!options.filter || !options.metadata) return options;
-	const schemaTree = parquetSchema(options.metadata);
-	const schemaElements = {};
-	for (const child of schemaTree.children) schemaElements[child.element.name] = child.element;
-	const bloomFiltersByGroup = await prefetchBloomFilters({
-		file: options.file,
-		metadata: options.metadata,
-		filter: options.filter,
-		filterStrict: options.filterStrict
-	});
-	return {
-		...options,
-		bloomFiltersByGroup,
-		schemaElements
-	};
-}
-async function withPageIndexes(options) {
-	if (!options.usePageIndex) return options;
-	if (!options.filter || !options.metadata) return options;
-	const { pageRangesByGroup, pageLocationsByGroup } = await prefetchPageIndexes({
-		file: options.file,
-		metadata: options.metadata,
-		filter: options.filter,
-		filterStrict: options.filterStrict,
-		rowStart: options.rowStart,
-		rowEnd: options.rowEnd,
-		columns: options.columns,
-		bloomFiltersByGroup: options.bloomFiltersByGroup,
-		schemaElements: options.schemaElements,
-		parsers: options.parsers
-	});
-	return {
-		...options,
-		pageRangesByGroup,
-		pageLocationsByGroup
-	};
-}
-function parquetReadObjects(options) {
-	return new Promise((onComplete, reject) => {
-		parquetRead({
-			...options,
-			rowFormat: "object",
-			onComplete
-		}).catch(reject);
-	});
-}
-export { parquetRead, parquetReadAsync, parquetReadObjects };

package/dist/node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/types/node.d.mts

@@ -1,3 +0,0 @@
-import "../src/types.mjs";
-import { AsyncBuffer, ParquetQueryFilter } from "./index.mjs";
-export { AsyncBuffer, ParquetQueryFilter };

package/dist/node_modules/.pnpm/icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/src/read.mjs

@@ -1,9 +0,0 @@
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/utils.mjs";
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
-import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
-import "./utils.mjs";
-import "./fetch.mjs";
-import "./metadata.mjs";
-import "./delete.mjs";
-import "./manifest.mjs";
-export {};