@gscdump/engine 3.5.0 → 3.6.1

package/dist/iceberg/schema.d.mts

@@ -2,23 +2,23 @@ import { SearchType } from "../storage.mjs";
 import { IcebergColumn, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1 } from "@gscdump/lakehouse/schema";
 import { TableName } from "@gscdump/contracts";
 /** The 9 fact tables that exist as global Iceberg tables. */
-type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
+export type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
 /** The 9 Iceberg table names, in canonical order. */
-declare const ICEBERG_TABLES: readonly IcebergTableName[];
+export declare const ICEBERG_TABLES: readonly IcebergTableName[];
 /**
  * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
  * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
  */
-declare const SEARCH_TYPE_INT: Record<SearchType, number>;
+export declare const SEARCH_TYPE_INT: Record<SearchType, number>;
 /** Reverse of {@link SEARCH_TYPE_INT} — int → `search_type`, for read-result mapping. */
-declare const INT_SEARCH_TYPE: Record<number, SearchType>;
+export declare const INT_SEARCH_TYPE: Record<number, SearchType>;
 /**
  * The two partition-identity columns Iceberg rows carry that the legacy
  * parquet path encoded in the object-key prefix instead. Field ids 1–2 are
  * the first two columns; per-table metric/dimension columns follow
  * contiguously from id 3 (see `ICEBERG_FIELD_ID_BASE`).
  */
-declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
+export declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
 /**
  * The two partition-identity columns for a given {@link PartitionKeyEncoding}.
  * `'string'` returns {@link ICEBERG_PARTITION_COLUMNS} verbatim; `'int'` swaps
@@ -27,7 +27,7 @@ declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
  * avoid R2 SQL's truncated-string-stats equality undercount and restore pruning.
  * Field ids are unchanged (1, 2) — only the column types differ.
  */
-declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): readonly IcebergColumn[];
+export declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): readonly IcebergColumn[];
 /**
  * First field id used for per-table (non-partition) columns — immediately
  * after the two partition-identity columns (`site_id`=1, `search_type`=2).
@@ -35,9 +35,9 @@ declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): reado
  * `defineIcebergDataset` itself, identity-then-dims-then-columns) — kept as a
  * documented, byte-identity-tested constant for existing external readers.
  */
-declare const ICEBERG_FIELD_ID_BASE = 3;
+export declare const ICEBERG_FIELD_ID_BASE = 3;
 /** Shared partition spec — identical across every table. */
-declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
+export declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
 /**
  * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
  * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a
@@ -47,17 +47,17 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
  * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
  * source of truth — do not hand-list columns elsewhere.
  */
-declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
+export declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
 /** All Iceberg table specs in legacy `'string'` encoding, keyed by table name. */
-declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
+export declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
 /** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
-declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
+export declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
 /** All Iceberg table specs for the default new-catalog encoding. */
-declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+export declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
 /** Table specs for the given encoding (`'int'` default). */
-declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
+export declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
 /** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
-declare function isIcebergTable(table: string): table is IcebergTableName;
+export declare function isIcebergTable(table: string): table is IcebergTableName;
 /**
  * Narrow an arbitrary table name to a canonical {@link IcebergTableName},
  * throwing a clear error otherwise. Guards write paths that index
@@ -65,5 +65,5 @@ declare function isIcebergTable(table: string): table is IcebergTableName;
  * silently yields `undefined` there, propagating a corrupt/empty spec into the
  * Iceberg job instead of failing loudly.
  */
-declare function assertIcebergTable(table: string): IcebergTableName;
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergS3Config, IcebergTableName, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, type SearchType, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export declare function assertIcebergTable(table: string): IcebergTableName;
+export type { IcebergS3Config, PartitionKeyEncoding$1 as PartitionKeyEncoding, SearchType };

package/dist/ingest-accumulator.d.mts

@@ -1,7 +1,7 @@
 import { SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { GscApiRow, RowAccumulatorOptions } from "./ingest.mjs";
 import { Grain, Row, TableName } from "@gscdump/contracts";
-interface IngestAccumulatorEngine {
+export interface IngestAccumulatorEngine {
   writeDay: (scope: TenantCtx$1 & {
     table: TableName;
     date: string;
@@ -25,7 +25,7 @@ interface IngestAccumulatorEngine {
     error?: string;
   }) => Promise<void>;
 }
-interface IngestAccumulatorCtx {
+export interface IngestAccumulatorCtx {
   userId: string | number;
   siteId: string;
   searchType?: SearchType;
@@ -36,7 +36,7 @@ interface IngestAccumulatorCtx {
    */
   grain?: Grain;
 }
-interface IngestAccumulatorHooks {
+export interface IngestAccumulatorHooks {
   /**
    * Called once per (table, date) when the job must abandon in-memory rows
    *  (overflow or `hasMore` continuation). Host queues a forced re-sync from
@@ -70,7 +70,7 @@ interface IngestAccumulatorHooks {
     rowsWritten: number;
   }) => Promise<void>;
 }
-interface FinalizeOptions {
+export interface FinalizeOptions {
   /**
    * The GSC `hasMore` flag for the whole job. When true, in-memory buckets
    *  only reflect this job's slice; we re-queue forced single-day re-syncs
@@ -78,20 +78,19 @@ interface FinalizeOptions {
    */
   hasMore: boolean;
 }
-interface FinalizeResult {
+export interface FinalizeResult {
   flushed: number;
   recovered: number;
   failed: number;
   rowsWritten: number;
 }
-interface IngestAccumulator {
+export interface IngestAccumulator {
   push: (table: TableName, rows: readonly GscApiRow[]) => boolean;
   finalize: (opts: FinalizeOptions) => Promise<FinalizeResult>;
 }
-interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
+export interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
   engine: IngestAccumulatorEngine;
   ctx: IngestAccumulatorCtx;
   hooks: IngestAccumulatorHooks;
 }
-declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;
-export { CreateIngestAccumulatorOptions, FinalizeOptions, FinalizeResult, IngestAccumulator, IngestAccumulatorCtx, IngestAccumulatorEngine, IngestAccumulatorHooks, createIngestAccumulator };
+export declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;

package/dist/ingest.d.mts

@@ -6,8 +6,8 @@ import { Row, TableName } from "./storage.mjs";
  * `keys[]` tuples. Storage-column names (e.g. `page` → `url`) are handled
  * inside `transformGscRow` — this record stays in GSC-API vocabulary.
  */
-declare const TABLE_DIMS: Record<TableName, string[]>;
-interface GscApiRow {
+export declare const TABLE_DIMS: Record<TableName, string[]>;
+export interface GscApiRow {
   keys: string[];
   clicks: number;
   impressions: number;
@@ -15,7 +15,7 @@ interface GscApiRow {
   ctr?: number;
   position: number;
 }
-interface IngestOptions {
+export interface IngestOptions {
   /** Date for one-day `searchAppearance` total queries, whose keys omit date. */
   date?: string;
   /** Search appearance filter used for contextual second-step rows. */
@@ -25,19 +25,21 @@ interface IngestOptions {
  * Strip a GSC URL to its pathname. Core analytics stores pages by path so
  * queries don't carry origin-prefix filters.
  */
-declare function toPath(gscUrl: string): string;
+export declare function toPath(gscUrl: string): string;
 /**
  * Encode weighted average position as `sum_position`. The raw GSC position
  * is 1-indexed; subtract 1 and weight by impressions so a downstream
  * `SUM(sum_position) / SUM(impressions) + 1` recovers the true mean without
  * ever materialising per-row position values.
  */
-declare function toSumPosition(apiPosition: number, impressions: number): number;
+export declare function toSumPosition(apiPosition: number, impressions: number): number;
 /**
  * Map one GSC API row into `{ date, row }` for the given table, or null if
  * the row has no keys (GSC occasionally emits empty-keys placeholders).
+ * Use `createRowAccumulator` for a complete slice so URL variants combine
+ * before the storage engine deduplicates stored keys.
  */
-declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?: IngestOptions): {
+export declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?: IngestOptions): {
   date: string;
   row: Row;
 } | null;
@@ -57,11 +59,11 @@ declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?:
  * page_grained_impressions`, where the page/date totals come from `totalsRow`.
  * Mirrors the legacy `dailyTotalsRollup` formula. Clamped to `[0, 1]`.
  */
-declare function assembleDatesRow(date: string, totalsRow: GscApiRow, deviceRows: readonly GscApiRow[], queryGrainedImpressions: number): {
+export declare function assembleDatesRow(date: string, totalsRow: GscApiRow, deviceRows: readonly GscApiRow[], queryGrainedImpressions: number): {
   date: string;
   row: Row;
 };
-interface RowAccumulator {
+export interface RowAccumulator {
   /**
    * Push a batch of GSC API rows into the accumulator. Returns `false` if
    * the batch pushed total row count past `maxRows`; subsequent pushes
@@ -71,6 +73,8 @@ interface RowAccumulator {
   /**
    * Consume accumulated rows, grouped by `table → date → rows`. Resets
    * internal state; subsequent pushes behave as on a fresh accumulator.
+   * Repeated source identities count once. Distinct URL variants that share
+   * stored dimensions contribute their combined metrics.
    */
   drain: () => Map<TableName, Map<string, Row[]>>;
   /**
@@ -85,12 +89,12 @@ interface RowAccumulator {
    * eventual `drain()` at job end.
    */
   drainCompleted: () => Map<TableName, Map<string, Row[]>>;
-  /** Total row count across all tables/dates since last drain. */
+  /** Distinct source row count across all tables/dates since last drain. */
   readonly totalRows: number;
   /** Whether the accumulator has overflowed since last drain. */
   readonly overflowed: boolean;
 }
-interface RowAccumulatorOptions extends IngestOptions {
+export interface RowAccumulatorOptions extends IngestOptions {
   /**
    * Soft cap on total accumulated rows before `push` starts returning
    * `false` and dropping rows. Defaults to 500_000 — matches the
@@ -109,5 +113,4 @@ interface RowAccumulatorOptions extends IngestOptions {
    */
   trackDateBoundary?: boolean;
 }
-declare function createRowAccumulator(options?: RowAccumulatorOptions): RowAccumulator;
-export { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, TABLE_DIMS, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow };
+export declare function createRowAccumulator(options?: RowAccumulatorOptions): RowAccumulator;

package/dist/ingest.mjs

@@ -233,6 +233,20 @@ function assembleDatesRow(date, totalsRow, deviceRows, queryGrainedImpressions)
 	};
 }
 const DEFAULT_MAX_ROWS = 5e5;
+function storedRows(table, sourceRows) {
+	if (!TABLE_DIMS[table].includes("page")) return [...sourceRows.values()];
+	const grouped = /* @__PURE__ */ new Map();
+	for (const row of sourceRows.values()) {
+		const key = JSON.stringify([row.searchAppearance, ...TABLE_DIMS[table].map((dimension) => row[dimension === "page" ? "url" : dimension])]);
+		const prior = grouped.get(key);
+		if (prior) {
+			prior.clicks = Number(prior.clicks) + Number(row.clicks);
+			prior.impressions = Number(prior.impressions) + Number(row.impressions);
+			prior.sum_position = Number(prior.sum_position) + Number(row.sum_position);
+		} else grouped.set(key, { ...row });
+	}
+	return [...grouped.values()];
+}
 function createRowAccumulator(options = {}) {
 	const maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
 	const trackDateBoundary = options.trackDateBoundary === true;
@@ -260,11 +274,12 @@ function createRowAccumulator(options = {}) {
 				}
 				let dateRows = byDate.get(t.date);
 				if (!dateRows) {
-					dateRows = [];
+					dateRows = /* @__PURE__ */ new Map();
 					byDate.set(t.date, dateRows);
 				}
-				dateRows.push(t.row);
-				total++;
+				const sourceKey = JSON.stringify(r.keys);
+				if (!dateRows.has(sourceKey)) total++;
+				dateRows.set(sourceKey, t.row);
 				if (trackDateBoundary && (!newestDate || t.date > newestDate)) newestDate = t.date;
 				if (total > maxRows) {
 					if (newestDate) latestDate.set(table, newestDate);
@@ -276,7 +291,8 @@ function createRowAccumulator(options = {}) {
 			return true;
 		},
 		drain() {
-			const out = buckets;
+			const out = /* @__PURE__ */ new Map();
+			for (const [table, byDate] of buckets) out.set(table, new Map([...byDate].map(([date, sourceRows]) => [date, storedRows(table, sourceRows)])));
 			buckets = /* @__PURE__ */ new Map();
 			latestDate.clear();
 			total = 0;
@@ -295,8 +311,8 @@ function createRowAccumulator(options = {}) {
 						outBy = /* @__PURE__ */ new Map();
 						out.set(table, outBy);
 					}
-					outBy.set(date, dateRows);
-					total -= dateRows.length;
+					outBy.set(date, storedRows(table, dateRows));
+					total -= dateRows.size;
 				}
 				if (outBy) for (const date of outBy.keys()) byDate.delete(date);
 			}

package/dist/layout.d.mts

@@ -6,18 +6,17 @@ import { SearchType } from "gscdump/query";
  * sync paths that don't request a specific type. GSC's own default; the
  * vast majority of stored data is web-search.
  */
-declare const DEFAULT_SEARCH_TYPE: SearchType;
-declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
+export declare const DEFAULT_SEARCH_TYPE: SearchType;
+export declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
 /**
  * Resolve the search type for an entry, defaulting legacy entries to `web`.
  * Use this anywhere code needs to bucket entries by searchType.
  */
-declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType;
+export declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType;
 /**
  * Infer the tier for an entry that pre-dates the `tier` field. Daily files
  * are `raw`; monthly files are `d30`. Anything else (already migrated, or
  * a partition shape we haven't seen) returns undefined and the caller must
  * decide how to handle it.
  */
-declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
-export { DEFAULT_SEARCH_TYPE, inferLegacyTier, inferSearchType, objectKey };
+export declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/datapage.mjs

@@ -1,6 +1,6 @@
-import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
+import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { writeRleBitPackedHybrid } from "./encoding.mjs";
 import { writePlain } from "./plain.mjs";

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/indexes.mjs

@@ -1,5 +1,5 @@
-import { BoundaryOrders } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
+import { BoundaryOrders } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 function writeIndexes(writer, pageIndexes) {
 	for (const { chunk, columnIndex } of pageIndexes) writeColumnIndex(writer, chunk, columnIndex);
 	for (const { chunk, offsetIndex } of pageIndexes) writeOffsetIndex(writer, chunk, offsetIndex);

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/metadata.mjs

@@ -1,6 +1,6 @@
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 import { ThriftStruct, serializeTCompactProtocol } from "./thrift.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { unconvertStatistics } from "./unconvert.mjs";
 function writeMetadata(writer, metadata) {
 	const compact = {

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/types.d.mts

@@ -1,5 +1,5 @@
 import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
-interface Writer {
+export interface Writer {
   buffer: ArrayBuffer;
   view: DataView;
   offset: number; // total bytes written
@@ -19,5 +19,4 @@ interface Writer {
   appendVarInt(value: number): void;
   appendVarBigInt(value: bigint): void;
   appendZigZag(value: number | bigint): void;
-}
-export { Writer };
+}

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/index.d.mts

@@ -6,7 +6,7 @@ import "./schema.mjs";
 import "./bytewriter.mjs";
 import "./parquet-writer.mjs";
 import "./wkb.mjs";
-type Writer = Writer$1;
+export type Writer = Writer$1;
 /**
  * @typedef {import('hyparquet').KeyValue} KeyValue
  * @typedef {import('hyparquet').SchemaElement} SchemaElement
@@ -14,5 +14,4 @@ type Writer = Writer$1;
  * @typedef {import('../src/types.d.ts').ColumnSource} ColumnSource
  * @typedef {import('../src/types.d.ts').ParquetWriteOptions} ParquetWriteOptions
  * @typedef {import('../src/types.d.ts').Writer} Writer
- */
-export { Writer };
+ */

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/assemble.mjs

@@ -1,144 +1,3 @@
-import { getMaxDefinitionLevel, isListLike, isMapLike } from "./schema.mjs";
-import { decodeVariantColumn } from "./variant.mjs";
-function assembleLists(output, definitionLevels, repetitionLevels, values, schemaPath) {
-	const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
-	if (!definitionLevels?.length && !repetitionLevels.length) {
-		if (!maxDefinitionLevel || !values.length) return values;
-		definitionLevels = new Array(values.length).fill(maxDefinitionLevel);
-	}
-	const n = definitionLevels?.length || repetitionLevels.length;
-	const repetitionPath = schemaPath.map(({ element }) => element.repetition_type);
-	let valueIndex = 0;
-	const containerStack = [output];
-	let currentContainer = output;
-	let currentDepth = 0;
-	let currentDefLevel = 0;
-	let currentRepLevel = 0;
-	if (repetitionLevels[0]) while (currentDepth < repetitionPath.length - 2 && currentRepLevel < repetitionLevels[0]) {
-		currentDepth++;
-		if (repetitionPath[currentDepth] !== "REQUIRED") {
-			currentContainer = currentContainer.at(-1);
-			containerStack.push(currentContainer);
-			currentDefLevel++;
-		}
-		if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel++;
-	}
-	for (let i = 0; i < n; i++) {
-		const def = definitionLevels?.length ? definitionLevels[i] : maxDefinitionLevel;
-		const rep = repetitionLevels[i];
-		while (currentDepth && (rep < currentRepLevel || repetitionPath[currentDepth] !== "REPEATED")) {
-			if (repetitionPath[currentDepth] !== "REQUIRED") {
-				containerStack.pop();
-				currentDefLevel--;
-			}
-			if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel--;
-			currentDepth--;
-		}
-		currentContainer = containerStack.at(-1);
-		while ((currentDepth < repetitionPath.length - 2 || repetitionPath[currentDepth + 1] === "REPEATED") && (currentDefLevel < def || repetitionPath[currentDepth + 1] === "REQUIRED")) {
-			currentDepth++;
-			if (repetitionPath[currentDepth] !== "REQUIRED") {
-				const newList = [];
-				currentContainer.push(newList);
-				currentContainer = newList;
-				containerStack.push(newList);
-				currentDefLevel++;
-			}
-			if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel++;
-		}
-		if (def === maxDefinitionLevel) currentContainer.push(values[valueIndex++]);
-		else if (currentDepth === repetitionPath.length - 2) currentContainer.push(null);
-		else currentContainer.push([]);
-	}
-	if (!output.length) for (let i = 0; i < maxDefinitionLevel; i++) {
-		const newList = [];
-		currentContainer.push(newList);
-		currentContainer = newList;
-	}
-	return output;
-}
-function assembleNested(subcolumnData, schema, parsers, depth = 0) {
-	const path = schema.path.join(".");
-	const optional = schema.element.repetition_type === "OPTIONAL";
-	const nextDepth = optional ? depth + 1 : depth;
-	if (isListLike(schema)) {
-		let sublist = schema.children[0];
-		let subDepth = nextDepth;
-		if (sublist.children.length === 1) {
-			sublist = sublist.children[0];
-			subDepth++;
-		}
-		assembleNested(subcolumnData, sublist, parsers, subDepth);
-		const subcolumn = sublist.path.join(".");
-		const values = subcolumnData.get(subcolumn);
-		if (!values) throw new Error("parquet list column missing values");
-		if (optional) flattenAtDepth(values, depth);
-		subcolumnData.set(path, values);
-		subcolumnData.delete(subcolumn);
-		return;
-	}
-	if (isMapLike(schema)) {
-		const mapName = schema.children[0].element.name;
-		assembleNested(subcolumnData, schema.children[0].children[0], parsers, nextDepth + 1);
-		assembleNested(subcolumnData, schema.children[0].children[1], parsers, nextDepth + 1);
-		const keys = subcolumnData.get(`${path}.${mapName}.key`);
-		const values = subcolumnData.get(`${path}.${mapName}.value`);
-		if (!keys) throw new Error("parquet map column missing keys");
-		if (!values) throw new Error("parquet map column missing values");
-		if (keys.length !== values.length) throw new Error("parquet map column key/value length mismatch");
-		const out = assembleMaps(keys, values, nextDepth);
-		if (optional) flattenAtDepth(out, depth);
-		subcolumnData.delete(`${path}.${mapName}.key`);
-		subcolumnData.delete(`${path}.${mapName}.value`);
-		subcolumnData.set(path, out);
-		return;
-	}
-	if (schema.children.length) {
-		const invertDepth = schema.element.repetition_type === "REQUIRED" ? depth : depth + 1;
-		const struct = {};
-		for (const child of schema.children) {
-			assembleNested(subcolumnData, child, parsers, invertDepth);
-			const childData = subcolumnData.get(child.path.join("."));
-			if (!childData) throw new Error("parquet struct missing child data");
-			struct[child.element.name] = childData;
-		}
-		for (const child of schema.children) subcolumnData.delete(child.path.join("."));
-		let inverted = invertStruct(struct, invertDepth);
-		if (schema.element.logical_type?.type === "VARIANT") inverted = decodeVariantColumn(inverted, parsers);
-		if (optional) flattenAtDepth(inverted, depth);
-		subcolumnData.set(path, inverted);
-	}
-}
-function flattenAtDepth(arr, depth) {
-	for (let i = 0; i < arr.length; i++) if (depth) flattenAtDepth(arr[i], depth - 1);
-	else arr[i] = arr[i][0];
-}
-function assembleMaps(keys, values, depth) {
-	const out = [];
-	for (let i = 0; i < keys.length; i++) if (depth) out.push(assembleMaps(keys[i], values[i], depth - 1));
-	else if (keys[i]) {
-		const obj = {};
-		for (let j = 0; j < keys[i].length; j++) {
-			const value = values[i][j];
-			obj[keys[i][j]] = value === void 0 ? null : value;
-		}
-		out.push(obj);
-	} else out.push(void 0);
-	return out;
-}
-function invertStruct(struct, depth) {
-	const keys = Object.keys(struct);
-	const length = struct[keys[0]]?.length;
-	const out = [];
-	for (let i = 0; i < length; i++) {
-		const obj = {};
-		for (const key of keys) {
-			if (struct[key].length !== length) throw new Error("parquet struct parsing error");
-			obj[key] = struct[key][i];
-		}
-		if (depth) out.push(invertStruct(obj, depth - 1));
-		else out.push(obj);
-	}
-	return out;
-}
-export { assembleLists, assembleNested };
+import "./schema.mjs";
+import "./variant.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs

@@ -1,7 +1,7 @@
-import { deserializeTCompactProtocol } from "./thrift.mjs";
+import "./thrift.mjs";
 import { xxhash64 } from "./xxhash.mjs";
 const textEncoder = new TextEncoder();
-const SALT = new Uint32Array([
+new Uint32Array([
 	1203114875,
 	1150766481,
 	2284105051,
@@ -11,38 +11,6 @@ const SALT = new Uint32Array([
 	2667333959,
 	1550580529
 ]);
-function blockIndex(hash, numBlocks) {
-	return Number((hash >> 32n) * BigInt(numBlocks) >> 32n);
-}
-function blockMask(hash) {
-	const m = /* @__PURE__ */ new Uint32Array(8);
-	const low = Number(hash & 4294967295n) | 0;
-	for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT[i]) >>> 27);
-	return m;
-}
-function sbbfContains(blocks, hash) {
-	const offset = blockIndex(hash, blocks.length >> 3) << 3;
-	const m = blockMask(hash);
-	for (let i = 0; i < 8; i++) if ((blocks[offset + i] & m[i]) === 0) return false;
-	return true;
-}
-function readBloomFilter(reader) {
-	const header = deserializeTCompactProtocol(reader);
-	const numBytes = header.field_1;
-	if (typeof numBytes !== "number" || numBytes <= 0 || numBytes % 32 !== 0) return void 0;
-	if (!header.field_2?.field_1) return void 0;
-	if (!header.field_3?.field_1) return void 0;
-	if (!header.field_4?.field_1) return void 0;
-	const { view, offset } = reader;
-	if (offset + numBytes > view.byteLength) throw new Error(`parquet bloom filter truncated: need ${numBytes} bytes, have ${view.byteLength - offset}`);
-	const blocks = new Uint32Array(numBytes >> 2);
-	for (let i = 0; i < blocks.length; i++) blocks[i] = view.getUint32(offset + i * 4, true);
-	reader.offset = offset + numBytes;
-	return {
-		numBytes,
-		blocks
-	};
-}
 function hashParquetValue(value, element) {
 	if (value === null || value === void 0) return void 0;
 	const { type, converted_type, logical_type } = element;
@@ -98,27 +66,4 @@ function hashParquetValue(value, element) {
 		return;
 	}
 }
-function bloomEligibleColumns(filter) {
-	const out = /* @__PURE__ */ new Set();
-	walkBloomEligible(filter, out);
-	return out;
-}
-function walkBloomEligible(filter, out) {
-	if (!filter) return;
-	if ("$and" in filter && Array.isArray(filter.$and)) {
-		for (const sub of filter.$and) walkBloomEligible(sub, out);
-		return;
-	}
-	if ("$or" in filter && Array.isArray(filter.$or)) {
-		for (const sub of filter.$or) walkBloomEligible(sub, out);
-		return;
-	}
-	if ("$nor" in filter) return;
-	for (const [field, condition] of Object.entries(filter)) {
-		if (field.startsWith("$")) continue;
-		if (typeof condition === "object" && condition !== null && !Array.isArray(condition)) {
-			if ("$eq" in condition || "$in" in condition) out.add(field);
-		} else out.add(field);
-	}
-}
-export { bloomEligibleColumns, hashParquetValue, readBloomFilter, sbbfContains };
+export { hashParquetValue };