@gscdump/engine 2.0.6 → 2.1.1

package/dist/adapters/duckdb-node.mjs

@@ -43,10 +43,14 @@ function createNodeDuckDBHandle(opts = {}) {
 	return {
 		async query(sql, params) {
 			const { conn } = await getSingleton(opts);
-			if (!params || params.length === 0) return arrowToRows(conn.query(sql));
+			if (!params || params.length === 0) {
+				const result = conn.query(sql);
+				return arrowToRows(result);
+			}
 			const stmt = conn.prepare(sql);
 			try {
-				return arrowToRows(stmt.query(...params));
+				const result = stmt.query(...params);
+				return arrowToRows(result);
 			} finally {
 				stmt.close();
 			}

package/dist/adapters/filesystem.mjs

@@ -79,7 +79,8 @@ function createFilesystemDataSource(opts) {
 			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
 		},
 		async head(key) {
-			return stat(pathFor(key)).then((s) => ({ bytes: s.size }), (err) => {
+			const path = pathFor(key);
+			return stat(path).then((s) => ({ bytes: s.size }), (err) => {
 				if (err.code === "ENOENT") return void 0;
 				throw err;
 			});
@@ -121,7 +122,8 @@ async function walk(dir, out) {
 	}
 }
 function lockFileFor(locksDir, scope) {
-	return join(locksDir, `${`${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_")}.lock`);
+	const safe = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_");
+	return join(locksDir, `${safe}.lock`);
 }
 function createFilesystemManifestStore(opts) {
 	const manifestPath = resolve(opts.path);

package/dist/adapters/hyparquet.mjs

@@ -73,13 +73,11 @@ function buildWriteSchema(columns) {
 					repetition_type
 				});
 				break;
-			case "DOUBLE":
-				schema.push({
-					name: col.name,
-					type: "DOUBLE",
-					repetition_type
-				});
-				break;
+			case "DOUBLE": schema.push({
+				name: col.name,
+				type: "DOUBLE",
+				repetition_type
+			});
 		}
 	}
 	return schema;

package/dist/analysis-range.mjs

@@ -17,7 +17,8 @@ function analysisRequestedRange(params, onInvalidFilter) {
 		const direct = state.filter;
 		addRange(ranges, direct?.startDate, direct?.endDate);
 		try {
-			const extracted = extractDateRange(normalizeFilter(state.filter) ?? state.filter);
+			const normalized = normalizeFilter(state.filter) ?? state.filter;
+			const extracted = extractDateRange(normalized);
 			addRange(ranges, extracted.startDate, extracted.endDate);
 		} catch (error) {
 			if (!onInvalidFilter) throw error;

package/dist/duckdb.mjs

@@ -168,7 +168,8 @@ function createDuckDBExecutor(factory, options = {}) {
 			await registerBufferedFiles(db, [...bufferedByName.values()], dataSource, registered, signal, options.bufferReadConcurrency);
 			endRegister?.({ buffered: registered.length });
 			signal?.throwIfAborted();
-			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
+			const rewritten = rewriteEmptyFileSets(sql, placeholders, table, placeholderTables);
+			const finalSql = substituteNamedFiles(rewritten, placeholders);
 			const endQuery = profiler?.start("query.run");
 			const rows = await db.query(finalSql, params);
 			endQuery?.({ rows: rows.length });

package/dist/engine.mjs

@@ -7,7 +7,7 @@ import { extractParquetPushdown } from "./parquet-pushdown.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
 const URL_PURGE_TABLES = ["pages", "page_queries"];
-const MAX_DAY_BYTES = 100 * 1024 * 1024;
+const MAX_DAY_BYTES = 104857600;
 const URL_COLUMNS = /* @__PURE__ */ new Set();
 for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
 function normalizeRow(table, row) {
@@ -179,13 +179,14 @@ function createStorageEngine(opts) {
 		const endList = profiler?.start("manifest.list", { fileSets: entries.length });
 		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
 			if (ref.keys !== void 0) return [name, ref.keys];
-			return [name, dedupeOverlappingTiers(await manifestStore.listLive({
+			const list = await manifestStore.listLive({
 				userId: opts.ctx.userId,
 				siteId: opts.ctx.siteId,
 				table: ref.table,
 				partitions: ref.partitions,
 				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-			}), queryRangeOf(ref.partitions)).map((e) => e.objectKey)];
+			});
+			return [name, dedupeOverlappingTiers(list, queryRangeOf(ref.partitions)).map((e) => e.objectKey)];
 		}));
 		opts.signal?.throwIfAborted();
 		const fileKeys = {};
@@ -250,11 +251,12 @@ function createStorageEngine(opts) {
 		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
 	}
 	async function reconcileSubsumed(ctx) {
-		const { subsumed } = splitOverlappingTiers(await manifestStore.listLive({
+		const live = await manifestStore.listLive({
 			userId: ctx.userId,
 			siteId: ctx.siteId,
 			table: ctx.table
-		}));
+		});
+		const { subsumed } = splitOverlappingTiers(live);
 		if (subsumed.length === 0) return {
 			retired: 0,
 			partitions: []

package/dist/entities/inspection.mjs

@@ -7,7 +7,7 @@ import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
 import { classifyCanonicalDifference } from "gscdump";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
-const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
+const INSPECTION_HISTORY_MAX_BYTES = 5242880;
 const INSPECTION_PARQUET_COLUMNS = [
 	{
 		name: "urlHash",
@@ -265,7 +265,7 @@ function createInspectionStore(opts) {
 				if (!byMonth.has(month)) byMonth.set(month, []);
 				byMonth.get(month).push(r);
 			}
-			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
+			const shards = [...byMonth].map(([yearMonth, batch]) => {
 				const bytes = encodeJsonBigintSafe({
 					version: 1,
 					records: batch
@@ -275,7 +275,8 @@ function createInspectionStore(opts) {
 					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
 					bytes
 				};
-			}), (shard) => ds.write(shard.key, shard.bytes));
+			});
+			await mapEntityIo(shards, (shard) => ds.write(shard.key, shard.bytes));
 		},
 		async loadHistory(ctx, yearMonth) {
 			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));

package/dist/entities/query-dim.mjs

@@ -71,12 +71,14 @@ function createQueryDimStore({ dataSource }) {
 			};
 		},
 		async loadMeta(ctx) {
-			const bytes = await readOptional(dataSource, queryDimMetaKey(ctx));
+			const key = queryDimMetaKey(ctx);
+			const bytes = await readOptional(dataSource, key);
 			if (!bytes) return null;
 			return JSON.parse(new TextDecoder().decode(bytes));
 		},
 		async loadRecords(ctx) {
-			const bytes = await readOptional(dataSource, queryDimParquetKey(ctx));
+			const key = queryDimParquetKey(ctx);
+			const bytes = await readOptional(dataSource, key);
 			if (!bytes) return [];
 			return (await decodeParquetToRows(bytes)).map((r) => ({
 				query: String(r.query),

package/dist/gc.mjs

@@ -1,5 +1,5 @@
 import { tenantPrefix } from "./layout.mjs";
-const DEFAULT_HOURLY_RETENTION_MS = 2160 * 60 * 60 * 1e3;
+const DEFAULT_HOURLY_RETENTION_MS = 7776e6;
 const VERSION_RE = /__v(\d+)\.parquet$/;
 function parseLockScope(key) {
 	const match = VERSION_RE.exec(key);

package/dist/iceberg/overwrite-writer.mjs

@@ -37,9 +37,11 @@ function subprocessBackend(opts = {}) {
 	return async (job) => {
 		const { dirname, join } = await import("node:path");
 		const { fileURLToPath } = await import("node:url");
+		const python = resolvePyIcebergPython(opts.python);
+		const script = opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py");
 		return runPyIcebergWriter({
-			python: resolvePyIcebergPython(opts.python),
-			script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
+			python,
+			script,
 			job,
 			label: "iceberg overwrite subprocess",
 			processErrorAsParseFailure: true

package/dist/iceberg/pyiceberg-runtime.mjs

@@ -7,7 +7,7 @@ function resolvePyIcebergPython(override) {
 async function runPyIcebergWriter(options) {
 	const { execFile } = await import("node:child_process");
 	return new Promise((resolve, reject) => {
-		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+		execFile(options.python, [options.script], { maxBuffer: 67108864 }, (err, stdout, stderr) => {
 			let parsed;
 			let parseError;
 			if (stdout.trim()) try {

package/dist/node_modules/.pnpm/fzstd@0.1.1/node_modules/fzstd/esm/index.mjs

@@ -145,7 +145,8 @@ var rfse = function(dat, bt, mal) {
 	if (sympos) err(0);
 	for (i = 0; i < sz; ++i) {
 		var ns = dstate[syms[i]]++;
-		nstate[i] = (ns << (nbits[i] = al - msb(ns))) - sz;
+		var nb = nbits[i] = al - msb(ns);
+		nstate[i] = (ns << nb) - sz;
 	}
 	return [tpos + 7 >> 3, {
 		b: al,
@@ -382,9 +383,10 @@ var rzb = function(dat, st, out) {
 	if (btype == 2) {
 		var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
 		var lss = b3 >> 4, lcs = 0, s4 = 0;
-		if (lbt < 2) if (sf & 1) lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
-		else lss = b3 >> 3;
-		else {
+		if (lbt < 2) {
+			if (sf & 1) lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
+			else lss = b3 >> 3;
+		} else {
 			s4 = sf;
 			if (sf < 2) lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
 			else if (sf == 2) lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.bitreader.mjs

@@ -64,7 +64,7 @@ BrotliBitReader.prototype.readMoreInput = function() {
 			for (let p = 0; p < 32; p++) this.buf_[dst + bytes_read + p] = 0;
 		}
 		if (dst === 0) {
-			for (let p = 0; p < 32; p++) this.buf_[(BROTLI_READ_SIZE << 1) + p] = this.buf_[p];
+			for (let p = 0; p < 32; p++) this.buf_[8192 + p] = this.buf_[p];
 			this.buf_ptr_ = BROTLI_READ_SIZE;
 		} else this.buf_ptr_ = 0;
 		this.bit_end_pos_ += bytes_read << 3;

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.blocks.mjs

@@ -114,7 +114,8 @@ function jumpToByteBoundary(br) {
 	return !br.readBits(new_bit_pos - br.bit_pos_);
 }
 function readBlockLength(table, index, br) {
-	const { offset, nbits } = kBlockLengthPrefixCode[readSymbol(table, index, br)];
+	const code = readSymbol(table, index, br);
+	const { offset, nbits } = kBlockLengthPrefixCode[code];
 	return offset + br.readBits(nbits);
 }
 export { copyUncompressedBlockToOutput, decodeBlockType, decodeMetaBlockLength, decodeVarLenUint8, decodeWindowBits, jumpToByteBoundary, readBlockLength };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.huffman.mjs

@@ -147,7 +147,6 @@ function readHuffmanCode(alphabet_size, tables, table, br) {
 					code_lengths[symbols[2]] = 3;
 					code_lengths[symbols[3]] = 3;
 				} else code_lengths[symbols[0]] = 2;
-				break;
 		}
 	} else {
 		const code_length_code_lengths = new Uint8Array(CODE_LENGTH_CODES);

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.mjs

@@ -240,9 +240,11 @@ function brotli(input, output) {
 				range_idx -= 2;
 				distance_code = -1;
 			} else distance_code = 0;
-			const insertPrefix = kInsertLengthPrefixCode[kInsertRangeLut[range_idx] + (cmd_code >> 3 & 7)];
+			const insertIndex = kInsertRangeLut[range_idx] + (cmd_code >> 3 & 7);
+			const insertPrefix = kInsertLengthPrefixCode[insertIndex];
 			const insertLength = insertPrefix.offset + br.readBits(insertPrefix.nbits);
-			const copyCode = kCopyLengthPrefixCode[kCopyRangeLut[range_idx] + (cmd_code & 7)];
+			const copyIndex = kCopyRangeLut[range_idx] + (cmd_code & 7);
+			const copyCode = kCopyLengthPrefixCode[copyIndex];
 			const copyLength = copyCode.offset + br.readBits(copyCode.nbits);
 			prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
 			prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
@@ -292,25 +294,26 @@ function brotli(input, output) {
 			if (pos < max_backward_distance && max_distance !== max_backward_distance) max_distance = pos;
 			else max_distance = max_backward_distance;
 			let copy_dst = pos & ringbuffer_mask;
-			if (distance > max_distance) if (copyLength >= minDictionaryWordLength && copyLength <= maxDictionaryWordLength) {
-				let offset = offsetsByLength[copyLength];
-				const word_id = distance - max_distance - 1;
-				const shift = sizeBitsByLength[copyLength];
-				const word_idx = word_id & (1 << shift) - 1;
-				const transform_idx = word_id >> shift;
-				offset += word_idx * copyLength;
-				if (transform_idx < kNumTransforms) {
-					const len = transformDictionaryWord(ringbuffer, copy_dst, offset, copyLength, transform_idx);
-					copy_dst += len;
-					pos += len;
-					meta_block_remaining_len -= len;
-					if (copy_dst >= ringbuffer_end) {
-						output.write(ringbuffer, ringbuffer_size);
-						for (let _x = 0; _x < copy_dst - ringbuffer_end; _x++) ringbuffer[_x] = ringbuffer[ringbuffer_end + _x];
-					}
+			if (distance > max_distance) {
+				if (copyLength >= minDictionaryWordLength && copyLength <= maxDictionaryWordLength) {
+					let offset = offsetsByLength[copyLength];
+					const word_id = distance - max_distance - 1;
+					const shift = sizeBitsByLength[copyLength];
+					const word_idx = word_id & (1 << shift) - 1;
+					const transform_idx = word_id >> shift;
+					offset += word_idx * copyLength;
+					if (transform_idx < kNumTransforms) {
+						const len = transformDictionaryWord(ringbuffer, copy_dst, offset, copyLength, transform_idx);
+						copy_dst += len;
+						pos += len;
+						meta_block_remaining_len -= len;
+						if (copy_dst >= ringbuffer_end) {
+							output.write(ringbuffer, ringbuffer_size);
+							for (let _x = 0; _x < copy_dst - ringbuffer_end; _x++) ringbuffer[_x] = ringbuffer[ringbuffer_end + _x];
+						}
+					} else throw new Error("Invalid backward reference");
 				} else throw new Error("Invalid backward reference");
-			} else throw new Error("Invalid backward reference");
-			else {
+			} else {
 				if (distance_code > 0) {
 					dist_rb[dist_rb_idx & 3] = distance;
 					dist_rb_idx++;

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.transform.mjs

@@ -167,7 +167,7 @@ function transformDictionaryWord(dst, idx, word, len, transform) {
 	const { prefix } = kTransforms[transform];
 	const { suffix } = kTransforms[transform];
 	const t = kTransforms[transform].transform;
-	let skip = t < kOmitFirst1 ? 0 : t - (kOmitFirst1 - 1);
+	let skip = t < kOmitFirst1 ? 0 : t - 11;
 	const start_idx = idx;
 	if (skip > len) skip = len;
 	let prefix_pos = 0;

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bloom.mjs

@@ -10,9 +10,7 @@ const SALT = new Uint32Array([
 	2667333959,
 	1550580529
 ]);
-const BYTES_PER_BLOCK = 32;
 const MIN_BYTES = 32;
-const MAX_BYTES = 128 * 1024 * 1024;
 function blockIndex(hash, numBlocks) {
 	return Number((hash >> 32n) * BigInt(numBlocks) >> 32n);
 }
@@ -37,8 +35,8 @@ function optimalNumBytes(ndv, fpp) {
 	if (!(ndv >= 0)) throw new Error(`bloom filter ndv must be >= 0, got ${ndv}`);
 	const m = -8 * ndv / Math.log(1 - fpp ** (1 / 8));
 	let numBits = Math.ceil(m);
-	if (!isFinite(numBits) || numBits > MAX_BYTES << 3) numBits = MAX_BYTES << 3;
-	const blockBits = BYTES_PER_BLOCK << 3;
+	if (!isFinite(numBits) || numBits > 1073741824) numBits = 1073741824;
+	const blockBits = 256;
 	numBits = Math.ceil(numBits / blockBits) * blockBits;
 	let numBytes = numBits >> 3;
 	if (numBytes < MIN_BYTES) numBytes = MIN_BYTES;
@@ -46,7 +44,7 @@ function optimalNumBytes(ndv, fpp) {
 	return numBytes;
 }
 var BloomBuilder = class {
-	constructor(element, { fpp = .01, maxBytes = 1024 * 1024 } = {}) {
+	constructor(element, { fpp = .01, maxBytes = 1048576 } = {}) {
 		this.element = element;
 		this.fpp = fpp;
 		this.maxBytes = maxBytes;

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/column.mjs

@@ -15,7 +15,8 @@ function writeColumn({ writer, column, pageData }) {
 	const geospatial_statistics = stats && isGeospatial ? geospatialStatistics(values) : void 0;
 	let bloomFilter;
 	if (column.bloomFilter) {
-		const builder = new BloomBuilder(element, typeof column.bloomFilter === "object" ? column.bloomFilter : void 0);
+		const opts = typeof column.bloomFilter === "object" ? column.bloomFilter : void 0;
+		const builder = new BloomBuilder(element, opts);
 		for (const v of values) builder.insert(v);
 		bloomFilter = builder.finalize();
 	}
@@ -29,7 +30,8 @@ function writeColumn({ writer, column, pageData }) {
 		writeType = "INT32";
 		encoding = "RLE_DICTIONARY";
 		dictionary_page_offset = BigInt(writer.offset);
-		writeDictionaryPage(writer, column, unconvert(element, dictionary));
+		const unconverted = unconvert(element, dictionary);
+		writeDictionaryPage(writer, column, unconverted);
 	} else {
 		writeValues = unconvert(element, values);
 		encoding = userEncoding ?? (type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN");

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/datapage.mjs

@@ -59,7 +59,7 @@ function writeDataPageV2({ writer, column, encoding, pageData }) {
 	writer.appendBytes(compressedBytes);
 }
 function writePageHeader(writer, header) {
-	serializeTCompactProtocol(writer, {
+	const compact = {
 		field_1: PageTypes.indexOf(header.type),
 		field_2: header.uncompressed_page_size,
 		field_3: header.compressed_page_size,
@@ -83,7 +83,8 @@ function writePageHeader(writer, header) {
 			field_6: header.data_page_header_v2.repetition_levels_byte_length,
 			field_7: header.data_page_header_v2.is_compressed ? void 0 : false
 		}
-	});
+	};
+	serializeTCompactProtocol(writer, compact);
 }
 function writeLevels(writer, column, dataPage) {
 	const { schemaPath } = column;
@@ -99,9 +100,15 @@ function writeLevels(writer, column, dataPage) {
 	}
 	const maxRepetitionLevel = getMaxRepetitionLevel(schemaPath);
 	let repetition_levels_byte_length = 0;
-	if (maxRepetitionLevel) repetition_levels_byte_length = writeRleBitPackedHybrid(writer, repetitionLevels, Math.ceil(Math.log2(maxRepetitionLevel + 1)));
+	if (maxRepetitionLevel) {
+		const bitWidth = Math.ceil(Math.log2(maxRepetitionLevel + 1));
+		repetition_levels_byte_length = writeRleBitPackedHybrid(writer, repetitionLevels, bitWidth);
+	}
 	let definition_levels_byte_length = 0;
-	if (maxDefinitionLevel) definition_levels_byte_length = writeRleBitPackedHybrid(writer, definitionLevels, Math.ceil(Math.log2(maxDefinitionLevel + 1)));
+	if (maxDefinitionLevel) {
+		const bitWidth = Math.ceil(Math.log2(maxDefinitionLevel + 1));
+		definition_levels_byte_length = writeRleBitPackedHybrid(writer, definitionLevels, bitWidth);
+	}
 	return {
 		definition_levels_byte_length,
 		repetition_levels_byte_length,

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/wkb.d.mts

@@ -1 +1 @@
-export { };
+export {}

package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/types.d.mts

@@ -1 +1 @@
-export { };
+export {}

package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/snappy.d.mts

@@ -1 +1 @@
-export { };
+export {}

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/column.mjs

@@ -56,7 +56,8 @@ function readPage(reader, header, columnDecoder, dictionary, previousChunk, page
 		const daph = header.data_page_header;
 		if (!daph) throw new Error("parquet data page header is undefined");
 		if (pageStart > daph.num_values && isFlatColumn(schemaPath)) return { skipped: daph.num_values };
-		const { definitionLevels, repetitionLevels, dataPage } = readDataPage(decompressPage(compressedBytes, Number(header.uncompressed_page_size), codec, compressors), daph, columnDecoder);
+		const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), codec, compressors);
+		const { definitionLevels, repetitionLevels, dataPage } = readDataPage(page, daph, columnDecoder);
 		const values = convertWithDictionary(dataPage, dictionary, daph.encoding, columnDecoder);
 		return {
 			skipped: 0,
@@ -76,12 +77,13 @@ function readPage(reader, header, columnDecoder, dictionary, previousChunk, page
 		const diph = header.dictionary_page_header;
 		if (!diph) throw new Error("parquet dictionary page header is undefined");
 		const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), codec, compressors);
+		const reader = {
+			view: new DataView(page.buffer, page.byteOffset, page.byteLength),
+			offset: 0
+		};
 		return {
 			skipped: 0,
-			data: readPlain({
-				view: new DataView(page.buffer, page.byteOffset, page.byteLength),
-				offset: 0
-			}, type, diph.num_values, element.type_length)
+			data: readPlain(reader, type, diph.num_values, element.type_length)
 		};
 	} else throw new Error(`parquet unsupported page type: ${header.type}`);
 }

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs

@@ -44,10 +44,11 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 	const metadataLengthOffset = view.byteLength - 8;
 	const metadataLength = view.getUint32(metadataLengthOffset, true);
 	if (metadataLength > view.byteLength - 8) throw new Error(`parquet metadata length ${metadataLength} exceeds available buffer ${view.byteLength - 8}`);
-	const metadata = deserializeTCompactProtocol({
+	const reader = {
 		view,
 		offset: metadataLengthOffset - metadataLength
-	});
+	};
+	const metadata = deserializeTCompactProtocol(reader);
 	const version = metadata.field_1;
 	const schema = metadata.field_2.map((field) => ({
 		type: ParquetTypes[field.field_1],

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plain.mjs

@@ -17,7 +17,8 @@ function readPlainBoolean(reader, count) {
 	for (let i = 0; i < count; i++) {
 		const byteOffset = reader.offset + (i / 8 | 0);
 		const bitOffset = i % 8;
-		values[i] = (reader.view.getUint8(byteOffset) & 1 << bitOffset) !== 0;
+		const byte = reader.view.getUint8(byteOffset);
+		values[i] = (byte & 1 << bitOffset) !== 0;
 	}
 	reader.offset += Math.ceil(count / 8);
 	return values;

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/rowgroup.mjs

@@ -25,10 +25,11 @@ function readRowGroup(options, { metadata }, groupPlan) {
 			asyncColumns.push({
 				pathInSchema,
 				data: Promise.resolve(options.file.slice(startByte, endByte)).then((buffer) => {
-					return readColumn({
+					const reader = {
 						view: new DataView(buffer),
 						offset: 0
-					}, groupPlan, columnDecoder, options.onPage);
+					};
+					return readColumn(reader, groupPlan, columnDecoder, options.onPage);
 				})
 			});
 			continue;
@@ -55,15 +56,17 @@ function readRowGroup(options, { metadata }, groupPlan) {
 				}
 				if (skipped < 0) skipped = 0;
 				const buffer = await options.file.slice(startByte, endByte);
-				const { data, skipped: columnSkipped } = readColumn({
+				const reader = {
 					view: new DataView(buffer),
 					offset: 0
-				}, skipped ? {
+				};
+				const adjustedGroupPlan = skipped ? {
 					...groupPlan,
 					groupStart: groupPlan.groupStart + skipped,
 					selectStart: groupPlan.selectStart - skipped,
 					selectEnd: groupPlan.selectEnd - skipped
-				} : groupPlan, columnDecoder, options.onPage);
+				} : groupPlan;
+				const { data, skipped: columnSkipped } = readColumn(reader, adjustedGroupPlan, columnDecoder, options.onPage);
 				return {
 					data,
 					skipped: skipped + columnSkipped

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs

@@ -56,8 +56,6 @@ function snappyUncompress(input, output) {
 					len = (c >>> 2) + 1;
 					offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
 					pos += 4;
-					break;
-				default: break;
 			}
 			if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
 			if (offset > outPos) throw new Error("cannot copy from before start of buffer");

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/variant.mjs

@@ -191,10 +191,13 @@ function readVariantArray(reader, header, metadata, parsers) {
 	for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetWidth);
 	const valuesStart = reader.offset;
 	const result = new Array(numElements);
-	for (let i = 0; i < numElements; i++) result[i] = readVariant({
-		view: reader.view,
-		offset: valuesStart + offsets[i]
-	}, metadata, parsers);
+	for (let i = 0; i < numElements; i++) {
+		const valueReader = {
+			view: reader.view,
+			offset: valuesStart + offsets[i]
+		};
+		result[i] = readVariant(valueReader, metadata, parsers);
+	}
 	reader.offset = valuesStart + offsets[offsets.length - 1];
 	return result;
 }

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/snappy.d.mts

@@ -1 +1 @@
-export { };
+export {}

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs

@@ -10,11 +10,12 @@ async function restCatalogCreateTable(ctx, { namespace, table, schema, location,
 	if (writeOrder !== void 0) body["write-order"] = writeOrder;
 	if (stageCreate !== void 0) body["stage-create"] = stageCreate;
 	if (properties !== void 0) body.properties = properties;
-	const responseBody = parseIcebergJson(await (await restFetch(ctx, `namespaces/${ns}/tables`, {
+	const res = await restFetch(ctx, `namespaces/${ns}/tables`, {
 		method: "POST",
 		headers: { "content-type": "application/json" },
 		body: stringifyIcebergJson(body)
-	})).text());
+	});
+	const responseBody = parseIcebergJson(await res.text());
 	return {
 		metadataLocation: responseBody["metadata-location"],
 		metadata: responseBody.metadata,

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs

@@ -12,7 +12,7 @@ Object.freeze({
 	initialMs: 50,
 	maxMs: 3e3,
 	factor: 2,
-	totalTimeoutMs: 1800 * 1e3
+	totalTimeoutMs: 18e5
 });
 async function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }) {
 	if (catalog.type === "rest") {

package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/ast.d.mts

@@ -1 +1 @@
-export { };
+export {}

package/dist/parquet-plan.mjs

@@ -32,7 +32,6 @@ function buildDimensionWhere(filters, table) {
 			case "excludingRegex":
 				clauses.push(`NOT regexp_matches(${column}, ?)`);
 				params.push(filter.expression);
-				break;
 		}
 	}
 	return {
@@ -73,7 +72,6 @@ function buildHaving(filters) {
 			case "metricBetween":
 				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
 				params.push(filter.expression, filter.expression2 ?? filter.expression);
-				break;
 		}
 	}
 	return {

package/dist/period/index.mjs

@@ -72,7 +72,6 @@ function resolveWindow(opts) {
 			if (!opts.start || !opts.end) throw new Error("resolveWindow: preset=custom requires start and end");
 			start = opts.start;
 			end = opts.end;
-			break;
 	}
 	const days = daysBetween(start, end);
 	const result = {

package/dist/resolver/compile.mjs

@@ -171,7 +171,6 @@ function resolveToSQLOptimized(state, options) {
 		case "position":
 			outerSelect.push(sql.raw("sum_position / NULLIF(impressions, 0) + 1 as position"));
 			outerTotals.push(sql.raw("SUM(sum_position) OVER() / NULLIF(SUM(impressions) OVER(), 0) + 1 as totalPosition"));
-			break;
 	}
 	let orderByColumnOverride;
 	const orderColumn = state.orderBy?.column;
@@ -187,9 +186,7 @@ function resolveToSQLOptimized(state, options) {
 			case "ctr":
 				outerSelect.push(sql.raw(`CAST(clicks AS REAL) / NULLIF(impressions, 0) as "${orderByColumnOverride}"`));
 				break;
-			case "position":
-				outerSelect.push(sql.raw(`sum_position / NULLIF(impressions, 0) + 1 as "${orderByColumnOverride}"`));
-				break;
+			case "position": outerSelect.push(sql.raw(`sum_position / NULLIF(impressions, 0) + 1 as "${orderByColumnOverride}"`));
 		}
 	}
 	outerSelect.push(sql.raw("COUNT(*) OVER() as totalCount"));

package/dist/resolver/fragments.mjs

@@ -53,7 +53,9 @@ function createSqlFragments(config) {
 	function fromSql(tableKey, options = {}) {
 		const base = tableRef(tableKey);
 		if (!options.queryCanonical || queryCanonicalSource !== "queryDim") return base;
-		return sql`${base} LEFT JOIN ${queryDimTableRef?.() ?? sql.raw(`${quoteIdent(QUERY_DIM_ALIAS)}`)} ON ${queryDimSiteScoped ? sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")} AND ${colRef(tableKey, "site_id")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "site_id")}` : sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")}`}`;
+		const dimTable = queryDimTableRef?.() ?? sql.raw(`${quoteIdent(QUERY_DIM_ALIAS)}`);
+		const joinOn = queryDimSiteScoped ? sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")} AND ${colRef(tableKey, "site_id")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "site_id")}` : sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")}`;
+		return sql`${base} LEFT JOIN ${dimTable} ON ${joinOn}`;
 	}
 	function dateColRef(tableKey) {
 		return colRef(tableKey, "date");
@@ -169,9 +171,7 @@ function createSqlFragments(config) {
 				case "includingRegex":
 					preds.push(regexPredicate(patternExpr, f.expression, false));
 					break;
-				case "excludingRegex":
-					preds.push(regexPredicate(patternExpr, f.expression, true));
-					break;
+				case "excludingRegex": preds.push(regexPredicate(patternExpr, f.expression, true));
 			}
 		}
 		return preds;

package/dist/resolver/run-query.mjs

@@ -126,8 +126,9 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 	const canonicalDecision = decideCanonicalSource(state, probe.capabilities, options, dateRange.endDate);
 	const useCanonicalSource = canonicalDecision.kind === "canonical-rollup";
 	const source = useCanonicalSource ? canonicalDecision : decidePrimarySource(options, dateRange, sourceFallbacks(canonicalDecision));
+	const adapter = useCanonicalSource ? createParquetResolverAdapter({ queryCanonicalSource: "column" }) : probe;
 	const optimized = resolveToSQLOptimized(state, {
-		adapter: useCanonicalSource ? createParquetResolverAdapter({ queryCanonicalSource: "column" }) : probe,
+		adapter,
 		siteId: void 0
 	});
 	const extras = buildExtrasQueries(state, {

package/dist/resolver/schema-drift.mjs

@@ -2,7 +2,8 @@ import { SCHEMAS } from "../schema.mjs";
 function assertSchemaInSync(options) {
 	const { label, schema, tableKeyToName, mode } = options;
 	for (const [key, table] of Object.entries(schema)) {
-		const sourceCols = SCHEMAS[tableKeyToName(key)].columns.map((c) => c.name).sort();
+		const tableName = tableKeyToName(key);
+		const sourceCols = SCHEMAS[tableName].columns.map((c) => c.name).sort();
 		const drizzleCols = Object.keys(table[Symbol.for("drizzle:Columns")] ?? {}).sort();
 		const missing = sourceCols.filter((c) => !drizzleCols.includes(c));
 		const extra = mode === "exact" ? drizzleCols.filter((c) => !sourceCols.includes(c)) : [];

package/dist/rollups/canonical.mjs

@@ -221,11 +221,12 @@ async function rebuildCanonicalDailyResumable(opts) {
 	const pageRows = opts.pageRows ?? 7e4;
 	const maxWindowDays = opts.maxWindowDays ?? 7;
 	const shardCount = Math.max(1, Math.floor(opts.shardCount ?? 1));
-	const windows = planRollupWindows((await engine.listPartitions({
+	const parts = await engine.listPartitions({
 		ctx,
 		table: "queries",
 		...sType
-	})).map((p) => ({
+	});
+	const windows = planRollupWindows(parts.map((p) => ({
 		partition: p.partition,
 		bytes: p.bytes
 	})), void 0, maxWindowDays);

package/dist/rollups/core.mjs

@@ -67,13 +67,14 @@ async function rebuildRollups(opts) {
 					parquetKey,
 					rowCount: rows.length
 				};
-				const envelopeBytes = encodeJsonBigintSafe({
+				const envelope = {
 					version: 1,
 					id: def.id,
 					builtAt,
 					windowDays: def.windowDays,
 					payload: pointer
-				});
+				};
+				const envelopeBytes = encodeJsonBigintSafe(envelope);
 				const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
 				await opts.dataSource.write(key, envelopeBytes);
 				results.push({
@@ -86,13 +87,14 @@ async function rebuildRollups(opts) {
 				});
 				continue;
 			}
-			const bytes = encodeJsonBigintSafe({
+			const envelope = {
 				version: 1,
 				id: def.id,
 				builtAt,
 				windowDays: def.windowDays,
 				payload
-			});
+			};
+			const bytes = encodeJsonBigintSafe(envelope);
 			const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
 			await opts.dataSource.write(key, bytes);
 			results.push({

package/dist/rollups/indexing.mjs

@@ -116,11 +116,12 @@ const indexPercentRollup = {
 		const currentMembership = "read_parquet({{SITEMAP_BASES}}, union_by_name = true)";
 		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const factSearchType = searchType ?? "web";
-		const pagesPartitions = partitionsInRange(await engine.listPartitions({
+		const pagesParts = await engine.listPartitions({
 			ctx,
 			table: "pages",
 			searchType: factSearchType
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		});
+		const pagesPartitions = partitionsInRange(pagesParts, cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		const numerator = await engine.runSQL({
 			ctx,
 			table: "pages",

package/dist/rollups/traffic.mjs

@@ -111,11 +111,12 @@ const topPages28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, windowAnchorMs, searchType }) {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
+		const parts = await engine.listPartitions({
 			ctx,
 			table: "pages",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		});
+		const partitions = partitionsInRange(parts, cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -150,11 +151,12 @@ const topCountries28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, windowAnchorMs, searchType }) {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
+		const parts = await engine.listPartitions({
 			ctx,
 			table: "countries",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		});
+		const partitions = partitionsInRange(parts, cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -189,11 +191,12 @@ const topKeywords28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, windowAnchorMs, searchType }) {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
+		const parts = await engine.listPartitions({
 			ctx,
 			table: "queries",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		});
+		const partitions = partitionsInRange(parts, cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -252,11 +255,12 @@ const topKeywords28dParquetRollup = {
 	parquetSortKey: ["clicks"],
 	async build({ engine, ctx, windowAnchorMs, searchType }) {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
+		const parts = await engine.listPartitions({
 			ctx,
 			table: "queries",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		});
+		const partitions = partitionsInRange(parts, cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,

package/dist/rollups/windows.mjs

@@ -1,5 +1,5 @@
 import { MS_PER_DAY } from "gscdump/dates";
-const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
+const WINDOW_BYTE_BUDGET = 10485760;
 const ROLLUP_PAGE_ROWS = 5e4;
 const ROLLUP_PAGE_ROWS_WIDE = 2e4;
 const ROLLUP_PAGE_ROWS_DAILY = 7e4;

package/dist/schedule.mjs

@@ -1,4 +1,4 @@
-const DAY = 24 * (3600 * 1e3);
+const DAY = 864e5;
 function isDue(state, now) {
 	return now >= state.nextAt;
 }

package/dist/source/attached-table.mjs

@@ -48,7 +48,8 @@ function createAttachedTableSource(runner, options) {
 				if (missing.length > 0) throw new AttachedTableMissingError(missing);
 			}
 			const rewritten = rewriteForTableSource(sql, schema, fileSets);
-			return coerceRows(await runner.query(rewritten, params ?? [], signal));
+			const rows = await runner.query(rewritten, params ?? [], signal);
+			return coerceRows(rows);
 		}
 	};
 }

package/dist/source/create-sql-query-source.mjs

@@ -18,10 +18,12 @@ function createSqlQuerySource(options) {
 				siteId,
 				searchType
 			});
-			return coerceRows(await execute(resolved.sql, resolved.params));
+			const rows = await execute(resolved.sql, resolved.params);
+			return coerceRows(rows);
 		},
 		async executeSql(sql, params) {
-			return coerceRows(await execute(sql, params ?? []));
+			const rows = await execute(sql, params ?? []);
+			return coerceRows(rows);
 		}
 	};
 }

package/dist/source/index.mjs

@@ -37,10 +37,11 @@ function createEngineQuerySource(options) {
 			const filterDims = getFilterDimensions(state.filter, isMetricDimension);
 			assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
 			if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
-			return coerceRows((await engine.query({
+			const result = await engine.query({
 				...ctx,
 				...searchType !== void 0 ? { searchType } : {}
-			}, state)).rows);
+			}, state);
+			return coerceRows(result.rows);
 		},
 		async executeSql(sql, params, opts) {
 			const fileSets = opts?.fileSets;

package/dist/sync-config.mjs

@@ -74,7 +74,7 @@ function getTablesForTier(tier) {
 }
 function getDateWeight(date, now = /* @__PURE__ */ new Date()) {
 	const target = new Date(date);
-	const daysAgo = Math.floor((now.getTime() - target.getTime()) / (1e3 * 60 * 60 * 24));
+	const daysAgo = Math.floor((now.getTime() - target.getTime()) / 864e5);
 	if (daysAgo <= 3) return "fresh";
 	if (daysAgo <= 60) return "recent";
 	return "historical";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "2.0.6",
+  "version": "2.1.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -186,13 +186,13 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.4",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^2.0.6",
-    "@gscdump/lakehouse": "^2.0.6",
-    "gscdump": "^2.0.6"
+    "@gscdump/lakehouse": "^2.1.1",
+    "gscdump": "^2.1.1",
+    "@gscdump/contracts": "^2.1.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
-    "@types/node": "^26.1.2",
+    "@types/node": "^26.2.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.26.2",