@gscdump/engine 0.35.2 → 0.35.4

package/dist/THIRD-PARTY-LICENSES.md

@@ -35,11 +35,11 @@ Repository: https://github.com/101arrowz/fzstd
 
 ---------------------------------------
 
-## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 
 > The MIT License (MIT)
 > 

package/dist/_chunks/libs/hyparquet.mjs

@@ -1,4 +1,4 @@
-const ParquetTypes = [
+const ParquetTypes$1 = [
 	"BOOLEAN",
 	"INT32",
 	"INT64",
@@ -8,7 +8,7 @@ const ParquetTypes = [
 	"BYTE_ARRAY",
 	"FIXED_LEN_BYTE_ARRAY"
 ];
-const Encodings = [
+const Encodings$1 = [
 	"PLAIN",
 	"GROUP_VAR_INT",
 	"PLAIN_DICTIONARY",
@@ -20,12 +20,12 @@ const Encodings = [
 	"RLE_DICTIONARY",
 	"BYTE_STREAM_SPLIT"
 ];
-const FieldRepetitionTypes = [
+const FieldRepetitionTypes$1 = [
 	"REQUIRED",
 	"OPTIONAL",
 	"REPEATED"
 ];
-const ConvertedTypes = [
+const ConvertedTypes$1 = [
 	"UTF8",
 	"MAP",
 	"MAP_KEY_VALUE",
@@ -49,7 +49,7 @@ const ConvertedTypes = [
 	"BSON",
 	"INTERVAL"
 ];
-const CompressionCodecs = [
+const CompressionCodecs$1 = [
 	"UNCOMPRESSED",
 	"SNAPPY",
 	"GZIP",
@@ -59,18 +59,13 @@ const CompressionCodecs = [
 	"ZSTD",
 	"LZ4_RAW"
 ];
-const PageTypes = [
+const PageTypes$1 = [
 	"DATA_PAGE",
 	"INDEX_PAGE",
 	"DICTIONARY_PAGE",
 	"DATA_PAGE_V2"
 ];
-const BoundaryOrders = [
-	"UNORDERED",
-	"ASCENDING",
-	"DESCENDING"
-];
-const EdgeInterpolationAlgorithms = [
+const EdgeInterpolationAlgorithms$1 = [
 	"SPHERICAL",
 	"VINCENTY",
 	"THOMAS",
@@ -170,7 +165,7 @@ function readPolygon(reader, flags) {
 	}
 	return rings;
 }
-const decoder$2 = new TextDecoder();
+const decoder$5 = new TextDecoder();
 const DEFAULT_PARSERS = {
 	timestampFromMilliseconds(millis) {
 		return new Date(Number(millis));
@@ -185,7 +180,10 @@ const DEFAULT_PARSERS = {
 		return /* @__PURE__ */ new Date(days * 864e5);
 	},
 	stringFromBytes(bytes) {
-		return bytes && decoder$2.decode(bytes);
+		return bytes && decoder$5.decode(bytes);
+	},
+	jsonFromBytes(bytes) {
+		return bytes && JSON.parse(decoder$5.decode(bytes));
 	},
 	geometryFromBytes(bytes) {
 		return bytes && wkbToGeojson({
@@ -229,7 +227,7 @@ function convert(data, columnDecoder) {
 	if (ctype === "DATE") return Array.from(data).map((v) => parsers.dateFromDays(v));
 	if (ctype === "TIMESTAMP_MILLIS") return Array.from(data).map((v) => parsers.timestampFromMilliseconds(v));
 	if (ctype === "TIMESTAMP_MICROS") return Array.from(data).map((v) => parsers.timestampFromMicroseconds(v));
-	if (ctype === "JSON") return data.map((v) => JSON.parse(decoder$2.decode(v)));
+	if (ctype === "JSON") return data.map((v) => parsers.jsonFromBytes(v));
 	if (ctype === "BSON") throw new Error("parquet bson not supported");
 	if (ctype === "INTERVAL") throw new Error("parquet interval not supported");
 	if (ltype?.type === "GEOMETRY") return data.map((v) => parsers.geometryFromBytes(v));
@@ -283,13 +281,13 @@ function parseFloat16(bytes) {
 	if (exp === 31) return frac ? NaN : sign * Infinity;
 	return sign * 2 ** (exp - 15) * (1 + frac / 1024);
 }
-function schemaTree(schema, rootIndex, path) {
+function schemaTree$1(schema, rootIndex, path) {
 	const element = schema[rootIndex];
 	const children = [];
 	let count = 1;
 	if (element.num_children) while (children.length < element.num_children) {
 		const childElement = schema[rootIndex + count];
-		const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
+		const child = schemaTree$1(schema, rootIndex + count, [...path, childElement.name]);
 		count += child.count;
 		children.push(child);
 	}
@@ -300,8 +298,8 @@ function schemaTree(schema, rootIndex, path) {
 		path
 	};
 }
-function getSchemaPath(schema, name) {
-	let tree = schemaTree(schema, 0, []);
+function getSchemaPath$1(schema, name) {
+	let tree = schemaTree$1(schema, 0, []);
 	const path = [tree];
 	for (const part of name) {
 		const child = tree.children.find((child) => child.element.name === part);
@@ -325,12 +323,12 @@ function getMaxRepetitionLevel(schemaPath) {
 	for (const { element } of schemaPath) if (element.repetition_type === "REPEATED") maxLevel++;
 	return maxLevel;
 }
-function getMaxDefinitionLevel(schemaPath) {
+function getMaxDefinitionLevel$1(schemaPath) {
 	let maxLevel = 0;
 	for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
 	return maxLevel;
 }
-function isListLike(schema) {
+function isListLike$1(schema) {
 	if (!schema) return false;
 	if (schema.element.converted_type !== "LIST") return false;
 	if (schema.children.length > 1) return false;
@@ -339,7 +337,7 @@ function isListLike(schema) {
 	if (firstChild.element.repetition_type !== "REPEATED") return false;
 	return true;
 }
-function isMapLike(schema) {
+function isMapLike$1(schema) {
 	if (!schema) return false;
 	if (schema.element.converted_type !== "MAP") return false;
 	if (schema.children.length > 1) return false;
@@ -466,9 +464,9 @@ function markGeoColumns(schema, key_value_metadata) {
 	}
 }
 const defaultInitialFetchSize = 1 << 19;
-const decoder$1 = new TextDecoder();
+const decoder$4 = new TextDecoder();
 function decode(value) {
-	return value && decoder$1.decode(value);
+	return value && decoder$4.decode(value);
 }
 async function parquetMetadataAsync(asyncBuffer, { parsers, initialFetchSize = defaultInitialFetchSize, geoparquet = true } = {}) {
 	if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error("parquet expected AsyncBuffer");
@@ -512,12 +510,12 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 	});
 	const version = metadata.field_1;
 	const schema = metadata.field_2.map((field) => ({
-		type: ParquetTypes[field.field_1],
+		type: ParquetTypes$1[field.field_1],
 		type_length: field.field_2,
-		repetition_type: FieldRepetitionTypes[field.field_3],
+		repetition_type: FieldRepetitionTypes$1[field.field_3],
 		name: decode(field.field_4),
 		num_children: field.field_5,
-		converted_type: ConvertedTypes[field.field_6],
+		converted_type: ConvertedTypes$1[field.field_6],
 		scale: field.field_7,
 		precision: field.field_8,
 		field_id: field.field_9,
@@ -530,10 +528,10 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 			file_path: decode(column.field_1),
 			file_offset: column.field_2,
 			meta_data: column.field_3 && {
-				type: ParquetTypes[column.field_3.field_1],
-				encodings: column.field_3.field_2?.map((e) => Encodings[e]),
+				type: ParquetTypes$1[column.field_3.field_1],
+				encodings: column.field_3.field_2?.map((e) => Encodings$1[e]),
 				path_in_schema: column.field_3.field_3.map(decode),
-				codec: CompressionCodecs[column.field_3.field_4],
+				codec: CompressionCodecs$1[column.field_3.field_4],
 				num_values: column.field_3.field_5,
 				total_uncompressed_size: column.field_3.field_6,
 				total_compressed_size: column.field_3.field_7,
@@ -546,8 +544,8 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 				dictionary_page_offset: column.field_3.field_11,
 				statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], parsers),
 				encoding_stats: column.field_3.field_13?.map((encodingStat) => ({
-					page_type: PageTypes[encodingStat.field_1],
-					encoding: Encodings[encodingStat.field_2],
+					page_type: PageTypes$1[encodingStat.field_1],
+					encoding: Encodings$1[encodingStat.field_2],
 					count: encodingStat.field_3
 				})),
 				bloom_filter_offset: column.field_3.field_14,
@@ -606,7 +604,7 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 	};
 }
 function parquetSchema({ schema }) {
-	return getSchemaPath(schema, [])[0];
+	return getSchemaPath$1(schema, [])[0];
 }
 function logicalType(logicalType) {
 	if (logicalType?.field_1) return { type: "STRING" };
@@ -650,7 +648,7 @@ function logicalType(logicalType) {
 	if (logicalType?.field_18) return {
 		type: "GEOGRAPHY",
 		crs: decode(logicalType.field_18.field_1),
-		algorithm: EdgeInterpolationAlgorithms[logicalType.field_18.field_2]
+		algorithm: EdgeInterpolationAlgorithms$1[logicalType.field_18.field_2]
 	};
 	return logicalType;
 }
@@ -705,75 +703,75 @@ function readOffsetIndex(reader) {
 		unencoded_byte_array_data_bytes: thrift.field_2
 	};
 }
-const MASK = 18446744073709551615n;
-const PRIME1 = 11400714785074694791n;
-const PRIME2 = 14029467366897019727n;
-const PRIME3 = 1609587929392839161n;
-const PRIME4 = 9650029242287828579n;
-const PRIME5 = 2870177450012600261n;
-function rotl64(x, r) {
-	return (x << r | x >> 64n - r) & MASK;
-}
-function round(acc, val) {
-	acc = acc + val * PRIME2 & MASK;
-	acc = rotl64(acc, 31n);
-	return acc * PRIME1 & MASK;
-}
-function mergeRound(acc, val) {
-	acc ^= round(0n, val);
-	return acc * PRIME1 + PRIME4 & MASK;
-}
-function xxhash64(input, seed = 0n) {
+const MASK$1 = 18446744073709551615n;
+const PRIME1$1 = 11400714785074694791n;
+const PRIME2$1 = 14029467366897019727n;
+const PRIME3$1 = 1609587929392839161n;
+const PRIME4$1 = 9650029242287828579n;
+const PRIME5$1 = 2870177450012600261n;
+function rotl64$1(x, r) {
+	return (x << r | x >> 64n - r) & MASK$1;
+}
+function round$1(acc, val) {
+	acc = acc + val * PRIME2$1 & MASK$1;
+	acc = rotl64$1(acc, 31n);
+	return acc * PRIME1$1 & MASK$1;
+}
+function mergeRound$1(acc, val) {
+	acc ^= round$1(0n, val);
+	return acc * PRIME1$1 + PRIME4$1 & MASK$1;
+}
+function xxhash64$1(input, seed = 0n) {
 	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
 	const len = input.byteLength;
 	let offset = 0;
 	let h64;
 	if (len >= 32) {
-		let v1 = seed + PRIME1 + PRIME2 & MASK;
-		let v2 = seed + PRIME2 & MASK;
+		let v1 = seed + PRIME1$1 + PRIME2$1 & MASK$1;
+		let v2 = seed + PRIME2$1 & MASK$1;
 		let v3 = seed;
-		let v4 = seed - PRIME1 & MASK;
+		let v4 = seed - PRIME1$1 & MASK$1;
 		while (offset + 32 <= len) {
-			v1 = round(v1, view.getBigUint64(offset, true));
+			v1 = round$1(v1, view.getBigUint64(offset, true));
 			offset += 8;
-			v2 = round(v2, view.getBigUint64(offset, true));
+			v2 = round$1(v2, view.getBigUint64(offset, true));
 			offset += 8;
-			v3 = round(v3, view.getBigUint64(offset, true));
+			v3 = round$1(v3, view.getBigUint64(offset, true));
 			offset += 8;
-			v4 = round(v4, view.getBigUint64(offset, true));
+			v4 = round$1(v4, view.getBigUint64(offset, true));
 			offset += 8;
 		}
-		h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
-		h64 = mergeRound(h64, v1);
-		h64 = mergeRound(h64, v2);
-		h64 = mergeRound(h64, v3);
-		h64 = mergeRound(h64, v4);
-	} else h64 = seed + PRIME5 & MASK;
-	h64 = h64 + BigInt(len) & MASK;
+		h64 = rotl64$1(v1, 1n) + rotl64$1(v2, 7n) + rotl64$1(v3, 12n) + rotl64$1(v4, 18n) & MASK$1;
+		h64 = mergeRound$1(h64, v1);
+		h64 = mergeRound$1(h64, v2);
+		h64 = mergeRound$1(h64, v3);
+		h64 = mergeRound$1(h64, v4);
+	} else h64 = seed + PRIME5$1 & MASK$1;
+	h64 = h64 + BigInt(len) & MASK$1;
 	while (offset + 8 <= len) {
-		h64 ^= round(0n, view.getBigUint64(offset, true));
-		h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
+		h64 ^= round$1(0n, view.getBigUint64(offset, true));
+		h64 = rotl64$1(h64, 27n) * PRIME1$1 + PRIME4$1 & MASK$1;
 		offset += 8;
 	}
 	if (offset + 4 <= len) {
-		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
-		h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
+		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1$1 & MASK$1;
+		h64 = rotl64$1(h64, 23n) * PRIME2$1 + PRIME3$1 & MASK$1;
 		offset += 4;
 	}
 	while (offset < len) {
-		h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
-		h64 = rotl64(h64, 11n) * PRIME1 & MASK;
+		h64 ^= BigInt(view.getUint8(offset)) * PRIME5$1 & MASK$1;
+		h64 = rotl64$1(h64, 11n) * PRIME1$1 & MASK$1;
 		offset += 1;
 	}
 	h64 ^= h64 >> 33n;
-	h64 = h64 * PRIME2 & MASK;
+	h64 = h64 * PRIME2$1 & MASK$1;
 	h64 ^= h64 >> 29n;
-	h64 = h64 * PRIME3 & MASK;
+	h64 = h64 * PRIME3$1 & MASK$1;
 	h64 ^= h64 >> 32n;
 	return h64;
 }
-const textEncoder = new TextEncoder();
-const SALT = new Uint32Array([
+const textEncoder$1 = new TextEncoder();
+const SALT$1 = new Uint32Array([
 	1203114875,
 	1150766481,
 	2284105051,
@@ -789,7 +787,7 @@ function blockIndex(hash, numBlocks) {
 function blockMask(hash) {
 	const m = /* @__PURE__ */ new Uint32Array(8);
 	const low = Number(hash & 4294967295n) | 0;
-	for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT[i]) >>> 27);
+	for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT$1[i]) >>> 27);
 	return m;
 }
 function sbbfContains(blocks, hash) {
@@ -815,24 +813,24 @@ function readBloomFilter(reader) {
 		blocks
 	};
 }
-function hashParquetValue(value, element) {
+function hashParquetValue$1(value, element) {
 	if (value === null || value === void 0) return void 0;
 	const { type, converted_type, logical_type } = element;
 	if (type === "BOOLEAN") {
 		if (typeof value !== "boolean") return void 0;
-		return xxhash64(new Uint8Array([value ? 1 : 0]));
+		return xxhash64$1(new Uint8Array([value ? 1 : 0]));
 	}
 	if (type === "FLOAT") {
 		if (typeof value !== "number") return void 0;
 		const buf = /* @__PURE__ */ new ArrayBuffer(4);
 		new DataView(buf).setFloat32(0, value, true);
-		return xxhash64(new Uint8Array(buf));
+		return xxhash64$1(new Uint8Array(buf));
 	}
 	if (type === "DOUBLE") {
 		if (typeof value !== "number") return void 0;
 		const buf = /* @__PURE__ */ new ArrayBuffer(8);
 		new DataView(buf).setFloat64(0, value, true);
-		return xxhash64(new Uint8Array(buf));
+		return xxhash64$1(new Uint8Array(buf));
 	}
 	if (type === "INT32") {
 		if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
@@ -840,7 +838,7 @@ function hashParquetValue(value, element) {
 		if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
 		const buf = /* @__PURE__ */ new ArrayBuffer(4);
 		new DataView(buf).setInt32(0, value | 0, true);
-		return xxhash64(new Uint8Array(buf));
+		return xxhash64$1(new Uint8Array(buf));
 	}
 	if (type === "INT64") {
 		if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
@@ -852,21 +850,21 @@ function hashParquetValue(value, element) {
 		else return void 0;
 		const buf = /* @__PURE__ */ new ArrayBuffer(8);
 		new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
-		return xxhash64(new Uint8Array(buf));
+		return xxhash64$1(new Uint8Array(buf));
 	}
 	if (type === "BYTE_ARRAY") {
 		if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
 		if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
 		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
-		if (typeof value === "string") return xxhash64(textEncoder.encode(value));
-		if (value instanceof Uint8Array) return xxhash64(value);
+		if (typeof value === "string") return xxhash64$1(textEncoder$1.encode(value));
+		if (value instanceof Uint8Array) return xxhash64$1(value);
 		return;
 	}
 	if (type === "FIXED_LEN_BYTE_ARRAY") {
 		if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
 		if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
 		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
-		if (value instanceof Uint8Array) return xxhash64(value);
+		if (value instanceof Uint8Array) return xxhash64$1(value);
 		return;
 	}
 }
@@ -893,23 +891,6 @@ function walkBloomEligible(filter, out) {
 		} else out.add(field);
 	}
 }
-function toJson(obj) {
-	if (obj === void 0) return null;
-	if (typeof obj === "bigint") return Number(obj);
-	if (Object.is(obj, -0)) return 0;
-	if (Array.isArray(obj)) return obj.map(toJson);
-	if (obj instanceof Uint8Array) return Array.from(obj);
-	if (obj instanceof Date) return obj.toISOString();
-	if (obj instanceof Object) {
-		const newObj = {};
-		for (const key of Object.keys(obj)) {
-			if (obj[key] === void 0) continue;
-			newObj[key] = toJson(obj[key]);
-		}
-		return newObj;
-	}
-	return obj;
-}
 function concat(aaa, bbb) {
 	const chunk = 1e4;
 	for (let i = 0; i < bbb.length; i += chunk) aaa.push(...bbb.slice(i, i + chunk));
@@ -1072,13 +1053,13 @@ function canSkipRowGroup({ rowGroup, physicalColumns, filter, strict = true, blo
 			}
 			if (bloom && element) {
 				if (operator === "$eq") {
-					const hash = hashParquetValue(target, element);
+					const hash = hashParquetValue$1(target, element);
 					if (hash !== void 0 && !sbbfContains(bloom.blocks, hash)) return true;
 				}
 				if (operator === "$in" && Array.isArray(target) && target.length > 0) {
 					let allAbsent = true;
 					for (const v of target) {
-						const h = hashParquetValue(v, element);
+						const h = hashParquetValue$1(v, element);
 						if (h === void 0 || sbbfContains(bloom.blocks, h)) {
 							allAbsent = false;
 							break;
@@ -1238,7 +1219,7 @@ function prefetchAsyncBuffer(file, { fetches }) {
 		}
 	};
 }
-const decoder = new TextDecoder();
+const decoder$3 = new TextDecoder();
 const metadataCache = /* @__PURE__ */ new WeakMap();
 function decodeVariantColumn(value, parsers = DEFAULT_PARSERS) {
 	if (Array.isArray(value)) return value.map((entry) => decodeVariantColumn(entry, parsers));
@@ -1302,7 +1283,7 @@ function parseVariantMetadata(bytes) {
 		const start = offsets[i];
 		const end = offsets[i + 1];
 		const strBytes = new Uint8Array(bytes.buffer, bytes.byteOffset + base + start, end - start);
-		dictionary[i] = decoder.decode(strBytes);
+		dictionary[i] = decoder$3.decode(strBytes);
 	}
 	const metadata = {
 		dictionary,
@@ -1326,7 +1307,7 @@ function readVariant(reader, metadata, parsers) {
 	if (basicType === 3) return readVariantArray(reader, header, metadata, parsers);
 	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, header);
 	reader.offset += header;
-	return decoder.decode(bytes);
+	return decoder$3.decode(bytes);
 }
 function readVariantPrimitive(reader, typeId, parsers) {
 	switch (typeId) {
@@ -1380,7 +1361,7 @@ function readVariantPrimitive(reader, typeId, parsers) {
 		case 15: return readVariantBinary(reader);
 		case 16: {
 			const bytes = readVariantBinary(reader);
-			return decoder.decode(bytes);
+			return decoder$3.decode(bytes);
 		}
 		case 17: {
 			const value = reader.view.getBigInt64(reader.offset, true);
@@ -1462,7 +1443,7 @@ function readVariantBinary(reader) {
 	return bytes;
 }
 function assembleLists(output, definitionLevels, repetitionLevels, values, schemaPath) {
-	const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
+	const maxDefinitionLevel = getMaxDefinitionLevel$1(schemaPath);
 	if (!definitionLevels?.length && !repetitionLevels.length) {
 		if (!maxDefinitionLevel || !values.length) return values;
 		definitionLevels = new Array(values.length).fill(maxDefinitionLevel);
@@ -1522,7 +1503,7 @@ function assembleNested(subcolumnData, schema, parsers, depth = 0) {
 	const path = schema.path.join(".");
 	const optional = schema.element.repetition_type === "OPTIONAL";
 	const nextDepth = optional ? depth + 1 : depth;
-	if (isListLike(schema)) {
+	if (isListLike$1(schema)) {
 		let sublist = schema.children[0];
 		let subDepth = nextDepth;
 		if (sublist.children.length === 1) {
@@ -1538,7 +1519,7 @@ function assembleNested(subcolumnData, schema, parsers, depth = 0) {
 		subcolumnData.delete(subcolumn);
 		return;
 	}
-	if (isMapLike(schema)) {
+	if (isMapLike$1(schema)) {
 		const mapName = schema.children[0].element.name;
 		assembleNested(subcolumnData, schema.children[0].children[0], parsers, nextDepth + 1);
 		assembleNested(subcolumnData, schema.children[0].children[1], parsers, nextDepth + 1);
@@ -1931,7 +1912,7 @@ function readRepetitionLevels(reader, daph, schemaPath) {
 	return [];
 }
 function readDefinitionLevels(reader, daph, schemaPath) {
-	const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
+	const maxDefinitionLevel = getMaxDefinitionLevel$1(schemaPath);
 	if (!maxDefinitionLevel) return {
 		definitionLevels: [],
 		numNulls: 0
@@ -2013,7 +1994,7 @@ function readRepetitionLevelsV2(reader, daph2, schemaPath) {
 	return values;
 }
 function readDefinitionLevelsV2(reader, daph2, schemaPath) {
-	const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
+	const maxDefinitionLevel = getMaxDefinitionLevel$1(schemaPath);
 	if (maxDefinitionLevel) {
 		const values = new Array(daph2.num_values);
 		readRleBitPackedHybrid(reader, bitWidth(maxDefinitionLevel), values, daph2.definition_levels_byte_length);
@@ -2106,15 +2087,15 @@ function readPage(reader, header, columnDecoder, dictionary, previousChunk, page
 function parquetHeader(reader) {
 	const header = deserializeTCompactProtocol(reader);
 	return {
-		type: PageTypes[header.field_1],
+		type: PageTypes$1[header.field_1],
 		uncompressed_page_size: header.field_2,
 		compressed_page_size: header.field_3,
 		crc: header.field_4,
 		data_page_header: header.field_5 && {
 			num_values: header.field_5.field_1,
-			encoding: Encodings[header.field_5.field_2],
-			definition_level_encoding: Encodings[header.field_5.field_3],
-			repetition_level_encoding: Encodings[header.field_5.field_4],
+			encoding: Encodings$1[header.field_5.field_2],
+			definition_level_encoding: Encodings$1[header.field_5.field_3],
+			repetition_level_encoding: Encodings$1[header.field_5.field_4],
 			statistics: header.field_5.field_5 && {
 				max: header.field_5.field_5.field_1,
 				min: header.field_5.field_5.field_2,
@@ -2127,14 +2108,14 @@ function parquetHeader(reader) {
 		index_page_header: header.field_6,
 		dictionary_page_header: header.field_7 && {
 			num_values: header.field_7.field_1,
-			encoding: Encodings[header.field_7.field_2],
+			encoding: Encodings$1[header.field_7.field_2],
 			is_sorted: header.field_7.field_3
 		},
 		data_page_header_v2: header.field_8 && {
 			num_values: header.field_8.field_1,
 			num_nulls: header.field_8.field_2,
 			num_rows: header.field_8.field_3,
-			encoding: Encodings[header.field_8.field_4],
+			encoding: Encodings$1[header.field_8.field_4],
 			definition_levels_byte_length: header.field_8.field_5,
 			repetition_levels_byte_length: header.field_8.field_6,
 			is_compressed: header.field_8.field_7 === void 0 ? true : header.field_8.field_7,
@@ -2146,7 +2127,7 @@ function readRowGroup(options, { metadata }, groupPlan) {
 	const asyncColumns = [];
 	for (const chunk of groupPlan.chunks) {
 		const { data_page_offset, dictionary_page_offset, path_in_schema: pathInSchema } = chunk.columnMetadata;
-		const schemaPath = getSchemaPath(metadata.schema, pathInSchema);
+		const schemaPath = getSchemaPath$1(metadata.schema, pathInSchema);
 		const columnDecoder = {
 			pathInSchema,
 			element: schemaPath[schemaPath.length - 1].element,
@@ -2389,4 +2370,289 @@ function parquetReadObjects(options) {
 		}).catch(reject);
 	});
 }
+function schemaTree(schema, rootIndex, path) {
+	const element = schema[rootIndex];
+	const children = [];
+	let count = 1;
+	if (element.num_children) while (children.length < element.num_children) {
+		const childElement = schema[rootIndex + count];
+		const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
+		count += child.count;
+		children.push(child);
+	}
+	return {
+		count,
+		element,
+		children,
+		path
+	};
+}
+function getSchemaPath(schema, name) {
+	let tree = schemaTree(schema, 0, []);
+	const path = [tree];
+	for (const part of name) {
+		const child = tree.children.find((child) => child.element.name === part);
+		if (!child) throw new Error(`parquet schema element not found: ${name}`);
+		path.push(child);
+		tree = child;
+	}
+	return path;
+}
+function getMaxDefinitionLevel(schemaPath) {
+	let maxLevel = 0;
+	for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
+	return maxLevel;
+}
+function isListLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "LIST") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length > 1) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	return true;
+}
+function isMapLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "MAP") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length !== 2) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "key")?.element.repetition_type === "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
+	return true;
+}
+const MASK = 18446744073709551615n;
+const PRIME1 = 11400714785074694791n;
+const PRIME2 = 14029467366897019727n;
+const PRIME3 = 1609587929392839161n;
+const PRIME4 = 9650029242287828579n;
+const PRIME5 = 2870177450012600261n;
+function rotl64(x, r) {
+	return (x << r | x >> 64n - r) & MASK;
+}
+function round(acc, val) {
+	acc = acc + val * PRIME2 & MASK;
+	acc = rotl64(acc, 31n);
+	return acc * PRIME1 & MASK;
+}
+function mergeRound(acc, val) {
+	acc ^= round(0n, val);
+	return acc * PRIME1 + PRIME4 & MASK;
+}
+function xxhash64(input, seed = 0n) {
+	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
+	const len = input.byteLength;
+	let offset = 0;
+	let h64;
+	if (len >= 32) {
+		let v1 = seed + PRIME1 + PRIME2 & MASK;
+		let v2 = seed + PRIME2 & MASK;
+		let v3 = seed;
+		let v4 = seed - PRIME1 & MASK;
+		while (offset + 32 <= len) {
+			v1 = round(v1, view.getBigUint64(offset, true));
+			offset += 8;
+			v2 = round(v2, view.getBigUint64(offset, true));
+			offset += 8;
+			v3 = round(v3, view.getBigUint64(offset, true));
+			offset += 8;
+			v4 = round(v4, view.getBigUint64(offset, true));
+			offset += 8;
+		}
+		h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
+		h64 = mergeRound(h64, v1);
+		h64 = mergeRound(h64, v2);
+		h64 = mergeRound(h64, v3);
+		h64 = mergeRound(h64, v4);
+	} else h64 = seed + PRIME5 & MASK;
+	h64 = h64 + BigInt(len) & MASK;
+	while (offset + 8 <= len) {
+		h64 ^= round(0n, view.getBigUint64(offset, true));
+		h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
+		offset += 8;
+	}
+	if (offset + 4 <= len) {
+		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
+		h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
+		offset += 4;
+	}
+	while (offset < len) {
+		h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
+		h64 = rotl64(h64, 11n) * PRIME1 & MASK;
+		offset += 1;
+	}
+	h64 ^= h64 >> 33n;
+	h64 = h64 * PRIME2 & MASK;
+	h64 ^= h64 >> 29n;
+	h64 = h64 * PRIME3 & MASK;
+	h64 ^= h64 >> 32n;
+	return h64;
+}
+const textEncoder = new TextEncoder();
+new Uint32Array([
+	1203114875,
+	1150766481,
+	2284105051,
+	2729912477,
+	1884591559,
+	770785867,
+	2667333959,
+	1550580529
+]);
+function hashParquetValue(value, element) {
+	if (value === null || value === void 0) return void 0;
+	const { type, converted_type, logical_type } = element;
+	if (type === "BOOLEAN") {
+		if (typeof value !== "boolean") return void 0;
+		return xxhash64(new Uint8Array([value ? 1 : 0]));
+	}
+	if (type === "FLOAT") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setFloat32(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "DOUBLE") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setFloat64(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT32") {
+		if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
+		if (logical_type?.type === "DATE" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setInt32(0, value | 0, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT64") {
+		if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
+		if (converted_type === "TIME_MICROS" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "TIMESTAMP" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		let bigValue;
+		if (typeof value === "bigint") bigValue = value;
+		else if (typeof value === "number" && Number.isSafeInteger(value)) bigValue = BigInt(value);
+		else return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "BYTE_ARRAY") {
+		if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (typeof value === "string") return xxhash64(textEncoder.encode(value));
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+	if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
+		if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+}
+const ParquetTypes = [
+	"BOOLEAN",
+	"INT32",
+	"INT64",
+	"INT96",
+	"FLOAT",
+	"DOUBLE",
+	"BYTE_ARRAY",
+	"FIXED_LEN_BYTE_ARRAY"
+];
+const Encodings = [
+	"PLAIN",
+	"GROUP_VAR_INT",
+	"PLAIN_DICTIONARY",
+	"RLE",
+	"BIT_PACKED",
+	"DELTA_BINARY_PACKED",
+	"DELTA_LENGTH_BYTE_ARRAY",
+	"DELTA_BYTE_ARRAY",
+	"RLE_DICTIONARY",
+	"BYTE_STREAM_SPLIT"
+];
+const FieldRepetitionTypes = [
+	"REQUIRED",
+	"OPTIONAL",
+	"REPEATED"
+];
+const ConvertedTypes = [
+	"UTF8",
+	"MAP",
+	"MAP_KEY_VALUE",
+	"LIST",
+	"ENUM",
+	"DECIMAL",
+	"DATE",
+	"TIME_MILLIS",
+	"TIME_MICROS",
+	"TIMESTAMP_MILLIS",
+	"TIMESTAMP_MICROS",
+	"UINT_8",
+	"UINT_16",
+	"UINT_32",
+	"UINT_64",
+	"INT_8",
+	"INT_16",
+	"INT_32",
+	"INT_64",
+	"JSON",
+	"BSON",
+	"INTERVAL"
+];
+const CompressionCodecs = [
+	"UNCOMPRESSED",
+	"SNAPPY",
+	"GZIP",
+	"LZO",
+	"BROTLI",
+	"LZ4",
+	"ZSTD",
+	"LZ4_RAW"
+];
+const PageTypes = [
+	"DATA_PAGE",
+	"INDEX_PAGE",
+	"DICTIONARY_PAGE",
+	"DATA_PAGE_V2"
+];
+const BoundaryOrders = [
+	"UNORDERED",
+	"ASCENDING",
+	"DESCENDING"
+];
+const EdgeInterpolationAlgorithms = [
+	"SPHERICAL",
+	"VINCENTY",
+	"THOMAS",
+	"ANDOYER",
+	"KARNEY"
+];
+new TextDecoder();
+new TextDecoder();
+function toJson(obj) {
+	if (obj === void 0) return null;
+	if (typeof obj === "bigint") return Number(obj);
+	if (Object.is(obj, -0)) return 0;
+	if (Array.isArray(obj)) return obj.map(toJson);
+	if (obj instanceof Uint8Array) return Array.from(obj);
+	if (obj instanceof Date) return obj.toISOString();
+	if (obj instanceof Object) {
+		const newObj = {};
+		for (const key of Object.keys(obj)) {
+			if (obj[key] === void 0) continue;
+			newObj[key] = toJson(obj[key]);
+		}
+		return newObj;
+	}
+	return obj;
+}
+new TextDecoder();
 export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parquetReadObjects, parseDecimal, snappyUncompress, toJson };

package/dist/_chunks/libs/icebird.mjs

@@ -241,32 +241,6 @@ async function fetchAvroRecords(url, resolver, byteLength) {
 	});
 }
 const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
-function stringifyIcebergJson(value, indent) {
-	const sp = indent ? " ".repeat(indent) : "";
-	function emit(v, depth) {
-		if (typeof v === "bigint") return v.toString();
-		if (v === null) return "null";
-		if (typeof v === "string") return JSON.stringify(v);
-		if (typeof v === "number" || typeof v === "boolean") return JSON.stringify(v);
-		if (Array.isArray(v)) {
-			if (v.length === 0) return "[]";
-			const inner = v.map((x) => emit(x, depth + 1));
-			if (!sp) return "[" + inner.join(",") + "]";
-			const pad = sp.repeat(depth + 1), close = sp.repeat(depth);
-			return "[\n" + pad + inner.join(",\n" + pad) + "\n" + close + "]";
-		}
-		if (typeof v === "object") {
-			const keys = Object.keys(v).filter((k) => v[k] !== void 0);
-			if (keys.length === 0) return "{}";
-			const inner = keys.map((k) => JSON.stringify(k) + (sp ? ": " : ":") + emit(v[k], depth + 1));
-			if (!sp) return "{" + inner.join(",") + "}";
-			const pad = sp.repeat(depth + 1), close = sp.repeat(depth);
-			return "{\n" + pad + inner.join(",\n" + pad) + "\n" + close + "}";
-		}
-		return JSON.stringify(v);
-	}
-	return emit(value, 0);
-}
 function parseIcebergJson(text) {
 	let i = 0;
 	function skipWs() {
@@ -395,6 +369,23 @@ function parseIcebergJson(text) {
 	if (i !== text.length) throw new Error(`unexpected trailing input at ${i}`);
 	return value;
 }
+function stringifyIcebergJson(value, indent = 2) {
+	const pad = " ".repeat(indent);
+	function serialize(val, depth) {
+		if (typeof val === "bigint") return val.toString();
+		if (val === null || typeof val !== "object") return JSON.stringify(val);
+		const inner = pad.repeat(depth + 1);
+		const outer = pad.repeat(depth);
+		if (Array.isArray(val)) {
+			if (val.length === 0) return "[]";
+			return `[\n${val.map((v) => inner + serialize(v === void 0 ? null : v, depth + 1)).join(",\n")}\n${outer}]`;
+		}
+		const keys = Object.keys(val).filter((k) => val[k] !== void 0);
+		if (keys.length === 0) return "{}";
+		return `{\n${keys.map((k) => `${inner}${JSON.stringify(k)}: ${serialize(val[k], depth + 1)}`).join(",\n")}\n${outer}}`;
+	}
+	return serialize(value, 0);
+}
 async function restCatalogLoadTable(ctx, { namespace, table }) {
 	const body = parseIcebergJson(await (await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}`)).text());
 	return {
@@ -737,7 +728,7 @@ async function icebergCreate({ tableUrl, resolver, schema, formatVersion, partit
 	if (formatVersion >= 3) metadata["next-row-id"] = 0;
 	if (!resolver.writer) throw new Error("resolver.writer is required");
 	const metadataWriter = conditionalCommits ? resolver.writer(metadataUrl, { ifNoneMatch: "*" }) : resolver.writer(metadataUrl);
-	const metadataBytes = new TextEncoder().encode(stringifyIcebergJson(metadata, 2));
+	const metadataBytes = new TextEncoder().encode(stringifyIcebergJson(metadata));
 	metadataWriter.appendBytes(metadataBytes);
 	await metadataWriter.finish();
 	const versionHintUrl = `${tableUrl}/metadata/version-hint.text`;

package/dist/_chunks/resolver.mjs

@@ -282,7 +282,6 @@ function createSqlFragments(config) {
 			let col;
 			if (metric === "clicks") col = t.clicks;
 			else if (metric === "impressions") col = t.impressions;
-			else if (metric === "position") col = t.sum_position;
 			if (!col) continue;
 			const v = Number(f.expression);
 			switch (f.operator) {

package/dist/_chunks/sink.d.mts

@@ -1,7 +1,8 @@
 import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { EngineError } from "./errors.mjs";
+import { icebergCreateTable } from "./libs/icebird.mjs";
 import { Result } from "gscdump/result";
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, connectIcebergCatalog, ensureIcebergNamespace, listIcebergTables } from "@gscdump/lakehouse";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergListedDataFile as IcebergListedDataFile$1, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, connectIcebergCatalog, ensureIcebergNamespace, listIcebergTables } from "@gscdump/lakehouse";
 import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
 import { TableName } from "@gscdump/contracts";
 /** The 9 fact tables that exist as global Iceberg tables. */
@@ -69,47 +70,26 @@ declare function isIcebergTable(table: string): table is IcebergTableName;
  * Iceberg job instead of failing loudly.
  */
 declare function assertIcebergTable(table: string): IcebergTableName;
+type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];
+type IcebirdSchema = NonNullable<IcebirdCreateTableOptions['schema']>;
+type IcebirdPartitionSpec = NonNullable<IcebirdCreateTableOptions['partitionSpec']>;
+type IcebirdSortOrder = NonNullable<IcebirdCreateTableOptions['sortOrder']>;
 /**
  * Build the icebird `Schema` for one of the 9 fact tables from the frozen
  * `ICEBERG_SCHEMAS` contract. Field ids are advisory — R2 Data Catalog
  * re-assigns them on `createTable`.
  */
-declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
-  type: "struct";
-  'schema-id': number;
-  fields: {
-    id: number;
-    name: string;
-    required: boolean;
-    type: import("@gscdump/lakehouse").IcebergPrimitiveType;
-  }[];
-};
+declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdSchema;
 /**
  * Build the icebird `PartitionSpec` for one of the 9 fact tables: the locked
  * spec `identity(site_id) + identity(search_type) + month(date)`.
  */
-declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
-  'spec-id': number;
-  fields: {
-    'source-id': number;
-    'field-id': number;
-    name: string;
-    transform: import("@gscdump/lakehouse").IcebergPartitionTransform;
-  }[];
-};
+declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdPartitionSpec;
 /**
  * Build the icebird `SortOrder` for a fact table from its `clusterKey`
  * (dimension-first, then `date`).
  */
-declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
-  'order-id': number;
-  fields: {
-    'source-id': number;
-    transform: "identity";
-    direction: "asc";
-    'null-order': "nulls-last";
-  }[];
-};
+declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdSortOrder;
 /**
  * Outcome of a single table create/drop: the table name plus a `Result` —
  * `Ok(void)` on success, `Err(iceberg-table-op-failed)` carrying the failure
@@ -153,7 +133,7 @@ interface ListIcebergDataFilesOptions {
  * List the parquet data files in the current snapshot of `table`, filtered to
  * a single partition slice `(siteId, searchType, month(date) ∈ range)`.
  */
-declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<import("@gscdump/lakehouse").IcebergListedDataFile[]>;
+declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
 /**
  * Identifies one fact slice — the atomic unit a sink emits.
  * `(table, site, searchType, date)`. `userId` rides along on `ctx` for
@@ -268,4 +248,4 @@ interface IcebergAppendSinkOptions extends SinkOptions {
    */
   encoding?: PartitionKeyEncoding$1;
 }
-export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn$1 as IcebergColumn, type IcebergColumnType, type IcebergConnection$1 as IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField$1 as IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, type IcebergTableSpec$1 as IcebergTableSpec, ListIcebergDataFilesOptions, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn$1 as IcebergColumn, type IcebergColumnType, type IcebergConnection$1 as IcebergConnection, type IcebergListedDataFile$1 as IcebergListedDataFile, type IcebergPartitionField$1 as IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, type IcebergTableSpec$1 as IcebergTableSpec, ListIcebergDataFilesOptions, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/iceberg/index.mjs

@@ -84,7 +84,15 @@ async function listIcebergDataFiles(conn, opts) {
 		field: "search_type",
 		value: opts.searchType,
 		encoding: "string"
-	}] : [];
+	}] : [{
+		field: "site_id",
+		value: opts.siteId,
+		encoding: "int32"
+	}, {
+		field: "search_type",
+		value: opts.searchType,
+		encoding: "int32"
+	}];
 	return resolveIcebergDataFiles(conn, {
 		namespace: conn.namespace,
 		table: opts.table,

package/dist/ingest.mjs

@@ -26,7 +26,7 @@ function toPath(gscUrl) {
 	}
 }
 function toSumPosition(apiPosition, impressions) {
-	return ((apiPosition >= 1 ? apiPosition : 1) - 1) * Math.max(impressions, 1);
+	return ((apiPosition >= 1 ? apiPosition : 1) - 1) * impressions;
 }
 function transformGscRow(table, apiRow, options = {}) {
 	const keys = apiRow.keys;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.35.2",
+  "version": "0.35.4",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,18 +181,18 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.35.2",
-    "@gscdump/lakehouse": "0.35.2",
-    "gscdump": "0.35.2"
+    "@gscdump/lakehouse": "0.35.4",
+    "gscdump": "0.35.4",
+    "@gscdump/contracts": "0.35.4"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
-    "hyparquet": "^1.26.1",
+    "hyparquet": "^1.26.2",
     "hyparquet-writer": "^0.16.1",
-    "icebird": "^0.8.11",
-    "tsx": "^4.22.4",
+    "icebird": "^0.8.12",
+    "tsx": "^4.22.5",
     "unstorage": "^1.17.5",
     "vitest": "^4.1.9"
   },