@gscdump/engine 0.38.1 → 0.39.0

package/dist/_chunks/compaction.mjs

@@ -189,42 +189,46 @@ function splitOverlappingTiers(entries, queryRange) {
 	const spanned = [];
 	const kept = [];
 	const subsumed = [];
+	const clampToRange = Number.isFinite(rangeStartMs) && Number.isFinite(rangeEndMs);
 	for (const entry of entries) {
 		const span = partitionSpan(entry.partition);
 		if (!span) {
 			kept.push(entry);
 			continue;
 		}
-		const days = [];
-		for (let t = span.startMs; t <= span.endMs; t += MS_PER_DAY) {
-			if (rangeStartMs !== void 0 && (t < rangeStartMs || t > rangeEndMs)) continue;
-			days.push(t);
-		}
-		if (queryRange && days.length === 0) {
+		const startMs = clampToRange ? Math.max(span.startMs, rangeStartMs) : span.startMs;
+		const endMs = clampToRange ? Math.min(span.endMs, rangeEndMs) : span.endMs;
+		if (queryRange && startMs > endMs) {
 			subsumed.push(entry);
 			continue;
 		}
 		spanned.push({
 			entry,
 			rank: span.rank,
-			days
+			startMs,
+			endMs
 		});
 	}
 	spanned.sort((a, b) => a.rank - b.rank || b.entry.createdAt - a.entry.createdAt);
 	const coveredBySearchType = /* @__PURE__ */ new Map();
-	for (const { entry, days } of spanned) {
+	for (const { entry, startMs, endMs } of spanned) {
 		const slice = inferSearchType(entry);
 		let covered = coveredBySearchType.get(slice);
 		if (!covered) {
 			covered = /* @__PURE__ */ new Set();
 			coveredBySearchType.set(slice, covered);
 		}
-		if (days.every((d) => covered.has(d))) {
+		let fullyCovered = true;
+		for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) if (!covered.has(dayMs)) {
+			fullyCovered = false;
+			break;
+		}
+		if (fullyCovered) {
 			subsumed.push(entry);
 			continue;
 		}
 		kept.push(entry);
-		for (const d of days) covered.add(d);
+		for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) covered.add(dayMs);
 	}
 	return {
 		kept,

package/dist/_chunks/duckdb.d.mts

@@ -14,8 +14,16 @@ interface DuckDBHandle {
 interface DuckDBFactory {
   getDuckDB: () => Promise<DuckDBHandle>;
 }
-declare function createDuckDBCodec(factory: DuckDBFactory): ParquetCodec;
-declare function createDuckDBExecutor(factory: DuckDBFactory): QueryExecutor;
+interface DuckDBReadOptions {
+  /**
+   * Maximum non-URI parquet reads held in flight while registering DuckDB
+   * buffers. Defaults to 16; callers handling unusually large files can lower
+   * it to trade latency for peak JS memory.
+   */
+  bufferReadConcurrency?: number;
+}
+declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
+declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
 /**
  * Canonical "empty-file" SELECT clause for a table. Codecs that need to
  * emit a schema-correct empty Parquet can wrap this in:

package/dist/_chunks/engine.mjs

@@ -8,11 +8,16 @@ import { sqlEscape } from "../sql-bind.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
-const BUFFER_READ_BATCH_SIZE = 8;
-async function registerBufferedFiles(db, files, dataSource, registered, signal) {
-	for (let i = 0; i < files.length; i += BUFFER_READ_BATCH_SIZE) {
+const DEFAULT_BUFFER_READ_CONCURRENCY = 16;
+const MAX_BUFFER_READ_CONCURRENCY = 64;
+function bufferReadConcurrency(value, fileCount) {
+	return Math.max(1, Math.min(fileCount || 1, typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : DEFAULT_BUFFER_READ_CONCURRENCY, MAX_BUFFER_READ_CONCURRENCY));
+}
+async function registerBufferedFiles(db, files, dataSource, registered, signal, maxConcurrency) {
+	const batchSize = bufferReadConcurrency(maxConcurrency, files.length);
+	for (let i = 0; i < files.length; i += batchSize) {
 		signal?.throwIfAborted();
-		const batch = files.slice(i, i + BUFFER_READ_BATCH_SIZE);
+		const batch = files.slice(i, i + batchSize);
 		const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
 		for (let j = 0; j < batch.length; j++) {
 			signal?.throwIfAborted();
@@ -47,7 +52,7 @@ async function decodeBytes(db, bytes, table) {
 		await db.dropFiles([name]);
 	}
 }
-function createDuckDBCodec(factory) {
+function createDuckDBCodec(factory, options = {}) {
 	return {
 		async writeRows(ctx, rows, key, dataSource) {
 			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
@@ -100,7 +105,7 @@ function createDuckDBCodec(factory) {
 				};
 			});
 			try {
-				await registerBufferedFiles(db, bufferedInputs, dataSource, registered);
+				await registerBufferedFiles(db, bufferedInputs, dataSource, registered, void 0, options.bufferReadConcurrency);
 				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
 				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
 				registered.push(outName);
@@ -137,7 +142,7 @@ function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables
 	}
 	return out;
 }
-function createDuckDBExecutor(factory) {
+function createDuckDBExecutor(factory, options = {}) {
 	return { async execute({ sql, params, fileKeys, placeholderTables, dataSource, table, signal, profiler }) {
 		signal?.throwIfAborted();
 		const db = await factory.getDuckDB();
@@ -146,24 +151,24 @@ function createDuckDBExecutor(factory) {
 		const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
 		const endRegister = profiler?.start("files.register", { files: totalFiles });
 		try {
-			await Promise.all(Object.entries(fileKeys).map(async ([name, keys]) => {
+			const bufferedByName = /* @__PURE__ */ new Map();
+			for (const [name, keys] of Object.entries(fileKeys)) {
 				const resolved = [];
-				const buffered = [];
 				for (let i = 0; i < keys.length; i++) {
 					const key = keys[i];
 					const uri = dataSource.uri?.(key);
 					if (uri !== void 0) resolved.push(uri);
 					else {
-						buffered.push({
+						if (!bufferedByName.has(key)) bufferedByName.set(key, {
 							key,
 							name: key
 						});
 						resolved.push(key);
 					}
 				}
-				await registerBufferedFiles(db, buffered, dataSource, registered, signal);
 				placeholders[name] = resolved;
-			}));
+			}
+			await registerBufferedFiles(db, [...bufferedByName.values()], dataSource, registered, signal, options.bufferReadConcurrency);
 			endRegister?.({ buffered: registered.length });
 			signal?.throwIfAborted();
 			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);

package/dist/_chunks/entities.mjs

@@ -9,9 +9,6 @@ function isMissingKeyError(e) {
 	return /\bnot found\b|\bENOENT\b|\bmissing key\b/i.test(message);
 }
 async function readOptional(ds, key, signal) {
-	if (ds.head) {
-		if (await ds.head(key) === void 0) return void 0;
-	}
 	return await ds.read(key, void 0, signal).catch((e) => {
 		if (isMissingKeyError(e)) return void 0;
 		throw e;
@@ -72,9 +69,6 @@ function buildQueryDimRecords(queries, deps) {
 	return out;
 }
 function createQueryDimStore({ dataSource }) {
-	async function exists(key, prefix) {
-		return (await dataSource.list(prefix)).includes(key);
-	}
 	return {
 		parquetKey: queryDimParquetKey,
 		async write(ctx, records, builtAt) {
@@ -98,15 +92,14 @@ function createQueryDimStore({ dataSource }) {
 			};
 		},
 		async loadMeta(ctx) {
-			const key = queryDimMetaKey(ctx);
-			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return null;
-			const bytes = await dataSource.read(key);
+			const bytes = await readOptional(dataSource, queryDimMetaKey(ctx));
+			if (!bytes) return null;
 			return JSON.parse(new TextDecoder().decode(bytes));
 		},
 		async loadRecords(ctx) {
-			const key = queryDimParquetKey(ctx);
-			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return [];
-			return (await decodeParquetToRows(await dataSource.read(key))).map((r) => ({
+			const bytes = await readOptional(dataSource, queryDimParquetKey(ctx));
+			if (!bytes) return [];
+			return (await decodeParquetToRows(bytes)).map((r) => ({
 				query: String(r.query),
 				query_canonical: String(r.query_canonical),
 				intent_code: Number(r.intent_code),
@@ -117,6 +110,21 @@ function createQueryDimStore({ dataSource }) {
 	};
 }
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
+const ENTITY_IO_CONCURRENCY = 8;
+async function mapEntityIo(items, fn) {
+	if (items.length === 0) return [];
+	const results = Array.from({ length: items.length }, () => void 0);
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const index = next++;
+			if (index >= items.length) return;
+			results[index] = await fn(items[index], index);
+		}
+	}
+	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
+	return results;
+}
 function inspectionIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
 }
@@ -311,33 +319,33 @@ function createInspectionStore(opts) {
 				if (!byMonth.has(month)) byMonth.set(month, []);
 				byMonth.get(month).push(r);
 			}
-			for (const [yearMonth, batch] of byMonth) {
+			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
 				const bytes = encodeJsonBigintSafe({
 					version: 1,
 					records: batch
 				});
 				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
-				await ds.write(inspectionHistoryShardKey(ctx, yearMonth, batchId), bytes);
-			}
+				return {
+					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
+					bytes
+				};
+			}), (shard) => ds.write(shard.key, shard.bytes));
 		},
 		async loadHistory(ctx, yearMonth) {
 			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
 			if (keys.length === 0) return void 0;
-			const out = [];
-			for (const key of keys) {
-				const bytes = await readOptional(ds, key);
-				if (!bytes) continue;
-				const shard = await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
-					console.warn("[inspection.loadHistory] failed to decode shard", {
-						key,
-						error: err.message
-					});
-				});
-				if (shard?.records) out.push(...shard.records);
-			}
 			return {
 				version: 1,
-				records: out
+				records: (await mapEntityIo(keys, async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return [];
+					return (await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
+						console.warn("[inspection.loadHistory] failed to decode shard", {
+							key,
+							error: err.message
+						});
+					}))?.records ?? [];
+				})).flat()
 			};
 		},
 		async materialize(ctx, rowIter) {
@@ -369,18 +377,19 @@ function createInspectionStore(opts) {
 				bucket.push(r);
 				byMonth.set(month, bucket);
 			}
-			const keys = [];
-			for (const [month, batch] of byMonth) {
+			const files = [...byMonth].map(([month, batch]) => {
 				const bytes = encodeRowsToParquetFlex(batch, {
 					columns: INSPECTION_EVENT_COLUMNS,
 					sortKey: ["urlHash"]
 				});
-				const key = inspectionEventKey(ctx, month, batchId);
-				await ds.write(key, bytes);
-				keys.push(key);
-			}
+				return {
+					key: inspectionEventKey(ctx, month, batchId),
+					bytes
+				};
+			});
+			await mapEntityIo(files, (file) => ds.write(file.key, file.bytes));
 			return {
-				keys,
+				keys: files.map((file) => file.key),
 				rowCount: rows.length
 			};
 		},
@@ -410,11 +419,18 @@ function createInspectionStore(opts) {
 			for (const row of baseRows) consider(row);
 			let eventsFolded = 0;
 			const consumed = [];
-			for (const key of eventKeys.sort()) {
+			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
 				const bytes = await readOptional(ds, key);
-				if (!bytes) continue;
+				if (!bytes) return void 0;
+				return {
+					key,
+					rows: await decodeParquetToRows(bytes)
+				};
+			});
+			for (const file of eventFiles) {
+				if (!file) continue;
+				const { key, rows } = file;
 				consumed.push(key);
-				const rows = await decodeParquetToRows(bytes);
 				for (const row of rows) {
 					consider(row);
 					eventsFolded++;
@@ -610,16 +626,18 @@ function createSitemapStore(opts) {
 				records: {}
 			};
 			const stamp = now();
+			const historyDocs = /* @__PURE__ */ new Map();
 			for (const r of records) {
 				const h = hash(r.path);
 				index.records[h] = r;
-				await writeJson(sitemapHistoryKey(ctx, h, stamp), {
+				historyDocs.set(sitemapHistoryKey(ctx, h, stamp), {
 					version: 1,
 					path: r.path,
 					capturedAt: r.capturedAt,
 					record: r
 				});
 			}
+			await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
 			await writeJson(indexKey, index);
 		},
 		async loadIndex(ctx) {
@@ -702,9 +720,10 @@ function createSitemapStore(opts) {
 		async *loadUrls(ctx, feedpath, opts) {
 			const fpHash = hash(feedpath);
 			const includeRemoved = opts?.includeRemoved ?? false;
-			const indexBytes = await readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash));
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
-			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const [indexRows, listedDeltaKeys] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+			const deltaKeys = listedDeltaKeys.filter((key) => {
+				return SITEMAP_URLS_DELTA_PREFIX_RE.exec(key)?.[2] === fpHash;
+			}).sort();
 			const live = /* @__PURE__ */ new Map();
 			const removedMap = /* @__PURE__ */ new Map();
 			for (const row of indexRows) {
@@ -712,36 +731,34 @@ function createSitemapStore(opts) {
 				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
 				else live.set(rec.urlHash, rec);
 			}
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m || m[2] !== fpHash) continue;
+			const deltas = await mapEntityIo(deltaKeys, async (key) => {
 				const dBytes = await readOptional(ds, key);
-				if (!dBytes) continue;
-				const dRows = await decodeParquetToRows(dBytes);
-				for (const r of dRows) {
-					const op = String(r.op);
-					const urlHash = String(r.url_hash);
-					const at = Number(r.at);
-					if (op === "added") {
-						const prev = live.get(urlHash) ?? removedMap.get(urlHash);
-						removedMap.delete(urlHash);
-						live.set(urlHash, {
-							feedpath,
-							feedpathHash: fpHash,
-							urlHash,
-							loc: String(r.loc),
-							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-							firstSeenAt: prev?.firstSeenAt ?? at,
-							lastSeenAt: at
-						});
-					} else if (op === "removed") {
-						const prev = live.get(urlHash);
-						live.delete(urlHash);
-						if (prev) removedMap.set(urlHash, {
-							...prev,
-							removedAt: at
-						});
-					}
+				if (!dBytes) return [];
+				return decodeParquetToRows(dBytes);
+			});
+			for (const dRows of deltas) for (const r of dRows) {
+				const op = String(r.op);
+				const urlHash = String(r.url_hash);
+				const at = Number(r.at);
+				if (op === "added") {
+					const prev = live.get(urlHash) ?? removedMap.get(urlHash);
+					removedMap.delete(urlHash);
+					live.set(urlHash, {
+						feedpath,
+						feedpathHash: fpHash,
+						urlHash,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						firstSeenAt: prev?.firstSeenAt ?? at,
+						lastSeenAt: at
+					});
+				} else if (op === "removed") {
+					const prev = live.get(urlHash);
+					live.delete(urlHash);
+					if (prev) removedMap.set(urlHash, {
+						...prev,
+						removedAt: at
+					});
 				}
 			}
 			for (const rec of live.values()) yield rec;
@@ -796,8 +813,14 @@ function createSitemapStore(opts) {
 			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
 				if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
 				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const indexBytes = await readOptional(ds, indexKey);
-				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo(feedDeltaKeys.sort(), async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return void 0;
+					return {
+						key,
+						rows: await decodeParquetToRows(bytes)
+					};
+				})]);
 				const live = /* @__PURE__ */ new Map();
 				const removed = /* @__PURE__ */ new Map();
 				for (const row of indexRows) {
@@ -806,11 +829,10 @@ function createSitemapStore(opts) {
 					else live.set(rec.urlHash, rec);
 				}
 				const consumed = [];
-				for (const key of feedDeltaKeys.sort()) {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) continue;
+				for (const file of deltaFiles) {
+					if (!file) continue;
+					const { key, rows } = file;
 					consumed.push(key);
-					const rows = await decodeParquetToRows(bytes);
 					for (const r of rows) {
 						const urlHash = String(r.url_hash);
 						const at = Number(r.at);
@@ -856,12 +878,13 @@ function createSitemapStore(opts) {
 			const at = atOpt ?? now();
 			const liveHashes = new Set(liveFeedpaths.map((fp) => hash(fp)));
 			const present = /* @__PURE__ */ new Set();
-			for (const key of await ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`)) {
+			const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+			for (const key of indexKeys) {
 				const m = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
 				if (m) present.add(m[1]);
 			}
 			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)) {
+			for (const key of deltaKeys) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
 				present.add(m[2]);
@@ -874,8 +897,14 @@ function createSitemapStore(opts) {
 			for (const fpHash of present) {
 				if (liveHashes.has(fpHash)) continue;
 				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const indexBytes = await readOptional(ds, indexKey);
-				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo((deltasByFeed.get(fpHash) ?? []).sort(), async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return void 0;
+					return {
+						key,
+						rows: await decodeParquetToRows(bytes)
+					};
+				})]);
 				const live = /* @__PURE__ */ new Map();
 				const removed = /* @__PURE__ */ new Map();
 				for (const row of indexRows) {
@@ -884,11 +913,10 @@ function createSitemapStore(opts) {
 					else live.set(r.urlHash, r);
 				}
 				const consumed = [];
-				for (const key of (deltasByFeed.get(fpHash) ?? []).sort()) {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) continue;
+				for (const file of deltaFiles) {
+					if (!file) continue;
+					const { key, rows } = file;
 					consumed.push(key);
-					const rows = await decodeParquetToRows(bytes);
 					for (const r of rows) {
 						const urlHash = String(r.url_hash);
 						const dat = Number(r.at);

package/dist/_chunks/resolver.mjs

@@ -822,65 +822,67 @@ function buildExtrasQueries(state, options) {
 function mergeExtras(rows, extrasResults) {
 	if (extrasResults.length === 0) return rows;
 	const lookups = [];
+	const parseVariants = (raw) => {
+		if (raw.length === 0) return [];
+		const encoded = raw.split("||");
+		const variants = [];
+		for (const value of encoded) {
+			if (value.length === 0) continue;
+			const parts = value.split(":::");
+			variants.push({
+				query: parts[0],
+				clicks: Number(parts[1] || 0),
+				impressions: Number(parts[2] || 0),
+				position: Number(parts[3] || 0)
+			});
+		}
+		return variants;
+	};
 	for (const { key, results } of extrasResults) {
 		if (key === "canonicalExtras") {
-			const variantCountMap = /* @__PURE__ */ new Map();
-			const variantsMap = /* @__PURE__ */ new Map();
-			const canonicalNameMap = /* @__PURE__ */ new Map();
+			const map = /* @__PURE__ */ new Map();
 			for (const r of results) {
 				const jk = String(r.joinKey);
-				variantCountMap.set(jk, r.variantCount);
-				canonicalNameMap.set(jk, r.canonicalName);
 				const raw = r.variants;
-				variantsMap.set(jk, typeof raw === "string" ? raw.split("||").filter(Boolean).map((v) => {
-					const parts = v.split(":::");
-					return {
-						query: parts[0],
-						clicks: Number(parts[1] || 0),
-						impressions: Number(parts[2] || 0),
-						position: Number(parts[3] || 0)
-					};
-				}) : []);
+				map.set(jk, {
+					variantCount: r.variantCount,
+					variants: typeof raw === "string" ? parseVariants(raw) : [],
+					canonicalName: r.canonicalName
+				});
 			}
 			lookups.push({
-				key: "variantCount",
-				map: variantCountMap
-			});
-			lookups.push({
-				key: "variants",
-				map: variantsMap
-			});
-			lookups.push({
-				key: "canonicalName",
-				map: canonicalNameMap
+				kind: "canonical",
+				map
 			});
 			continue;
 		}
-		const filtered = results.filter((r) => r.rn === void 0 || r.rn === 1);
 		const map = /* @__PURE__ */ new Map();
-		for (const r of filtered) {
+		for (const r of results) {
+			if (r.rn !== void 0 && r.rn !== 1) continue;
 			let val = r[key];
-			if (key === "variants" && typeof val === "string") val = val.split("||").filter(Boolean).map((v) => {
-				const parts = v.split(":::");
-				return {
-					query: parts[0],
-					clicks: Number(parts[1] || 0),
-					impressions: Number(parts[2] || 0),
-					position: Number(parts[3] || 0)
-				};
-			});
+			if (key === "variants" && typeof val === "string") val = parseVariants(val);
 			map.set(String(r.joinKey), val);
 		}
 		lookups.push({
+			kind: "generic",
 			key,
 			map
 		});
 	}
 	return rows.map((row) => {
 		const enriched = { ...row };
-		for (const { key, map } of lookups) {
-			let joinValue;
-			if (key === "variantCount" || key === "variants" || key === "canonicalName") joinValue = String(row.queryCanonical ?? row.query_canonical ?? "");
+		for (const lookup of lookups) {
+			if (lookup.kind === "canonical") {
+				const joinValue = String(row.queryCanonical ?? row.query_canonical ?? "");
+				const extra = joinValue ? lookup.map.get(joinValue) : void 0;
+				enriched.variantCount = extra?.variantCount ?? null;
+				enriched.variants = extra?.variants ?? [];
+				enriched.canonicalName = extra?.canonicalName ?? null;
+				if (enriched.canonicalName) enriched.queryCanonical = enriched.canonicalName;
+				continue;
+			}
+			const { key, map } = lookup;
+			const joinValue = key === "variantCount" || key === "variants" || key === "canonicalName" ? String(row.queryCanonical ?? row.query_canonical ?? "") : void 0;
 			enriched[key] = (joinValue && map.get(joinValue)) ?? (key === "variants" ? [] : null);
 			if (key === "canonicalName" && enriched[key]) enriched.queryCanonical = enriched[key];
 		}
@@ -955,7 +957,10 @@ function matchesMetricFilter(row, filter) {
 	}
 }
 function matchesTopLevelPage(row) {
-	return (normalizeUrl(dimensionValue(row, "page")).match(/\//g)?.length ?? 0) <= 1;
+	const path = normalizeUrl(dimensionValue(row, "page"));
+	let slashes = 0;
+	for (let index = 0; index < path.length; index++) if (path.charCodeAt(index) === 47 && ++slashes > 1) return false;
+	return true;
 }
 var QuerySourceCoverageError = class extends Error {
 	fallback;

package/dist/adapters/filesystem.mjs

@@ -2,31 +2,71 @@ import { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, m
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
-import { mkdir, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
 import { lock } from "proper-lockfile";
 function createFilesystemDataSource(opts) {
 	const root = resolve(opts.rootDir);
+	const readyDirectories = /* @__PURE__ */ new Map();
 	function pathFor(key) {
 		const resolved = resolve(root, key);
 		if (!resolved.startsWith(`${root}/`) && resolved !== root) throw new Error(`path escapes root: ${key}`);
 		return resolved;
 	}
+	async function ensureDirectory(path) {
+		let pending = readyDirectories.get(path);
+		if (!pending) {
+			pending = mkdir(path, { recursive: true }).then(() => void 0);
+			readyDirectories.set(path, pending);
+			pending.catch(() => readyDirectories.delete(path));
+		}
+		await pending;
+	}
 	return {
 		async read(key, range, signal) {
-			const bytes = await readFile(pathFor(key), { signal });
-			if (!range) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-			const sliced = bytes.subarray(range.offset, range.offset + range.length);
-			return new Uint8Array(sliced.buffer, sliced.byteOffset, sliced.byteLength);
+			const path = pathFor(key);
+			if (!range) {
+				const bytes = await readFile(path, { signal });
+				return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+			}
+			signal?.throwIfAborted();
+			const handle = await open(path, "r");
+			try {
+				const bytes = Buffer.allocUnsafe(range.length);
+				let totalRead = 0;
+				while (totalRead < range.length) {
+					signal?.throwIfAborted();
+					const { bytesRead } = await handle.read(bytes, totalRead, range.length - totalRead, range.offset + totalRead);
+					if (bytesRead === 0) break;
+					totalRead += bytesRead;
+				}
+				return new Uint8Array(bytes.buffer, bytes.byteOffset, totalRead);
+			} finally {
+				await handle.close();
+			}
 		},
 		async write(key, bytes) {
 			const path = pathFor(key);
-			await mkdir(dirname(path), { recursive: true });
-			await writeFile(path, Buffer.from(bytes));
+			const dir = dirname(path);
+			await ensureDirectory(dir);
+			try {
+				await writeFile(path, bytes);
+			} catch (error) {
+				if (error.code !== "ENOENT") throw error;
+				readyDirectories.delete(dir);
+				await ensureDirectory(dir);
+				await writeFile(path, bytes);
+			}
 		},
 		async delete(keys) {
-			await Promise.all(keys.map(async (k) => {
-				await rm(pathFor(k), { force: true });
-			}));
+			let next = 0;
+			async function worker() {
+				while (true) {
+					const index = next++;
+					if (index >= keys.length) return;
+					await rm(pathFor(keys[index]), { force: true });
+				}
+			}
+			await Promise.all(Array.from({ length: Math.min(32, keys.length) }, worker));
 		},
 		async list(prefix) {
 			const full = pathFor(prefix);
@@ -61,14 +101,23 @@ async function* walkStream(dir) {
 	}
 }
 async function walk(dir, out) {
-	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
-		if (err.code === "ENOENT") return [];
-		throw err;
-	});
-	for (const entry of entries) {
-		const p = join(dir, String(entry.name));
-		if (entry.isDirectory()) await walk(p, out);
-		else out.push(p);
+	const pending = [dir];
+	let next = 0;
+	while (next < pending.length) {
+		const batch = pending.slice(next, next + 32);
+		next += batch.length;
+		const listings = await Promise.all(batch.map(async (current) => ({
+			current,
+			entries: await readdir(current, { withFileTypes: true }).catch((err) => {
+				if (err.code === "ENOENT") return [];
+				throw err;
+			})
+		})));
+		for (const { current, entries } of listings) for (const entry of entries) {
+			const p = join(current, String(entry.name));
+			if (entry.isDirectory()) pending.push(p);
+			else out.push(p);
+		}
 	}
 }
 function lockFileFor(locksDir, scope) {
@@ -77,6 +126,16 @@ function lockFileFor(locksDir, scope) {
 function createFilesystemManifestStore(opts) {
 	const manifestPath = resolve(opts.path);
 	const locksDir = join(dirname(manifestPath), "locks");
+	const readyDirectories = /* @__PURE__ */ new Map();
+	async function ensureDirectory(path) {
+		let pending = readyDirectories.get(path);
+		if (!pending) {
+			pending = mkdir(path, { recursive: true }).then(() => void 0);
+			readyDirectories.set(path, pending);
+			pending.catch(() => readyDirectories.delete(path));
+		}
+		await pending;
+	}
 	async function load() {
 		const content = await readFile(manifestPath, "utf8").catch((err) => {
 			if (err.code === "ENOENT") return null;
@@ -91,9 +150,16 @@ function createFilesystemManifestStore(opts) {
 		return parsed;
 	}
 	async function save(data) {
-		await mkdir(dirname(manifestPath), { recursive: true });
+		const manifestDir = dirname(manifestPath);
+		await ensureDirectory(manifestDir);
 		const tmp = `${manifestPath}.${randomBytes(6).toString("hex")}.tmp`;
-		await writeFile(tmp, JSON.stringify(data), "utf8");
+		const content = JSON.stringify(data);
+		await writeFile(tmp, content, "utf8").catch(async (error) => {
+			if (error.code !== "ENOENT") throw error;
+			readyDirectories.delete(manifestDir);
+			await ensureDirectory(manifestDir);
+			await writeFile(tmp, content, "utf8");
+		});
 		await rename(tmp, manifestPath).catch(async (err) => {
 			try {
 				await unlink(tmp);
@@ -175,9 +241,14 @@ function createFilesystemManifestStore(opts) {
 			});
 		},
 		async withLock(scope, fn) {
-			await mkdir(locksDir, { recursive: true });
+			await ensureDirectory(locksDir);
 			const path = lockFileFor(locksDir, scope);
-			await writeFile(path, "", { flag: "a" });
+			await writeFile(path, "", { flag: "a" }).catch(async (error) => {
+				if (error.code !== "ENOENT") throw error;
+				readyDirectories.delete(locksDir);
+				await ensureDirectory(locksDir);
+				await writeFile(path, "", { flag: "a" });
+			});
 			const release = await lock(path, {
 				realpath: false,
 				stale: 3e4,
@@ -244,24 +315,22 @@ async function filesystemStats(rootDir) {
 	const keys = [];
 	await walkForStats(resolve(rootDir), keys);
 	let bytes = 0;
-	for (const k of keys) {
-		const s = await stat(k);
-		bytes += s.size;
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const index = next++;
+			if (index >= keys.length) return;
+			const s = await stat(keys[index]);
+			bytes += s.size;
+		}
 	}
+	await Promise.all(Array.from({ length: Math.min(32, keys.length) }, worker));
 	return {
 		files: keys.length,
 		bytes
 	};
 }
 async function walkForStats(dir, out) {
-	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
-		if (err.code === "ENOENT") return [];
-		throw err;
-	});
-	for (const entry of entries) {
-		const p = join(dir, String(entry.name));
-		if (entry.isDirectory()) await walkForStats(p, out);
-		else out.push(p);
-	}
+	await walk(dir, out);
 }
 export { createFilesystemDataSource, createFilesystemManifestStore, filesystemStats };

package/dist/adapters/hyparquet.d.mts

@@ -45,6 +45,11 @@ interface HyparquetCodecOptions {
    * hyparquet to avoid WASM linear-memory growth. Defaults to hyparquet.
    */
   readRows?: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row[]>;
+  /**
+   * Maximum input-file reads held in flight during compaction. Defaults to 8;
+   * lower it for unusually large files to trade latency for peak JS memory.
+   */
+  compactionReadConcurrency?: number;
 }
 declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;
 export { DecodeParquetOptions, EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };

package/dist/adapters/hyparquet.mjs

@@ -2,6 +2,8 @@ import { SCHEMAS, TABLE_METADATA } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
 import { ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
+const DEFAULT_COMPACTION_READ_CONCURRENCY = 8;
+const MAX_COMPACTION_READ_CONCURRENCY = 32;
 function basicTypeFor(colType) {
 	if (colType === "VARCHAR") return "STRING";
 	if (colType === "BIGINT") return "INT64";
@@ -120,7 +122,13 @@ function sortRowsByClusterKey(table, rows) {
 	return copy;
 }
 function naturalKeyFor(table, row) {
-	return TABLE_METADATA[table].sortKey.map((col) => `${row[col] ?? ""}`).join("\0");
+	const key = TABLE_METADATA[table].sortKey;
+	let value = "";
+	for (let index = 0; index < key.length; index++) {
+		if (index > 0) value += "\0";
+		value += `${row[key[index]] ?? ""}`;
+	}
+	return value;
 }
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
@@ -204,6 +212,11 @@ function normalizeDecodedDates(rows) {
 	return rows;
 }
 function createHyparquetCodec(options = {}) {
+	const readRows = options.readRows ?? (async (_ctx, key, dataSource) => {
+		return decodeParquetToRows(await dataSource.read(key));
+	});
+	const requestedConcurrency = options.compactionReadConcurrency;
+	const compactionReadConcurrency = typeof requestedConcurrency === "number" && Number.isFinite(requestedConcurrency) ? Math.max(1, Math.min(MAX_COMPACTION_READ_CONCURRENCY, Math.floor(requestedConcurrency))) : DEFAULT_COMPACTION_READ_CONCURRENCY;
 	return {
 		async writeRows(ctx, rows, key, dataSource) {
 			const bytes = encodeRowsToParquet(ctx.table, rows);
@@ -213,9 +226,7 @@ function createHyparquetCodec(options = {}) {
 				rowCount: rows.length
 			};
 		},
-		readRows: options.readRows ?? (async (_ctx, key, dataSource) => {
-			return decodeParquetToRows(await dataSource.read(key));
-		}),
+		readRows,
 		async compactRows(ctx, inputKeys, outputKey, dataSource) {
 			if (inputKeys.length === 0) {
 				const bytes = encodeRowsToParquet(ctx.table, []);
@@ -226,9 +237,13 @@ function createHyparquetCodec(options = {}) {
 				};
 			}
 			const byNaturalKey = /* @__PURE__ */ new Map();
-			for (const key of inputKeys) {
-				const rows = await decodeParquetToRows(await dataSource.read(key));
-				for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
+			for (let offset = 0; offset < inputKeys.length; offset += compactionReadConcurrency) {
+				const batch = inputKeys.slice(offset, offset + compactionReadConcurrency);
+				const inputs = await Promise.all(batch.map((key) => dataSource.read(key)));
+				for (const input of inputs) {
+					const rows = await decodeParquetToRows(input);
+					for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
+				}
 			}
 			const rows = [...byNaturalKey.values()];
 			const bytes = encodeRowsToParquet(ctx.table, rows);

package/dist/adapters/r2-manifest.mjs

@@ -6,6 +6,7 @@ const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;
 const SHARD_IO_CONCURRENCY = 8;
+const R2_DELETE_BATCH_SIZE = 1e3;
 async function casBackoff(attempt) {
 	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
 	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
@@ -331,15 +332,8 @@ function createR2ManifestStore(opts) {
 		},
 		async purgeTenant(filter) {
 			if (filter.userId !== userId) throw new Error(`purgeTenant: store is scoped to userId=${userId}, got ${filter.userId}`);
-			const shards = await shardsForFilter({ siteId: filter.siteId });
-			let entriesRemoved = 0;
-			let watermarksRemoved = 0;
-			let syncStatesRemoved = 0;
-			for (const { siteId, table } of shards) {
+			const purged = await mapWithConcurrency(await shardsForFilter({ siteId: filter.siteId }), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				entriesRemoved += snapshot.entries.length;
-				watermarksRemoved += snapshot.watermarks.length;
-				syncStatesRemoved += snapshot.syncStates.length;
 				const prefix = shardPrefix(userId, siteId, table);
 				const keys = [];
 				let cursor;
@@ -352,12 +346,17 @@ function createR2ManifestStore(opts) {
 					for (const obj of res.objects) keys.push(obj.key);
 					cursor = res.truncated ? res.cursor : void 0;
 				} while (cursor);
-				if (keys.length > 0) await bucket.delete(keys);
-			}
+				for (let offset = 0; offset < keys.length; offset += R2_DELETE_BATCH_SIZE) await bucket.delete(keys.slice(offset, offset + R2_DELETE_BATCH_SIZE));
+				return {
+					entriesRemoved: snapshot.entries.length,
+					watermarksRemoved: snapshot.watermarks.length,
+					syncStatesRemoved: snapshot.syncStates.length
+				};
+			});
 			return {
-				entriesRemoved,
-				watermarksRemoved,
-				syncStatesRemoved
+				entriesRemoved: purged.reduce((sum, result) => sum + result.entriesRemoved, 0),
+				watermarksRemoved: purged.reduce((sum, result) => sum + result.watermarksRemoved, 0),
+				syncStatesRemoved: purged.reduce((sum, result) => sum + result.syncStatesRemoved, 0)
 			};
 		}
 	};

package/dist/adapters/r2.mjs

@@ -1,5 +1,7 @@
 const BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/;
 const KEY_RE = /^[\w./=-]+$/;
+const DELETE_CHUNK_SIZE = 1e3;
+const DELETE_CONCURRENCY = 4;
 function assertKey(key) {
 	if (!KEY_RE.test(key)) throw new Error(`createR2DataSource: refusing unsafe key ${JSON.stringify(key)}`);
 }
@@ -22,8 +24,17 @@ function createR2DataSource(options) {
 		},
 		async delete(keys) {
 			if (keys.length === 0) return;
-			const CHUNK = 1e3;
-			for (let i = 0; i < keys.length; i += CHUNK) await bucket.delete(keys.slice(i, i + CHUNK));
+			let next = 0;
+			async function worker() {
+				while (true) {
+					const offset = next;
+					next += DELETE_CHUNK_SIZE;
+					if (offset >= keys.length) return;
+					await bucket.delete(keys.slice(offset, offset + DELETE_CHUNK_SIZE));
+				}
+			}
+			const chunkCount = Math.ceil(keys.length / DELETE_CHUNK_SIZE);
+			await Promise.all(Array.from({ length: Math.min(DELETE_CONCURRENCY, chunkCount) }, worker));
 		},
 		async list(prefix) {
 			const out = [];

package/dist/iceberg/index.mjs

@@ -137,8 +137,12 @@ function dedupeByIdentity(table, records) {
 	const key = gscDataset(table, "int").tableSpec.identityColumns;
 	const seen = /* @__PURE__ */ new Map();
 	for (const rec of records) {
-		const k = key.map((col) => `${rec[col] ?? ""}`).join("\0");
-		seen.set(k, rec);
+		let identity = "";
+		for (let index = 0; index < key.length; index++) {
+			if (index > 0) identity += "\0";
+			identity += `${rec[key[index]] ?? ""}`;
+		}
+		seen.set(identity, rec);
 	}
 	return seen.size === records.length ? records : [...seen.values()];
 }

package/dist/rollups.mjs

@@ -195,26 +195,42 @@ function planRollupWindows(parts, clampRange, maxWindowDays) {
 		});
 	}
 	if (spans.length === 0) return [];
-	let rangeStartMs = Math.min(...spans.map((s) => s.startMs));
-	let rangeEndMs = Math.max(...spans.map((s) => s.endMs));
+	let rangeStartMs = Number.POSITIVE_INFINITY;
+	let rangeEndMs = Number.NEGATIVE_INFINITY;
+	let totalBytes = 0;
+	for (const span of spans) {
+		if (span.startMs < rangeStartMs) rangeStartMs = span.startMs;
+		if (span.endMs > rangeEndMs) rangeEndMs = span.endMs;
+		totalBytes += span.bytes;
+	}
 	if (clampStartMs !== void 0) rangeStartMs = Math.max(rangeStartMs, clampStartMs);
 	if (clampEndMs !== void 0) rangeEndMs = Math.min(rangeEndMs, clampEndMs);
-	const totalBytes = spans.reduce((a, s) => a + s.bytes, 0);
 	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
 	const bytesPerDay = Math.max(1, totalBytes / spanDays);
 	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
 	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
+	if (!Number.isFinite(windowDays) || !Number.isFinite(rangeStartMs) || !Number.isFinite(rangeEndMs) || rangeEndMs < rangeStartMs) return [];
+	const windowWidthMs = windowDays * MS_PER_DAY;
+	const windowCount = Math.floor((rangeEndMs - rangeStartMs) / windowWidthMs) + 1;
+	const partitionsByWindow = Array.from({ length: windowCount }, () => []);
+	for (const span of spans) {
+		const overlapStartMs = Math.max(span.startMs, rangeStartMs);
+		const overlapEndMs = Math.min(span.endMs, rangeEndMs);
+		if (overlapStartMs > overlapEndMs) continue;
+		const firstWindow = Math.max(0, Math.ceil((overlapStartMs - rangeStartMs - (windowDays - 1) * MS_PER_DAY) / windowWidthMs));
+		const lastWindow = Math.min(windowCount - 1, Math.floor((overlapEndMs - rangeStartMs) / windowWidthMs));
+		for (let index = firstWindow; index <= lastWindow; index++) partitionsByWindow[index].push(span.partition);
+	}
 	const windows = [];
-	let cursorMs = rangeStartMs;
-	while (cursorMs <= rangeEndMs) {
+	for (let index = 0; index < windowCount; index++) {
+		const cursorMs = rangeStartMs + index * windowWidthMs;
 		const windowEndMs = Math.min(cursorMs + (windowDays - 1) * MS_PER_DAY, rangeEndMs);
-		const partitions = spans.filter((s) => s.endMs >= cursorMs && s.startMs <= windowEndMs).map((s) => s.partition);
+		const partitions = partitionsByWindow[index];
 		if (partitions.length > 0) windows.push({
 			start: isoDate(cursorMs),
 			end: isoDate(windowEndMs),
 			partitions
 		});
-		cursorMs = windowEndMs + MS_PER_DAY;
 	}
 	return windows;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.38.1",
+  "version": "0.39.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.38.1",
-    "@gscdump/lakehouse": "0.38.1",
-    "gscdump": "0.38.1"
+    "@gscdump/contracts": "0.39.0",
+    "@gscdump/lakehouse": "0.39.0",
+    "gscdump": "0.39.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",